{
  "model_type": "zonos3",
  "backbone": {
    "n_layers": 11,
    "max_seqlen": 32768,
    "dim": 1024,
    "head_dim": 32,
    "n_heads": 16,
    "n_kv_heads": 4,
    "ffn_hidden": 2048,
    "n_experts": 64,
    "moe_topk": 2,
    "n_codebooks": 10,
    "codebook_size": 4096,
    "audio_vocab": 4098,
    "eoa_id": 4096,
    "audio_pad_id": 4097,
    "span_end_id": null,
    "text_vocab": 8196,
    "text_extra_tokens": 0,
    "gdn_heads": 4,
    "gdn_head_dim": 128,
    "gdn_kernel": 4,
    "speaker_embedding_dim": 192,
    "condition_count": 3,
    "condition_bucket_counts": [
      10,
      10,
      30
    ],
    "condition_names": [
      "loudness",
      "fmax",
      "speaking_rate"
    ],
    "condition_units": {
      "loudness": "LUFS",
      "fmax": "Hz",
      "speaking_rate": "utf-8 bytes per second"
    },
    "condition_edges": {
      "loudness": [
        -29.777999877929688,
        -26.94700050354004,
        -25.047000885009766,
        -23.299999237060547,
        -21.743000030517578,
        -20.38599967956543,
        -19.18600082397461,
        -17.85099983215332,
        -15.967000007629395
      ],
      "fmax": [
        8437.5,
        10242.2001953125,
        11507.7998046875,
        13359.400390625,
        15164.099609375,
        15585.900390625,
        15750.0,
        17765.599609375,
        19945.30078125
      ],
      "speaking_rate": [
        6.546000003814697,
        8.182000160217285,
        9.27299976348877,
        10.3100004196167,
        11.053999900817871,
        11.666999816894531,
        12.192999839782715,
        12.656000137329102,
        13.083000183105469,
        13.47599983215332,
        13.845999717712402,
        14.196000099182129,
        14.529999732971191,
        14.88700008392334,
        15.192999839782715,
        15.517000198364258,
        15.864999771118164,
        16.20800018310547,
        16.57900047302246,
        16.972000122070312,
        17.410999298095703,
        17.898000717163086,
        18.43400001525879,
        19.069000244140625,
        19.81800079345703,
        20.81599998474121,
        22.285999298095703,
        24.722000122070312,
        29.25
      ]
    },
    "rope_theta": 10000.0,
    "norm_eps": 9.999999747378752e-06,
    "qk_eps": 9.999999974752427e-07,
    "loss_softcap": 15.0,
    "gdn_period": 4,
    "silence_frame": [
      1918,
      746,
      3544,
      234,
      1267,
      2914,
      3944,
      2849,
      326,
      1337
    ],
    "text_tokenizer_spec": {
      "kind": "superbpe",
      "path": "tokenizer.json",
      "sha256": "90322f910600027dee7fe63d400d1d9dc1035c167d560aadf0871e3661d08bf3",
      "vocab_size": 8192,
      "offset": 4,
      "bos": 2,
      "eos": 3,
      "_original_path": "/data/groups/audio/gabriel/bingua2-gdn-fullattnres-sbpe8k-rawclone-moealpha025-x15-e64-6node-100k-20261006/tokenizers/superbpe8192.json"
    },
    "tag_cfg_trained": false,
    "audio_codec": "opus_dual_3n_600k_sched600k_r4096_step600000",
    "speaker_embedding_codec": "ownmect_s2",
    "speaker_reference_prefix_enabled": true,
    "speaker_reference_max_seconds": 60.0
  },
  "tensors": {
    "audio_embd.0.weight": {
      "shape": [
        4098,
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "audio_embd.1.weight": {
      "shape": [
        4098,
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "audio_embd.2.weight": {
      "shape": [
        4098,
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "audio_embd.3.weight": {
      "shape": [
        4098,
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "audio_embd.4.weight": {
      "shape": [
        4098,
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "audio_embd.5.weight": {
      "shape": [
        4098,
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "audio_embd.6.weight": {
      "shape": [
        4098,
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "audio_embd.7.weight": {
      "shape": [
        4098,
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "audio_embd.8.weight": {
      "shape": [
        4098,
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "audio_embd.9.weight": {
      "shape": [
        4098,
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "text_embd.weight": {
      "shape": [
        8197,
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "output_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "output.weight": {
      "shape": [
        40980,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "attnres_q.weight": {
      "shape": [
        23,
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "speaker_proj.weight": {
      "shape": [
        1024,
        192
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "speaker_proj.bias": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "cond_embd.0.weight": {
      "shape": [
        11,
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "cond_embd.1.weight": {
      "shape": [
        11,
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "cond_embd.2.weight": {
      "shape": [
        31,
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.0.attn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.0.ffn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.0.gdn_in.weight": {
      "shape": [
        2056,
        1024
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.0.gdn_norm.weight": {
      "shape": [
        128
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.0.gdn_A_log": {
      "shape": [
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.0.gdn_dt_bias": {
      "shape": [
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.0.gdn_conv.weight": {
      "shape": [
        1536,
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.0.attn_output.weight": {
      "shape": [
        1024,
        512
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.0.ffn_up.weight": {
      "shape": [
        2048,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.0.ffn_gate.weight": {
      "shape": [
        2048,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.0.ffn_down.weight": {
      "shape": [
        1024,
        2048
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.1.attn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.1.ffn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.1.gdn_in.weight": {
      "shape": [
        2056,
        1024
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.1.gdn_norm.weight": {
      "shape": [
        128
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.1.gdn_A_log": {
      "shape": [
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.1.gdn_dt_bias": {
      "shape": [
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.1.gdn_conv.weight": {
      "shape": [
        1536,
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.1.attn_output.weight": {
      "shape": [
        1024,
        512
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.1.ffn_gate_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.1.ffn_up_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.1.ffn_down_exps.weight": {
      "shape": [
        64,
        256,
        1536
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.1.ffn_latent_down.weight": {
      "shape": [
        256,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.1.ffn_latent_up.weight": {
      "shape": [
        1024,
        256
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.1.router.weight": {
      "shape": [
        64,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.1.router_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.1.router_bias": {
      "shape": [
        64
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.2.attn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.2.ffn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.2.gdn_in.weight": {
      "shape": [
        2056,
        1024
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.2.gdn_norm.weight": {
      "shape": [
        128
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.2.gdn_A_log": {
      "shape": [
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.2.gdn_dt_bias": {
      "shape": [
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.2.gdn_conv.weight": {
      "shape": [
        1536,
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.2.attn_output.weight": {
      "shape": [
        1024,
        512
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.2.ffn_gate_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.2.ffn_up_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.2.ffn_down_exps.weight": {
      "shape": [
        64,
        256,
        1536
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.2.ffn_latent_down.weight": {
      "shape": [
        256,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.2.ffn_latent_up.weight": {
      "shape": [
        1024,
        256
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.2.router.weight": {
      "shape": [
        64,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.2.router_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.2.router_bias": {
      "shape": [
        64
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.3.attn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.3.ffn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.3.attn_q.weight": {
      "shape": [
        512,
        1024
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.3.attn_k.weight": {
      "shape": [
        128,
        1024
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.3.attn_v.weight": {
      "shape": [
        128,
        1024
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.3.attn_temp": {
      "shape": [
        16
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.3.attn_gate.weight": {
      "shape": [
        16,
        1024
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.3.attn_output.weight": {
      "shape": [
        1024,
        512
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.3.ffn_gate_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.3.ffn_up_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.3.ffn_down_exps.weight": {
      "shape": [
        64,
        256,
        1536
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.3.ffn_latent_down.weight": {
      "shape": [
        256,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.3.ffn_latent_up.weight": {
      "shape": [
        1024,
        256
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.3.router.weight": {
      "shape": [
        64,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.3.router_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.3.router_bias": {
      "shape": [
        64
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.4.attn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.4.ffn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.4.gdn_in.weight": {
      "shape": [
        2056,
        1024
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.4.gdn_norm.weight": {
      "shape": [
        128
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.4.gdn_A_log": {
      "shape": [
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.4.gdn_dt_bias": {
      "shape": [
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.4.gdn_conv.weight": {
      "shape": [
        1536,
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.4.attn_output.weight": {
      "shape": [
        1024,
        512
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.4.ffn_gate_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.4.ffn_up_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.4.ffn_down_exps.weight": {
      "shape": [
        64,
        256,
        1536
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.4.ffn_latent_down.weight": {
      "shape": [
        256,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.4.ffn_latent_up.weight": {
      "shape": [
        1024,
        256
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.4.router.weight": {
      "shape": [
        64,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.4.router_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.4.router_bias": {
      "shape": [
        64
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.5.attn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.5.ffn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.5.gdn_in.weight": {
      "shape": [
        2056,
        1024
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.5.gdn_norm.weight": {
      "shape": [
        128
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.5.gdn_A_log": {
      "shape": [
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.5.gdn_dt_bias": {
      "shape": [
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.5.gdn_conv.weight": {
      "shape": [
        1536,
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.5.attn_output.weight": {
      "shape": [
        1024,
        512
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.5.ffn_gate_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.5.ffn_up_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.5.ffn_down_exps.weight": {
      "shape": [
        64,
        256,
        1536
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.5.ffn_latent_down.weight": {
      "shape": [
        256,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.5.ffn_latent_up.weight": {
      "shape": [
        1024,
        256
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.5.router.weight": {
      "shape": [
        64,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.5.router_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.5.router_bias": {
      "shape": [
        64
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.6.attn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.6.ffn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.6.gdn_in.weight": {
      "shape": [
        2056,
        1024
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.6.gdn_norm.weight": {
      "shape": [
        128
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.6.gdn_A_log": {
      "shape": [
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.6.gdn_dt_bias": {
      "shape": [
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.6.gdn_conv.weight": {
      "shape": [
        1536,
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.6.attn_output.weight": {
      "shape": [
        1024,
        512
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.6.ffn_gate_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.6.ffn_up_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.6.ffn_down_exps.weight": {
      "shape": [
        64,
        256,
        1536
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.6.ffn_latent_down.weight": {
      "shape": [
        256,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.6.ffn_latent_up.weight": {
      "shape": [
        1024,
        256
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.6.router.weight": {
      "shape": [
        64,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.6.router_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.6.router_bias": {
      "shape": [
        64
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.7.attn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.7.ffn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.7.attn_q.weight": {
      "shape": [
        512,
        1024
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.7.attn_k.weight": {
      "shape": [
        128,
        1024
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.7.attn_v.weight": {
      "shape": [
        128,
        1024
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.7.attn_temp": {
      "shape": [
        16
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.7.attn_gate.weight": {
      "shape": [
        16,
        1024
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.7.attn_output.weight": {
      "shape": [
        1024,
        512
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.7.ffn_gate_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.7.ffn_up_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.7.ffn_down_exps.weight": {
      "shape": [
        64,
        256,
        1536
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.7.ffn_latent_down.weight": {
      "shape": [
        256,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.7.ffn_latent_up.weight": {
      "shape": [
        1024,
        256
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.7.router.weight": {
      "shape": [
        64,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.7.router_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.7.router_bias": {
      "shape": [
        64
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.8.attn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.8.ffn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.8.gdn_in.weight": {
      "shape": [
        2056,
        1024
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.8.gdn_norm.weight": {
      "shape": [
        128
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.8.gdn_A_log": {
      "shape": [
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.8.gdn_dt_bias": {
      "shape": [
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.8.gdn_conv.weight": {
      "shape": [
        1536,
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.8.attn_output.weight": {
      "shape": [
        1024,
        512
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.8.ffn_gate_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.8.ffn_up_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.8.ffn_down_exps.weight": {
      "shape": [
        64,
        256,
        1536
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.8.ffn_latent_down.weight": {
      "shape": [
        256,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.8.ffn_latent_up.weight": {
      "shape": [
        1024,
        256
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.8.router.weight": {
      "shape": [
        64,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.8.router_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.8.router_bias": {
      "shape": [
        64
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.9.attn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.9.ffn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.9.gdn_in.weight": {
      "shape": [
        2056,
        1024
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.9.gdn_norm.weight": {
      "shape": [
        128
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.9.gdn_A_log": {
      "shape": [
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.9.gdn_dt_bias": {
      "shape": [
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.9.gdn_conv.weight": {
      "shape": [
        1536,
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.9.attn_output.weight": {
      "shape": [
        1024,
        512
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.9.ffn_gate_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.9.ffn_up_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.9.ffn_down_exps.weight": {
      "shape": [
        64,
        256,
        1536
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.9.ffn_latent_down.weight": {
      "shape": [
        256,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.9.ffn_latent_up.weight": {
      "shape": [
        1024,
        256
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.9.router.weight": {
      "shape": [
        64,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.9.router_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.9.router_bias": {
      "shape": [
        64
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.10.attn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.10.ffn_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.10.gdn_in.weight": {
      "shape": [
        2056,
        1024
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.10.gdn_norm.weight": {
      "shape": [
        128
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.10.gdn_A_log": {
      "shape": [
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.10.gdn_dt_bias": {
      "shape": [
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.10.gdn_conv.weight": {
      "shape": [
        1536,
        4
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.10.attn_output.weight": {
      "shape": [
        1024,
        512
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 8
    },
    "blk.10.ffn_gate_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.10.ffn_up_exps.weight": {
      "shape": [
        64,
        1536,
        256
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.10.ffn_down_exps.weight": {
      "shape": [
        64,
        256,
        1536
      ],
      "matrix": true,
      "dtype": "uint32",
      "bits": 4
    },
    "blk.10.ffn_latent_down.weight": {
      "shape": [
        256,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.10.ffn_latent_up.weight": {
      "shape": [
        1024,
        256
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.10.router.weight": {
      "shape": [
        64,
        1024
      ],
      "matrix": true,
      "dtype": "float16",
      "bits": null
    },
    "blk.10.router_norm.weight": {
      "shape": [
        1024
      ],
      "matrix": false,
      "dtype": "float32"
    },
    "blk.10.router_bias": {
      "shape": [
        64
      ],
      "matrix": false,
      "dtype": "float32"
    }
  }
}