| { |
| "model_type": "zonos3", |
| "backbone": { |
| "n_layers": 11, |
| "max_seqlen": 32768, |
| "dim": 1024, |
| "head_dim": 32, |
| "n_heads": 16, |
| "n_kv_heads": 4, |
| "ffn_hidden": 2048, |
| "n_experts": 64, |
| "moe_topk": 2, |
| "n_codebooks": 10, |
| "codebook_size": 4096, |
| "audio_vocab": 4098, |
| "eoa_id": 4096, |
| "audio_pad_id": 4097, |
| "span_end_id": null, |
| "text_vocab": 8196, |
| "text_extra_tokens": 0, |
| "gdn_heads": 4, |
| "gdn_head_dim": 128, |
| "gdn_kernel": 4, |
| "speaker_embedding_dim": 192, |
| "condition_count": 3, |
| "condition_bucket_counts": [ |
| 10, |
| 10, |
| 30 |
| ], |
| "condition_names": [ |
| "loudness", |
| "fmax", |
| "speaking_rate" |
| ], |
| "condition_units": { |
| "loudness": "LUFS", |
| "fmax": "Hz", |
| "speaking_rate": "utf-8 bytes per second" |
| }, |
| "condition_edges": { |
| "loudness": [ |
| -29.777999877929688, |
| -26.94700050354004, |
| -25.047000885009766, |
| -23.299999237060547, |
| -21.743000030517578, |
| -20.38599967956543, |
| -19.18600082397461, |
| -17.85099983215332, |
| -15.967000007629395 |
| ], |
| "fmax": [ |
| 8437.5, |
| 10242.2001953125, |
| 11507.7998046875, |
| 13359.400390625, |
| 15164.099609375, |
| 15585.900390625, |
| 15750.0, |
| 17765.599609375, |
| 19945.30078125 |
| ], |
| "speaking_rate": [ |
| 6.546000003814697, |
| 8.182000160217285, |
| 9.27299976348877, |
| 10.3100004196167, |
| 11.053999900817871, |
| 11.666999816894531, |
| 12.192999839782715, |
| 12.656000137329102, |
| 13.083000183105469, |
| 13.47599983215332, |
| 13.845999717712402, |
| 14.196000099182129, |
| 14.529999732971191, |
| 14.88700008392334, |
| 15.192999839782715, |
| 15.517000198364258, |
| 15.864999771118164, |
| 16.20800018310547, |
| 16.57900047302246, |
| 16.972000122070312, |
| 17.410999298095703, |
| 17.898000717163086, |
| 18.43400001525879, |
| 19.069000244140625, |
| 19.81800079345703, |
| 20.81599998474121, |
| 22.285999298095703, |
| 24.722000122070312, |
| 29.25 |
| ] |
| }, |
| "rope_theta": 10000.0, |
| "norm_eps": 9.999999747378752e-06, |
| "qk_eps": 9.999999974752427e-07, |
| "loss_softcap": 15.0, |
| "gdn_period": 4, |
| "silence_frame": [ |
| 1918, |
| 746, |
| 3544, |
| 234, |
| 1267, |
| 2914, |
| 3944, |
| 2849, |
| 326, |
| 1337 |
| ], |
| "text_tokenizer_spec": { |
| "kind": "superbpe", |
| "path": "tokenizer.json", |
| "sha256": "90322f910600027dee7fe63d400d1d9dc1035c167d560aadf0871e3661d08bf3", |
| "vocab_size": 8192, |
| "offset": 4, |
| "bos": 2, |
| "eos": 3, |
| "_original_path": "/data/groups/audio/gabriel/bingua2-gdn-fullattnres-sbpe8k-rawclone-moealpha025-x15-e64-6node-100k-20261006/tokenizers/superbpe8192.json" |
| }, |
| "tag_cfg_trained": false, |
| "audio_codec": "opus_dual_3n_600k_sched600k_r4096_step600000", |
| "speaker_embedding_codec": "ownmect_s2", |
| "speaker_reference_prefix_enabled": true, |
| "speaker_reference_max_seconds": 60.0 |
| }, |
| "tensors": { |
| "audio_embd.0.weight": { |
| "shape": [ |
| 4098, |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "audio_embd.1.weight": { |
| "shape": [ |
| 4098, |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "audio_embd.2.weight": { |
| "shape": [ |
| 4098, |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "audio_embd.3.weight": { |
| "shape": [ |
| 4098, |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "audio_embd.4.weight": { |
| "shape": [ |
| 4098, |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "audio_embd.5.weight": { |
| "shape": [ |
| 4098, |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "audio_embd.6.weight": { |
| "shape": [ |
| 4098, |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "audio_embd.7.weight": { |
| "shape": [ |
| 4098, |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "audio_embd.8.weight": { |
| "shape": [ |
| 4098, |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "audio_embd.9.weight": { |
| "shape": [ |
| 4098, |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "text_embd.weight": { |
| "shape": [ |
| 8197, |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "output_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "output.weight": { |
| "shape": [ |
| 40980, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "attnres_q.weight": { |
| "shape": [ |
| 23, |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "speaker_proj.weight": { |
| "shape": [ |
| 1024, |
| 192 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "speaker_proj.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "cond_embd.0.weight": { |
| "shape": [ |
| 11, |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "cond_embd.1.weight": { |
| "shape": [ |
| 11, |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "cond_embd.2.weight": { |
| "shape": [ |
| 31, |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.0.attn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.0.ffn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.0.gdn_in.weight": { |
| "shape": [ |
| 2056, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.0.gdn_norm.weight": { |
| "shape": [ |
| 128 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.0.gdn_A_log": { |
| "shape": [ |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.0.gdn_dt_bias": { |
| "shape": [ |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.0.gdn_conv.weight": { |
| "shape": [ |
| 1536, |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.0.attn_output.weight": { |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.0.ffn_up.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.0.ffn_gate.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.0.ffn_down.weight": { |
| "shape": [ |
| 1024, |
| 2048 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.1.attn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.1.ffn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.1.gdn_in.weight": { |
| "shape": [ |
| 2056, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.1.gdn_norm.weight": { |
| "shape": [ |
| 128 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.1.gdn_A_log": { |
| "shape": [ |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.1.gdn_dt_bias": { |
| "shape": [ |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.1.gdn_conv.weight": { |
| "shape": [ |
| 1536, |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.1.attn_output.weight": { |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.1.ffn_gate_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.1.ffn_up_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.1.ffn_down_exps.weight": { |
| "shape": [ |
| 64, |
| 256, |
| 1536 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.1.ffn_latent_down.weight": { |
| "shape": [ |
| 256, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.1.ffn_latent_up.weight": { |
| "shape": [ |
| 1024, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.1.router.weight": { |
| "shape": [ |
| 64, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.1.router_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.1.router_bias": { |
| "shape": [ |
| 64 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.2.attn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.2.ffn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.2.gdn_in.weight": { |
| "shape": [ |
| 2056, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.2.gdn_norm.weight": { |
| "shape": [ |
| 128 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.2.gdn_A_log": { |
| "shape": [ |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.2.gdn_dt_bias": { |
| "shape": [ |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.2.gdn_conv.weight": { |
| "shape": [ |
| 1536, |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.2.attn_output.weight": { |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.2.ffn_gate_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.2.ffn_up_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.2.ffn_down_exps.weight": { |
| "shape": [ |
| 64, |
| 256, |
| 1536 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.2.ffn_latent_down.weight": { |
| "shape": [ |
| 256, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.2.ffn_latent_up.weight": { |
| "shape": [ |
| 1024, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.2.router.weight": { |
| "shape": [ |
| 64, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.2.router_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.2.router_bias": { |
| "shape": [ |
| 64 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.3.attn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.3.ffn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.3.attn_q.weight": { |
| "shape": [ |
| 512, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.3.attn_k.weight": { |
| "shape": [ |
| 128, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.3.attn_v.weight": { |
| "shape": [ |
| 128, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.3.attn_temp": { |
| "shape": [ |
| 16 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.3.attn_gate.weight": { |
| "shape": [ |
| 16, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.3.attn_output.weight": { |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.3.ffn_gate_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.3.ffn_up_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.3.ffn_down_exps.weight": { |
| "shape": [ |
| 64, |
| 256, |
| 1536 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.3.ffn_latent_down.weight": { |
| "shape": [ |
| 256, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.3.ffn_latent_up.weight": { |
| "shape": [ |
| 1024, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.3.router.weight": { |
| "shape": [ |
| 64, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.3.router_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.3.router_bias": { |
| "shape": [ |
| 64 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.4.attn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.4.ffn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.4.gdn_in.weight": { |
| "shape": [ |
| 2056, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.4.gdn_norm.weight": { |
| "shape": [ |
| 128 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.4.gdn_A_log": { |
| "shape": [ |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.4.gdn_dt_bias": { |
| "shape": [ |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.4.gdn_conv.weight": { |
| "shape": [ |
| 1536, |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.4.attn_output.weight": { |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.4.ffn_gate_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.4.ffn_up_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.4.ffn_down_exps.weight": { |
| "shape": [ |
| 64, |
| 256, |
| 1536 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.4.ffn_latent_down.weight": { |
| "shape": [ |
| 256, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.4.ffn_latent_up.weight": { |
| "shape": [ |
| 1024, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.4.router.weight": { |
| "shape": [ |
| 64, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.4.router_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.4.router_bias": { |
| "shape": [ |
| 64 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.5.attn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.5.ffn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.5.gdn_in.weight": { |
| "shape": [ |
| 2056, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.5.gdn_norm.weight": { |
| "shape": [ |
| 128 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.5.gdn_A_log": { |
| "shape": [ |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.5.gdn_dt_bias": { |
| "shape": [ |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.5.gdn_conv.weight": { |
| "shape": [ |
| 1536, |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.5.attn_output.weight": { |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.5.ffn_gate_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.5.ffn_up_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.5.ffn_down_exps.weight": { |
| "shape": [ |
| 64, |
| 256, |
| 1536 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.5.ffn_latent_down.weight": { |
| "shape": [ |
| 256, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.5.ffn_latent_up.weight": { |
| "shape": [ |
| 1024, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.5.router.weight": { |
| "shape": [ |
| 64, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.5.router_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.5.router_bias": { |
| "shape": [ |
| 64 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.6.attn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.6.ffn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.6.gdn_in.weight": { |
| "shape": [ |
| 2056, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.6.gdn_norm.weight": { |
| "shape": [ |
| 128 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.6.gdn_A_log": { |
| "shape": [ |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.6.gdn_dt_bias": { |
| "shape": [ |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.6.gdn_conv.weight": { |
| "shape": [ |
| 1536, |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.6.attn_output.weight": { |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.6.ffn_gate_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.6.ffn_up_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.6.ffn_down_exps.weight": { |
| "shape": [ |
| 64, |
| 256, |
| 1536 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.6.ffn_latent_down.weight": { |
| "shape": [ |
| 256, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.6.ffn_latent_up.weight": { |
| "shape": [ |
| 1024, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.6.router.weight": { |
| "shape": [ |
| 64, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.6.router_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.6.router_bias": { |
| "shape": [ |
| 64 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.7.attn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.7.ffn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.7.attn_q.weight": { |
| "shape": [ |
| 512, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.7.attn_k.weight": { |
| "shape": [ |
| 128, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.7.attn_v.weight": { |
| "shape": [ |
| 128, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.7.attn_temp": { |
| "shape": [ |
| 16 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.7.attn_gate.weight": { |
| "shape": [ |
| 16, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.7.attn_output.weight": { |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.7.ffn_gate_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.7.ffn_up_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.7.ffn_down_exps.weight": { |
| "shape": [ |
| 64, |
| 256, |
| 1536 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.7.ffn_latent_down.weight": { |
| "shape": [ |
| 256, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.7.ffn_latent_up.weight": { |
| "shape": [ |
| 1024, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.7.router.weight": { |
| "shape": [ |
| 64, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.7.router_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.7.router_bias": { |
| "shape": [ |
| 64 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.8.attn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.8.ffn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.8.gdn_in.weight": { |
| "shape": [ |
| 2056, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.8.gdn_norm.weight": { |
| "shape": [ |
| 128 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.8.gdn_A_log": { |
| "shape": [ |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.8.gdn_dt_bias": { |
| "shape": [ |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.8.gdn_conv.weight": { |
| "shape": [ |
| 1536, |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.8.attn_output.weight": { |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.8.ffn_gate_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.8.ffn_up_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.8.ffn_down_exps.weight": { |
| "shape": [ |
| 64, |
| 256, |
| 1536 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.8.ffn_latent_down.weight": { |
| "shape": [ |
| 256, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.8.ffn_latent_up.weight": { |
| "shape": [ |
| 1024, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.8.router.weight": { |
| "shape": [ |
| 64, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.8.router_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.8.router_bias": { |
| "shape": [ |
| 64 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.9.attn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.9.ffn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.9.gdn_in.weight": { |
| "shape": [ |
| 2056, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.9.gdn_norm.weight": { |
| "shape": [ |
| 128 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.9.gdn_A_log": { |
| "shape": [ |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.9.gdn_dt_bias": { |
| "shape": [ |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.9.gdn_conv.weight": { |
| "shape": [ |
| 1536, |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.9.attn_output.weight": { |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.9.ffn_gate_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.9.ffn_up_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.9.ffn_down_exps.weight": { |
| "shape": [ |
| 64, |
| 256, |
| 1536 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.9.ffn_latent_down.weight": { |
| "shape": [ |
| 256, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.9.ffn_latent_up.weight": { |
| "shape": [ |
| 1024, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.9.router.weight": { |
| "shape": [ |
| 64, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.9.router_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.9.router_bias": { |
| "shape": [ |
| 64 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.10.attn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.10.ffn_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.10.gdn_in.weight": { |
| "shape": [ |
| 2056, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.10.gdn_norm.weight": { |
| "shape": [ |
| 128 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.10.gdn_A_log": { |
| "shape": [ |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.10.gdn_dt_bias": { |
| "shape": [ |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.10.gdn_conv.weight": { |
| "shape": [ |
| 1536, |
| 4 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.10.attn_output.weight": { |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 8 |
| }, |
| "blk.10.ffn_gate_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.10.ffn_up_exps.weight": { |
| "shape": [ |
| 64, |
| 1536, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.10.ffn_down_exps.weight": { |
| "shape": [ |
| 64, |
| 256, |
| 1536 |
| ], |
| "matrix": true, |
| "dtype": "uint32", |
| "bits": 4 |
| }, |
| "blk.10.ffn_latent_down.weight": { |
| "shape": [ |
| 256, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.10.ffn_latent_up.weight": { |
| "shape": [ |
| 1024, |
| 256 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.10.router.weight": { |
| "shape": [ |
| 64, |
| 1024 |
| ], |
| "matrix": true, |
| "dtype": "float16", |
| "bits": null |
| }, |
| "blk.10.router_norm.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| }, |
| "blk.10.router_bias": { |
| "shape": [ |
| 64 |
| ], |
| "matrix": false, |
| "dtype": "float32" |
| } |
| } |
| } |
|
|
