fantastisk's picture
Add files using upload-large-folder tool
b6d2234 verified
Raw History Blame Contribute Delete
39.4 kB
{ "config": { "attention_bias": true, "attention_dropout": 0.0, "depth": 24, "hidden_act": "silu", "hidden_size": 1024, "image_size": 448, "in_channels": 3, "initializer_range": 0.02, "intermediate_size": 4096, "model_type": "glm5_next_vision", "num_heads": 16, "out_hidden_size": 4096, "patch_size": 14, "projection_intermediate_size": 10240, "rms_norm_eps": 1e-05, "spatial_merge_size": 2, "swiglu_limit": 10.0, "temporal_patch_size": 2, "image_token_id": 154854, "image_start_token_id": 154830, "image_end_token_id": 154831, "rope_theta": 10000.0 }, "tensors": { "patch.w": { "kind": "f32", "shape": [ 1024, 1176 ], "off": 0, "bytes": 4816896 }, "patch.b": { "kind": "f32", "shape": [ 1024 ], "off": 4816896, "bytes": 4096 }, "B0.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 4820992, "bytes": 4096 }, "B0.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 4825088, "bytes": 4096 }, "B0.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 4829184, "bytes": 6291456 }, "B0.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 11120640, "bytes": 12288 }, "B0.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 11132928, "bytes": 2097152 }, "B0.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 13230080, "bytes": 4096 }, "B0.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 13234176, "bytes": 256 }, "B0.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 13234432, "bytes": 256 }, "B0.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 13234688, "bytes": 8388608 }, "B0.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 21623296, "bytes": 16384 }, "B0.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 21639680, "bytes": 8388608 }, "B0.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 30028288, "bytes": 16384 }, "B0.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 30044672, "bytes": 8388608 }, "B0.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 38433280, "bytes": 4096 }, "B1.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 38437376, "bytes": 4096 }, "B1.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 38441472, "bytes": 4096 }, "B1.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 38445568, "bytes": 6291456 }, "B1.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 44737024, "bytes": 12288 }, "B1.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 44749312, "bytes": 2097152 }, "B1.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 46846464, "bytes": 4096 }, "B1.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 46850560, "bytes": 256 }, "B1.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 46850816, "bytes": 256 }, "B1.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 46851072, "bytes": 8388608 }, "B1.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 55239680, "bytes": 16384 }, "B1.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 55256064, "bytes": 8388608 }, "B1.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 63644672, "bytes": 16384 }, "B1.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 63661056, "bytes": 8388608 }, "B1.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 72049664, "bytes": 4096 }, "B2.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 72053760, "bytes": 4096 }, "B2.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 72057856, "bytes": 4096 }, "B2.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 72061952, "bytes": 6291456 }, "B2.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 78353408, "bytes": 12288 }, "B2.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 78365696, "bytes": 2097152 }, "B2.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 80462848, "bytes": 4096 }, "B2.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 80466944, "bytes": 256 }, "B2.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 80467200, "bytes": 256 }, "B2.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 80467456, "bytes": 8388608 }, "B2.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 88856064, "bytes": 16384 }, "B2.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 88872448, "bytes": 8388608 }, "B2.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 97261056, "bytes": 16384 }, "B2.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 97277440, "bytes": 8388608 }, "B2.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 105666048, "bytes": 4096 }, "B3.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 105670144, "bytes": 4096 }, "B3.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 105674240, "bytes": 4096 }, "B3.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 105678336, "bytes": 6291456 }, "B3.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 111969792, "bytes": 12288 }, "B3.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 111982080, "bytes": 2097152 }, "B3.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 114079232, "bytes": 4096 }, "B3.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 114083328, "bytes": 256 }, "B3.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 114083584, "bytes": 256 }, "B3.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 114083840, "bytes": 8388608 }, "B3.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 122472448, "bytes": 16384 }, "B3.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 122488832, "bytes": 8388608 }, "B3.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 130877440, "bytes": 16384 }, "B3.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 130893824, "bytes": 8388608 }, "B3.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 139282432, "bytes": 4096 }, "B4.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 139286528, "bytes": 4096 }, "B4.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 139290624, "bytes": 4096 }, "B4.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 139294720, "bytes": 6291456 }, "B4.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 145586176, "bytes": 12288 }, "B4.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 145598464, "bytes": 2097152 }, "B4.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 147695616, "bytes": 4096 }, "B4.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 147699712, "bytes": 256 }, "B4.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 147699968, "bytes": 256 }, "B4.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 147700224, "bytes": 8388608 }, "B4.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 156088832, "bytes": 16384 }, "B4.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 156105216, "bytes": 8388608 }, "B4.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 164493824, "bytes": 16384 }, "B4.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 164510208, "bytes": 8388608 }, "B4.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 172898816, "bytes": 4096 }, "B5.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 172902912, "bytes": 4096 }, "B5.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 172907008, "bytes": 4096 }, "B5.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 172911104, "bytes": 6291456 }, "B5.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 179202560, "bytes": 12288 }, "B5.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 179214848, "bytes": 2097152 }, "B5.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 181312000, "bytes": 4096 }, "B5.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 181316096, "bytes": 256 }, "B5.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 181316352, "bytes": 256 }, "B5.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 181316608, "bytes": 8388608 }, "B5.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 189705216, "bytes": 16384 }, "B5.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 189721600, "bytes": 8388608 }, "B5.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 198110208, "bytes": 16384 }, "B5.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 198126592, "bytes": 8388608 }, "B5.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 206515200, "bytes": 4096 }, "B6.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 206519296, "bytes": 4096 }, "B6.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 206523392, "bytes": 4096 }, "B6.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 206527488, "bytes": 6291456 }, "B6.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 212818944, "bytes": 12288 }, "B6.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 212831232, "bytes": 2097152 }, "B6.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 214928384, "bytes": 4096 }, "B6.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 214932480, "bytes": 256 }, "B6.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 214932736, "bytes": 256 }, "B6.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 214932992, "bytes": 8388608 }, "B6.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 223321600, "bytes": 16384 }, "B6.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 223337984, "bytes": 8388608 }, "B6.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 231726592, "bytes": 16384 }, "B6.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 231742976, "bytes": 8388608 }, "B6.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 240131584, "bytes": 4096 }, "B7.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 240135680, "bytes": 4096 }, "B7.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 240139776, "bytes": 4096 }, "B7.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 240143872, "bytes": 6291456 }, "B7.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 246435328, "bytes": 12288 }, "B7.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 246447616, "bytes": 2097152 }, "B7.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 248544768, "bytes": 4096 }, "B7.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 248548864, "bytes": 256 }, "B7.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 248549120, "bytes": 256 }, "B7.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 248549376, "bytes": 8388608 }, "B7.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 256937984, "bytes": 16384 }, "B7.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 256954368, "bytes": 8388608 }, "B7.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 265342976, "bytes": 16384 }, "B7.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 265359360, "bytes": 8388608 }, "B7.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 273747968, "bytes": 4096 }, "B8.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 273752064, "bytes": 4096 }, "B8.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 273756160, "bytes": 4096 }, "B8.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 273760256, "bytes": 6291456 }, "B8.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 280051712, "bytes": 12288 }, "B8.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 280064000, "bytes": 2097152 }, "B8.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 282161152, "bytes": 4096 }, "B8.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 282165248, "bytes": 256 }, "B8.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 282165504, "bytes": 256 }, "B8.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 282165760, "bytes": 8388608 }, "B8.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 290554368, "bytes": 16384 }, "B8.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 290570752, "bytes": 8388608 }, "B8.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 298959360, "bytes": 16384 }, "B8.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 298975744, "bytes": 8388608 }, "B8.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 307364352, "bytes": 4096 }, "B9.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 307368448, "bytes": 4096 }, "B9.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 307372544, "bytes": 4096 }, "B9.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 307376640, "bytes": 6291456 }, "B9.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 313668096, "bytes": 12288 }, "B9.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 313680384, "bytes": 2097152 }, "B9.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 315777536, "bytes": 4096 }, "B9.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 315781632, "bytes": 256 }, "B9.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 315781888, "bytes": 256 }, "B9.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 315782144, "bytes": 8388608 }, "B9.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 324170752, "bytes": 16384 }, "B9.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 324187136, "bytes": 8388608 }, "B9.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 332575744, "bytes": 16384 }, "B9.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 332592128, "bytes": 8388608 }, "B9.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 340980736, "bytes": 4096 }, "B10.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 340984832, "bytes": 4096 }, "B10.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 340988928, "bytes": 4096 }, "B10.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 340993024, "bytes": 6291456 }, "B10.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 347284480, "bytes": 12288 }, "B10.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 347296768, "bytes": 2097152 }, "B10.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 349393920, "bytes": 4096 }, "B10.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 349398016, "bytes": 256 }, "B10.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 349398272, "bytes": 256 }, "B10.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 349398528, "bytes": 8388608 }, "B10.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 357787136, "bytes": 16384 }, "B10.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 357803520, "bytes": 8388608 }, "B10.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 366192128, "bytes": 16384 }, "B10.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 366208512, "bytes": 8388608 }, "B10.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 374597120, "bytes": 4096 }, "B11.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 374601216, "bytes": 4096 }, "B11.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 374605312, "bytes": 4096 }, "B11.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 374609408, "bytes": 6291456 }, "B11.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 380900864, "bytes": 12288 }, "B11.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 380913152, "bytes": 2097152 }, "B11.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 383010304, "bytes": 4096 }, "B11.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 383014400, "bytes": 256 }, "B11.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 383014656, "bytes": 256 }, "B11.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 383014912, "bytes": 8388608 }, "B11.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 391403520, "bytes": 16384 }, "B11.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 391419904, "bytes": 8388608 }, "B11.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 399808512, "bytes": 16384 }, "B11.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 399824896, "bytes": 8388608 }, "B11.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 408213504, "bytes": 4096 }, "B12.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 408217600, "bytes": 4096 }, "B12.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 408221696, "bytes": 4096 }, "B12.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 408225792, "bytes": 6291456 }, "B12.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 414517248, "bytes": 12288 }, "B12.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 414529536, "bytes": 2097152 }, "B12.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 416626688, "bytes": 4096 }, "B12.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 416630784, "bytes": 256 }, "B12.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 416631040, "bytes": 256 }, "B12.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 416631296, "bytes": 8388608 }, "B12.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 425019904, "bytes": 16384 }, "B12.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 425036288, "bytes": 8388608 }, "B12.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 433424896, "bytes": 16384 }, "B12.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 433441280, "bytes": 8388608 }, "B12.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 441829888, "bytes": 4096 }, "B13.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 441833984, "bytes": 4096 }, "B13.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 441838080, "bytes": 4096 }, "B13.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 441842176, "bytes": 6291456 }, "B13.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 448133632, "bytes": 12288 }, "B13.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 448145920, "bytes": 2097152 }, "B13.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 450243072, "bytes": 4096 }, "B13.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 450247168, "bytes": 256 }, "B13.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 450247424, "bytes": 256 }, "B13.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 450247680, "bytes": 8388608 }, "B13.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 458636288, "bytes": 16384 }, "B13.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 458652672, "bytes": 8388608 }, "B13.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 467041280, "bytes": 16384 }, "B13.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 467057664, "bytes": 8388608 }, "B13.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 475446272, "bytes": 4096 }, "B14.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 475450368, "bytes": 4096 }, "B14.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 475454464, "bytes": 4096 }, "B14.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 475458560, "bytes": 6291456 }, "B14.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 481750016, "bytes": 12288 }, "B14.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 481762304, "bytes": 2097152 }, "B14.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 483859456, "bytes": 4096 }, "B14.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 483863552, "bytes": 256 }, "B14.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 483863808, "bytes": 256 }, "B14.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 483864064, "bytes": 8388608 }, "B14.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 492252672, "bytes": 16384 }, "B14.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 492269056, "bytes": 8388608 }, "B14.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 500657664, "bytes": 16384 }, "B14.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 500674048, "bytes": 8388608 }, "B14.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 509062656, "bytes": 4096 }, "B15.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 509066752, "bytes": 4096 }, "B15.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 509070848, "bytes": 4096 }, "B15.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 509074944, "bytes": 6291456 }, "B15.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 515366400, "bytes": 12288 }, "B15.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 515378688, "bytes": 2097152 }, "B15.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 517475840, "bytes": 4096 }, "B15.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 517479936, "bytes": 256 }, "B15.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 517480192, "bytes": 256 }, "B15.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 517480448, "bytes": 8388608 }, "B15.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 525869056, "bytes": 16384 }, "B15.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 525885440, "bytes": 8388608 }, "B15.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 534274048, "bytes": 16384 }, "B15.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 534290432, "bytes": 8388608 }, "B15.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 542679040, "bytes": 4096 }, "B16.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 542683136, "bytes": 4096 }, "B16.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 542687232, "bytes": 4096 }, "B16.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 542691328, "bytes": 6291456 }, "B16.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 548982784, "bytes": 12288 }, "B16.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 548995072, "bytes": 2097152 }, "B16.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 551092224, "bytes": 4096 }, "B16.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 551096320, "bytes": 256 }, "B16.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 551096576, "bytes": 256 }, "B16.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 551096832, "bytes": 8388608 }, "B16.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 559485440, "bytes": 16384 }, "B16.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 559501824, "bytes": 8388608 }, "B16.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 567890432, "bytes": 16384 }, "B16.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 567906816, "bytes": 8388608 }, "B16.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 576295424, "bytes": 4096 }, "B17.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 576299520, "bytes": 4096 }, "B17.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 576303616, "bytes": 4096 }, "B17.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 576307712, "bytes": 6291456 }, "B17.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 582599168, "bytes": 12288 }, "B17.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 582611456, "bytes": 2097152 }, "B17.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 584708608, "bytes": 4096 }, "B17.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 584712704, "bytes": 256 }, "B17.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 584712960, "bytes": 256 }, "B17.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 584713216, "bytes": 8388608 }, "B17.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 593101824, "bytes": 16384 }, "B17.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 593118208, "bytes": 8388608 }, "B17.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 601506816, "bytes": 16384 }, "B17.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 601523200, "bytes": 8388608 }, "B17.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 609911808, "bytes": 4096 }, "B18.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 609915904, "bytes": 4096 }, "B18.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 609920000, "bytes": 4096 }, "B18.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 609924096, "bytes": 6291456 }, "B18.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 616215552, "bytes": 12288 }, "B18.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 616227840, "bytes": 2097152 }, "B18.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 618324992, "bytes": 4096 }, "B18.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 618329088, "bytes": 256 }, "B18.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 618329344, "bytes": 256 }, "B18.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 618329600, "bytes": 8388608 }, "B18.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 626718208, "bytes": 16384 }, "B18.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 626734592, "bytes": 8388608 }, "B18.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 635123200, "bytes": 16384 }, "B18.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 635139584, "bytes": 8388608 }, "B18.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 643528192, "bytes": 4096 }, "B19.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 643532288, "bytes": 4096 }, "B19.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 643536384, "bytes": 4096 }, "B19.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 643540480, "bytes": 6291456 }, "B19.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 649831936, "bytes": 12288 }, "B19.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 649844224, "bytes": 2097152 }, "B19.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 651941376, "bytes": 4096 }, "B19.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 651945472, "bytes": 256 }, "B19.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 651945728, "bytes": 256 }, "B19.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 651945984, "bytes": 8388608 }, "B19.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 660334592, "bytes": 16384 }, "B19.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 660350976, "bytes": 8388608 }, "B19.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 668739584, "bytes": 16384 }, "B19.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 668755968, "bytes": 8388608 }, "B19.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 677144576, "bytes": 4096 }, "B20.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 677148672, "bytes": 4096 }, "B20.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 677152768, "bytes": 4096 }, "B20.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 677156864, "bytes": 6291456 }, "B20.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 683448320, "bytes": 12288 }, "B20.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 683460608, "bytes": 2097152 }, "B20.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 685557760, "bytes": 4096 }, "B20.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 685561856, "bytes": 256 }, "B20.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 685562112, "bytes": 256 }, "B20.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 685562368, "bytes": 8388608 }, "B20.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 693950976, "bytes": 16384 }, "B20.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 693967360, "bytes": 8388608 }, "B20.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 702355968, "bytes": 16384 }, "B20.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 702372352, "bytes": 8388608 }, "B20.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 710760960, "bytes": 4096 }, "B21.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 710765056, "bytes": 4096 }, "B21.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 710769152, "bytes": 4096 }, "B21.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 710773248, "bytes": 6291456 }, "B21.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 717064704, "bytes": 12288 }, "B21.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 717076992, "bytes": 2097152 }, "B21.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 719174144, "bytes": 4096 }, "B21.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 719178240, "bytes": 256 }, "B21.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 719178496, "bytes": 256 }, "B21.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 719178752, "bytes": 8388608 }, "B21.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 727567360, "bytes": 16384 }, "B21.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 727583744, "bytes": 8388608 }, "B21.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 735972352, "bytes": 16384 }, "B21.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 735988736, "bytes": 8388608 }, "B21.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 744377344, "bytes": 4096 }, "B22.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 744381440, "bytes": 4096 }, "B22.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 744385536, "bytes": 4096 }, "B22.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 744389632, "bytes": 6291456 }, "B22.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 750681088, "bytes": 12288 }, "B22.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 750693376, "bytes": 2097152 }, "B22.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 752790528, "bytes": 4096 }, "B22.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 752794624, "bytes": 256 }, "B22.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 752794880, "bytes": 256 }, "B22.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 752795136, "bytes": 8388608 }, "B22.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 761183744, "bytes": 16384 }, "B22.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 761200128, "bytes": 8388608 }, "B22.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 769588736, "bytes": 16384 }, "B22.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 769605120, "bytes": 8388608 }, "B22.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 777993728, "bytes": 4096 }, "B23.norm1": { "kind": "f32", "shape": [ 1024 ], "off": 777997824, "bytes": 4096 }, "B23.norm2": { "kind": "f32", "shape": [ 1024 ], "off": 778001920, "bytes": 4096 }, "B23.qkv.w": { "kind": "bf16", "shape": [ 3072, 1024 ], "off": 778006016, "bytes": 6291456 }, "B23.qkv.b": { "kind": "f32", "shape": [ 3072 ], "off": 784297472, "bytes": 12288 }, "B23.proj.w": { "kind": "bf16", "shape": [ 1024, 1024 ], "off": 784309760, "bytes": 2097152 }, "B23.proj.b": { "kind": "f32", "shape": [ 1024 ], "off": 786406912, "bytes": 4096 }, "B23.q_norm": { "kind": "f32", "shape": [ 64 ], "off": 786411008, "bytes": 256 }, "B23.k_norm": { "kind": "f32", "shape": [ 64 ], "off": 786411264, "bytes": 256 }, "B23.gate.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 786411520, "bytes": 8388608 }, "B23.gate.b": { "kind": "f32", "shape": [ 4096 ], "off": 794800128, "bytes": 16384 }, "B23.up.w": { "kind": "bf16", "shape": [ 4096, 1024 ], "off": 794816512, "bytes": 8388608 }, "B23.up.b": { "kind": "f32", "shape": [ 4096 ], "off": 803205120, "bytes": 16384 }, "B23.down.w": { "kind": "bf16", "shape": [ 1024, 4096 ], "off": 803221504, "bytes": 8388608 }, "B23.down.b": { "kind": "f32", "shape": [ 1024 ], "off": 811610112, "bytes": 4096 }, "post_norm": { "kind": "f32", "shape": [ 1024 ], "off": 811614208, "bytes": 4096 }, "down.w": { "kind": "bf16", "shape": [ 4096, 4096 ], "off": 811618304, "bytes": 33554432 }, "down.b": { "kind": "f32", "shape": [ 4096 ], "off": 845172736, "bytes": 16384 }, "merger.proj.w": { "kind": "bf16", "shape": [ 4096, 4096 ], "off": 845189120, "bytes": 33554432 }, "merger.ln.w": { "kind": "f32", "shape": [ 4096 ], "off": 878743552, "bytes": 16384 }, "merger.ln.b": { "kind": "f32", "shape": [ 4096 ], "off": 878759936, "bytes": 16384 }, "merger.gate.w": { "kind": "bf16", "shape": [ 10240, 4096 ], "off": 878776320, "bytes": 83886080 }, "merger.up.w": { "kind": "bf16", "shape": [ 10240, 4096 ], "off": 962662400, "bytes": 83886080 }, "merger.down.w": { "kind": "bf16", "shape": [ 4096, 10240 ], "off": 1046548480, "bytes": 83886080 } } }
{
"config": {
"attention_bias": true,
"attention_dropout": 0.0,
"depth": 24,
"hidden_act": "silu",
"hidden_size": 1024,
"image_size": 448,
"in_channels": 3,
"initializer_range": 0.02,
"intermediate_size": 4096,
"model_type": "glm5_next_vision",
"num_heads": 16,
"out_hidden_size": 4096,
"patch_size": 14,
"projection_intermediate_size": 10240,
"rms_norm_eps": 1e-05,
"spatial_merge_size": 2,
"swiglu_limit": 10.0,
"temporal_patch_size": 2,
"image_token_id": 154854,
"image_start_token_id": 154830,
"image_end_token_id": 154831,
"rope_theta": 10000.0
},
"tensors": {
"patch.w": {
"kind": "f32",
"shape": [
1024,
1176
],
"off": 0,
"bytes": 4816896
},
"patch.b": {
"kind": "f32",
"shape": [
1024
],
"off": 4816896,
"bytes": 4096
},
"B0.norm1": {
"kind": "f32",
"shape": [
1024
],
"off": 4820992,
"bytes": 4096
},
"B0.norm2": {
"kind": "f32",
"shape": [
1024
],
"off": 4825088,
"bytes": 4096
},
"B0.qkv.w": {
"kind": "bf16",
"shape": [
3072,
1024
],
"off": 4829184,
"bytes": 6291456
},
"B0.qkv.b": {
"kind": "f32",
"shape": [
3072
],
"off": 11120640,
"bytes": 12288
},
"B0.proj.w": {
"kind": "bf16",
"shape": [
1024,
1024
],
"off": 11132928,
"bytes": 2097152
},
"B0.proj.b": {
"kind": "f32",
"shape": [
1024
],
"off": 13230080,
"bytes": 4096
},
"B0.q_norm": {
"kind": "f32",
"shape": [
64
],
"off": 13234176,
"bytes": 256