| { |
| "schema": "trellis-webgpu-pack", |
| "version": 1, |
| "stage": "sparseStructureFlow", |
| "upstreamRevision": "25e0d31ffbebe4b5a97464dd851910efc3002d96", |
| "shards": [ |
| { |
| "file": "stage-000.bin", |
| "bytes": 66705408, |
| "sha256": "f0784b06e2b1de485881fb7382f4fb5f12ad1afb3bbf5a57cdbba9701b0a84b2" |
| }, |
| { |
| "file": "stage-001.bin", |
| "bytes": 64538624, |
| "sha256": "44b727fea9979fd8f0d93a9f8edb3e8c37d380bcabd5fb4bd6cdc05903fa84e7" |
| }, |
| { |
| "file": "stage-002.bin", |
| "bytes": 65628160, |
| "sha256": "2569e48e2700140e674038592de2f95440f1ed4f7897305300fe1cfd5f7b7e19" |
| }, |
| { |
| "file": "stage-003.bin", |
| "bytes": 65617920, |
| "sha256": "3913321cb4f6dd6fc47619fc5c16a1369d28a5342fde66afa6f55d570468966e" |
| }, |
| { |
| "file": "stage-004.bin", |
| "bytes": 62349312, |
| "sha256": "187edbe3a8d1258a2b2f15307a0c19a60608c9111aa98e13764935e7ba645f4f" |
| }, |
| { |
| "file": "stage-005.bin", |
| "bytes": 66693120, |
| "sha256": "a1b1300cd56ac0379cbde82c8e5cf7ee17b81610db5a9bc36a6a8d0c9a36f7a7" |
| }, |
| { |
| "file": "stage-006.bin", |
| "bytes": 64538624, |
| "sha256": "521267c28bfa739213e478c017db00eee2b42572c8b3f9c1e55808b65d74dd45" |
| }, |
| { |
| "file": "stage-007.bin", |
| "bytes": 65628160, |
| "sha256": "0a1e8e93c5f80670cc0bca860d26a4d92e82c307707a41ef1bc0ced957b5e56f" |
| }, |
| { |
| "file": "stage-008.bin", |
| "bytes": 65617920, |
| "sha256": "810a6c3f441ec717975ad7173e269178020f91ca307e703cb0d01f454a1c886e" |
| }, |
| { |
| "file": "stage-009.bin", |
| "bytes": 62349312, |
| "sha256": "215a57b9568b49fd7b049780ddd563bb109fb72dc8754c7798420e03bb744657" |
| }, |
| { |
| "file": "stage-010.bin", |
| "bytes": 66693120, |
| "sha256": "407fa67941be2024845c160fe3ccbe8c684cfe906e53b2c62704b97012a80898" |
| }, |
| { |
| "file": "stage-011.bin", |
| "bytes": 64538624, |
| "sha256": "3fb0f00dbe8b4a45adcab7b36f298d1f975fef4f9249ac35bc4c8f1b37bd800f" |
| }, |
| { |
| "file": "stage-012.bin", |
| "bytes": 65628160, |
| "sha256": "ea04edc07730f3dbd4f0df87a07d0027a57d7ed7513475f96878c90fc34d8f75" |
| }, |
| { |
| "file": "stage-013.bin", |
| "bytes": 26177792, |
| "sha256": "cb7118a3f8a9dbbd2b7a4a4f83ef707ac2dbea1ce3f6dedb51457d4e857d969a" |
| } |
| ], |
| "tensors": { |
| "blocks.0.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 0, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.0.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 0, |
| "offset": 12288, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 0, |
| "offset": 6303744, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 0, |
| "offset": 6500352, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 0, |
| "offset": 9646080, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.0.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 9842688, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.0.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 0, |
| "offset": 9846784, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 0, |
| "offset": 11943936, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 0, |
| "offset": 12009472, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 0, |
| "offset": 13058048, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.0.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 13123584, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.0.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 0, |
| "offset": 13125632, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 0, |
| "offset": 14174208, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 0, |
| "offset": 14206976, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 0, |
| "offset": 14731264, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.0.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 14764032, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.0.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 0, |
| "offset": 14766080, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 0, |
| "offset": 15814656, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 0, |
| "offset": 15847424, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 0, |
| "offset": 16371712, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.0.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 16404480, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.0.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 0, |
| "offset": 16412672, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 0, |
| "offset": 20606976, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 0, |
| "offset": 20738048, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 0, |
| "offset": 22835200, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.0.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 22966272, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.0.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 0, |
| "offset": 22968320, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 0, |
| "offset": 27162624, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 0, |
| "offset": 27293696, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 0, |
| "offset": 29390848, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.0.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 29521920, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.0.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 29523968, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.0.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 29526016, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.0.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 29528064, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.0.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 29530112, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.0.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 0, |
| "offset": 29532160, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 0, |
| "offset": 30580736, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 0, |
| "offset": 30613504, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 0, |
| "offset": 31137792, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.0.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 31170560, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.0.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 0, |
| "offset": 31176704, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 0, |
| "offset": 34322432, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 0, |
| "offset": 34420736, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 0, |
| "offset": 35993600, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.1.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 36091904, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.1.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 0, |
| "offset": 36104192, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 0, |
| "offset": 42395648, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 0, |
| "offset": 42592256, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 0, |
| "offset": 45737984, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.1.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 45934592, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.1.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 0, |
| "offset": 45938688, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 0, |
| "offset": 48035840, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 0, |
| "offset": 48101376, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 0, |
| "offset": 49149952, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.1.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 49215488, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.1.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 0, |
| "offset": 49217536, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 0, |
| "offset": 50266112, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 0, |
| "offset": 50298880, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 0, |
| "offset": 50823168, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.1.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 50855936, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.1.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 0, |
| "offset": 50857984, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 0, |
| "offset": 51906560, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 0, |
| "offset": 51939328, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 0, |
| "offset": 52463616, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.1.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 52496384, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.1.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 0, |
| "offset": 52504576, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 0, |
| "offset": 56698880, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 0, |
| "offset": 56829952, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 0, |
| "offset": 58927104, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.1.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 59058176, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.1.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 0, |
| "offset": 59060224, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 0, |
| "offset": 63254528, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 0, |
| "offset": 63385600, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 0, |
| "offset": 65482752, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.1.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 65613824, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.1.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 65615872, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.1.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 65617920, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.1.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 65619968, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.1.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 65622016, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.1.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 0, |
| "offset": 65624064, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 0, |
| "offset": 66672640, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 1, |
| "offset": 0, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 1, |
| "offset": 524288, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.1.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 557056, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.1.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 1, |
| "offset": 563200, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 1, |
| "offset": 3708928, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 1, |
| "offset": 3807232, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 1, |
| "offset": 5380096, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.10.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 5478400, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.10.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 1, |
| "offset": 5490688, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 1, |
| "offset": 11782144, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 1, |
| "offset": 11978752, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 1, |
| "offset": 15124480, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.10.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 15321088, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.10.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 1, |
| "offset": 15325184, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 1, |
| "offset": 17422336, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 1, |
| "offset": 17487872, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 1, |
| "offset": 18536448, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.10.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 18601984, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.10.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 1, |
| "offset": 18604032, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 1, |
| "offset": 19652608, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 1, |
| "offset": 19685376, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 1, |
| "offset": 20209664, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.10.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 20242432, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.10.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 1, |
| "offset": 20244480, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 1, |
| "offset": 21293056, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 1, |
| "offset": 21325824, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 1, |
| "offset": 21850112, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.10.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 21882880, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.10.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 1, |
| "offset": 21891072, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 1, |
| "offset": 26085376, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 1, |
| "offset": 26216448, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 1, |
| "offset": 28313600, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.10.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 28444672, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.10.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 1, |
| "offset": 28446720, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 1, |
| "offset": 32641024, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 1, |
| "offset": 32772096, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 1, |
| "offset": 34869248, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.10.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 35000320, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.10.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 35002368, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.10.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 35004416, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.10.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 35006464, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.10.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 35008512, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.10.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 1, |
| "offset": 35010560, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 1, |
| "offset": 36059136, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 1, |
| "offset": 36091904, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 1, |
| "offset": 36616192, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.10.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 36648960, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.10.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 1, |
| "offset": 36655104, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 1, |
| "offset": 39800832, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 1, |
| "offset": 39899136, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 1, |
| "offset": 41472000, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.11.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 41570304, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.11.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 1, |
| "offset": 41582592, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 1, |
| "offset": 47874048, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 1, |
| "offset": 48070656, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 1, |
| "offset": 51216384, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.11.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 51412992, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.11.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 1, |
| "offset": 51417088, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 1, |
| "offset": 53514240, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 1, |
| "offset": 53579776, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 1, |
| "offset": 54628352, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.11.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 54693888, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.11.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 1, |
| "offset": 54695936, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 1, |
| "offset": 55744512, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 1, |
| "offset": 55777280, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 1, |
| "offset": 56301568, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.11.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 56334336, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.11.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 1, |
| "offset": 56336384, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 1, |
| "offset": 57384960, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 1, |
| "offset": 57417728, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 1, |
| "offset": 57942016, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.11.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 57974784, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.11.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 1, |
| "offset": 57982976, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 1, |
| "offset": 62177280, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 1, |
| "offset": 62308352, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 1, |
| "offset": 64405504, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.11.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 64536576, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.11.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 2, |
| "offset": 0, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 2, |
| "offset": 4194304, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 2, |
| "offset": 4325376, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 2, |
| "offset": 6422528, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.11.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 6553600, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.11.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 6555648, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.11.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 6557696, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.11.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 6559744, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.11.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 6561792, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.11.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 2, |
| "offset": 6563840, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 2, |
| "offset": 7612416, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 2, |
| "offset": 7645184, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 2, |
| "offset": 8169472, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.11.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 8202240, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.11.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 2, |
| "offset": 8208384, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 2, |
| "offset": 11354112, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 2, |
| "offset": 11452416, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 2, |
| "offset": 13025280, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.12.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 13123584, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.12.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 2, |
| "offset": 13135872, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 2, |
| "offset": 19427328, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 2, |
| "offset": 19623936, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 2, |
| "offset": 22769664, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.12.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 22966272, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.12.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 2, |
| "offset": 22970368, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 2, |
| "offset": 25067520, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 2, |
| "offset": 25133056, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 2, |
| "offset": 26181632, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.12.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 26247168, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.12.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 2, |
| "offset": 26249216, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 2, |
| "offset": 27297792, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 2, |
| "offset": 27330560, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 2, |
| "offset": 27854848, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.12.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 27887616, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.12.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 2, |
| "offset": 27889664, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 2, |
| "offset": 28938240, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 2, |
| "offset": 28971008, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 2, |
| "offset": 29495296, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.12.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 29528064, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.12.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 2, |
| "offset": 29536256, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 2, |
| "offset": 33730560, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 2, |
| "offset": 33861632, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 2, |
| "offset": 35958784, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.12.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 36089856, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.12.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 2, |
| "offset": 36091904, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 2, |
| "offset": 40286208, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 2, |
| "offset": 40417280, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 2, |
| "offset": 42514432, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.12.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 42645504, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.12.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 42647552, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.12.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 42649600, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.12.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 42651648, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.12.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 42653696, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.12.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 2, |
| "offset": 42655744, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 2, |
| "offset": 43704320, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 2, |
| "offset": 43737088, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 2, |
| "offset": 44261376, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.12.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 44294144, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.12.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 2, |
| "offset": 44300288, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 2, |
| "offset": 47446016, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 2, |
| "offset": 47544320, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 2, |
| "offset": 49117184, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.13.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 49215488, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.13.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 2, |
| "offset": 49227776, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 2, |
| "offset": 55519232, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 2, |
| "offset": 55715840, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 2, |
| "offset": 58861568, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.13.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 59058176, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.13.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 2, |
| "offset": 59062272, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 2, |
| "offset": 61159424, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 2, |
| "offset": 61224960, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 2, |
| "offset": 62273536, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.13.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 62339072, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.13.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 2, |
| "offset": 62341120, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 2, |
| "offset": 63389696, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 2, |
| "offset": 63422464, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 2, |
| "offset": 63946752, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.13.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 63979520, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.13.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 2, |
| "offset": 63981568, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 2, |
| "offset": 65030144, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 2, |
| "offset": 65062912, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 2, |
| "offset": 65587200, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.13.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 65619968, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.13.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 3, |
| "offset": 0, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 3, |
| "offset": 4194304, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 3, |
| "offset": 4325376, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 3, |
| "offset": 6422528, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.13.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 6553600, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.13.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 3, |
| "offset": 6555648, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 3, |
| "offset": 10749952, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 3, |
| "offset": 10881024, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 3, |
| "offset": 12978176, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.13.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 13109248, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.13.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 13111296, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.13.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 13113344, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.13.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 13115392, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.13.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 13117440, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.13.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 3, |
| "offset": 13119488, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 3, |
| "offset": 14168064, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 3, |
| "offset": 14200832, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 3, |
| "offset": 14725120, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.13.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 14757888, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.13.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 3, |
| "offset": 14764032, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 3, |
| "offset": 17909760, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 3, |
| "offset": 18008064, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 3, |
| "offset": 19580928, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.14.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 19679232, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.14.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 3, |
| "offset": 19691520, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 3, |
| "offset": 25982976, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 3, |
| "offset": 26179584, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 3, |
| "offset": 29325312, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.14.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 29521920, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.14.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 3, |
| "offset": 29526016, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 3, |
| "offset": 31623168, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 3, |
| "offset": 31688704, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 3, |
| "offset": 32737280, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.14.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 32802816, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.14.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 3, |
| "offset": 32804864, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 3, |
| "offset": 33853440, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 3, |
| "offset": 33886208, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 3, |
| "offset": 34410496, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.14.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 34443264, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.14.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 3, |
| "offset": 34445312, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 3, |
| "offset": 35493888, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 3, |
| "offset": 35526656, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 3, |
| "offset": 36050944, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.14.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 36083712, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.14.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 3, |
| "offset": 36091904, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 3, |
| "offset": 40286208, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 3, |
| "offset": 40417280, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 3, |
| "offset": 42514432, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.14.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 42645504, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.14.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 3, |
| "offset": 42647552, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 3, |
| "offset": 46841856, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 3, |
| "offset": 46972928, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 3, |
| "offset": 49070080, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.14.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 49201152, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.14.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 49203200, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.14.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 49205248, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.14.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 49207296, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.14.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 49209344, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.14.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 3, |
| "offset": 49211392, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 3, |
| "offset": 50259968, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 3, |
| "offset": 50292736, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 3, |
| "offset": 50817024, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.14.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 50849792, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.14.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 3, |
| "offset": 50855936, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 3, |
| "offset": 54001664, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 3, |
| "offset": 54099968, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 3, |
| "offset": 55672832, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.15.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 55771136, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.15.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 3, |
| "offset": 55783424, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 3, |
| "offset": 62074880, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 3, |
| "offset": 62271488, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 3, |
| "offset": 65417216, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.15.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 65613824, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.15.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 4, |
| "offset": 0, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 4, |
| "offset": 2097152, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 4, |
| "offset": 2162688, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 4, |
| "offset": 3211264, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.15.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 3276800, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.15.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 4, |
| "offset": 3278848, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 4, |
| "offset": 4327424, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 4, |
| "offset": 4360192, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 4, |
| "offset": 4884480, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.15.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 4917248, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.15.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 4, |
| "offset": 4919296, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 4, |
| "offset": 5967872, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 4, |
| "offset": 6000640, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 4, |
| "offset": 6524928, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.15.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 6557696, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.15.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 4, |
| "offset": 6565888, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 4, |
| "offset": 10760192, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 4, |
| "offset": 10891264, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 4, |
| "offset": 12988416, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.15.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 13119488, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.15.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 4, |
| "offset": 13121536, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 4, |
| "offset": 17315840, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 4, |
| "offset": 17446912, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 4, |
| "offset": 19544064, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.15.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 19675136, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.15.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 19677184, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.15.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 19679232, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.15.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 19681280, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.15.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 19683328, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.15.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 4, |
| "offset": 19685376, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 4, |
| "offset": 20733952, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 4, |
| "offset": 20766720, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 4, |
| "offset": 21291008, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.15.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 21323776, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.15.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 4, |
| "offset": 21329920, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 4, |
| "offset": 24475648, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 4, |
| "offset": 24573952, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 4, |
| "offset": 26146816, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.16.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 26245120, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.16.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 4, |
| "offset": 26257408, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 4, |
| "offset": 32548864, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 4, |
| "offset": 32745472, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 4, |
| "offset": 35891200, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.16.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 36087808, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.16.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 4, |
| "offset": 36091904, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 4, |
| "offset": 38189056, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 4, |
| "offset": 38254592, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 4, |
| "offset": 39303168, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.16.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 39368704, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.16.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 4, |
| "offset": 39370752, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 4, |
| "offset": 40419328, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 4, |
| "offset": 40452096, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 4, |
| "offset": 40976384, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.16.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 41009152, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.16.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 4, |
| "offset": 41011200, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 4, |
| "offset": 42059776, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 4, |
| "offset": 42092544, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 4, |
| "offset": 42616832, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.16.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 42649600, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.16.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 4, |
| "offset": 42657792, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 4, |
| "offset": 46852096, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 4, |
| "offset": 46983168, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 4, |
| "offset": 49080320, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.16.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 49211392, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.16.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 4, |
| "offset": 49213440, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 4, |
| "offset": 53407744, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 4, |
| "offset": 53538816, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 4, |
| "offset": 55635968, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.16.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 55767040, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.16.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 55769088, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.16.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 55771136, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.16.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 55773184, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.16.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 55775232, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.16.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 4, |
| "offset": 55777280, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 4, |
| "offset": 56825856, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 4, |
| "offset": 56858624, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 4, |
| "offset": 57382912, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.16.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 57415680, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.16.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 4, |
| "offset": 57421824, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 4, |
| "offset": 60567552, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 4, |
| "offset": 60665856, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 4, |
| "offset": 62238720, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.17.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 62337024, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.17.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 5, |
| "offset": 0, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 5, |
| "offset": 6291456, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 5, |
| "offset": 6488064, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 5, |
| "offset": 9633792, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.17.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 9830400, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.17.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 5, |
| "offset": 9834496, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 5, |
| "offset": 11931648, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 5, |
| "offset": 11997184, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 5, |
| "offset": 13045760, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.17.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 13111296, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.17.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 5, |
| "offset": 13113344, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 5, |
| "offset": 14161920, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 5, |
| "offset": 14194688, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 5, |
| "offset": 14718976, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.17.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 14751744, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.17.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 5, |
| "offset": 14753792, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 5, |
| "offset": 15802368, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 5, |
| "offset": 15835136, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 5, |
| "offset": 16359424, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.17.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 16392192, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.17.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 5, |
| "offset": 16400384, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 5, |
| "offset": 20594688, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 5, |
| "offset": 20725760, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 5, |
| "offset": 22822912, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.17.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 22953984, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.17.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 5, |
| "offset": 22956032, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 5, |
| "offset": 27150336, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 5, |
| "offset": 27281408, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 5, |
| "offset": 29378560, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.17.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 29509632, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.17.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 29511680, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.17.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 29513728, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.17.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 29515776, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.17.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 29517824, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.17.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 5, |
| "offset": 29519872, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 5, |
| "offset": 30568448, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 5, |
| "offset": 30601216, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 5, |
| "offset": 31125504, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.17.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 31158272, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.17.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 5, |
| "offset": 31164416, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 5, |
| "offset": 34310144, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 5, |
| "offset": 34408448, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 5, |
| "offset": 35981312, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.18.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 36079616, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.18.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 5, |
| "offset": 36091904, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 5, |
| "offset": 42383360, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 5, |
| "offset": 42579968, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 5, |
| "offset": 45725696, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.18.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 45922304, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.18.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 5, |
| "offset": 45926400, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 5, |
| "offset": 48023552, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 5, |
| "offset": 48089088, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 5, |
| "offset": 49137664, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.18.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 49203200, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.18.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 5, |
| "offset": 49205248, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 5, |
| "offset": 50253824, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 5, |
| "offset": 50286592, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 5, |
| "offset": 50810880, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.18.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 50843648, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.18.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 5, |
| "offset": 50845696, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 5, |
| "offset": 51894272, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 5, |
| "offset": 51927040, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 5, |
| "offset": 52451328, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.18.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 52484096, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.18.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 5, |
| "offset": 52492288, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 5, |
| "offset": 56686592, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 5, |
| "offset": 56817664, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 5, |
| "offset": 58914816, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.18.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 59045888, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.18.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 5, |
| "offset": 59047936, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 5, |
| "offset": 63242240, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 5, |
| "offset": 63373312, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 5, |
| "offset": 65470464, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.18.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 65601536, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.18.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 65603584, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.18.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 65605632, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.18.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 65607680, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.18.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 65609728, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.18.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 5, |
| "offset": 65611776, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 5, |
| "offset": 66660352, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 6, |
| "offset": 0, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 6, |
| "offset": 524288, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.18.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 557056, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.18.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 6, |
| "offset": 563200, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 6, |
| "offset": 3708928, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 6, |
| "offset": 3807232, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 6, |
| "offset": 5380096, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.19.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 5478400, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.19.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 6, |
| "offset": 5490688, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 6, |
| "offset": 11782144, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 6, |
| "offset": 11978752, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 6, |
| "offset": 15124480, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.19.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 15321088, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.19.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 6, |
| "offset": 15325184, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 6, |
| "offset": 17422336, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 6, |
| "offset": 17487872, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 6, |
| "offset": 18536448, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.19.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 18601984, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.19.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 6, |
| "offset": 18604032, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 6, |
| "offset": 19652608, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 6, |
| "offset": 19685376, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 6, |
| "offset": 20209664, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.19.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 20242432, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.19.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 6, |
| "offset": 20244480, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 6, |
| "offset": 21293056, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 6, |
| "offset": 21325824, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 6, |
| "offset": 21850112, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.19.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 21882880, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.19.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 6, |
| "offset": 21891072, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 6, |
| "offset": 26085376, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 6, |
| "offset": 26216448, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 6, |
| "offset": 28313600, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.19.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 28444672, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.19.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 6, |
| "offset": 28446720, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 6, |
| "offset": 32641024, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 6, |
| "offset": 32772096, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 6, |
| "offset": 34869248, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.19.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 35000320, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.19.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 35002368, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.19.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 35004416, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.19.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 35006464, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.19.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 35008512, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.19.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 6, |
| "offset": 35010560, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 6, |
| "offset": 36059136, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 6, |
| "offset": 36091904, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 6, |
| "offset": 36616192, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.19.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 36648960, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.19.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 6, |
| "offset": 36655104, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 6, |
| "offset": 39800832, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 6, |
| "offset": 39899136, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 6, |
| "offset": 41472000, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.2.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 41570304, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.2.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 6, |
| "offset": 41582592, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 6, |
| "offset": 47874048, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 6, |
| "offset": 48070656, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 6, |
| "offset": 51216384, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.2.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 51412992, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.2.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 6, |
| "offset": 51417088, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 6, |
| "offset": 53514240, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 6, |
| "offset": 53579776, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 6, |
| "offset": 54628352, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.2.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 54693888, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.2.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 6, |
| "offset": 54695936, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 6, |
| "offset": 55744512, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 6, |
| "offset": 55777280, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 6, |
| "offset": 56301568, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.2.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 56334336, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.2.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 6, |
| "offset": 56336384, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 6, |
| "offset": 57384960, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 6, |
| "offset": 57417728, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 6, |
| "offset": 57942016, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.2.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 57974784, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.2.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 6, |
| "offset": 57982976, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 6, |
| "offset": 62177280, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 6, |
| "offset": 62308352, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 6, |
| "offset": 64405504, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.2.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 64536576, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.2.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 7, |
| "offset": 0, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 7, |
| "offset": 4194304, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 7, |
| "offset": 4325376, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 7, |
| "offset": 6422528, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.2.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 6553600, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.2.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 6555648, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.2.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 6557696, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.2.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 6559744, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.2.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 6561792, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.2.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 7, |
| "offset": 6563840, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 7, |
| "offset": 7612416, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 7, |
| "offset": 7645184, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 7, |
| "offset": 8169472, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.2.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 8202240, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.2.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 7, |
| "offset": 8208384, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 7, |
| "offset": 11354112, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 7, |
| "offset": 11452416, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 7, |
| "offset": 13025280, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.20.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 13123584, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.20.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 7, |
| "offset": 13135872, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 7, |
| "offset": 19427328, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 7, |
| "offset": 19623936, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 7, |
| "offset": 22769664, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.20.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 22966272, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.20.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 7, |
| "offset": 22970368, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 7, |
| "offset": 25067520, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 7, |
| "offset": 25133056, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 7, |
| "offset": 26181632, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.20.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 26247168, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.20.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 7, |
| "offset": 26249216, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 7, |
| "offset": 27297792, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 7, |
| "offset": 27330560, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 7, |
| "offset": 27854848, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.20.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 27887616, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.20.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 7, |
| "offset": 27889664, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 7, |
| "offset": 28938240, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 7, |
| "offset": 28971008, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 7, |
| "offset": 29495296, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.20.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 29528064, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.20.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 7, |
| "offset": 29536256, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 7, |
| "offset": 33730560, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 7, |
| "offset": 33861632, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 7, |
| "offset": 35958784, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.20.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 36089856, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.20.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 7, |
| "offset": 36091904, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 7, |
| "offset": 40286208, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 7, |
| "offset": 40417280, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 7, |
| "offset": 42514432, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.20.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 42645504, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.20.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 42647552, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.20.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 42649600, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.20.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 42651648, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.20.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 42653696, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.20.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 7, |
| "offset": 42655744, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 7, |
| "offset": 43704320, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 7, |
| "offset": 43737088, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 7, |
| "offset": 44261376, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.20.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 44294144, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.20.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 7, |
| "offset": 44300288, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 7, |
| "offset": 47446016, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 7, |
| "offset": 47544320, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 7, |
| "offset": 49117184, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.21.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 49215488, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.21.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 7, |
| "offset": 49227776, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 7, |
| "offset": 55519232, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 7, |
| "offset": 55715840, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 7, |
| "offset": 58861568, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.21.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 59058176, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.21.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 7, |
| "offset": 59062272, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 7, |
| "offset": 61159424, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 7, |
| "offset": 61224960, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 7, |
| "offset": 62273536, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.21.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 62339072, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.21.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 7, |
| "offset": 62341120, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 7, |
| "offset": 63389696, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 7, |
| "offset": 63422464, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 7, |
| "offset": 63946752, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.21.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 63979520, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.21.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 7, |
| "offset": 63981568, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 7, |
| "offset": 65030144, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 7, |
| "offset": 65062912, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 7, |
| "offset": 65587200, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.21.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 65619968, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.21.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 8, |
| "offset": 0, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 8, |
| "offset": 4194304, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 8, |
| "offset": 4325376, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 8, |
| "offset": 6422528, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.21.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 6553600, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.21.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 8, |
| "offset": 6555648, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 8, |
| "offset": 10749952, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 8, |
| "offset": 10881024, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 8, |
| "offset": 12978176, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.21.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 13109248, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.21.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 13111296, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.21.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 13113344, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.21.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 13115392, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.21.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 13117440, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.21.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 8, |
| "offset": 13119488, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 8, |
| "offset": 14168064, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 8, |
| "offset": 14200832, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 8, |
| "offset": 14725120, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.21.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 14757888, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.21.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 8, |
| "offset": 14764032, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 8, |
| "offset": 17909760, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 8, |
| "offset": 18008064, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 8, |
| "offset": 19580928, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.22.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 19679232, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.22.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 8, |
| "offset": 19691520, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 8, |
| "offset": 25982976, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 8, |
| "offset": 26179584, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 8, |
| "offset": 29325312, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.22.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 29521920, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.22.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 8, |
| "offset": 29526016, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 8, |
| "offset": 31623168, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 8, |
| "offset": 31688704, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 8, |
| "offset": 32737280, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.22.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 32802816, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.22.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 8, |
| "offset": 32804864, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 8, |
| "offset": 33853440, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 8, |
| "offset": 33886208, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 8, |
| "offset": 34410496, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.22.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 34443264, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.22.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 8, |
| "offset": 34445312, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 8, |
| "offset": 35493888, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 8, |
| "offset": 35526656, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 8, |
| "offset": 36050944, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.22.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 36083712, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.22.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 8, |
| "offset": 36091904, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 8, |
| "offset": 40286208, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 8, |
| "offset": 40417280, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 8, |
| "offset": 42514432, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.22.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 42645504, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.22.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 8, |
| "offset": 42647552, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 8, |
| "offset": 46841856, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 8, |
| "offset": 46972928, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 8, |
| "offset": 49070080, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.22.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 49201152, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.22.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 49203200, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.22.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 49205248, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.22.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 49207296, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.22.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 49209344, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.22.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 8, |
| "offset": 49211392, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 8, |
| "offset": 50259968, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 8, |
| "offset": 50292736, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 8, |
| "offset": 50817024, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.22.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 50849792, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.22.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 8, |
| "offset": 50855936, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 8, |
| "offset": 54001664, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 8, |
| "offset": 54099968, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 8, |
| "offset": 55672832, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.23.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 55771136, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.23.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 8, |
| "offset": 55783424, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 8, |
| "offset": 62074880, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 8, |
| "offset": 62271488, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 8, |
| "offset": 65417216, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.23.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 65613824, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.23.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 9, |
| "offset": 0, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 9, |
| "offset": 2097152, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 9, |
| "offset": 2162688, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 9, |
| "offset": 3211264, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.23.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 3276800, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.23.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 9, |
| "offset": 3278848, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 9, |
| "offset": 4327424, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 9, |
| "offset": 4360192, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 9, |
| "offset": 4884480, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.23.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 4917248, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.23.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 9, |
| "offset": 4919296, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 9, |
| "offset": 5967872, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 9, |
| "offset": 6000640, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 9, |
| "offset": 6524928, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.23.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 6557696, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.23.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 9, |
| "offset": 6565888, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 9, |
| "offset": 10760192, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 9, |
| "offset": 10891264, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 9, |
| "offset": 12988416, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.23.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 13119488, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.23.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 9, |
| "offset": 13121536, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 9, |
| "offset": 17315840, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 9, |
| "offset": 17446912, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 9, |
| "offset": 19544064, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.23.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 19675136, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.23.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 19677184, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.23.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 19679232, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.23.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 19681280, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.23.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 19683328, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.23.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 9, |
| "offset": 19685376, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 9, |
| "offset": 20733952, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 9, |
| "offset": 20766720, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 9, |
| "offset": 21291008, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.23.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 21323776, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.23.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 9, |
| "offset": 21329920, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 9, |
| "offset": 24475648, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 9, |
| "offset": 24573952, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 9, |
| "offset": 26146816, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.3.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 26245120, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.3.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 9, |
| "offset": 26257408, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 9, |
| "offset": 32548864, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 9, |
| "offset": 32745472, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 9, |
| "offset": 35891200, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.3.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 36087808, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.3.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 9, |
| "offset": 36091904, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 9, |
| "offset": 38189056, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 9, |
| "offset": 38254592, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 9, |
| "offset": 39303168, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.3.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 39368704, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.3.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 9, |
| "offset": 39370752, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 9, |
| "offset": 40419328, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 9, |
| "offset": 40452096, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 9, |
| "offset": 40976384, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.3.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 41009152, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.3.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 9, |
| "offset": 41011200, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 9, |
| "offset": 42059776, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 9, |
| "offset": 42092544, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 9, |
| "offset": 42616832, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.3.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 42649600, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.3.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 9, |
| "offset": 42657792, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 9, |
| "offset": 46852096, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 9, |
| "offset": 46983168, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 9, |
| "offset": 49080320, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.3.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 49211392, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.3.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 9, |
| "offset": 49213440, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 9, |
| "offset": 53407744, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 9, |
| "offset": 53538816, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 9, |
| "offset": 55635968, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.3.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 55767040, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.3.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 55769088, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.3.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 55771136, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.3.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 55773184, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.3.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 55775232, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.3.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 9, |
| "offset": 55777280, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 9, |
| "offset": 56825856, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 9, |
| "offset": 56858624, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 9, |
| "offset": 57382912, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.3.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 57415680, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.3.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 9, |
| "offset": 57421824, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 9, |
| "offset": 60567552, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 9, |
| "offset": 60665856, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 9, |
| "offset": 62238720, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.4.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 62337024, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.4.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 10, |
| "offset": 0, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 10, |
| "offset": 6291456, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 10, |
| "offset": 6488064, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 10, |
| "offset": 9633792, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.4.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 9830400, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.4.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 10, |
| "offset": 9834496, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 10, |
| "offset": 11931648, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 10, |
| "offset": 11997184, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 10, |
| "offset": 13045760, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.4.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 13111296, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.4.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 10, |
| "offset": 13113344, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 10, |
| "offset": 14161920, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 10, |
| "offset": 14194688, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 10, |
| "offset": 14718976, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.4.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 14751744, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.4.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 10, |
| "offset": 14753792, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 10, |
| "offset": 15802368, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 10, |
| "offset": 15835136, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 10, |
| "offset": 16359424, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.4.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 16392192, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.4.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 10, |
| "offset": 16400384, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 10, |
| "offset": 20594688, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 10, |
| "offset": 20725760, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 10, |
| "offset": 22822912, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.4.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 22953984, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.4.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 10, |
| "offset": 22956032, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 10, |
| "offset": 27150336, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 10, |
| "offset": 27281408, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 10, |
| "offset": 29378560, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.4.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 29509632, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.4.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 29511680, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.4.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 29513728, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.4.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 29515776, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.4.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 29517824, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.4.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 10, |
| "offset": 29519872, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 10, |
| "offset": 30568448, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 10, |
| "offset": 30601216, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 10, |
| "offset": 31125504, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.4.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 31158272, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.4.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 10, |
| "offset": 31164416, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 10, |
| "offset": 34310144, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 10, |
| "offset": 34408448, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 10, |
| "offset": 35981312, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.5.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 36079616, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.5.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 10, |
| "offset": 36091904, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 10, |
| "offset": 42383360, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 10, |
| "offset": 42579968, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 10, |
| "offset": 45725696, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.5.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 45922304, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.5.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 10, |
| "offset": 45926400, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 10, |
| "offset": 48023552, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 10, |
| "offset": 48089088, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 10, |
| "offset": 49137664, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.5.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 49203200, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.5.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 10, |
| "offset": 49205248, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 10, |
| "offset": 50253824, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 10, |
| "offset": 50286592, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 10, |
| "offset": 50810880, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.5.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 50843648, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.5.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 10, |
| "offset": 50845696, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 10, |
| "offset": 51894272, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 10, |
| "offset": 51927040, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 10, |
| "offset": 52451328, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.5.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 52484096, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.5.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 10, |
| "offset": 52492288, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 10, |
| "offset": 56686592, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 10, |
| "offset": 56817664, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 10, |
| "offset": 58914816, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.5.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 59045888, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.5.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 10, |
| "offset": 59047936, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 10, |
| "offset": 63242240, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 10, |
| "offset": 63373312, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 10, |
| "offset": 65470464, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.5.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 65601536, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.5.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 65603584, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.5.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 65605632, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.5.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 65607680, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.5.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 65609728, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.5.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 10, |
| "offset": 65611776, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 10, |
| "offset": 66660352, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 11, |
| "offset": 0, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 11, |
| "offset": 524288, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.5.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 557056, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.5.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 11, |
| "offset": 563200, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 11, |
| "offset": 3708928, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 11, |
| "offset": 3807232, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 11, |
| "offset": 5380096, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.6.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 5478400, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.6.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 11, |
| "offset": 5490688, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 11, |
| "offset": 11782144, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 11, |
| "offset": 11978752, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 11, |
| "offset": 15124480, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.6.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 15321088, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.6.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 11, |
| "offset": 15325184, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 11, |
| "offset": 17422336, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 11, |
| "offset": 17487872, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 11, |
| "offset": 18536448, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.6.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 18601984, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.6.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 11, |
| "offset": 18604032, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 11, |
| "offset": 19652608, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 11, |
| "offset": 19685376, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 11, |
| "offset": 20209664, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.6.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 20242432, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.6.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 11, |
| "offset": 20244480, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 11, |
| "offset": 21293056, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 11, |
| "offset": 21325824, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 11, |
| "offset": 21850112, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.6.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 21882880, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.6.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 11, |
| "offset": 21891072, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 11, |
| "offset": 26085376, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 11, |
| "offset": 26216448, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 11, |
| "offset": 28313600, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.6.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 28444672, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.6.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 11, |
| "offset": 28446720, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 11, |
| "offset": 32641024, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 11, |
| "offset": 32772096, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 11, |
| "offset": 34869248, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.6.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 35000320, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.6.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 35002368, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.6.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 35004416, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.6.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 35006464, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.6.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 35008512, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.6.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 11, |
| "offset": 35010560, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 11, |
| "offset": 36059136, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 11, |
| "offset": 36091904, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 11, |
| "offset": 36616192, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.6.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 36648960, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.6.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 11, |
| "offset": 36655104, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 11, |
| "offset": 39800832, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 11, |
| "offset": 39899136, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 11, |
| "offset": 41472000, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.7.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 41570304, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.7.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 11, |
| "offset": 41582592, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 11, |
| "offset": 47874048, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 11, |
| "offset": 48070656, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 11, |
| "offset": 51216384, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.7.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 51412992, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.7.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 11, |
| "offset": 51417088, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 11, |
| "offset": 53514240, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 11, |
| "offset": 53579776, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 11, |
| "offset": 54628352, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.7.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 54693888, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.7.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 11, |
| "offset": 54695936, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 11, |
| "offset": 55744512, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 11, |
| "offset": 55777280, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 11, |
| "offset": 56301568, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.7.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 56334336, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.7.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 11, |
| "offset": 56336384, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 11, |
| "offset": 57384960, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 11, |
| "offset": 57417728, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 11, |
| "offset": 57942016, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.7.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 57974784, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.7.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 11, |
| "offset": 57982976, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 11, |
| "offset": 62177280, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 11, |
| "offset": 62308352, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 11, |
| "offset": 64405504, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.7.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 64536576, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.7.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 12, |
| "offset": 0, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 12, |
| "offset": 4194304, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 12, |
| "offset": 4325376, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 12, |
| "offset": 6422528, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.7.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 6553600, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.7.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 6555648, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.7.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 6557696, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.7.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 6559744, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.7.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 6561792, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.7.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 12, |
| "offset": 6563840, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 12, |
| "offset": 7612416, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 12, |
| "offset": 7645184, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 12, |
| "offset": 8169472, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.7.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 8202240, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.7.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 12, |
| "offset": 8208384, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 12, |
| "offset": 11354112, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 12, |
| "offset": 11452416, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 12, |
| "offset": 13025280, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.8.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 13123584, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.8.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 12, |
| "offset": 13135872, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 12, |
| "offset": 19427328, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 12, |
| "offset": 19623936, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 12, |
| "offset": 22769664, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.8.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 22966272, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.8.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 12, |
| "offset": 22970368, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 12, |
| "offset": 25067520, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 12, |
| "offset": 25133056, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 12, |
| "offset": 26181632, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.8.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 26247168, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.8.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 12, |
| "offset": 26249216, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 12, |
| "offset": 27297792, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 12, |
| "offset": 27330560, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 12, |
| "offset": 27854848, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.8.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 27887616, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.8.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 12, |
| "offset": 27889664, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 12, |
| "offset": 28938240, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 12, |
| "offset": 28971008, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 12, |
| "offset": 29495296, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.8.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 29528064, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.8.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 12, |
| "offset": 29536256, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 12, |
| "offset": 33730560, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 12, |
| "offset": 33861632, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 12, |
| "offset": 35958784, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.8.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 36089856, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.8.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 12, |
| "offset": 36091904, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 12, |
| "offset": 40286208, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 12, |
| "offset": 40417280, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 12, |
| "offset": 42514432, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.8.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 42645504, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.8.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 42647552, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.8.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 42649600, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.8.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 42651648, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.8.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 42653696, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.8.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 12, |
| "offset": 42655744, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 12, |
| "offset": 43704320, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 12, |
| "offset": 43737088, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 12, |
| "offset": 44261376, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.8.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 44294144, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.8.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 12, |
| "offset": 44300288, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 12, |
| "offset": 47446016, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 12, |
| "offset": 47544320, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 12, |
| "offset": 49117184, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.9.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 49215488, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.9.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 12, |
| "offset": 49227776, |
| "bytes": 6291456 |
| }, |
| "base_scales": { |
| "shard": 12, |
| "offset": 55519232, |
| "bytes": 196608 |
| }, |
| "residual_data": { |
| "shard": 12, |
| "offset": 55715840, |
| "bytes": 3145728 |
| }, |
| "residual_scales": { |
| "shard": 12, |
| "offset": 58861568, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.9.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 59058176, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.9.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 1024, |
| "padded_rows": 2048, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 12, |
| "offset": 59062272, |
| "bytes": 2097152 |
| }, |
| "base_scales": { |
| "shard": 12, |
| "offset": 61159424, |
| "bytes": 65536 |
| }, |
| "residual_data": { |
| "shard": 12, |
| "offset": 61224960, |
| "bytes": 1048576 |
| }, |
| "residual_scales": { |
| "shard": 12, |
| "offset": 62273536, |
| "bytes": 65536 |
| } |
| }, |
| "blocks.9.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 62339072, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.9.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 12, |
| "offset": 62341120, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 12, |
| "offset": 63389696, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 12, |
| "offset": 63422464, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 12, |
| "offset": 63946752, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.9.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 63979520, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.9.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 12, |
| "offset": 63981568, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 12, |
| "offset": 65030144, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 12, |
| "offset": 65062912, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 12, |
| "offset": 65587200, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.9.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 65619968, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.9.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 13, |
| "offset": 0, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 13, |
| "offset": 4194304, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 13, |
| "offset": 4325376, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 13, |
| "offset": 6422528, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.9.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 6553600, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.9.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "base_data": { |
| "shard": 13, |
| "offset": 6555648, |
| "bytes": 4194304 |
| }, |
| "base_scales": { |
| "shard": 13, |
| "offset": 10749952, |
| "bytes": 131072 |
| }, |
| "residual_data": { |
| "shard": 13, |
| "offset": 10881024, |
| "bytes": 2097152 |
| }, |
| "residual_scales": { |
| "shard": 13, |
| "offset": 12978176, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.9.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 13109248, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.9.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 13111296, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.9.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 13113344, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.9.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 13115392, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.9.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 13117440, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.9.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 13, |
| "offset": 13119488, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 13, |
| "offset": 14168064, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 13, |
| "offset": 14200832, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 13, |
| "offset": 14725120, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.9.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 14757888, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.9.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 13, |
| "offset": 14764032, |
| "bytes": 3145728 |
| }, |
| "base_scales": { |
| "shard": 13, |
| "offset": 17909760, |
| "bytes": 98304 |
| }, |
| "residual_data": { |
| "shard": 13, |
| "offset": 18008064, |
| "bytes": 1572864 |
| }, |
| "residual_scales": { |
| "shard": 13, |
| "offset": 19580928, |
| "bytes": 98304 |
| } |
| }, |
| "input_layer.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 19679232, |
| "bytes": 2048 |
| } |
| }, |
| "input_layer.weight": { |
| "shape": [ |
| 1024, |
| 8 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 8, |
| "padded_rows": 1024, |
| "padded_columns": 64, |
| "base_data": { |
| "shard": 13, |
| "offset": 19681280, |
| "bytes": 65536 |
| }, |
| "base_scales": { |
| "shard": 13, |
| "offset": 19746816, |
| "bytes": 2048 |
| }, |
| "residual_data": { |
| "shard": 13, |
| "offset": 19748864, |
| "bytes": 32768 |
| }, |
| "residual_scales": { |
| "shard": 13, |
| "offset": 19781632, |
| "bytes": 2048 |
| } |
| }, |
| "out_layer.bias": { |
| "shape": [ |
| 8 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 19783680, |
| "bytes": 16 |
| } |
| }, |
| "out_layer.weight": { |
| "shape": [ |
| 8, |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 19783936, |
| "bytes": 16384 |
| } |
| }, |
| "pos_emb": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 13, |
| "offset": 19800320, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 13, |
| "offset": 23994624, |
| "bytes": 131072 |
| } |
| }, |
| "t_embedder.mlp.0.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 24125696, |
| "bytes": 2048 |
| } |
| }, |
| "t_embedder.mlp.0.weight": { |
| "shape": [ |
| 1024, |
| 256 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 256, |
| "padded_rows": 1024, |
| "padded_columns": 256, |
| "base_data": { |
| "shard": 13, |
| "offset": 24127744, |
| "bytes": 262144 |
| }, |
| "base_scales": { |
| "shard": 13, |
| "offset": 24389888, |
| "bytes": 8192 |
| }, |
| "residual_data": { |
| "shard": 13, |
| "offset": 24398080, |
| "bytes": 131072 |
| }, |
| "residual_scales": { |
| "shard": 13, |
| "offset": 24529152, |
| "bytes": 8192 |
| } |
| }, |
| "t_embedder.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 24537344, |
| "bytes": 2048 |
| } |
| }, |
| "t_embedder.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "residual-quantized", |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "base_data": { |
| "shard": 13, |
| "offset": 24539392, |
| "bytes": 1048576 |
| }, |
| "base_scales": { |
| "shard": 13, |
| "offset": 25587968, |
| "bytes": 32768 |
| }, |
| "residual_data": { |
| "shard": 13, |
| "offset": 25620736, |
| "bytes": 524288 |
| }, |
| "residual_scales": { |
| "shard": 13, |
| "offset": 26145024, |
| "bytes": 32768 |
| } |
| } |
| } |
| } |
|
|
