File size: 6,654 Bytes
47e2685 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 | {
"name": "leapquant_kda_flush_r4_int8_h32_v128_k128",
"op_type": "leapquant_kda_flush",
"description": "LeapQuant window flush (arXiv:2609.38166) for Kimi Delta Attention (per-key-channel gate): at the end of a 16-token window, rebuild each head's V x K fp32 recurrent state from its quantized checkpoint (4 fp16 Compensator Tokens + smoothed int8 residual) decayed per channel and the window's 16 buffered rank-1 updates with their per-channel decay histories, then re-quantize it (one round of subspace iteration for the new Compensator Tokens, smoothing scales and int8 codes for the residual). Correctness is judged on the decoded state (see the request README), not elementwise.",
"axes": {
"batch_size": {
"type": "var"
},
"num_heads": {
"type": "const",
"value": 32
},
"value_dim": {
"type": "const",
"value": 128
},
"key_dim": {
"type": "const",
"value": 128
},
"window": {
"type": "const",
"value": 16
},
"rank": {
"type": "const",
"value": 4
}
},
"inputs": {
"codes": {
"shape": [
"batch_size",
"num_heads",
"value_dim",
"key_dim"
],
"dtype": "int8"
},
"s_k": {
"shape": [
"batch_size",
"num_heads",
"key_dim"
],
"dtype": "float32"
},
"s_v": {
"shape": [
"batch_size",
"num_heads",
"value_dim"
],
"dtype": "float32"
},
"u": {
"shape": [
"batch_size",
"num_heads",
"rank",
"key_dim"
],
"dtype": "float16"
},
"q": {
"shape": [
"batch_size",
"num_heads",
"rank",
"value_dim"
],
"dtype": "float16"
},
"kbuf": {
"shape": [
"batch_size",
"num_heads",
"window",
"key_dim"
],
"dtype": "bfloat16"
},
"ubuf": {
"shape": [
"batch_size",
"num_heads",
"window",
"value_dim"
],
"dtype": "bfloat16"
},
"gbuf": {
"shape": [
"batch_size",
"num_heads",
"window",
"key_dim"
],
"dtype": "float16"
},
"w": {
"shape": [
"batch_size",
"num_heads",
"window"
],
"dtype": "float32"
},
"pcum": {
"shape": [
"batch_size",
"num_heads",
"key_dim"
],
"dtype": "float32"
},
"q0": {
"shape": [
"value_dim",
"rank"
],
"dtype": "float32"
}
},
"outputs": {
"codes_new": {
"shape": [
"batch_size",
"num_heads",
"value_dim",
"key_dim"
],
"dtype": "int8"
},
"s_k_new": {
"shape": [
"batch_size",
"num_heads",
"key_dim"
],
"dtype": "float32"
},
"s_v_new": {
"shape": [
"batch_size",
"num_heads",
"value_dim"
],
"dtype": "float32"
},
"u_new": {
"shape": [
"batch_size",
"num_heads",
"rank",
"key_dim"
],
"dtype": "float16"
},
"q_new": {
"shape": [
"batch_size",
"num_heads",
"rank",
"value_dim"
],
"dtype": "float16"
}
},
"constraints": [
"batch_size >= 1"
],
"reference": "# SPDX-License-Identifier: Apache-2.0\nimport torch\n\nQMAX = 127.0\nEPS = 1e-12\n\n\ndef _mgs(m):\n \"\"\"Modified Gram-Schmidt on the columns of m, projections applied twice (\"twice is enough\"): a column that is\n numerically dependent on the previous ones (a rank-deficient head) still comes out orthogonal to them.\"\"\"\n cols = []\n for j in range(m.shape[-1]):\n v = m[..., j]\n for _ in range(2):\n for c in cols:\n v = v - c * (c * v).sum(-1, keepdim=True)\n cols.append(v / v.norm(dim=-1, keepdim=True).clamp_min(1e-30))\n return torch.stack(cols, -1)\n\n\ndef run(codes, s_k, s_v, u, q, kbuf, ubuf, gbuf, w, pcum, q0):\n \"\"\"LeapQuant window flush for Kimi Delta Attention (per-key-channel gate): fold the buffered updates of one window into a new\n quantized checkpoint.\n\n codes int8 [B, H, V, K], s_k f32 [B, H, K], s_v f32 [B, H, V]: smoothed int8 residual of the old checkpoint\n u f16 [B, H, R, K], q f16 [B, H, R, V]: the R Compensator Tokens of the old checkpoint (rank-R part = q^T u)\n kbuf bf16 [B, H, L, K], ubuf bf16 [B, H, L, V]: the window's buffered updates; w f32 [B, H, L]: their weights\n gbuf f16 [B, H, L, K]: per-channel decay factor of the step that appended each entry; pcum f32 [B, H, K]: log-gate summed\n over the window (the checkpoint's decay is exp(pcum))\n q0 f32 [V, R]: fixed start of the subspace iteration\n Returns the new (codes, s_k, s_v, u, q). fp32 arithmetic; the order of operations is not part of the contract.\n \"\"\"\n # 1. rebuild the fp32 state: S = (q^T u + dequant(codes)) diag(exp(pcum)) + sum_j w_j ubuf_j (kbuf_j * D_j)^T [V, K]\n # with D_j = prod_{i > j} gbuf_i (the decay entry j has seen since it was appended)\n s_old = torch.einsum(\"bhrv,bhrk->bhvk\", q.float(), u.float())\n s_old = s_old + codes.float() * (s_k[..., None, :] / QMAX) * s_v[..., :, None]\n after = torch.flip(torch.cumprod(torch.flip(gbuf.float(), [-2]), -2), [-2]) # prod_{i >= j}\n D = torch.cat([after[..., 1:, :], torch.ones_like(after[..., :1, :])], -2) # prod_{i > j}\n s = s_old * torch.exp(pcum)[..., None, :] + torch.einsum(\"bhj,bhjv,bhjk->bhvk\", w, ubuf.float(), kbuf.float() * D)\n # 2. Compensator Tokens: ONE round of subspace iteration from q0, stored as fp16\n x = s.transpose(-1, -2) # [K, V]\n qk = _mgs(x @ q0) # [K, R]\n qv = _mgs(x.transpose(-1, -2) @ qk) # [V, R], orthonormal columns\n uk = x @ qv # [K, R]\n u_new = uk.transpose(-1, -2).to(torch.float16)\n q_new = qv.transpose(-1, -2).to(torch.float16)\n # 3. residual, smoothing scales (column scale = sqrt of the mean |E| over rows, row scale = max of the smoothed row), int8\n e = s - torch.einsum(\"bhrv,bhrk->bhvk\", q_new.float(), u_new.float())\n sk = e.abs().mean(-2).sqrt().clamp_min(EPS)\n sv = (e.abs() / sk[..., None, :]).amax(-1).clamp_min(EPS)\n c = torch.round(e / (sk[..., None, :] * sv[..., :, None]) * QMAX).clamp(-QMAX, QMAX).to(torch.int8)\n return c, sk, sv, u_new, q_new\n"
}
|