ajh-code commited on
Commit
4ccf216
·
verified ·
1 Parent(s): 3085e9a

Add runtime/spas_sage_attn/quant_per_block.py

Browse files
runtime/spas_sage_attn/quant_per_block.py ADDED
@@ -0,0 +1,148 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Copyright (c) 2025 by SpargeAttn team.
3
+
4
+ Licensed under the Apache License, Version 2.0 (the "License");
5
+ you may not use this file except in compliance with the License.
6
+ You may obtain a copy of the License at
7
+
8
+ http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+ Unless required by applicable law or agreed to in writing, software
11
+ distributed under the License is distributed on an "AS IS" BASIS,
12
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ See the License for the specific language governing permissions and
14
+ limitations under the License.
15
+ """
16
+
17
+ import torch
18
+ import triton
19
+ import triton.language as tl
20
+
21
+ @triton.jit
22
+ def quant_per_block_int8_kernel(Input, Output, Scale, L,
23
+ stride_iz, stride_ih, stride_in,
24
+ stride_oz, stride_oh, stride_on,
25
+ stride_sz, stride_sh,
26
+ sm_scale,
27
+ C: tl.constexpr, BLK: tl.constexpr):
28
+ off_blk = tl.program_id(0)
29
+ off_h = tl.program_id(1)
30
+ off_b = tl.program_id(2)
31
+
32
+ offs_n = off_blk * BLK + tl.arange(0, BLK)
33
+ offs_k = tl.arange(0, C)
34
+
35
+ input_ptrs = Input + off_b * stride_iz + off_h * stride_ih + offs_n[:, None] * stride_in + offs_k[None, :]
36
+ output_ptrs = Output + off_b * stride_oz + off_h * stride_oh + offs_n[:, None] * stride_on + offs_k[None, :]
37
+ scale_ptrs = Scale + off_b * stride_sz + off_h * stride_sh + off_blk
38
+
39
+ x = tl.load(input_ptrs, mask=offs_n[:, None] < L)
40
+ x = x.to(tl.float32)
41
+ x *= sm_scale
42
+ scale = tl.max(tl.abs(x)) / 127.
43
+ scale += 0.0000001
44
+ x_int8 = x / scale
45
+ x_int8 += 0.5 * tl.where(x_int8 >= 0, 1, -1)
46
+ x_int8 = x_int8.to(tl.int8)
47
+ tl.store(output_ptrs, x_int8, mask=offs_n[:, None] < L)
48
+ tl.store(scale_ptrs, scale)
49
+
50
+
51
+ def per_block_int8(q, k, BLKQ=128, BLKK=64, sm_scale=None, tensor_layout="HND"):
52
+ q_int8 = torch.empty(q.shape, dtype=torch.int8, device=q.device)
53
+ k_int8 = torch.empty(k.shape, dtype=torch.int8, device=k.device)
54
+
55
+ if tensor_layout == "HND":
56
+ b, h_qo, qo_len, head_dim = q.shape
57
+ _, h_kv, kv_len, _ = k.shape
58
+
59
+ stride_bz_q, stride_h_q, stride_seq_q = q.stride(0), q.stride(1), q.stride(2)
60
+ stride_bz_qo, stride_h_qo, stride_seq_qo = q_int8.stride(0), q_int8.stride(1), q_int8.stride(2)
61
+ stride_bz_k, stride_h_k, stride_seq_k = k.stride(0), k.stride(1), k.stride(2)
62
+ stride_bz_ko, stride_h_ko, stride_seq_ko = k_int8.stride(0), k_int8.stride(1), k_int8.stride(2)
63
+ elif tensor_layout == "NHD":
64
+ b, qo_len, h_qo, head_dim = q.shape
65
+ _, kv_len, h_kv, _ = k.shape
66
+
67
+ stride_bz_q, stride_h_q, stride_seq_q = q.stride(0), q.stride(2), q.stride(1)
68
+ stride_bz_qo, stride_h_qo, stride_seq_qo = q_int8.stride(0), q_int8.stride(2), q_int8.stride(1)
69
+ stride_bz_k, stride_h_k, stride_seq_k = k.stride(0), k.stride(2), k.stride(1)
70
+ stride_bz_ko, stride_h_ko, stride_seq_ko = k_int8.stride(0), k_int8.stride(2), k_int8.stride(1)
71
+ else:
72
+ raise ValueError(f"Unknown tensor layout: {tensor_layout}")
73
+
74
+ q_scale = torch.empty((b, h_qo, (qo_len + BLKQ - 1) // BLKQ, 1), device=q.device, dtype=torch.float32)
75
+ k_scale = torch.empty((b, h_kv, (kv_len + BLKK - 1) // BLKK, 1), device=q.device, dtype=torch.float32)
76
+
77
+ if sm_scale is None:
78
+ sm_scale = head_dim**-0.5
79
+
80
+ grid = ((qo_len + BLKQ - 1) // BLKQ, h_qo, b)
81
+ quant_per_block_int8_kernel[grid](
82
+ q, q_int8, q_scale, qo_len,
83
+ stride_bz_q, stride_h_q, stride_seq_q,
84
+ stride_bz_qo, stride_h_qo, stride_seq_qo,
85
+ q_scale.stride(0), q_scale.stride(1),
86
+ sm_scale=(sm_scale * 1.44269504),
87
+ C=head_dim, BLK=BLKQ
88
+ )
89
+
90
+ grid = ((kv_len + BLKK - 1) // BLKK, h_kv, b)
91
+ quant_per_block_int8_kernel[grid](
92
+ k, k_int8, k_scale, kv_len,
93
+ stride_bz_k, stride_h_k, stride_seq_k,
94
+ stride_bz_ko, stride_h_ko, stride_seq_ko,
95
+ k_scale.stride(0), k_scale.stride(1),
96
+ sm_scale=1.0,
97
+ C=head_dim, BLK=BLKK
98
+ )
99
+
100
+ return q_int8, q_scale, k_int8, k_scale
101
+
102
+ def per_warp_int8(q, k, BLKQ=128, WARPQ=32, BLKK=64, WARPK=64, tensor_layout="HND"):
103
+ q_int8 = torch.empty(q.shape, dtype=torch.int8, device=q.device)
104
+ k_int8 = torch.empty(k.shape, dtype=torch.int8, device=k.device)
105
+
106
+ if tensor_layout == "HND":
107
+ b, h_qo, qo_len, head_dim = q.shape
108
+ _, h_kv, kv_len, _ = k.shape
109
+
110
+ stride_bz_q, stride_h_q, stride_seq_q = q.stride(0), q.stride(1), q.stride(2)
111
+ stride_bz_qo, stride_h_qo, stride_seq_qo = q_int8.stride(0), q_int8.stride(1), q_int8.stride(2)
112
+ stride_bz_k, stride_h_k, stride_seq_k = k.stride(0), k.stride(1), k.stride(2)
113
+ stride_bz_ko, stride_h_ko, stride_seq_ko = k_int8.stride(0), k_int8.stride(1), k_int8.stride(2)
114
+ elif tensor_layout == "NHD":
115
+ b, qo_len, h_qo, head_dim = q.shape
116
+ _, kv_len, h_kv, _ = k.shape
117
+
118
+ stride_bz_q, stride_h_q, stride_seq_q = q.stride(0), q.stride(2), q.stride(1)
119
+ stride_bz_qo, stride_h_qo, stride_seq_qo = q_int8.stride(0), q_int8.stride(2), q_int8.stride(1)
120
+ stride_bz_k, stride_h_k, stride_seq_k = k.stride(0), k.stride(2), k.stride(1)
121
+ stride_bz_ko, stride_h_ko, stride_seq_ko = k_int8.stride(0), k_int8.stride(2), k_int8.stride(1)
122
+ else:
123
+ raise ValueError(f"Unknown tensor layout: {tensor_layout}")
124
+
125
+ q_scale = torch.empty((b, h_qo, (qo_len + BLKQ - 1) // BLKQ * (BLKQ // WARPQ), 1), device=q.device, dtype=torch.float32)
126
+ k_scale = torch.empty((b, h_kv, (kv_len + BLKK - 1) // BLKK * (BLKK // WARPK), 1), device=q.device, dtype=torch.float32)
127
+
128
+ grid = ((qo_len + BLKQ - 1) // BLKQ * (BLKQ // WARPQ), h_qo, b)
129
+ quant_per_block_int8_kernel[grid](
130
+ q, q_int8, q_scale, qo_len,
131
+ stride_bz_q, stride_h_q, stride_seq_q,
132
+ stride_bz_qo, stride_h_qo, stride_seq_qo,
133
+ q_scale.stride(0), q_scale.stride(1),
134
+ sm_scale=1.0,
135
+ C=head_dim, BLK=WARPQ
136
+ )
137
+
138
+ grid = ((kv_len + BLKK - 1) // BLKK * (BLKK // WARPK), h_kv, b)
139
+ quant_per_block_int8_kernel[grid](
140
+ k, k_int8, k_scale, kv_len,
141
+ stride_bz_k, stride_h_k, stride_seq_k,
142
+ stride_bz_ko, stride_h_ko, stride_seq_ko,
143
+ k_scale.stride(0), k_scale.stride(1),
144
+ sm_scale=1.0,
145
+ C=head_dim, BLK=WARPK
146
+ )
147
+
148
+ return q_int8, q_scale, k_int8, k_scale