liangsu9988 commited on
Commit
913bae1
·
verified ·
1 Parent(s): 62edca9

Uploaded using `kernel-builder`.

Browse files
benchmarks/benchmark.py ADDED
@@ -0,0 +1,105 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Benchmark INT8 transformer primitives."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import importlib
8
+ import sys
9
+ from pathlib import Path
10
+
11
+ import torch
12
+
13
+ ROOT = Path(__file__).resolve().parents[2]
14
+ sys.path.insert(0, str(ROOT / "int8-transformer-primitives" / "tests"))
15
+ from test_int8_transformer_primitives import load_source_ops # noqa: E402
16
+
17
+
18
+ def load_ops(backend: str, artifact: str | None):
19
+ if backend == "source":
20
+ return load_source_ops()
21
+ if artifact:
22
+ sys.path.insert(0, artifact)
23
+ try:
24
+ return importlib.import_module("int8_transformer_primitives")
25
+ finally:
26
+ if artifact:
27
+ sys.path.remove(artifact)
28
+
29
+
30
+ def time_us(fn, warmup: int, iters: int) -> float:
31
+ for _ in range(warmup):
32
+ fn()
33
+ torch.cuda.synchronize()
34
+ start = torch.cuda.Event(enable_timing=True)
35
+ end = torch.cuda.Event(enable_timing=True)
36
+ start.record()
37
+ for _ in range(iters):
38
+ fn()
39
+ end.record()
40
+ torch.cuda.synchronize()
41
+ return start.elapsed_time(end) * 1000.0 / iters
42
+
43
+
44
+ def main() -> int:
45
+ parser = argparse.ArgumentParser()
46
+ parser.add_argument("--backend", choices=["source", "installed"], default="source")
47
+ parser.add_argument("--artifact", default=None)
48
+ parser.add_argument("--mode", choices=["headline", "full"], default="headline")
49
+ parser.add_argument("--warmup", type=int, default=20)
50
+ parser.add_argument("--iters", type=int, default=100)
51
+ args = parser.parse_args()
52
+ ops = load_ops(args.backend, args.artifact)
53
+
54
+ shapes = [
55
+ ("decode_m8", 8, 1024, 2560),
56
+ ("small_batch", 64, 2048, 8192),
57
+ ("vision_prefill", 522, 2048, 2560),
58
+ ]
59
+ if args.mode == "full":
60
+ shapes += [
61
+ ("m1", 1, 1024, 1024),
62
+ ("m17", 17, 256, 256),
63
+ ("wide_ffn", 257, 2048, 8192),
64
+ ]
65
+
66
+ print("workload,M,K,N,op,flashrt_us,torch_eager_us,speedup")
67
+ for name, m, k, n in shapes:
68
+ x = (torch.randn((m, k), device="cuda") * 0.5).to(torch.bfloat16)
69
+ w = (torch.randn((n, k), device="cuda") * 0.5).to(torch.bfloat16)
70
+ x_i8, x_scale = ops.quantize_int8_rowwise_bf16(x)
71
+ w_i8, w_scale = ops.quantize_int8_rowwise_bf16(w)
72
+ torch.cuda.synchronize()
73
+ out = torch.empty((m, n), device="cuda", dtype=torch.bfloat16)
74
+
75
+ def flash():
76
+ ops.int8_rowwise_linear_bf16(x_i8, w_i8, x_scale, w_scale, out=out)
77
+
78
+ def eager():
79
+ ((x_i8.float() @ w_i8.float().t()) * x_scale[:, None] * w_scale[None, :]).to(torch.bfloat16)
80
+
81
+ fu = time_us(flash, args.warmup, args.iters)
82
+ eu = time_us(eager, max(5, args.warmup // 2), max(20, args.iters // 2))
83
+ print(f"{name},{m},{k},{n},int8_rowwise_linear_bf16,{fu:.3f},{eu:.3f},{eu/fu:.2f}x")
84
+
85
+ q = (torch.randn((522, 2048), device="cuda") * 0.5).to(torch.bfloat16)
86
+ weight = torch.randn((2048,), device="cuda", dtype=torch.bfloat16)
87
+ out = torch.empty_like(q, dtype=torch.int8)
88
+ scales = torch.empty((q.shape[0],), device="cuda", dtype=torch.float32)
89
+
90
+ def flash_rms():
91
+ ops.rms_norm_quantize_int8_rowwise_bf16(q, weight, out=out, scales=scales)
92
+
93
+ def eager_rms():
94
+ y = q.float() * torch.rsqrt((q.float() * q.float()).mean(dim=1, keepdim=True) + 1e-6) * weight.float()
95
+ s = torch.clamp(y.abs().amax(dim=1) / 127.0, min=1e-10)
96
+ torch.clamp(torch.round(y / s[:, None]), -127, 127).to(torch.int8)
97
+
98
+ fu = time_us(flash_rms, args.warmup, args.iters)
99
+ eu = time_us(eager_rms, max(5, args.warmup // 2), max(20, args.iters // 2))
100
+ print(f"vision_prefill,522,2048,0,rms_norm_quantize_int8_rowwise_bf16,{fu:.3f},{eu:.3f},{eu/fu:.2f}x")
101
+ return 0
102
+
103
+
104
+ if __name__ == "__main__":
105
+ raise SystemExit(main())
build/torch211-cxx11-cu128-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,179 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT INT8 transformer primitives."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Optional
6
+
7
+ import torch
8
+
9
+ from ._ops import add_op_namespace_prefix, ops
10
+
11
+
12
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_int8_static_bf16"))
13
+ def _quantize_int8_static_bf16_fake(input: torch.Tensor, scale: torch.Tensor, out: torch.Tensor) -> None:
14
+ if out.shape != input.shape:
15
+ raise RuntimeError("out must match input shape")
16
+ return None
17
+
18
+
19
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_int8_rowwise_bf16"))
20
+ def _quantize_int8_rowwise_bf16_fake(input: torch.Tensor, out: torch.Tensor, scales: torch.Tensor) -> None:
21
+ if input.dim() != 2 or out.shape != input.shape or scales.shape != (input.shape[0],):
22
+ raise RuntimeError("quantize_int8_rowwise_bf16 expects input/out (rows, cols), scales (rows,)")
23
+ return None
24
+
25
+
26
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_int8_rowwise_static_bf16"))
27
+ def _quantize_int8_rowwise_static_bf16_fake(input: torch.Tensor, scales: torch.Tensor, out: torch.Tensor) -> None:
28
+ return _quantize_int8_rowwise_bf16_fake(input, out, scales)
29
+
30
+
31
+ @torch.library.register_fake(add_op_namespace_prefix("rms_norm_quantize_int8_rowwise_bf16"))
32
+ def _rms_norm_quantize_int8_rowwise_bf16_fake(
33
+ x: torch.Tensor, weight: torch.Tensor, eps: float, out: torch.Tensor, scales: torch.Tensor
34
+ ) -> None:
35
+ return _quantize_int8_rowwise_bf16_fake(x, out, scales)
36
+
37
+
38
+ @torch.library.register_fake(add_op_namespace_prefix("residual_add_rms_norm_quantize_int8_rowwise_bf16"))
39
+ def _residual_add_rms_norm_quantize_int8_rowwise_bf16_fake(
40
+ residual: torch.Tensor,
41
+ x: torch.Tensor,
42
+ weight: torch.Tensor,
43
+ eps: float,
44
+ out: torch.Tensor,
45
+ scales: torch.Tensor,
46
+ ) -> None:
47
+ return _quantize_int8_rowwise_bf16_fake(residual, out, scales)
48
+
49
+
50
+ @torch.library.register_fake(add_op_namespace_prefix("int8_rowwise_linear_bf16"))
51
+ def _int8_rowwise_linear_bf16_fake(
52
+ input_i8: torch.Tensor,
53
+ weight_i8: torch.Tensor,
54
+ input_scale: torch.Tensor,
55
+ weight_scale: torch.Tensor,
56
+ out: torch.Tensor,
57
+ variant: int,
58
+ ) -> None:
59
+ if input_i8.dim() != 2 or weight_i8.dim() != 2 or out.shape != (input_i8.shape[0], weight_i8.shape[0]):
60
+ raise RuntimeError("int8_rowwise_linear_bf16 expects input (M,K), weight (N,K), out (M,N)")
61
+ return None
62
+
63
+
64
+ @torch.library.register_fake(add_op_namespace_prefix("int8_silu_gated_linear_bf16"))
65
+ def _int8_silu_gated_linear_bf16_fake(
66
+ input_i8: torch.Tensor,
67
+ up_weight_i8: torch.Tensor,
68
+ input_scale: torch.Tensor,
69
+ weight_scale: torch.Tensor,
70
+ gate: torch.Tensor,
71
+ out: torch.Tensor,
72
+ ) -> None:
73
+ return _int8_rowwise_linear_bf16_fake(input_i8, up_weight_i8, input_scale, weight_scale, out, 0)
74
+
75
+
76
+ def quantize_int8_static_bf16(
77
+ input: torch.Tensor, scale: torch.Tensor, *, out: Optional[torch.Tensor] = None
78
+ ) -> torch.Tensor:
79
+ if out is None:
80
+ out = torch.empty_like(input, dtype=torch.int8)
81
+ ops.quantize_int8_static_bf16(input, scale, out)
82
+ return out
83
+
84
+
85
+ def quantize_int8_rowwise_bf16(
86
+ input: torch.Tensor,
87
+ *,
88
+ out: Optional[torch.Tensor] = None,
89
+ scales: Optional[torch.Tensor] = None,
90
+ ) -> tuple[torch.Tensor, torch.Tensor]:
91
+ if out is None:
92
+ out = torch.empty_like(input, dtype=torch.int8)
93
+ if scales is None:
94
+ scales = torch.empty((input.shape[0],), device=input.device, dtype=torch.float32)
95
+ ops.quantize_int8_rowwise_bf16(input, out, scales)
96
+ return out, scales
97
+
98
+
99
+ def quantize_int8_rowwise_static_bf16(
100
+ input: torch.Tensor, scales: torch.Tensor, *, out: Optional[torch.Tensor] = None
101
+ ) -> torch.Tensor:
102
+ if out is None:
103
+ out = torch.empty_like(input, dtype=torch.int8)
104
+ ops.quantize_int8_rowwise_static_bf16(input, scales, out)
105
+ return out
106
+
107
+
108
+ def rms_norm_quantize_int8_rowwise_bf16(
109
+ x: torch.Tensor,
110
+ weight: torch.Tensor,
111
+ eps: float = 1e-6,
112
+ *,
113
+ out: Optional[torch.Tensor] = None,
114
+ scales: Optional[torch.Tensor] = None,
115
+ ) -> tuple[torch.Tensor, torch.Tensor]:
116
+ if out is None:
117
+ out = torch.empty_like(x, dtype=torch.int8)
118
+ if scales is None:
119
+ scales = torch.empty((x.shape[0],), device=x.device, dtype=torch.float32)
120
+ ops.rms_norm_quantize_int8_rowwise_bf16(x, weight, float(eps), out, scales)
121
+ return out, scales
122
+
123
+
124
+ def residual_add_rms_norm_quantize_int8_rowwise_bf16(
125
+ residual: torch.Tensor,
126
+ x: torch.Tensor,
127
+ weight: torch.Tensor,
128
+ eps: float = 1e-6,
129
+ *,
130
+ out: Optional[torch.Tensor] = None,
131
+ scales: Optional[torch.Tensor] = None,
132
+ ) -> tuple[torch.Tensor, torch.Tensor]:
133
+ if out is None:
134
+ out = torch.empty_like(residual, dtype=torch.int8)
135
+ if scales is None:
136
+ scales = torch.empty((residual.shape[0],), device=residual.device, dtype=torch.float32)
137
+ ops.residual_add_rms_norm_quantize_int8_rowwise_bf16(residual, x, weight, float(eps), out, scales)
138
+ return out, scales
139
+
140
+
141
+ def int8_rowwise_linear_bf16(
142
+ input_i8: torch.Tensor,
143
+ weight_i8: torch.Tensor,
144
+ input_scale: torch.Tensor,
145
+ weight_scale: torch.Tensor,
146
+ variant: int = 0,
147
+ *,
148
+ out: Optional[torch.Tensor] = None,
149
+ ) -> torch.Tensor:
150
+ if out is None:
151
+ out = torch.empty((input_i8.shape[0], weight_i8.shape[0]), device=input_i8.device, dtype=torch.bfloat16)
152
+ ops.int8_rowwise_linear_bf16(input_i8, weight_i8, input_scale, weight_scale, out, int(variant))
153
+ return out
154
+
155
+
156
+ def int8_silu_gated_linear_bf16(
157
+ input_i8: torch.Tensor,
158
+ up_weight_i8: torch.Tensor,
159
+ input_scale: torch.Tensor,
160
+ weight_scale: torch.Tensor,
161
+ gate: torch.Tensor,
162
+ *,
163
+ out: Optional[torch.Tensor] = None,
164
+ ) -> torch.Tensor:
165
+ if out is None:
166
+ out = torch.empty((input_i8.shape[0], up_weight_i8.shape[0]), device=input_i8.device, dtype=torch.bfloat16)
167
+ ops.int8_silu_gated_linear_bf16(input_i8, up_weight_i8, input_scale, weight_scale, gate, out)
168
+ return out
169
+
170
+
171
+ __all__ = [
172
+ "quantize_int8_static_bf16",
173
+ "quantize_int8_rowwise_bf16",
174
+ "quantize_int8_rowwise_static_bf16",
175
+ "rms_norm_quantize_int8_rowwise_bf16",
176
+ "residual_add_rms_norm_quantize_int8_rowwise_bf16",
177
+ "int8_rowwise_linear_bf16",
178
+ "int8_silu_gated_linear_bf16",
179
+ ]
build/torch211-cxx11-cu128-x86_64-linux/_int8_transformer_primitives_cuda_f5e354d.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5d645ffda89e6448384e5abd28f736b4a1ad049805fcb2e405f12dc6b1890345
3
+ size 1336536
build/torch211-cxx11-cu128-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _int8_transformer_primitives_cuda_f5e354d
3
+ ops = torch.ops._int8_transformer_primitives_cuda_f5e354d
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_int8_transformer_primitives_cuda_f5e354d::{op_name}"
build/torch211-cxx11-cu128-x86_64-linux/int8_transformer_primitives/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-cu128-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "int8-transformer-primitives",
3
+ "id": "_int8_transformer_primitives_cuda_f5e354d",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "12.0a",
11
+ "8.0",
12
+ "8.6",
13
+ "8.9",
14
+ "9.0"
15
+ ]
16
+ }
17
+ }
build/torch211-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,179 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT INT8 transformer primitives."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Optional
6
+
7
+ import torch
8
+
9
+ from ._ops import add_op_namespace_prefix, ops
10
+
11
+
12
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_int8_static_bf16"))
13
+ def _quantize_int8_static_bf16_fake(input: torch.Tensor, scale: torch.Tensor, out: torch.Tensor) -> None:
14
+ if out.shape != input.shape:
15
+ raise RuntimeError("out must match input shape")
16
+ return None
17
+
18
+
19
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_int8_rowwise_bf16"))
20
+ def _quantize_int8_rowwise_bf16_fake(input: torch.Tensor, out: torch.Tensor, scales: torch.Tensor) -> None:
21
+ if input.dim() != 2 or out.shape != input.shape or scales.shape != (input.shape[0],):
22
+ raise RuntimeError("quantize_int8_rowwise_bf16 expects input/out (rows, cols), scales (rows,)")
23
+ return None
24
+
25
+
26
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_int8_rowwise_static_bf16"))
27
+ def _quantize_int8_rowwise_static_bf16_fake(input: torch.Tensor, scales: torch.Tensor, out: torch.Tensor) -> None:
28
+ return _quantize_int8_rowwise_bf16_fake(input, out, scales)
29
+
30
+
31
+ @torch.library.register_fake(add_op_namespace_prefix("rms_norm_quantize_int8_rowwise_bf16"))
32
+ def _rms_norm_quantize_int8_rowwise_bf16_fake(
33
+ x: torch.Tensor, weight: torch.Tensor, eps: float, out: torch.Tensor, scales: torch.Tensor
34
+ ) -> None:
35
+ return _quantize_int8_rowwise_bf16_fake(x, out, scales)
36
+
37
+
38
+ @torch.library.register_fake(add_op_namespace_prefix("residual_add_rms_norm_quantize_int8_rowwise_bf16"))
39
+ def _residual_add_rms_norm_quantize_int8_rowwise_bf16_fake(
40
+ residual: torch.Tensor,
41
+ x: torch.Tensor,
42
+ weight: torch.Tensor,
43
+ eps: float,
44
+ out: torch.Tensor,
45
+ scales: torch.Tensor,
46
+ ) -> None:
47
+ return _quantize_int8_rowwise_bf16_fake(residual, out, scales)
48
+
49
+
50
+ @torch.library.register_fake(add_op_namespace_prefix("int8_rowwise_linear_bf16"))
51
+ def _int8_rowwise_linear_bf16_fake(
52
+ input_i8: torch.Tensor,
53
+ weight_i8: torch.Tensor,
54
+ input_scale: torch.Tensor,
55
+ weight_scale: torch.Tensor,
56
+ out: torch.Tensor,
57
+ variant: int,
58
+ ) -> None:
59
+ if input_i8.dim() != 2 or weight_i8.dim() != 2 or out.shape != (input_i8.shape[0], weight_i8.shape[0]):
60
+ raise RuntimeError("int8_rowwise_linear_bf16 expects input (M,K), weight (N,K), out (M,N)")
61
+ return None
62
+
63
+
64
+ @torch.library.register_fake(add_op_namespace_prefix("int8_silu_gated_linear_bf16"))
65
+ def _int8_silu_gated_linear_bf16_fake(
66
+ input_i8: torch.Tensor,
67
+ up_weight_i8: torch.Tensor,
68
+ input_scale: torch.Tensor,
69
+ weight_scale: torch.Tensor,
70
+ gate: torch.Tensor,
71
+ out: torch.Tensor,
72
+ ) -> None:
73
+ return _int8_rowwise_linear_bf16_fake(input_i8, up_weight_i8, input_scale, weight_scale, out, 0)
74
+
75
+
76
+ def quantize_int8_static_bf16(
77
+ input: torch.Tensor, scale: torch.Tensor, *, out: Optional[torch.Tensor] = None
78
+ ) -> torch.Tensor:
79
+ if out is None:
80
+ out = torch.empty_like(input, dtype=torch.int8)
81
+ ops.quantize_int8_static_bf16(input, scale, out)
82
+ return out
83
+
84
+
85
+ def quantize_int8_rowwise_bf16(
86
+ input: torch.Tensor,
87
+ *,
88
+ out: Optional[torch.Tensor] = None,
89
+ scales: Optional[torch.Tensor] = None,
90
+ ) -> tuple[torch.Tensor, torch.Tensor]:
91
+ if out is None:
92
+ out = torch.empty_like(input, dtype=torch.int8)
93
+ if scales is None:
94
+ scales = torch.empty((input.shape[0],), device=input.device, dtype=torch.float32)
95
+ ops.quantize_int8_rowwise_bf16(input, out, scales)
96
+ return out, scales
97
+
98
+
99
+ def quantize_int8_rowwise_static_bf16(
100
+ input: torch.Tensor, scales: torch.Tensor, *, out: Optional[torch.Tensor] = None
101
+ ) -> torch.Tensor:
102
+ if out is None:
103
+ out = torch.empty_like(input, dtype=torch.int8)
104
+ ops.quantize_int8_rowwise_static_bf16(input, scales, out)
105
+ return out
106
+
107
+
108
+ def rms_norm_quantize_int8_rowwise_bf16(
109
+ x: torch.Tensor,
110
+ weight: torch.Tensor,
111
+ eps: float = 1e-6,
112
+ *,
113
+ out: Optional[torch.Tensor] = None,
114
+ scales: Optional[torch.Tensor] = None,
115
+ ) -> tuple[torch.Tensor, torch.Tensor]:
116
+ if out is None:
117
+ out = torch.empty_like(x, dtype=torch.int8)
118
+ if scales is None:
119
+ scales = torch.empty((x.shape[0],), device=x.device, dtype=torch.float32)
120
+ ops.rms_norm_quantize_int8_rowwise_bf16(x, weight, float(eps), out, scales)
121
+ return out, scales
122
+
123
+
124
+ def residual_add_rms_norm_quantize_int8_rowwise_bf16(
125
+ residual: torch.Tensor,
126
+ x: torch.Tensor,
127
+ weight: torch.Tensor,
128
+ eps: float = 1e-6,
129
+ *,
130
+ out: Optional[torch.Tensor] = None,
131
+ scales: Optional[torch.Tensor] = None,
132
+ ) -> tuple[torch.Tensor, torch.Tensor]:
133
+ if out is None:
134
+ out = torch.empty_like(residual, dtype=torch.int8)
135
+ if scales is None:
136
+ scales = torch.empty((residual.shape[0],), device=residual.device, dtype=torch.float32)
137
+ ops.residual_add_rms_norm_quantize_int8_rowwise_bf16(residual, x, weight, float(eps), out, scales)
138
+ return out, scales
139
+
140
+
141
+ def int8_rowwise_linear_bf16(
142
+ input_i8: torch.Tensor,
143
+ weight_i8: torch.Tensor,
144
+ input_scale: torch.Tensor,
145
+ weight_scale: torch.Tensor,
146
+ variant: int = 0,
147
+ *,
148
+ out: Optional[torch.Tensor] = None,
149
+ ) -> torch.Tensor:
150
+ if out is None:
151
+ out = torch.empty((input_i8.shape[0], weight_i8.shape[0]), device=input_i8.device, dtype=torch.bfloat16)
152
+ ops.int8_rowwise_linear_bf16(input_i8, weight_i8, input_scale, weight_scale, out, int(variant))
153
+ return out
154
+
155
+
156
+ def int8_silu_gated_linear_bf16(
157
+ input_i8: torch.Tensor,
158
+ up_weight_i8: torch.Tensor,
159
+ input_scale: torch.Tensor,
160
+ weight_scale: torch.Tensor,
161
+ gate: torch.Tensor,
162
+ *,
163
+ out: Optional[torch.Tensor] = None,
164
+ ) -> torch.Tensor:
165
+ if out is None:
166
+ out = torch.empty((input_i8.shape[0], up_weight_i8.shape[0]), device=input_i8.device, dtype=torch.bfloat16)
167
+ ops.int8_silu_gated_linear_bf16(input_i8, up_weight_i8, input_scale, weight_scale, gate, out)
168
+ return out
169
+
170
+
171
+ __all__ = [
172
+ "quantize_int8_static_bf16",
173
+ "quantize_int8_rowwise_bf16",
174
+ "quantize_int8_rowwise_static_bf16",
175
+ "rms_norm_quantize_int8_rowwise_bf16",
176
+ "residual_add_rms_norm_quantize_int8_rowwise_bf16",
177
+ "int8_rowwise_linear_bf16",
178
+ "int8_silu_gated_linear_bf16",
179
+ ]
build/torch211-cxx11-cu130-x86_64-linux/_int8_transformer_primitives_cuda_f5e354d.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ef01bcdf0ccbb8d2781aa01465696f1ef0866ecdf3eb20dd7dcd331fdcfc9278
3
+ size 1389176
build/torch211-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _int8_transformer_primitives_cuda_f5e354d
3
+ ops = torch.ops._int8_transformer_primitives_cuda_f5e354d
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_int8_transformer_primitives_cuda_f5e354d::{op_name}"
build/torch211-cxx11-cu130-x86_64-linux/int8_transformer_primitives/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "int8-transformer-primitives",
3
+ "id": "_int8_transformer_primitives_cuda_f5e354d",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "12.0a",
11
+ "8.0",
12
+ "8.6",
13
+ "8.9",
14
+ "9.0"
15
+ ]
16
+ }
17
+ }
build/torch212-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,179 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT INT8 transformer primitives."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Optional
6
+
7
+ import torch
8
+
9
+ from ._ops import add_op_namespace_prefix, ops
10
+
11
+
12
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_int8_static_bf16"))
13
+ def _quantize_int8_static_bf16_fake(input: torch.Tensor, scale: torch.Tensor, out: torch.Tensor) -> None:
14
+ if out.shape != input.shape:
15
+ raise RuntimeError("out must match input shape")
16
+ return None
17
+
18
+
19
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_int8_rowwise_bf16"))
20
+ def _quantize_int8_rowwise_bf16_fake(input: torch.Tensor, out: torch.Tensor, scales: torch.Tensor) -> None:
21
+ if input.dim() != 2 or out.shape != input.shape or scales.shape != (input.shape[0],):
22
+ raise RuntimeError("quantize_int8_rowwise_bf16 expects input/out (rows, cols), scales (rows,)")
23
+ return None
24
+
25
+
26
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_int8_rowwise_static_bf16"))
27
+ def _quantize_int8_rowwise_static_bf16_fake(input: torch.Tensor, scales: torch.Tensor, out: torch.Tensor) -> None:
28
+ return _quantize_int8_rowwise_bf16_fake(input, out, scales)
29
+
30
+
31
+ @torch.library.register_fake(add_op_namespace_prefix("rms_norm_quantize_int8_rowwise_bf16"))
32
+ def _rms_norm_quantize_int8_rowwise_bf16_fake(
33
+ x: torch.Tensor, weight: torch.Tensor, eps: float, out: torch.Tensor, scales: torch.Tensor
34
+ ) -> None:
35
+ return _quantize_int8_rowwise_bf16_fake(x, out, scales)
36
+
37
+
38
+ @torch.library.register_fake(add_op_namespace_prefix("residual_add_rms_norm_quantize_int8_rowwise_bf16"))
39
+ def _residual_add_rms_norm_quantize_int8_rowwise_bf16_fake(
40
+ residual: torch.Tensor,
41
+ x: torch.Tensor,
42
+ weight: torch.Tensor,
43
+ eps: float,
44
+ out: torch.Tensor,
45
+ scales: torch.Tensor,
46
+ ) -> None:
47
+ return _quantize_int8_rowwise_bf16_fake(residual, out, scales)
48
+
49
+
50
+ @torch.library.register_fake(add_op_namespace_prefix("int8_rowwise_linear_bf16"))
51
+ def _int8_rowwise_linear_bf16_fake(
52
+ input_i8: torch.Tensor,
53
+ weight_i8: torch.Tensor,
54
+ input_scale: torch.Tensor,
55
+ weight_scale: torch.Tensor,
56
+ out: torch.Tensor,
57
+ variant: int,
58
+ ) -> None:
59
+ if input_i8.dim() != 2 or weight_i8.dim() != 2 or out.shape != (input_i8.shape[0], weight_i8.shape[0]):
60
+ raise RuntimeError("int8_rowwise_linear_bf16 expects input (M,K), weight (N,K), out (M,N)")
61
+ return None
62
+
63
+
64
+ @torch.library.register_fake(add_op_namespace_prefix("int8_silu_gated_linear_bf16"))
65
+ def _int8_silu_gated_linear_bf16_fake(
66
+ input_i8: torch.Tensor,
67
+ up_weight_i8: torch.Tensor,
68
+ input_scale: torch.Tensor,
69
+ weight_scale: torch.Tensor,
70
+ gate: torch.Tensor,
71
+ out: torch.Tensor,
72
+ ) -> None:
73
+ return _int8_rowwise_linear_bf16_fake(input_i8, up_weight_i8, input_scale, weight_scale, out, 0)
74
+
75
+
76
+ def quantize_int8_static_bf16(
77
+ input: torch.Tensor, scale: torch.Tensor, *, out: Optional[torch.Tensor] = None
78
+ ) -> torch.Tensor:
79
+ if out is None:
80
+ out = torch.empty_like(input, dtype=torch.int8)
81
+ ops.quantize_int8_static_bf16(input, scale, out)
82
+ return out
83
+
84
+
85
+ def quantize_int8_rowwise_bf16(
86
+ input: torch.Tensor,
87
+ *,
88
+ out: Optional[torch.Tensor] = None,
89
+ scales: Optional[torch.Tensor] = None,
90
+ ) -> tuple[torch.Tensor, torch.Tensor]:
91
+ if out is None:
92
+ out = torch.empty_like(input, dtype=torch.int8)
93
+ if scales is None:
94
+ scales = torch.empty((input.shape[0],), device=input.device, dtype=torch.float32)
95
+ ops.quantize_int8_rowwise_bf16(input, out, scales)
96
+ return out, scales
97
+
98
+
99
+ def quantize_int8_rowwise_static_bf16(
100
+ input: torch.Tensor, scales: torch.Tensor, *, out: Optional[torch.Tensor] = None
101
+ ) -> torch.Tensor:
102
+ if out is None:
103
+ out = torch.empty_like(input, dtype=torch.int8)
104
+ ops.quantize_int8_rowwise_static_bf16(input, scales, out)
105
+ return out
106
+
107
+
108
+ def rms_norm_quantize_int8_rowwise_bf16(
109
+ x: torch.Tensor,
110
+ weight: torch.Tensor,
111
+ eps: float = 1e-6,
112
+ *,
113
+ out: Optional[torch.Tensor] = None,
114
+ scales: Optional[torch.Tensor] = None,
115
+ ) -> tuple[torch.Tensor, torch.Tensor]:
116
+ if out is None:
117
+ out = torch.empty_like(x, dtype=torch.int8)
118
+ if scales is None:
119
+ scales = torch.empty((x.shape[0],), device=x.device, dtype=torch.float32)
120
+ ops.rms_norm_quantize_int8_rowwise_bf16(x, weight, float(eps), out, scales)
121
+ return out, scales
122
+
123
+
124
+ def residual_add_rms_norm_quantize_int8_rowwise_bf16(
125
+ residual: torch.Tensor,
126
+ x: torch.Tensor,
127
+ weight: torch.Tensor,
128
+ eps: float = 1e-6,
129
+ *,
130
+ out: Optional[torch.Tensor] = None,
131
+ scales: Optional[torch.Tensor] = None,
132
+ ) -> tuple[torch.Tensor, torch.Tensor]:
133
+ if out is None:
134
+ out = torch.empty_like(residual, dtype=torch.int8)
135
+ if scales is None:
136
+ scales = torch.empty((residual.shape[0],), device=residual.device, dtype=torch.float32)
137
+ ops.residual_add_rms_norm_quantize_int8_rowwise_bf16(residual, x, weight, float(eps), out, scales)
138
+ return out, scales
139
+
140
+
141
+ def int8_rowwise_linear_bf16(
142
+ input_i8: torch.Tensor,
143
+ weight_i8: torch.Tensor,
144
+ input_scale: torch.Tensor,
145
+ weight_scale: torch.Tensor,
146
+ variant: int = 0,
147
+ *,
148
+ out: Optional[torch.Tensor] = None,
149
+ ) -> torch.Tensor:
150
+ if out is None:
151
+ out = torch.empty((input_i8.shape[0], weight_i8.shape[0]), device=input_i8.device, dtype=torch.bfloat16)
152
+ ops.int8_rowwise_linear_bf16(input_i8, weight_i8, input_scale, weight_scale, out, int(variant))
153
+ return out
154
+
155
+
156
+ def int8_silu_gated_linear_bf16(
157
+ input_i8: torch.Tensor,
158
+ up_weight_i8: torch.Tensor,
159
+ input_scale: torch.Tensor,
160
+ weight_scale: torch.Tensor,
161
+ gate: torch.Tensor,
162
+ *,
163
+ out: Optional[torch.Tensor] = None,
164
+ ) -> torch.Tensor:
165
+ if out is None:
166
+ out = torch.empty((input_i8.shape[0], up_weight_i8.shape[0]), device=input_i8.device, dtype=torch.bfloat16)
167
+ ops.int8_silu_gated_linear_bf16(input_i8, up_weight_i8, input_scale, weight_scale, gate, out)
168
+ return out
169
+
170
+
171
+ __all__ = [
172
+ "quantize_int8_static_bf16",
173
+ "quantize_int8_rowwise_bf16",
174
+ "quantize_int8_rowwise_static_bf16",
175
+ "rms_norm_quantize_int8_rowwise_bf16",
176
+ "residual_add_rms_norm_quantize_int8_rowwise_bf16",
177
+ "int8_rowwise_linear_bf16",
178
+ "int8_silu_gated_linear_bf16",
179
+ ]
build/torch212-cxx11-cu130-x86_64-linux/_int8_transformer_primitives_cuda_f5e354d.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:34315262fe72d1edc99517434e91def1b4a4c7897c2e04871a731de9156ca5d5
3
+ size 1395440
build/torch212-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _int8_transformer_primitives_cuda_f5e354d
3
+ ops = torch.ops._int8_transformer_primitives_cuda_f5e354d
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_int8_transformer_primitives_cuda_f5e354d::{op_name}"
build/torch212-cxx11-cu130-x86_64-linux/int8_transformer_primitives/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "int8-transformer-primitives",
3
+ "id": "_int8_transformer_primitives_cuda_f5e354d",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "12.0a",
11
+ "8.0",
12
+ "8.6",
13
+ "8.9",
14
+ "9.0"
15
+ ]
16
+ }
17
+ }
build/torch212-cxx11-cu132-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,179 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT INT8 transformer primitives."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Optional
6
+
7
+ import torch
8
+
9
+ from ._ops import add_op_namespace_prefix, ops
10
+
11
+
12
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_int8_static_bf16"))
13
+ def _quantize_int8_static_bf16_fake(input: torch.Tensor, scale: torch.Tensor, out: torch.Tensor) -> None:
14
+ if out.shape != input.shape:
15
+ raise RuntimeError("out must match input shape")
16
+ return None
17
+
18
+
19
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_int8_rowwise_bf16"))
20
+ def _quantize_int8_rowwise_bf16_fake(input: torch.Tensor, out: torch.Tensor, scales: torch.Tensor) -> None:
21
+ if input.dim() != 2 or out.shape != input.shape or scales.shape != (input.shape[0],):
22
+ raise RuntimeError("quantize_int8_rowwise_bf16 expects input/out (rows, cols), scales (rows,)")
23
+ return None
24
+
25
+
26
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_int8_rowwise_static_bf16"))
27
+ def _quantize_int8_rowwise_static_bf16_fake(input: torch.Tensor, scales: torch.Tensor, out: torch.Tensor) -> None:
28
+ return _quantize_int8_rowwise_bf16_fake(input, out, scales)
29
+
30
+
31
+ @torch.library.register_fake(add_op_namespace_prefix("rms_norm_quantize_int8_rowwise_bf16"))
32
+ def _rms_norm_quantize_int8_rowwise_bf16_fake(
33
+ x: torch.Tensor, weight: torch.Tensor, eps: float, out: torch.Tensor, scales: torch.Tensor
34
+ ) -> None:
35
+ return _quantize_int8_rowwise_bf16_fake(x, out, scales)
36
+
37
+
38
+ @torch.library.register_fake(add_op_namespace_prefix("residual_add_rms_norm_quantize_int8_rowwise_bf16"))
39
+ def _residual_add_rms_norm_quantize_int8_rowwise_bf16_fake(
40
+ residual: torch.Tensor,
41
+ x: torch.Tensor,
42
+ weight: torch.Tensor,
43
+ eps: float,
44
+ out: torch.Tensor,
45
+ scales: torch.Tensor,
46
+ ) -> None:
47
+ return _quantize_int8_rowwise_bf16_fake(residual, out, scales)
48
+
49
+
50
+ @torch.library.register_fake(add_op_namespace_prefix("int8_rowwise_linear_bf16"))
51
+ def _int8_rowwise_linear_bf16_fake(
52
+ input_i8: torch.Tensor,
53
+ weight_i8: torch.Tensor,
54
+ input_scale: torch.Tensor,
55
+ weight_scale: torch.Tensor,
56
+ out: torch.Tensor,
57
+ variant: int,
58
+ ) -> None:
59
+ if input_i8.dim() != 2 or weight_i8.dim() != 2 or out.shape != (input_i8.shape[0], weight_i8.shape[0]):
60
+ raise RuntimeError("int8_rowwise_linear_bf16 expects input (M,K), weight (N,K), out (M,N)")
61
+ return None
62
+
63
+
64
+ @torch.library.register_fake(add_op_namespace_prefix("int8_silu_gated_linear_bf16"))
65
+ def _int8_silu_gated_linear_bf16_fake(
66
+ input_i8: torch.Tensor,
67
+ up_weight_i8: torch.Tensor,
68
+ input_scale: torch.Tensor,
69
+ weight_scale: torch.Tensor,
70
+ gate: torch.Tensor,
71
+ out: torch.Tensor,
72
+ ) -> None:
73
+ return _int8_rowwise_linear_bf16_fake(input_i8, up_weight_i8, input_scale, weight_scale, out, 0)
74
+
75
+
76
+ def quantize_int8_static_bf16(
77
+ input: torch.Tensor, scale: torch.Tensor, *, out: Optional[torch.Tensor] = None
78
+ ) -> torch.Tensor:
79
+ if out is None:
80
+ out = torch.empty_like(input, dtype=torch.int8)
81
+ ops.quantize_int8_static_bf16(input, scale, out)
82
+ return out
83
+
84
+
85
+ def quantize_int8_rowwise_bf16(
86
+ input: torch.Tensor,
87
+ *,
88
+ out: Optional[torch.Tensor] = None,
89
+ scales: Optional[torch.Tensor] = None,
90
+ ) -> tuple[torch.Tensor, torch.Tensor]:
91
+ if out is None:
92
+ out = torch.empty_like(input, dtype=torch.int8)
93
+ if scales is None:
94
+ scales = torch.empty((input.shape[0],), device=input.device, dtype=torch.float32)
95
+ ops.quantize_int8_rowwise_bf16(input, out, scales)
96
+ return out, scales
97
+
98
+
99
+ def quantize_int8_rowwise_static_bf16(
100
+ input: torch.Tensor, scales: torch.Tensor, *, out: Optional[torch.Tensor] = None
101
+ ) -> torch.Tensor:
102
+ if out is None:
103
+ out = torch.empty_like(input, dtype=torch.int8)
104
+ ops.quantize_int8_rowwise_static_bf16(input, scales, out)
105
+ return out
106
+
107
+
108
+ def rms_norm_quantize_int8_rowwise_bf16(
109
+ x: torch.Tensor,
110
+ weight: torch.Tensor,
111
+ eps: float = 1e-6,
112
+ *,
113
+ out: Optional[torch.Tensor] = None,
114
+ scales: Optional[torch.Tensor] = None,
115
+ ) -> tuple[torch.Tensor, torch.Tensor]:
116
+ if out is None:
117
+ out = torch.empty_like(x, dtype=torch.int8)
118
+ if scales is None:
119
+ scales = torch.empty((x.shape[0],), device=x.device, dtype=torch.float32)
120
+ ops.rms_norm_quantize_int8_rowwise_bf16(x, weight, float(eps), out, scales)
121
+ return out, scales
122
+
123
+
124
+ def residual_add_rms_norm_quantize_int8_rowwise_bf16(
125
+ residual: torch.Tensor,
126
+ x: torch.Tensor,
127
+ weight: torch.Tensor,
128
+ eps: float = 1e-6,
129
+ *,
130
+ out: Optional[torch.Tensor] = None,
131
+ scales: Optional[torch.Tensor] = None,
132
+ ) -> tuple[torch.Tensor, torch.Tensor]:
133
+ if out is None:
134
+ out = torch.empty_like(residual, dtype=torch.int8)
135
+ if scales is None:
136
+ scales = torch.empty((residual.shape[0],), device=residual.device, dtype=torch.float32)
137
+ ops.residual_add_rms_norm_quantize_int8_rowwise_bf16(residual, x, weight, float(eps), out, scales)
138
+ return out, scales
139
+
140
+
141
+ def int8_rowwise_linear_bf16(
142
+ input_i8: torch.Tensor,
143
+ weight_i8: torch.Tensor,
144
+ input_scale: torch.Tensor,
145
+ weight_scale: torch.Tensor,
146
+ variant: int = 0,
147
+ *,
148
+ out: Optional[torch.Tensor] = None,
149
+ ) -> torch.Tensor:
150
+ if out is None:
151
+ out = torch.empty((input_i8.shape[0], weight_i8.shape[0]), device=input_i8.device, dtype=torch.bfloat16)
152
+ ops.int8_rowwise_linear_bf16(input_i8, weight_i8, input_scale, weight_scale, out, int(variant))
153
+ return out
154
+
155
+
156
+ def int8_silu_gated_linear_bf16(
157
+ input_i8: torch.Tensor,
158
+ up_weight_i8: torch.Tensor,
159
+ input_scale: torch.Tensor,
160
+ weight_scale: torch.Tensor,
161
+ gate: torch.Tensor,
162
+ *,
163
+ out: Optional[torch.Tensor] = None,
164
+ ) -> torch.Tensor:
165
+ if out is None:
166
+ out = torch.empty((input_i8.shape[0], up_weight_i8.shape[0]), device=input_i8.device, dtype=torch.bfloat16)
167
+ ops.int8_silu_gated_linear_bf16(input_i8, up_weight_i8, input_scale, weight_scale, gate, out)
168
+ return out
169
+
170
+
171
+ __all__ = [
172
+ "quantize_int8_static_bf16",
173
+ "quantize_int8_rowwise_bf16",
174
+ "quantize_int8_rowwise_static_bf16",
175
+ "rms_norm_quantize_int8_rowwise_bf16",
176
+ "residual_add_rms_norm_quantize_int8_rowwise_bf16",
177
+ "int8_rowwise_linear_bf16",
178
+ "int8_silu_gated_linear_bf16",
179
+ ]
build/torch212-cxx11-cu132-x86_64-linux/_int8_transformer_primitives_cuda_f5e354d.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0984aad643982134f36895a25bc0b5a5c3846c8bd8565d5044ff370326a651ed
3
+ size 1387304
build/torch212-cxx11-cu132-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _int8_transformer_primitives_cuda_f5e354d
3
+ ops = torch.ops._int8_transformer_primitives_cuda_f5e354d
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_int8_transformer_primitives_cuda_f5e354d::{op_name}"
build/torch212-cxx11-cu132-x86_64-linux/int8_transformer_primitives/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-cu132-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "int8-transformer-primitives",
3
+ "id": "_int8_transformer_primitives_cuda_f5e354d",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "12.0a",
11
+ "8.0",
12
+ "8.6",
13
+ "8.9",
14
+ "9.0"
15
+ ]
16
+ }
17
+ }