Uploaded using `kernel-builder`.
Browse files- benchmarks/benchmark.py +105 -0
- build/torch211-cxx11-cu128-x86_64-linux/__init__.py +179 -0
- build/torch211-cxx11-cu128-x86_64-linux/_int8_transformer_primitives_cuda_f5e354d.abi3.so +3 -0
- build/torch211-cxx11-cu128-x86_64-linux/_ops.py +9 -0
- build/torch211-cxx11-cu128-x86_64-linux/int8_transformer_primitives/__init__.py +26 -0
- build/torch211-cxx11-cu128-x86_64-linux/metadata.json +17 -0
- build/torch211-cxx11-cu130-x86_64-linux/__init__.py +179 -0
- build/torch211-cxx11-cu130-x86_64-linux/_int8_transformer_primitives_cuda_f5e354d.abi3.so +3 -0
- build/torch211-cxx11-cu130-x86_64-linux/_ops.py +9 -0
- build/torch211-cxx11-cu130-x86_64-linux/int8_transformer_primitives/__init__.py +26 -0
- build/torch211-cxx11-cu130-x86_64-linux/metadata.json +17 -0
- build/torch212-cxx11-cu130-x86_64-linux/__init__.py +179 -0
- build/torch212-cxx11-cu130-x86_64-linux/_int8_transformer_primitives_cuda_f5e354d.abi3.so +3 -0
- build/torch212-cxx11-cu130-x86_64-linux/_ops.py +9 -0
- build/torch212-cxx11-cu130-x86_64-linux/int8_transformer_primitives/__init__.py +26 -0
- build/torch212-cxx11-cu130-x86_64-linux/metadata.json +17 -0
- build/torch212-cxx11-cu132-x86_64-linux/__init__.py +179 -0
- build/torch212-cxx11-cu132-x86_64-linux/_int8_transformer_primitives_cuda_f5e354d.abi3.so +3 -0
- build/torch212-cxx11-cu132-x86_64-linux/_ops.py +9 -0
- build/torch212-cxx11-cu132-x86_64-linux/int8_transformer_primitives/__init__.py +26 -0
- build/torch212-cxx11-cu132-x86_64-linux/metadata.json +17 -0
benchmarks/benchmark.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Benchmark INT8 transformer primitives."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import importlib
|
| 8 |
+
import sys
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
|
| 13 |
+
ROOT = Path(__file__).resolve().parents[2]
|
| 14 |
+
sys.path.insert(0, str(ROOT / "int8-transformer-primitives" / "tests"))
|
| 15 |
+
from test_int8_transformer_primitives import load_source_ops # noqa: E402
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def load_ops(backend: str, artifact: str | None):
|
| 19 |
+
if backend == "source":
|
| 20 |
+
return load_source_ops()
|
| 21 |
+
if artifact:
|
| 22 |
+
sys.path.insert(0, artifact)
|
| 23 |
+
try:
|
| 24 |
+
return importlib.import_module("int8_transformer_primitives")
|
| 25 |
+
finally:
|
| 26 |
+
if artifact:
|
| 27 |
+
sys.path.remove(artifact)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def time_us(fn, warmup: int, iters: int) -> float:
|
| 31 |
+
for _ in range(warmup):
|
| 32 |
+
fn()
|
| 33 |
+
torch.cuda.synchronize()
|
| 34 |
+
start = torch.cuda.Event(enable_timing=True)
|
| 35 |
+
end = torch.cuda.Event(enable_timing=True)
|
| 36 |
+
start.record()
|
| 37 |
+
for _ in range(iters):
|
| 38 |
+
fn()
|
| 39 |
+
end.record()
|
| 40 |
+
torch.cuda.synchronize()
|
| 41 |
+
return start.elapsed_time(end) * 1000.0 / iters
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def main() -> int:
|
| 45 |
+
parser = argparse.ArgumentParser()
|
| 46 |
+
parser.add_argument("--backend", choices=["source", "installed"], default="source")
|
| 47 |
+
parser.add_argument("--artifact", default=None)
|
| 48 |
+
parser.add_argument("--mode", choices=["headline", "full"], default="headline")
|
| 49 |
+
parser.add_argument("--warmup", type=int, default=20)
|
| 50 |
+
parser.add_argument("--iters", type=int, default=100)
|
| 51 |
+
args = parser.parse_args()
|
| 52 |
+
ops = load_ops(args.backend, args.artifact)
|
| 53 |
+
|
| 54 |
+
shapes = [
|
| 55 |
+
("decode_m8", 8, 1024, 2560),
|
| 56 |
+
("small_batch", 64, 2048, 8192),
|
| 57 |
+
("vision_prefill", 522, 2048, 2560),
|
| 58 |
+
]
|
| 59 |
+
if args.mode == "full":
|
| 60 |
+
shapes += [
|
| 61 |
+
("m1", 1, 1024, 1024),
|
| 62 |
+
("m17", 17, 256, 256),
|
| 63 |
+
("wide_ffn", 257, 2048, 8192),
|
| 64 |
+
]
|
| 65 |
+
|
| 66 |
+
print("workload,M,K,N,op,flashrt_us,torch_eager_us,speedup")
|
| 67 |
+
for name, m, k, n in shapes:
|
| 68 |
+
x = (torch.randn((m, k), device="cuda") * 0.5).to(torch.bfloat16)
|
| 69 |
+
w = (torch.randn((n, k), device="cuda") * 0.5).to(torch.bfloat16)
|
| 70 |
+
x_i8, x_scale = ops.quantize_int8_rowwise_bf16(x)
|
| 71 |
+
w_i8, w_scale = ops.quantize_int8_rowwise_bf16(w)
|
| 72 |
+
torch.cuda.synchronize()
|
| 73 |
+
out = torch.empty((m, n), device="cuda", dtype=torch.bfloat16)
|
| 74 |
+
|
| 75 |
+
def flash():
|
| 76 |
+
ops.int8_rowwise_linear_bf16(x_i8, w_i8, x_scale, w_scale, out=out)
|
| 77 |
+
|
| 78 |
+
def eager():
|
| 79 |
+
((x_i8.float() @ w_i8.float().t()) * x_scale[:, None] * w_scale[None, :]).to(torch.bfloat16)
|
| 80 |
+
|
| 81 |
+
fu = time_us(flash, args.warmup, args.iters)
|
| 82 |
+
eu = time_us(eager, max(5, args.warmup // 2), max(20, args.iters // 2))
|
| 83 |
+
print(f"{name},{m},{k},{n},int8_rowwise_linear_bf16,{fu:.3f},{eu:.3f},{eu/fu:.2f}x")
|
| 84 |
+
|
| 85 |
+
q = (torch.randn((522, 2048), device="cuda") * 0.5).to(torch.bfloat16)
|
| 86 |
+
weight = torch.randn((2048,), device="cuda", dtype=torch.bfloat16)
|
| 87 |
+
out = torch.empty_like(q, dtype=torch.int8)
|
| 88 |
+
scales = torch.empty((q.shape[0],), device="cuda", dtype=torch.float32)
|
| 89 |
+
|
| 90 |
+
def flash_rms():
|
| 91 |
+
ops.rms_norm_quantize_int8_rowwise_bf16(q, weight, out=out, scales=scales)
|
| 92 |
+
|
| 93 |
+
def eager_rms():
|
| 94 |
+
y = q.float() * torch.rsqrt((q.float() * q.float()).mean(dim=1, keepdim=True) + 1e-6) * weight.float()
|
| 95 |
+
s = torch.clamp(y.abs().amax(dim=1) / 127.0, min=1e-10)
|
| 96 |
+
torch.clamp(torch.round(y / s[:, None]), -127, 127).to(torch.int8)
|
| 97 |
+
|
| 98 |
+
fu = time_us(flash_rms, args.warmup, args.iters)
|
| 99 |
+
eu = time_us(eager_rms, max(5, args.warmup // 2), max(20, args.iters // 2))
|
| 100 |
+
print(f"vision_prefill,522,2048,0,rms_norm_quantize_int8_rowwise_bf16,{fu:.3f},{eu:.3f},{eu/fu:.2f}x")
|
| 101 |
+
return 0
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
if __name__ == "__main__":
|
| 105 |
+
raise SystemExit(main())
|
build/torch211-cxx11-cu128-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT INT8 transformer primitives."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Optional
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
from ._ops import add_op_namespace_prefix, ops
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@torch.library.register_fake(add_op_namespace_prefix("quantize_int8_static_bf16"))
|
| 13 |
+
def _quantize_int8_static_bf16_fake(input: torch.Tensor, scale: torch.Tensor, out: torch.Tensor) -> None:
|
| 14 |
+
if out.shape != input.shape:
|
| 15 |
+
raise RuntimeError("out must match input shape")
|
| 16 |
+
return None
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@torch.library.register_fake(add_op_namespace_prefix("quantize_int8_rowwise_bf16"))
|
| 20 |
+
def _quantize_int8_rowwise_bf16_fake(input: torch.Tensor, out: torch.Tensor, scales: torch.Tensor) -> None:
|
| 21 |
+
if input.dim() != 2 or out.shape != input.shape or scales.shape != (input.shape[0],):
|
| 22 |
+
raise RuntimeError("quantize_int8_rowwise_bf16 expects input/out (rows, cols), scales (rows,)")
|
| 23 |
+
return None
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
@torch.library.register_fake(add_op_namespace_prefix("quantize_int8_rowwise_static_bf16"))
|
| 27 |
+
def _quantize_int8_rowwise_static_bf16_fake(input: torch.Tensor, scales: torch.Tensor, out: torch.Tensor) -> None:
|
| 28 |
+
return _quantize_int8_rowwise_bf16_fake(input, out, scales)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
@torch.library.register_fake(add_op_namespace_prefix("rms_norm_quantize_int8_rowwise_bf16"))
|
| 32 |
+
def _rms_norm_quantize_int8_rowwise_bf16_fake(
|
| 33 |
+
x: torch.Tensor, weight: torch.Tensor, eps: float, out: torch.Tensor, scales: torch.Tensor
|
| 34 |
+
) -> None:
|
| 35 |
+
return _quantize_int8_rowwise_bf16_fake(x, out, scales)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
@torch.library.register_fake(add_op_namespace_prefix("residual_add_rms_norm_quantize_int8_rowwise_bf16"))
|
| 39 |
+
def _residual_add_rms_norm_quantize_int8_rowwise_bf16_fake(
|
| 40 |
+
residual: torch.Tensor,
|
| 41 |
+
x: torch.Tensor,
|
| 42 |
+
weight: torch.Tensor,
|
| 43 |
+
eps: float,
|
| 44 |
+
out: torch.Tensor,
|
| 45 |
+
scales: torch.Tensor,
|
| 46 |
+
) -> None:
|
| 47 |
+
return _quantize_int8_rowwise_bf16_fake(residual, out, scales)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
@torch.library.register_fake(add_op_namespace_prefix("int8_rowwise_linear_bf16"))
|
| 51 |
+
def _int8_rowwise_linear_bf16_fake(
|
| 52 |
+
input_i8: torch.Tensor,
|
| 53 |
+
weight_i8: torch.Tensor,
|
| 54 |
+
input_scale: torch.Tensor,
|
| 55 |
+
weight_scale: torch.Tensor,
|
| 56 |
+
out: torch.Tensor,
|
| 57 |
+
variant: int,
|
| 58 |
+
) -> None:
|
| 59 |
+
if input_i8.dim() != 2 or weight_i8.dim() != 2 or out.shape != (input_i8.shape[0], weight_i8.shape[0]):
|
| 60 |
+
raise RuntimeError("int8_rowwise_linear_bf16 expects input (M,K), weight (N,K), out (M,N)")
|
| 61 |
+
return None
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
@torch.library.register_fake(add_op_namespace_prefix("int8_silu_gated_linear_bf16"))
|
| 65 |
+
def _int8_silu_gated_linear_bf16_fake(
|
| 66 |
+
input_i8: torch.Tensor,
|
| 67 |
+
up_weight_i8: torch.Tensor,
|
| 68 |
+
input_scale: torch.Tensor,
|
| 69 |
+
weight_scale: torch.Tensor,
|
| 70 |
+
gate: torch.Tensor,
|
| 71 |
+
out: torch.Tensor,
|
| 72 |
+
) -> None:
|
| 73 |
+
return _int8_rowwise_linear_bf16_fake(input_i8, up_weight_i8, input_scale, weight_scale, out, 0)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def quantize_int8_static_bf16(
|
| 77 |
+
input: torch.Tensor, scale: torch.Tensor, *, out: Optional[torch.Tensor] = None
|
| 78 |
+
) -> torch.Tensor:
|
| 79 |
+
if out is None:
|
| 80 |
+
out = torch.empty_like(input, dtype=torch.int8)
|
| 81 |
+
ops.quantize_int8_static_bf16(input, scale, out)
|
| 82 |
+
return out
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def quantize_int8_rowwise_bf16(
|
| 86 |
+
input: torch.Tensor,
|
| 87 |
+
*,
|
| 88 |
+
out: Optional[torch.Tensor] = None,
|
| 89 |
+
scales: Optional[torch.Tensor] = None,
|
| 90 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 91 |
+
if out is None:
|
| 92 |
+
out = torch.empty_like(input, dtype=torch.int8)
|
| 93 |
+
if scales is None:
|
| 94 |
+
scales = torch.empty((input.shape[0],), device=input.device, dtype=torch.float32)
|
| 95 |
+
ops.quantize_int8_rowwise_bf16(input, out, scales)
|
| 96 |
+
return out, scales
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def quantize_int8_rowwise_static_bf16(
|
| 100 |
+
input: torch.Tensor, scales: torch.Tensor, *, out: Optional[torch.Tensor] = None
|
| 101 |
+
) -> torch.Tensor:
|
| 102 |
+
if out is None:
|
| 103 |
+
out = torch.empty_like(input, dtype=torch.int8)
|
| 104 |
+
ops.quantize_int8_rowwise_static_bf16(input, scales, out)
|
| 105 |
+
return out
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def rms_norm_quantize_int8_rowwise_bf16(
|
| 109 |
+
x: torch.Tensor,
|
| 110 |
+
weight: torch.Tensor,
|
| 111 |
+
eps: float = 1e-6,
|
| 112 |
+
*,
|
| 113 |
+
out: Optional[torch.Tensor] = None,
|
| 114 |
+
scales: Optional[torch.Tensor] = None,
|
| 115 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 116 |
+
if out is None:
|
| 117 |
+
out = torch.empty_like(x, dtype=torch.int8)
|
| 118 |
+
if scales is None:
|
| 119 |
+
scales = torch.empty((x.shape[0],), device=x.device, dtype=torch.float32)
|
| 120 |
+
ops.rms_norm_quantize_int8_rowwise_bf16(x, weight, float(eps), out, scales)
|
| 121 |
+
return out, scales
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def residual_add_rms_norm_quantize_int8_rowwise_bf16(
|
| 125 |
+
residual: torch.Tensor,
|
| 126 |
+
x: torch.Tensor,
|
| 127 |
+
weight: torch.Tensor,
|
| 128 |
+
eps: float = 1e-6,
|
| 129 |
+
*,
|
| 130 |
+
out: Optional[torch.Tensor] = None,
|
| 131 |
+
scales: Optional[torch.Tensor] = None,
|
| 132 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 133 |
+
if out is None:
|
| 134 |
+
out = torch.empty_like(residual, dtype=torch.int8)
|
| 135 |
+
if scales is None:
|
| 136 |
+
scales = torch.empty((residual.shape[0],), device=residual.device, dtype=torch.float32)
|
| 137 |
+
ops.residual_add_rms_norm_quantize_int8_rowwise_bf16(residual, x, weight, float(eps), out, scales)
|
| 138 |
+
return out, scales
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def int8_rowwise_linear_bf16(
|
| 142 |
+
input_i8: torch.Tensor,
|
| 143 |
+
weight_i8: torch.Tensor,
|
| 144 |
+
input_scale: torch.Tensor,
|
| 145 |
+
weight_scale: torch.Tensor,
|
| 146 |
+
variant: int = 0,
|
| 147 |
+
*,
|
| 148 |
+
out: Optional[torch.Tensor] = None,
|
| 149 |
+
) -> torch.Tensor:
|
| 150 |
+
if out is None:
|
| 151 |
+
out = torch.empty((input_i8.shape[0], weight_i8.shape[0]), device=input_i8.device, dtype=torch.bfloat16)
|
| 152 |
+
ops.int8_rowwise_linear_bf16(input_i8, weight_i8, input_scale, weight_scale, out, int(variant))
|
| 153 |
+
return out
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def int8_silu_gated_linear_bf16(
|
| 157 |
+
input_i8: torch.Tensor,
|
| 158 |
+
up_weight_i8: torch.Tensor,
|
| 159 |
+
input_scale: torch.Tensor,
|
| 160 |
+
weight_scale: torch.Tensor,
|
| 161 |
+
gate: torch.Tensor,
|
| 162 |
+
*,
|
| 163 |
+
out: Optional[torch.Tensor] = None,
|
| 164 |
+
) -> torch.Tensor:
|
| 165 |
+
if out is None:
|
| 166 |
+
out = torch.empty((input_i8.shape[0], up_weight_i8.shape[0]), device=input_i8.device, dtype=torch.bfloat16)
|
| 167 |
+
ops.int8_silu_gated_linear_bf16(input_i8, up_weight_i8, input_scale, weight_scale, gate, out)
|
| 168 |
+
return out
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
__all__ = [
|
| 172 |
+
"quantize_int8_static_bf16",
|
| 173 |
+
"quantize_int8_rowwise_bf16",
|
| 174 |
+
"quantize_int8_rowwise_static_bf16",
|
| 175 |
+
"rms_norm_quantize_int8_rowwise_bf16",
|
| 176 |
+
"residual_add_rms_norm_quantize_int8_rowwise_bf16",
|
| 177 |
+
"int8_rowwise_linear_bf16",
|
| 178 |
+
"int8_silu_gated_linear_bf16",
|
| 179 |
+
]
|
build/torch211-cxx11-cu128-x86_64-linux/_int8_transformer_primitives_cuda_f5e354d.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5d645ffda89e6448384e5abd28f736b4a1ad049805fcb2e405f12dc6b1890345
|
| 3 |
+
size 1336536
|
build/torch211-cxx11-cu128-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _int8_transformer_primitives_cuda_f5e354d
|
| 3 |
+
ops = torch.ops._int8_transformer_primitives_cuda_f5e354d
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_int8_transformer_primitives_cuda_f5e354d::{op_name}"
|
build/torch211-cxx11-cu128-x86_64-linux/int8_transformer_primitives/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch211-cxx11-cu128-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "int8-transformer-primitives",
|
| 3 |
+
"id": "_int8_transformer_primitives_cuda_f5e354d",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"12.0a",
|
| 11 |
+
"8.0",
|
| 12 |
+
"8.6",
|
| 13 |
+
"8.9",
|
| 14 |
+
"9.0"
|
| 15 |
+
]
|
| 16 |
+
}
|
| 17 |
+
}
|
build/torch211-cxx11-cu130-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT INT8 transformer primitives."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Optional
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
from ._ops import add_op_namespace_prefix, ops
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@torch.library.register_fake(add_op_namespace_prefix("quantize_int8_static_bf16"))
|
| 13 |
+
def _quantize_int8_static_bf16_fake(input: torch.Tensor, scale: torch.Tensor, out: torch.Tensor) -> None:
|
| 14 |
+
if out.shape != input.shape:
|
| 15 |
+
raise RuntimeError("out must match input shape")
|
| 16 |
+
return None
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@torch.library.register_fake(add_op_namespace_prefix("quantize_int8_rowwise_bf16"))
|
| 20 |
+
def _quantize_int8_rowwise_bf16_fake(input: torch.Tensor, out: torch.Tensor, scales: torch.Tensor) -> None:
|
| 21 |
+
if input.dim() != 2 or out.shape != input.shape or scales.shape != (input.shape[0],):
|
| 22 |
+
raise RuntimeError("quantize_int8_rowwise_bf16 expects input/out (rows, cols), scales (rows,)")
|
| 23 |
+
return None
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
@torch.library.register_fake(add_op_namespace_prefix("quantize_int8_rowwise_static_bf16"))
|
| 27 |
+
def _quantize_int8_rowwise_static_bf16_fake(input: torch.Tensor, scales: torch.Tensor, out: torch.Tensor) -> None:
|
| 28 |
+
return _quantize_int8_rowwise_bf16_fake(input, out, scales)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
@torch.library.register_fake(add_op_namespace_prefix("rms_norm_quantize_int8_rowwise_bf16"))
|
| 32 |
+
def _rms_norm_quantize_int8_rowwise_bf16_fake(
|
| 33 |
+
x: torch.Tensor, weight: torch.Tensor, eps: float, out: torch.Tensor, scales: torch.Tensor
|
| 34 |
+
) -> None:
|
| 35 |
+
return _quantize_int8_rowwise_bf16_fake(x, out, scales)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
@torch.library.register_fake(add_op_namespace_prefix("residual_add_rms_norm_quantize_int8_rowwise_bf16"))
|
| 39 |
+
def _residual_add_rms_norm_quantize_int8_rowwise_bf16_fake(
|
| 40 |
+
residual: torch.Tensor,
|
| 41 |
+
x: torch.Tensor,
|
| 42 |
+
weight: torch.Tensor,
|
| 43 |
+
eps: float,
|
| 44 |
+
out: torch.Tensor,
|
| 45 |
+
scales: torch.Tensor,
|
| 46 |
+
) -> None:
|
| 47 |
+
return _quantize_int8_rowwise_bf16_fake(residual, out, scales)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
@torch.library.register_fake(add_op_namespace_prefix("int8_rowwise_linear_bf16"))
|
| 51 |
+
def _int8_rowwise_linear_bf16_fake(
|
| 52 |
+
input_i8: torch.Tensor,
|
| 53 |
+
weight_i8: torch.Tensor,
|
| 54 |
+
input_scale: torch.Tensor,
|
| 55 |
+
weight_scale: torch.Tensor,
|
| 56 |
+
out: torch.Tensor,
|
| 57 |
+
variant: int,
|
| 58 |
+
) -> None:
|
| 59 |
+
if input_i8.dim() != 2 or weight_i8.dim() != 2 or out.shape != (input_i8.shape[0], weight_i8.shape[0]):
|
| 60 |
+
raise RuntimeError("int8_rowwise_linear_bf16 expects input (M,K), weight (N,K), out (M,N)")
|
| 61 |
+
return None
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
@torch.library.register_fake(add_op_namespace_prefix("int8_silu_gated_linear_bf16"))
|
| 65 |
+
def _int8_silu_gated_linear_bf16_fake(
|
| 66 |
+
input_i8: torch.Tensor,
|
| 67 |
+
up_weight_i8: torch.Tensor,
|
| 68 |
+
input_scale: torch.Tensor,
|
| 69 |
+
weight_scale: torch.Tensor,
|
| 70 |
+
gate: torch.Tensor,
|
| 71 |
+
out: torch.Tensor,
|
| 72 |
+
) -> None:
|
| 73 |
+
return _int8_rowwise_linear_bf16_fake(input_i8, up_weight_i8, input_scale, weight_scale, out, 0)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def quantize_int8_static_bf16(
|
| 77 |
+
input: torch.Tensor, scale: torch.Tensor, *, out: Optional[torch.Tensor] = None
|
| 78 |
+
) -> torch.Tensor:
|
| 79 |
+
if out is None:
|
| 80 |
+
out = torch.empty_like(input, dtype=torch.int8)
|
| 81 |
+
ops.quantize_int8_static_bf16(input, scale, out)
|
| 82 |
+
return out
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def quantize_int8_rowwise_bf16(
|
| 86 |
+
input: torch.Tensor,
|
| 87 |
+
*,
|
| 88 |
+
out: Optional[torch.Tensor] = None,
|
| 89 |
+
scales: Optional[torch.Tensor] = None,
|
| 90 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 91 |
+
if out is None:
|
| 92 |
+
out = torch.empty_like(input, dtype=torch.int8)
|
| 93 |
+
if scales is None:
|
| 94 |
+
scales = torch.empty((input.shape[0],), device=input.device, dtype=torch.float32)
|
| 95 |
+
ops.quantize_int8_rowwise_bf16(input, out, scales)
|
| 96 |
+
return out, scales
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def quantize_int8_rowwise_static_bf16(
|
| 100 |
+
input: torch.Tensor, scales: torch.Tensor, *, out: Optional[torch.Tensor] = None
|
| 101 |
+
) -> torch.Tensor:
|
| 102 |
+
if out is None:
|
| 103 |
+
out = torch.empty_like(input, dtype=torch.int8)
|
| 104 |
+
ops.quantize_int8_rowwise_static_bf16(input, scales, out)
|
| 105 |
+
return out
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def rms_norm_quantize_int8_rowwise_bf16(
|
| 109 |
+
x: torch.Tensor,
|
| 110 |
+
weight: torch.Tensor,
|
| 111 |
+
eps: float = 1e-6,
|
| 112 |
+
*,
|
| 113 |
+
out: Optional[torch.Tensor] = None,
|
| 114 |
+
scales: Optional[torch.Tensor] = None,
|
| 115 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 116 |
+
if out is None:
|
| 117 |
+
out = torch.empty_like(x, dtype=torch.int8)
|
| 118 |
+
if scales is None:
|
| 119 |
+
scales = torch.empty((x.shape[0],), device=x.device, dtype=torch.float32)
|
| 120 |
+
ops.rms_norm_quantize_int8_rowwise_bf16(x, weight, float(eps), out, scales)
|
| 121 |
+
return out, scales
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def residual_add_rms_norm_quantize_int8_rowwise_bf16(
|
| 125 |
+
residual: torch.Tensor,
|
| 126 |
+
x: torch.Tensor,
|
| 127 |
+
weight: torch.Tensor,
|
| 128 |
+
eps: float = 1e-6,
|
| 129 |
+
*,
|
| 130 |
+
out: Optional[torch.Tensor] = None,
|
| 131 |
+
scales: Optional[torch.Tensor] = None,
|
| 132 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 133 |
+
if out is None:
|
| 134 |
+
out = torch.empty_like(residual, dtype=torch.int8)
|
| 135 |
+
if scales is None:
|
| 136 |
+
scales = torch.empty((residual.shape[0],), device=residual.device, dtype=torch.float32)
|
| 137 |
+
ops.residual_add_rms_norm_quantize_int8_rowwise_bf16(residual, x, weight, float(eps), out, scales)
|
| 138 |
+
return out, scales
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def int8_rowwise_linear_bf16(
|
| 142 |
+
input_i8: torch.Tensor,
|
| 143 |
+
weight_i8: torch.Tensor,
|
| 144 |
+
input_scale: torch.Tensor,
|
| 145 |
+
weight_scale: torch.Tensor,
|
| 146 |
+
variant: int = 0,
|
| 147 |
+
*,
|
| 148 |
+
out: Optional[torch.Tensor] = None,
|
| 149 |
+
) -> torch.Tensor:
|
| 150 |
+
if out is None:
|
| 151 |
+
out = torch.empty((input_i8.shape[0], weight_i8.shape[0]), device=input_i8.device, dtype=torch.bfloat16)
|
| 152 |
+
ops.int8_rowwise_linear_bf16(input_i8, weight_i8, input_scale, weight_scale, out, int(variant))
|
| 153 |
+
return out
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def int8_silu_gated_linear_bf16(
|
| 157 |
+
input_i8: torch.Tensor,
|
| 158 |
+
up_weight_i8: torch.Tensor,
|
| 159 |
+
input_scale: torch.Tensor,
|
| 160 |
+
weight_scale: torch.Tensor,
|
| 161 |
+
gate: torch.Tensor,
|
| 162 |
+
*,
|
| 163 |
+
out: Optional[torch.Tensor] = None,
|
| 164 |
+
) -> torch.Tensor:
|
| 165 |
+
if out is None:
|
| 166 |
+
out = torch.empty((input_i8.shape[0], up_weight_i8.shape[0]), device=input_i8.device, dtype=torch.bfloat16)
|
| 167 |
+
ops.int8_silu_gated_linear_bf16(input_i8, up_weight_i8, input_scale, weight_scale, gate, out)
|
| 168 |
+
return out
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
__all__ = [
|
| 172 |
+
"quantize_int8_static_bf16",
|
| 173 |
+
"quantize_int8_rowwise_bf16",
|
| 174 |
+
"quantize_int8_rowwise_static_bf16",
|
| 175 |
+
"rms_norm_quantize_int8_rowwise_bf16",
|
| 176 |
+
"residual_add_rms_norm_quantize_int8_rowwise_bf16",
|
| 177 |
+
"int8_rowwise_linear_bf16",
|
| 178 |
+
"int8_silu_gated_linear_bf16",
|
| 179 |
+
]
|
build/torch211-cxx11-cu130-x86_64-linux/_int8_transformer_primitives_cuda_f5e354d.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ef01bcdf0ccbb8d2781aa01465696f1ef0866ecdf3eb20dd7dcd331fdcfc9278
|
| 3 |
+
size 1389176
|
build/torch211-cxx11-cu130-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _int8_transformer_primitives_cuda_f5e354d
|
| 3 |
+
ops = torch.ops._int8_transformer_primitives_cuda_f5e354d
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_int8_transformer_primitives_cuda_f5e354d::{op_name}"
|
build/torch211-cxx11-cu130-x86_64-linux/int8_transformer_primitives/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch211-cxx11-cu130-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "int8-transformer-primitives",
|
| 3 |
+
"id": "_int8_transformer_primitives_cuda_f5e354d",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"12.0a",
|
| 11 |
+
"8.0",
|
| 12 |
+
"8.6",
|
| 13 |
+
"8.9",
|
| 14 |
+
"9.0"
|
| 15 |
+
]
|
| 16 |
+
}
|
| 17 |
+
}
|
build/torch212-cxx11-cu130-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT INT8 transformer primitives."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Optional
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
from ._ops import add_op_namespace_prefix, ops
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@torch.library.register_fake(add_op_namespace_prefix("quantize_int8_static_bf16"))
|
| 13 |
+
def _quantize_int8_static_bf16_fake(input: torch.Tensor, scale: torch.Tensor, out: torch.Tensor) -> None:
|
| 14 |
+
if out.shape != input.shape:
|
| 15 |
+
raise RuntimeError("out must match input shape")
|
| 16 |
+
return None
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@torch.library.register_fake(add_op_namespace_prefix("quantize_int8_rowwise_bf16"))
|
| 20 |
+
def _quantize_int8_rowwise_bf16_fake(input: torch.Tensor, out: torch.Tensor, scales: torch.Tensor) -> None:
|
| 21 |
+
if input.dim() != 2 or out.shape != input.shape or scales.shape != (input.shape[0],):
|
| 22 |
+
raise RuntimeError("quantize_int8_rowwise_bf16 expects input/out (rows, cols), scales (rows,)")
|
| 23 |
+
return None
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
@torch.library.register_fake(add_op_namespace_prefix("quantize_int8_rowwise_static_bf16"))
|
| 27 |
+
def _quantize_int8_rowwise_static_bf16_fake(input: torch.Tensor, scales: torch.Tensor, out: torch.Tensor) -> None:
|
| 28 |
+
return _quantize_int8_rowwise_bf16_fake(input, out, scales)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
@torch.library.register_fake(add_op_namespace_prefix("rms_norm_quantize_int8_rowwise_bf16"))
|
| 32 |
+
def _rms_norm_quantize_int8_rowwise_bf16_fake(
|
| 33 |
+
x: torch.Tensor, weight: torch.Tensor, eps: float, out: torch.Tensor, scales: torch.Tensor
|
| 34 |
+
) -> None:
|
| 35 |
+
return _quantize_int8_rowwise_bf16_fake(x, out, scales)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
@torch.library.register_fake(add_op_namespace_prefix("residual_add_rms_norm_quantize_int8_rowwise_bf16"))
|
| 39 |
+
def _residual_add_rms_norm_quantize_int8_rowwise_bf16_fake(
|
| 40 |
+
residual: torch.Tensor,
|
| 41 |
+
x: torch.Tensor,
|
| 42 |
+
weight: torch.Tensor,
|
| 43 |
+
eps: float,
|
| 44 |
+
out: torch.Tensor,
|
| 45 |
+
scales: torch.Tensor,
|
| 46 |
+
) -> None:
|
| 47 |
+
return _quantize_int8_rowwise_bf16_fake(residual, out, scales)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
@torch.library.register_fake(add_op_namespace_prefix("int8_rowwise_linear_bf16"))
|
| 51 |
+
def _int8_rowwise_linear_bf16_fake(
|
| 52 |
+
input_i8: torch.Tensor,
|
| 53 |
+
weight_i8: torch.Tensor,
|
| 54 |
+
input_scale: torch.Tensor,
|
| 55 |
+
weight_scale: torch.Tensor,
|
| 56 |
+
out: torch.Tensor,
|
| 57 |
+
variant: int,
|
| 58 |
+
) -> None:
|
| 59 |
+
if input_i8.dim() != 2 or weight_i8.dim() != 2 or out.shape != (input_i8.shape[0], weight_i8.shape[0]):
|
| 60 |
+
raise RuntimeError("int8_rowwise_linear_bf16 expects input (M,K), weight (N,K), out (M,N)")
|
| 61 |
+
return None
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
@torch.library.register_fake(add_op_namespace_prefix("int8_silu_gated_linear_bf16"))
|
| 65 |
+
def _int8_silu_gated_linear_bf16_fake(
|
| 66 |
+
input_i8: torch.Tensor,
|
| 67 |
+
up_weight_i8: torch.Tensor,
|
| 68 |
+
input_scale: torch.Tensor,
|
| 69 |
+
weight_scale: torch.Tensor,
|
| 70 |
+
gate: torch.Tensor,
|
| 71 |
+
out: torch.Tensor,
|
| 72 |
+
) -> None:
|
| 73 |
+
return _int8_rowwise_linear_bf16_fake(input_i8, up_weight_i8, input_scale, weight_scale, out, 0)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def quantize_int8_static_bf16(
|
| 77 |
+
input: torch.Tensor, scale: torch.Tensor, *, out: Optional[torch.Tensor] = None
|
| 78 |
+
) -> torch.Tensor:
|
| 79 |
+
if out is None:
|
| 80 |
+
out = torch.empty_like(input, dtype=torch.int8)
|
| 81 |
+
ops.quantize_int8_static_bf16(input, scale, out)
|
| 82 |
+
return out
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def quantize_int8_rowwise_bf16(
|
| 86 |
+
input: torch.Tensor,
|
| 87 |
+
*,
|
| 88 |
+
out: Optional[torch.Tensor] = None,
|
| 89 |
+
scales: Optional[torch.Tensor] = None,
|
| 90 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 91 |
+
if out is None:
|
| 92 |
+
out = torch.empty_like(input, dtype=torch.int8)
|
| 93 |
+
if scales is None:
|
| 94 |
+
scales = torch.empty((input.shape[0],), device=input.device, dtype=torch.float32)
|
| 95 |
+
ops.quantize_int8_rowwise_bf16(input, out, scales)
|
| 96 |
+
return out, scales
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def quantize_int8_rowwise_static_bf16(
|
| 100 |
+
input: torch.Tensor, scales: torch.Tensor, *, out: Optional[torch.Tensor] = None
|
| 101 |
+
) -> torch.Tensor:
|
| 102 |
+
if out is None:
|
| 103 |
+
out = torch.empty_like(input, dtype=torch.int8)
|
| 104 |
+
ops.quantize_int8_rowwise_static_bf16(input, scales, out)
|
| 105 |
+
return out
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def rms_norm_quantize_int8_rowwise_bf16(
|
| 109 |
+
x: torch.Tensor,
|
| 110 |
+
weight: torch.Tensor,
|
| 111 |
+
eps: float = 1e-6,
|
| 112 |
+
*,
|
| 113 |
+
out: Optional[torch.Tensor] = None,
|
| 114 |
+
scales: Optional[torch.Tensor] = None,
|
| 115 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 116 |
+
if out is None:
|
| 117 |
+
out = torch.empty_like(x, dtype=torch.int8)
|
| 118 |
+
if scales is None:
|
| 119 |
+
scales = torch.empty((x.shape[0],), device=x.device, dtype=torch.float32)
|
| 120 |
+
ops.rms_norm_quantize_int8_rowwise_bf16(x, weight, float(eps), out, scales)
|
| 121 |
+
return out, scales
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def residual_add_rms_norm_quantize_int8_rowwise_bf16(
|
| 125 |
+
residual: torch.Tensor,
|
| 126 |
+
x: torch.Tensor,
|
| 127 |
+
weight: torch.Tensor,
|
| 128 |
+
eps: float = 1e-6,
|
| 129 |
+
*,
|
| 130 |
+
out: Optional[torch.Tensor] = None,
|
| 131 |
+
scales: Optional[torch.Tensor] = None,
|
| 132 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 133 |
+
if out is None:
|
| 134 |
+
out = torch.empty_like(residual, dtype=torch.int8)
|
| 135 |
+
if scales is None:
|
| 136 |
+
scales = torch.empty((residual.shape[0],), device=residual.device, dtype=torch.float32)
|
| 137 |
+
ops.residual_add_rms_norm_quantize_int8_rowwise_bf16(residual, x, weight, float(eps), out, scales)
|
| 138 |
+
return out, scales
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def int8_rowwise_linear_bf16(
|
| 142 |
+
input_i8: torch.Tensor,
|
| 143 |
+
weight_i8: torch.Tensor,
|
| 144 |
+
input_scale: torch.Tensor,
|
| 145 |
+
weight_scale: torch.Tensor,
|
| 146 |
+
variant: int = 0,
|
| 147 |
+
*,
|
| 148 |
+
out: Optional[torch.Tensor] = None,
|
| 149 |
+
) -> torch.Tensor:
|
| 150 |
+
if out is None:
|
| 151 |
+
out = torch.empty((input_i8.shape[0], weight_i8.shape[0]), device=input_i8.device, dtype=torch.bfloat16)
|
| 152 |
+
ops.int8_rowwise_linear_bf16(input_i8, weight_i8, input_scale, weight_scale, out, int(variant))
|
| 153 |
+
return out
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def int8_silu_gated_linear_bf16(
|
| 157 |
+
input_i8: torch.Tensor,
|
| 158 |
+
up_weight_i8: torch.Tensor,
|
| 159 |
+
input_scale: torch.Tensor,
|
| 160 |
+
weight_scale: torch.Tensor,
|
| 161 |
+
gate: torch.Tensor,
|
| 162 |
+
*,
|
| 163 |
+
out: Optional[torch.Tensor] = None,
|
| 164 |
+
) -> torch.Tensor:
|
| 165 |
+
if out is None:
|
| 166 |
+
out = torch.empty((input_i8.shape[0], up_weight_i8.shape[0]), device=input_i8.device, dtype=torch.bfloat16)
|
| 167 |
+
ops.int8_silu_gated_linear_bf16(input_i8, up_weight_i8, input_scale, weight_scale, gate, out)
|
| 168 |
+
return out
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
__all__ = [
|
| 172 |
+
"quantize_int8_static_bf16",
|
| 173 |
+
"quantize_int8_rowwise_bf16",
|
| 174 |
+
"quantize_int8_rowwise_static_bf16",
|
| 175 |
+
"rms_norm_quantize_int8_rowwise_bf16",
|
| 176 |
+
"residual_add_rms_norm_quantize_int8_rowwise_bf16",
|
| 177 |
+
"int8_rowwise_linear_bf16",
|
| 178 |
+
"int8_silu_gated_linear_bf16",
|
| 179 |
+
]
|
build/torch212-cxx11-cu130-x86_64-linux/_int8_transformer_primitives_cuda_f5e354d.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:34315262fe72d1edc99517434e91def1b4a4c7897c2e04871a731de9156ca5d5
|
| 3 |
+
size 1395440
|
build/torch212-cxx11-cu130-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _int8_transformer_primitives_cuda_f5e354d
|
| 3 |
+
ops = torch.ops._int8_transformer_primitives_cuda_f5e354d
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_int8_transformer_primitives_cuda_f5e354d::{op_name}"
|
build/torch212-cxx11-cu130-x86_64-linux/int8_transformer_primitives/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch212-cxx11-cu130-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "int8-transformer-primitives",
|
| 3 |
+
"id": "_int8_transformer_primitives_cuda_f5e354d",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"12.0a",
|
| 11 |
+
"8.0",
|
| 12 |
+
"8.6",
|
| 13 |
+
"8.9",
|
| 14 |
+
"9.0"
|
| 15 |
+
]
|
| 16 |
+
}
|
| 17 |
+
}
|
build/torch212-cxx11-cu132-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT INT8 transformer primitives."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Optional
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
from ._ops import add_op_namespace_prefix, ops
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@torch.library.register_fake(add_op_namespace_prefix("quantize_int8_static_bf16"))
|
| 13 |
+
def _quantize_int8_static_bf16_fake(input: torch.Tensor, scale: torch.Tensor, out: torch.Tensor) -> None:
|
| 14 |
+
if out.shape != input.shape:
|
| 15 |
+
raise RuntimeError("out must match input shape")
|
| 16 |
+
return None
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@torch.library.register_fake(add_op_namespace_prefix("quantize_int8_rowwise_bf16"))
|
| 20 |
+
def _quantize_int8_rowwise_bf16_fake(input: torch.Tensor, out: torch.Tensor, scales: torch.Tensor) -> None:
|
| 21 |
+
if input.dim() != 2 or out.shape != input.shape or scales.shape != (input.shape[0],):
|
| 22 |
+
raise RuntimeError("quantize_int8_rowwise_bf16 expects input/out (rows, cols), scales (rows,)")
|
| 23 |
+
return None
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
@torch.library.register_fake(add_op_namespace_prefix("quantize_int8_rowwise_static_bf16"))
|
| 27 |
+
def _quantize_int8_rowwise_static_bf16_fake(input: torch.Tensor, scales: torch.Tensor, out: torch.Tensor) -> None:
|
| 28 |
+
return _quantize_int8_rowwise_bf16_fake(input, out, scales)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
@torch.library.register_fake(add_op_namespace_prefix("rms_norm_quantize_int8_rowwise_bf16"))
|
| 32 |
+
def _rms_norm_quantize_int8_rowwise_bf16_fake(
|
| 33 |
+
x: torch.Tensor, weight: torch.Tensor, eps: float, out: torch.Tensor, scales: torch.Tensor
|
| 34 |
+
) -> None:
|
| 35 |
+
return _quantize_int8_rowwise_bf16_fake(x, out, scales)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
@torch.library.register_fake(add_op_namespace_prefix("residual_add_rms_norm_quantize_int8_rowwise_bf16"))
|
| 39 |
+
def _residual_add_rms_norm_quantize_int8_rowwise_bf16_fake(
|
| 40 |
+
residual: torch.Tensor,
|
| 41 |
+
x: torch.Tensor,
|
| 42 |
+
weight: torch.Tensor,
|
| 43 |
+
eps: float,
|
| 44 |
+
out: torch.Tensor,
|
| 45 |
+
scales: torch.Tensor,
|
| 46 |
+
) -> None:
|
| 47 |
+
return _quantize_int8_rowwise_bf16_fake(residual, out, scales)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
@torch.library.register_fake(add_op_namespace_prefix("int8_rowwise_linear_bf16"))
|
| 51 |
+
def _int8_rowwise_linear_bf16_fake(
|
| 52 |
+
input_i8: torch.Tensor,
|
| 53 |
+
weight_i8: torch.Tensor,
|
| 54 |
+
input_scale: torch.Tensor,
|
| 55 |
+
weight_scale: torch.Tensor,
|
| 56 |
+
out: torch.Tensor,
|
| 57 |
+
variant: int,
|
| 58 |
+
) -> None:
|
| 59 |
+
if input_i8.dim() != 2 or weight_i8.dim() != 2 or out.shape != (input_i8.shape[0], weight_i8.shape[0]):
|
| 60 |
+
raise RuntimeError("int8_rowwise_linear_bf16 expects input (M,K), weight (N,K), out (M,N)")
|
| 61 |
+
return None
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
@torch.library.register_fake(add_op_namespace_prefix("int8_silu_gated_linear_bf16"))
|
| 65 |
+
def _int8_silu_gated_linear_bf16_fake(
|
| 66 |
+
input_i8: torch.Tensor,
|
| 67 |
+
up_weight_i8: torch.Tensor,
|
| 68 |
+
input_scale: torch.Tensor,
|
| 69 |
+
weight_scale: torch.Tensor,
|
| 70 |
+
gate: torch.Tensor,
|
| 71 |
+
out: torch.Tensor,
|
| 72 |
+
) -> None:
|
| 73 |
+
return _int8_rowwise_linear_bf16_fake(input_i8, up_weight_i8, input_scale, weight_scale, out, 0)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def quantize_int8_static_bf16(
|
| 77 |
+
input: torch.Tensor, scale: torch.Tensor, *, out: Optional[torch.Tensor] = None
|
| 78 |
+
) -> torch.Tensor:
|
| 79 |
+
if out is None:
|
| 80 |
+
out = torch.empty_like(input, dtype=torch.int8)
|
| 81 |
+
ops.quantize_int8_static_bf16(input, scale, out)
|
| 82 |
+
return out
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def quantize_int8_rowwise_bf16(
|
| 86 |
+
input: torch.Tensor,
|
| 87 |
+
*,
|
| 88 |
+
out: Optional[torch.Tensor] = None,
|
| 89 |
+
scales: Optional[torch.Tensor] = None,
|
| 90 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 91 |
+
if out is None:
|
| 92 |
+
out = torch.empty_like(input, dtype=torch.int8)
|
| 93 |
+
if scales is None:
|
| 94 |
+
scales = torch.empty((input.shape[0],), device=input.device, dtype=torch.float32)
|
| 95 |
+
ops.quantize_int8_rowwise_bf16(input, out, scales)
|
| 96 |
+
return out, scales
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def quantize_int8_rowwise_static_bf16(
|
| 100 |
+
input: torch.Tensor, scales: torch.Tensor, *, out: Optional[torch.Tensor] = None
|
| 101 |
+
) -> torch.Tensor:
|
| 102 |
+
if out is None:
|
| 103 |
+
out = torch.empty_like(input, dtype=torch.int8)
|
| 104 |
+
ops.quantize_int8_rowwise_static_bf16(input, scales, out)
|
| 105 |
+
return out
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def rms_norm_quantize_int8_rowwise_bf16(
|
| 109 |
+
x: torch.Tensor,
|
| 110 |
+
weight: torch.Tensor,
|
| 111 |
+
eps: float = 1e-6,
|
| 112 |
+
*,
|
| 113 |
+
out: Optional[torch.Tensor] = None,
|
| 114 |
+
scales: Optional[torch.Tensor] = None,
|
| 115 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 116 |
+
if out is None:
|
| 117 |
+
out = torch.empty_like(x, dtype=torch.int8)
|
| 118 |
+
if scales is None:
|
| 119 |
+
scales = torch.empty((x.shape[0],), device=x.device, dtype=torch.float32)
|
| 120 |
+
ops.rms_norm_quantize_int8_rowwise_bf16(x, weight, float(eps), out, scales)
|
| 121 |
+
return out, scales
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def residual_add_rms_norm_quantize_int8_rowwise_bf16(
|
| 125 |
+
residual: torch.Tensor,
|
| 126 |
+
x: torch.Tensor,
|
| 127 |
+
weight: torch.Tensor,
|
| 128 |
+
eps: float = 1e-6,
|
| 129 |
+
*,
|
| 130 |
+
out: Optional[torch.Tensor] = None,
|
| 131 |
+
scales: Optional[torch.Tensor] = None,
|
| 132 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 133 |
+
if out is None:
|
| 134 |
+
out = torch.empty_like(residual, dtype=torch.int8)
|
| 135 |
+
if scales is None:
|
| 136 |
+
scales = torch.empty((residual.shape[0],), device=residual.device, dtype=torch.float32)
|
| 137 |
+
ops.residual_add_rms_norm_quantize_int8_rowwise_bf16(residual, x, weight, float(eps), out, scales)
|
| 138 |
+
return out, scales
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def int8_rowwise_linear_bf16(
|
| 142 |
+
input_i8: torch.Tensor,
|
| 143 |
+
weight_i8: torch.Tensor,
|
| 144 |
+
input_scale: torch.Tensor,
|
| 145 |
+
weight_scale: torch.Tensor,
|
| 146 |
+
variant: int = 0,
|
| 147 |
+
*,
|
| 148 |
+
out: Optional[torch.Tensor] = None,
|
| 149 |
+
) -> torch.Tensor:
|
| 150 |
+
if out is None:
|
| 151 |
+
out = torch.empty((input_i8.shape[0], weight_i8.shape[0]), device=input_i8.device, dtype=torch.bfloat16)
|
| 152 |
+
ops.int8_rowwise_linear_bf16(input_i8, weight_i8, input_scale, weight_scale, out, int(variant))
|
| 153 |
+
return out
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def int8_silu_gated_linear_bf16(
|
| 157 |
+
input_i8: torch.Tensor,
|
| 158 |
+
up_weight_i8: torch.Tensor,
|
| 159 |
+
input_scale: torch.Tensor,
|
| 160 |
+
weight_scale: torch.Tensor,
|
| 161 |
+
gate: torch.Tensor,
|
| 162 |
+
*,
|
| 163 |
+
out: Optional[torch.Tensor] = None,
|
| 164 |
+
) -> torch.Tensor:
|
| 165 |
+
if out is None:
|
| 166 |
+
out = torch.empty((input_i8.shape[0], up_weight_i8.shape[0]), device=input_i8.device, dtype=torch.bfloat16)
|
| 167 |
+
ops.int8_silu_gated_linear_bf16(input_i8, up_weight_i8, input_scale, weight_scale, gate, out)
|
| 168 |
+
return out
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
__all__ = [
|
| 172 |
+
"quantize_int8_static_bf16",
|
| 173 |
+
"quantize_int8_rowwise_bf16",
|
| 174 |
+
"quantize_int8_rowwise_static_bf16",
|
| 175 |
+
"rms_norm_quantize_int8_rowwise_bf16",
|
| 176 |
+
"residual_add_rms_norm_quantize_int8_rowwise_bf16",
|
| 177 |
+
"int8_rowwise_linear_bf16",
|
| 178 |
+
"int8_silu_gated_linear_bf16",
|
| 179 |
+
]
|
build/torch212-cxx11-cu132-x86_64-linux/_int8_transformer_primitives_cuda_f5e354d.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0984aad643982134f36895a25bc0b5a5c3846c8bd8565d5044ff370326a651ed
|
| 3 |
+
size 1387304
|
build/torch212-cxx11-cu132-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _int8_transformer_primitives_cuda_f5e354d
|
| 3 |
+
ops = torch.ops._int8_transformer_primitives_cuda_f5e354d
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_int8_transformer_primitives_cuda_f5e354d::{op_name}"
|
build/torch212-cxx11-cu132-x86_64-linux/int8_transformer_primitives/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch212-cxx11-cu132-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "int8-transformer-primitives",
|
| 3 |
+
"id": "_int8_transformer_primitives_cuda_f5e354d",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"12.0a",
|
| 11 |
+
"8.0",
|
| 12 |
+
"8.6",
|
| 13 |
+
"8.9",
|
| 14 |
+
"9.0"
|
| 15 |
+
]
|
| 16 |
+
}
|
| 17 |
+
}
|