liangsu9988 commited on
Commit
bfcf853
·
verified ·
1 Parent(s): 0cf8718

Promote latest kernel artifacts to main

Browse files
Files changed (38) hide show
  1. README.md +0 -9
  2. benchmarks/benchmark.py +299 -16
  3. build/torch211-cxx11-cu128-x86_64-linux/__init__.py +106 -0
  4. build/{torch212-cxx11-cu132-x86_64-linux/_fp8_gemm_cuda_1798e7f.abi3.so → torch211-cxx11-cu128-x86_64-linux/_fp8_gemm_cuda_bcb0782.abi3.so} +2 -2
  5. build/torch211-cxx11-cu128-x86_64-linux/_ops.py +3 -3
  6. build/torch211-cxx11-cu128-x86_64-linux/metadata.json +6 -5
  7. build/torch211-cxx11-cu130-aarch64-linux/__init__.py +254 -0
  8. build/{torch211-cxx11-cu128-x86_64-linux/_fp8_gemm_cuda_1798e7f.abi3.so → torch211-cxx11-cu130-aarch64-linux/_fp8_gemm_cuda_d31c69b1_sm110.abi3.so} +2 -2
  9. build/torch211-cxx11-cu130-aarch64-linux/_ops.py +6 -0
  10. build/torch211-cxx11-cu130-aarch64-linux/fp8_gemm/__init__.py +12 -0
  11. build/torch211-cxx11-cu130-aarch64-linux/metadata.json +31 -0
  12. build/torch211-cxx11-cu130-x86_64-linux/__init__.py +106 -0
  13. build/torch211-cxx11-cu130-x86_64-linux/{_fp8_gemm_cuda_1798e7f.abi3.so → _fp8_gemm_cuda_bcb0782.abi3.so} +2 -2
  14. build/torch211-cxx11-cu130-x86_64-linux/_ops.py +3 -3
  15. build/torch211-cxx11-cu130-x86_64-linux/metadata.json +6 -5
  16. build/torch212-cxx11-cu130-x86_64-linux/__init__.py +127 -1
  17. build/torch212-cxx11-cu130-x86_64-linux/{_fp8_gemm_cuda_1798e7f.abi3.so → _fp8_gemm_cuda_bf15968.abi3.so} +2 -2
  18. build/torch212-cxx11-cu130-x86_64-linux/_ops.py +3 -3
  19. build/torch212-cxx11-cu130-x86_64-linux/fp8_gemm/__init__.py +0 -26
  20. build/torch212-cxx11-cu130-x86_64-linux/metadata.json +18 -6
  21. build/torch212-cxx11-cu132-x86_64-linux/__init__.py +127 -1
  22. build/torch212-cxx11-cu132-x86_64-linux/_fp8_gemm_cuda_bf15968.abi3.so +3 -0
  23. build/torch212-cxx11-cu132-x86_64-linux/_ops.py +3 -3
  24. build/torch212-cxx11-cu132-x86_64-linux/fp8_gemm/__init__.py +0 -26
  25. build/torch212-cxx11-cu132-x86_64-linux/metadata.json +18 -6
  26. build/torch213-cxx11-cu130-aarch64-linux/__init__.py +254 -0
  27. build/torch213-cxx11-cu130-aarch64-linux/_ops.py +6 -0
  28. build/torch213-cxx11-cu130-aarch64-linux/fp8_gemm/__init__.py +14 -0
  29. build/torch213-cxx11-cu130-aarch64-linux/fp8_gemm_source_test.abi3.so +3 -0
  30. build/torch213-cxx11-cu130-aarch64-linux/metadata.json +32 -0
  31. build/torch213-cxx11-cu130-x86_64-linux/__init__.py +254 -0
  32. build/torch213-cxx11-cu130-x86_64-linux/_fp8_gemm_cuda_bf15968.abi3.so +3 -0
  33. build/torch213-cxx11-cu130-x86_64-linux/_ops.py +9 -0
  34. build/torch213-cxx11-cu130-x86_64-linux/metadata.json +34 -0
  35. build/torch213-cxx11-cu132-x86_64-linux/__init__.py +254 -0
  36. build/torch213-cxx11-cu132-x86_64-linux/_fp8_gemm_cuda_bf15968.abi3.so +3 -0
  37. build/torch213-cxx11-cu132-x86_64-linux/_ops.py +9 -0
  38. build/torch213-cxx11-cu132-x86_64-linux/metadata.json +34 -0
README.md DELETED
@@ -1,9 +0,0 @@
1
- # flashrt/fp8-gemm
2
-
3
- This repository is a compatibility mirror for older `kernels` clients
4
- that resolve repositories through the default Hugging Face model repo API.
5
-
6
- Canonical Kernel Hub repo: https://huggingface.co/kernels/flashrt/fp8-gemm
7
-
8
- Do not edit this mirror by hand. It is generated from the Kernel Hub
9
- `vN` branches and contains the same `build/**` artifacts.
 
 
 
 
 
 
 
 
 
 
benchmarks/benchmark.py CHANGED
@@ -7,6 +7,7 @@ import argparse
7
  import importlib
8
  import json
9
  import os
 
10
  import sys
11
  from dataclasses import asdict, dataclass
12
  from pathlib import Path
@@ -32,11 +33,32 @@ SHAPES = {
32
  "small_m16_k4096_n4096": (16, 4096, 4096),
33
  "small_m32_k4096_n8192": (32, 4096, 8192),
34
  "small_m64_k512_n1024": (64, 512, 1024),
 
 
 
 
 
 
 
 
 
 
 
 
35
  }
36
 
37
  MODES = {
38
  "smoke": ["decode_m1_k4096_n2048", "small_m16_k4096_n4096"],
39
- "headline": list(SHAPES),
 
 
 
 
 
 
 
 
 
40
  }
41
 
42
 
@@ -49,6 +71,18 @@ class Result:
49
  variant: int
50
  tile: str
51
  flashrt_us: float
 
 
 
 
 
 
 
 
 
 
 
 
52
  torch_eager_us: float
53
  torch_compile_us: float | None
54
  speedup_vs_eager: float
@@ -77,6 +111,8 @@ class SourceOps:
77
 
78
  def _current_arch_list() -> str:
79
  major, minor = torch.cuda.get_device_capability(0)
 
 
80
  if major >= 12:
81
  return "12.0a"
82
  return f"{major}.{minor}"
@@ -87,17 +123,40 @@ def load_source_ops() -> SourceOps:
87
 
88
  os.environ.setdefault("TORCH_CUDA_ARCH_LIST", _current_arch_list())
89
  namespace = "fp8_gemm_source_bench"
90
- load(
91
- name=namespace,
92
- sources=[
93
- str(PACKAGE / "torch-ext" / "torch_binding.cpp"),
 
 
 
 
 
 
 
 
 
94
  str(PACKAGE / "csrc" / "fp8_gemv_m1_sm120.cu"),
95
  str(PACKAGE / "csrc" / "fp8_smallM_handtuned_sm120.cu"),
96
  str(PACKAGE / "csrc" / "fp8_smallM_handtuned_ldmatrix_sm120.cu"),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97
  ],
98
- extra_include_paths=[str(PACKAGE / "csrc"), str(REGISTRATION_INCLUDE)],
99
- extra_cflags=["-O3", "-DCUDA_KERNEL"],
100
- extra_cuda_cflags=["-O3", "--expt-relaxed-constexpr", "-DCUDA_KERNEL"],
101
  verbose=False,
102
  )
103
  return SourceOps(namespace)
@@ -114,6 +173,21 @@ def load_installed_ops(artifact: str | None):
114
 
115
 
116
  def select_tile(m: int, n: int, k: int, variant: int = 0) -> str:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
117
  if m == 1:
118
  if variant == 4:
119
  return "gemv_fp8_m1_w4"
@@ -185,6 +259,51 @@ def measure(fn, warmup: int, iters: int) -> float:
185
  return float(start.elapsed_time(end) * 1000.0 / iters)
186
 
187
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
188
  def metrics(got, expected):
189
  diff = (got.float() - expected.float()).abs().flatten()
190
  return (
@@ -195,7 +314,18 @@ def metrics(got, expected):
195
  )
196
 
197
 
198
- def bench_case(ops, name: str, shape: tuple[int, int, int], variant: int, warmup: int, iters: int, compile_ref: bool):
 
 
 
 
 
 
 
 
 
 
 
199
  m, k, n = shape
200
  x, w = make_inputs(m, k, n, seed=3000 + m + k + n + variant)
201
  out = torch.empty((m, n), device="cuda", dtype=torch.bfloat16)
@@ -205,7 +335,140 @@ def bench_case(ops, name: str, shape: tuple[int, int, int], variant: int, warmup
205
  max_abs, mean_abs, p99_abs, cos = metrics(got, expected)
206
  tile = ops.select_fp8_linear_tile(m, n, k, variant)
207
 
208
- flashrt_us = measure(lambda: ops.fp8_linear_bf16(x, w, out=out, variant=variant), warmup, iters)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
209
  eager_us = measure(lambda: ref_fn(x, w), warmup, iters)
210
  compile_us = None
211
  if compile_ref:
@@ -225,6 +488,18 @@ def bench_case(ops, name: str, shape: tuple[int, int, int], variant: int, warmup
225
  variant=variant,
226
  tile=tile,
227
  flashrt_us=flashrt_us,
 
 
 
 
 
 
 
 
 
 
 
 
228
  torch_eager_us=eager_us,
229
  torch_compile_us=compile_us,
230
  speedup_vs_eager=eager_us / flashrt_us,
@@ -233,7 +508,11 @@ def bench_case(ops, name: str, shape: tuple[int, int, int], variant: int, warmup
233
  mean_abs=mean_abs,
234
  p99_abs=p99_abs,
235
  cosine=cos,
236
- status="pass" if max_abs <= 0.5 and p99_abs <= 0.25 and cos >= 0.999 else "fail",
 
 
 
 
237
  )
238
 
239
 
@@ -250,19 +529,23 @@ def main() -> None:
250
 
251
  if not torch.cuda.is_available():
252
  raise SystemExit("CUDA is required")
253
- major, _minor = torch.cuda.get_device_capability(0)
254
- if major < 12:
255
- raise SystemExit("fp8-gemm requires Blackwell/SM120 for this package")
256
 
257
  ops = load_source_ops() if args.backend == "source" else load_installed_ops(args.artifact)
 
258
  rows: list[Result] = []
259
  for name in MODES[args.mode]:
260
  shape = SHAPES[name]
261
  variants = [0]
262
- if shape[0] == 1:
263
  variants = [0, 4, 8, 16]
264
  for variant in variants:
265
- rows.append(bench_case(ops, name, shape, variant, args.warmup, args.iterations, args.compile_ref))
 
 
 
266
 
267
  payload = {"rows": [asdict(row) for row in rows]}
268
  print(json.dumps(payload, indent=2, sort_keys=True))
 
7
  import importlib
8
  import json
9
  import os
10
+ import statistics
11
  import sys
12
  from dataclasses import asdict, dataclass
13
  from pathlib import Path
 
33
  "small_m16_k4096_n4096": (16, 4096, 4096),
34
  "small_m32_k4096_n8192": (32, 4096, 8192),
35
  "small_m64_k512_n1024": (64, 512, 1024),
36
+ "pi05_action_qkv": (51, 2048, 2560),
37
+ "pi05_action_o": (51, 2048, 2048),
38
+ "pi05_action_gate_up": (51, 2048, 16384),
39
+ "pi05_action_down": (51, 8192, 2048),
40
+ "groot_dit_qkv": (51, 1536, 4608),
41
+ "groot_n17_llm_o": (277, 2048, 2048),
42
+ "groot_n17_llm_gate_up": (277, 2048, 16384),
43
+ "groot_n17_llm_down": (277, 8192, 2048),
44
+ "groot_n17_vit_o": (1024, 1024, 1024),
45
+ "cosmos_edge_action": (64, 2048, 9216),
46
+ "lingbot_vision_o": (1024, 1280, 1280),
47
+ "lingbot_action_gate_up": (105, 2048, 16384),
48
  }
49
 
50
  MODES = {
51
  "smoke": ["decode_m1_k4096_n2048", "small_m16_k4096_n4096"],
52
+ "headline": [
53
+ "decode_m1_k4096_n2048",
54
+ "pi05_action_qkv",
55
+ "pi05_action_gate_up",
56
+ "pi05_action_down",
57
+ "groot_n17_llm_o",
58
+ "cosmos_edge_action",
59
+ "lingbot_action_gate_up",
60
+ ],
61
+ "thor-full": list(SHAPES),
62
  }
63
 
64
 
 
71
  variant: int
72
  tile: str
73
  flashrt_us: float
74
+ flashrt_graph_us: float | None
75
+ package_best_graph_tile: str | None
76
+ package_best_graph_us: float | None
77
+ auto_over_package_best: float | None
78
+ package_tiles_us: dict[str, dict[str, float]] | None
79
+ native_best_tile: str | None
80
+ native_best_us: float | None
81
+ native_best_graph_tile: str | None
82
+ native_best_graph_us: float | None
83
+ native_tiles_us: dict[str, dict[str, float]] | None
84
+ wrapper_vs_native: float | None
85
+ graph_vs_native: float | None
86
  torch_eager_us: float
87
  torch_compile_us: float | None
88
  speedup_vs_eager: float
 
111
 
112
  def _current_arch_list() -> str:
113
  major, minor = torch.cuda.get_device_capability(0)
114
+ if (major, minor) == (11, 0):
115
+ return "11.0a"
116
  if major >= 12:
117
  return "12.0a"
118
  return f"{major}.{minor}"
 
123
 
124
  os.environ.setdefault("TORCH_CUDA_ARCH_LIST", _current_arch_list())
125
  namespace = "fp8_gemm_source_bench"
126
+ capability = torch.cuda.get_device_capability(0)
127
+ cutlass_include = Path(os.environ.get("CUTLASS_INCLUDE", ""))
128
+ if capability == (11, 0):
129
+ if not (cutlass_include / "cutlass" / "cutlass.h").is_file():
130
+ raise RuntimeError("set CUTLASS_INCLUDE for the SM110 source benchmark")
131
+ cuda_sources = [str(PACKAGE / "csrc" / "cutlass_sm110_fp8_gemm.cu")]
132
+ source_define = "-DFLASHRT_FP8_GEMM_SOURCE_SM110_ONLY"
133
+ extra_includes = [
134
+ str(cutlass_include),
135
+ str(cutlass_include.parent / "tools" / "util" / "include"),
136
+ ]
137
+ else:
138
+ cuda_sources = [
139
  str(PACKAGE / "csrc" / "fp8_gemv_m1_sm120.cu"),
140
  str(PACKAGE / "csrc" / "fp8_smallM_handtuned_sm120.cu"),
141
  str(PACKAGE / "csrc" / "fp8_smallM_handtuned_ldmatrix_sm120.cu"),
142
+ ]
143
+ source_define = "-DFLASHRT_FP8_GEMM_SOURCE_SM120_ONLY"
144
+ extra_includes = []
145
+ load(
146
+ name=namespace,
147
+ sources=[str(PACKAGE / "torch-ext" / "torch_binding.cpp"), *cuda_sources],
148
+ extra_include_paths=[
149
+ str(PACKAGE / "csrc"), str(REGISTRATION_INCLUDE), *extra_includes
150
+ ],
151
+ extra_cflags=["-O3", "-DNDEBUG", "-DCUDA_KERNEL", source_define],
152
+ extra_cuda_cflags=[
153
+ "-O3", "-DNDEBUG", "--expt-relaxed-constexpr", "--use_fast_math",
154
+ "-U__CUDA_NO_HALF_OPERATORS__",
155
+ "-U__CUDA_NO_HALF_CONVERSIONS__",
156
+ "-U__CUDA_NO_BFLOAT16_CONVERSIONS__",
157
+ "-U__CUDA_NO_HALF2_OPERATORS__",
158
+ "-DCUDA_KERNEL", source_define
159
  ],
 
 
 
160
  verbose=False,
161
  )
162
  return SourceOps(namespace)
 
173
 
174
 
175
  def select_tile(m: int, n: int, k: int, variant: int = 0) -> str:
176
+ if torch.cuda.get_device_capability(0) == (11, 0):
177
+ forced = {1: "sm110_sq_bf16", 2: "sm110_t1_bf16", 3: "sm110_wide_bf16"}
178
+ if variant not in {0, *forced}:
179
+ raise RuntimeError("SM110 variant must be in [0, 3]")
180
+ if variant:
181
+ return forced[variant]
182
+ if n >= 8 * k:
183
+ return "sm110_wide_bf16"
184
+ if m >= 128 and k >= 4 * n:
185
+ return "sm110_sq_bf16"
186
+ if n == k and m >= 512:
187
+ return "sm110_sq_bf16" if k <= 1024 else "sm110_wide_bf16"
188
+ if n == k and m >= 128:
189
+ return "sm110_wide_bf16"
190
+ return "sm110_t1_bf16"
191
  if m == 1:
192
  if variant == 4:
193
  return "gemv_fp8_m1_w4"
 
259
  return float(start.elapsed_time(end) * 1000.0 / iters)
260
 
261
 
262
+ def measure_median(fn, warmup: int, iters: int, rounds: int = 5) -> float:
263
+ """Reduce clock/order bias without hiding Python-side launch behavior."""
264
+ return float(statistics.median(measure(fn, warmup, iters) for _ in range(rounds)))
265
+
266
+
267
+ def capture_graph(fn, warmup: int) -> torch.cuda.CUDAGraph:
268
+ """Capture one static invocation and retain the graph for paired timing."""
269
+ for _ in range(warmup):
270
+ fn()
271
+ torch.cuda.synchronize()
272
+ graph = torch.cuda.CUDAGraph()
273
+ with torch.cuda.graph(graph):
274
+ fn()
275
+ torch.cuda.synchronize()
276
+ return graph
277
+
278
+
279
+ def measure_group(functions, warmup: int, iters: int, rounds: int = 7):
280
+ """Measure one launch per candidate per round to balance Thor DVFS drift."""
281
+ names = list(functions)
282
+ samples = {name: [] for name in names}
283
+ for _ in range(warmup):
284
+ for fn in functions.values():
285
+ fn()
286
+ torch.cuda.synchronize()
287
+ sample_count = max(iters, rounds * 16)
288
+ event_pairs = {name: [] for name in names}
289
+ for round_index in range(sample_count):
290
+ offset = round_index % len(names)
291
+ for name in names[offset:] + names[:offset]:
292
+ start = torch.cuda.Event(enable_timing=True)
293
+ end = torch.cuda.Event(enable_timing=True)
294
+ start.record()
295
+ functions[name]()
296
+ end.record()
297
+ event_pairs[name].append((start, end))
298
+ torch.cuda.synchronize()
299
+ for name, pairs in event_pairs.items():
300
+ samples[name] = [start.elapsed_time(end) * 1000.0 for start, end in pairs]
301
+ medians = {
302
+ name: float(statistics.median(values)) for name, values in samples.items()
303
+ }
304
+ return medians, samples
305
+
306
+
307
  def metrics(got, expected):
308
  diff = (got.float() - expected.float()).abs().flatten()
309
  return (
 
314
  )
315
 
316
 
317
+ def load_native_reference():
318
+ root = os.environ.get("FLASHRT_NATIVE_ROOT")
319
+ if not root:
320
+ return None
321
+ sys.path.insert(0, root)
322
+ try:
323
+ return importlib.import_module("flash_rt.flash_rt_kernels")
324
+ finally:
325
+ sys.path.remove(root)
326
+
327
+
328
+ def bench_case(ops, native, name: str, shape: tuple[int, int, int], variant: int, warmup: int, iters: int, compile_ref: bool):
329
  m, k, n = shape
330
  x, w = make_inputs(m, k, n, seed=3000 + m + k + n + variant)
331
  out = torch.empty((m, n), device="cuda", dtype=torch.bfloat16)
 
335
  max_abs, mean_abs, p99_abs, cos = metrics(got, expected)
336
  tile = ops.select_fp8_linear_tile(m, n, k, variant)
337
 
338
+ wrapper_invoke = lambda: ops.fp8_linear_bf16(x, w, out=out, variant=variant)
339
+ eager_functions = {"wrapper": wrapper_invoke}
340
+ graph_objects = {"wrapper": capture_graph(wrapper_invoke, warmup)}
341
+ package_forced_tiles = {}
342
+ if torch.cuda.get_device_capability(0) == (11, 0):
343
+ for forced_variant, forced_tile in {
344
+ 1: "sm110_sq_bf16",
345
+ 2: "sm110_t1_bf16",
346
+ 3: "sm110_wide_bf16",
347
+ }.items():
348
+ invoke = lambda forced_variant=forced_variant: ops.fp8_linear_bf16(
349
+ x, w, out=out, variant=forced_variant
350
+ )
351
+ invoke()
352
+ torch.cuda.synchronize()
353
+ fmax, fmean, fp99, fcos = metrics(out, expected)
354
+ if fmax > 0.5 or fmean > 0.02 or fp99 > 0.25 or fcos < 0.999:
355
+ raise RuntimeError(
356
+ f"package {forced_tile} failed correctness for {name}: "
357
+ f"{fmax=}, {fmean=}, {fp99=}, {fcos=}"
358
+ )
359
+ key = f"package:{forced_tile}"
360
+ package_forced_tiles[forced_tile] = invoke
361
+ eager_functions[key] = invoke
362
+ graph_objects[key] = capture_graph(invoke, warmup)
363
+ package_best_graph_tile = None
364
+ package_best_graph_us = None
365
+ package_tiles_us = None
366
+ native_best_tile = None
367
+ native_best_us = None
368
+ native_best_graph_tile = None
369
+ native_best_graph_us = None
370
+ native_tiles_us = None
371
+ native_invokes = {}
372
+ wrapper_vs_native = None
373
+ graph_vs_native = None
374
+ auto_over_package_best = None
375
+ if native is not None and torch.cuda.get_device_capability(0) == (11, 0):
376
+ native_out = torch.empty_like(out)
377
+ candidates = [
378
+ ("sm110_sq_bf16", native.cutlass_fp8_sq_bf16out),
379
+ ("sm110_t1_bf16", native.cutlass_fp8_t1_bf16out),
380
+ ("sm110_wide_bf16", native.cutlass_fp8_wide_bf16out),
381
+ ]
382
+ for tile_name, fn in candidates:
383
+ invoke = lambda fn=fn: fn(
384
+ x.data_ptr(), w.data_ptr(), native_out.data_ptr(),
385
+ m, n, k, 1.0, 0.0,
386
+ int(torch.cuda.current_stream().cuda_stream),
387
+ )
388
+ rc = invoke()
389
+ if rc != 0:
390
+ continue
391
+ torch.cuda.synchronize()
392
+ nmax, nmean, np99, ncos = metrics(native_out, expected)
393
+ if nmax > 0.5 or nmean > 0.02 or np99 > 0.25 or ncos < 0.999:
394
+ raise RuntimeError(
395
+ f"native {tile_name} failed correctness for {name}: "
396
+ f"{nmax=}, {nmean=}, {np99=}, {ncos=}"
397
+ )
398
+ key = f"native:{tile_name}"
399
+ native_invokes[tile_name] = invoke
400
+ eager_functions[key] = invoke
401
+ graph_objects[key] = capture_graph(invoke, warmup)
402
+ if package_forced_tiles:
403
+ eager_times, eager_samples = measure_group(eager_functions, warmup, iters)
404
+ graph_times, graph_samples = measure_group(
405
+ {
406
+ key: graph.replay
407
+ for key, graph in graph_objects.items()
408
+ if key != "wrapper"
409
+ },
410
+ warmup,
411
+ iters,
412
+ )
413
+ flashrt_us = eager_times["wrapper"]
414
+ package_tiles_us = {
415
+ tile_name: {
416
+ "eager": eager_times[f"package:{tile_name}"],
417
+ "graph": graph_times[f"package:{tile_name}"],
418
+ }
419
+ for tile_name in package_forced_tiles
420
+ }
421
+ package_best_graph_us, package_best_graph_tile = min(
422
+ (times["graph"], tile_name)
423
+ for tile_name, times in package_tiles_us.items()
424
+ )
425
+ # Auto and its matching forced variant resolve to the same native
426
+ # function. Use that single graph measurement for the tile gate; timing
427
+ # duplicate graph objects is vulnerable to Thor DVFS order bias.
428
+ flashrt_graph_us = package_tiles_us[tile]["graph"]
429
+ auto_over_package_best = float(statistics.median(
430
+ selected / best
431
+ for selected, best in zip(
432
+ graph_samples[f"package:{tile}"],
433
+ graph_samples[f"package:{package_best_graph_tile}"],
434
+ )
435
+ ))
436
+ if native_invokes:
437
+ native_tiles_us = {
438
+ tile_name: {
439
+ "eager": eager_times[f"native:{tile_name}"],
440
+ "graph": graph_times[f"native:{tile_name}"],
441
+ }
442
+ for tile_name in native_invokes
443
+ }
444
+ native_best_us, native_best_tile = min(
445
+ (times["eager"], tile_name)
446
+ for tile_name, times in native_tiles_us.items()
447
+ )
448
+ native_best_graph_us, native_best_graph_tile = min(
449
+ (times["graph"], tile_name)
450
+ for tile_name, times in native_tiles_us.items()
451
+ )
452
+ wrapper_vs_native = float(statistics.median(
453
+ wrapper / native_sample
454
+ for wrapper, native_sample in zip(
455
+ eager_samples["wrapper"],
456
+ eager_samples[f"native:{native_best_tile}"],
457
+ )
458
+ ))
459
+ graph_vs_native = float(statistics.median(
460
+ package_sample / native_sample
461
+ for package_sample, native_sample in zip(
462
+ graph_samples[f"package:{tile}"],
463
+ graph_samples[f"native:{native_best_graph_tile}"],
464
+ )
465
+ ))
466
+ else:
467
+ flashrt_us = measure_median(wrapper_invoke, warmup, iters)
468
+ flashrt_graph_us = measure_median(
469
+ graph_objects["wrapper"].replay, warmup, iters
470
+ )
471
+ tile_pass = auto_over_package_best is None or auto_over_package_best <= 1.10
472
  eager_us = measure(lambda: ref_fn(x, w), warmup, iters)
473
  compile_us = None
474
  if compile_ref:
 
488
  variant=variant,
489
  tile=tile,
490
  flashrt_us=flashrt_us,
491
+ flashrt_graph_us=flashrt_graph_us,
492
+ package_best_graph_tile=package_best_graph_tile,
493
+ package_best_graph_us=package_best_graph_us,
494
+ auto_over_package_best=auto_over_package_best,
495
+ package_tiles_us=package_tiles_us,
496
+ native_best_tile=native_best_tile,
497
+ native_best_us=native_best_us,
498
+ native_best_graph_tile=native_best_graph_tile,
499
+ native_best_graph_us=native_best_graph_us,
500
+ native_tiles_us=native_tiles_us,
501
+ wrapper_vs_native=wrapper_vs_native,
502
+ graph_vs_native=graph_vs_native,
503
  torch_eager_us=eager_us,
504
  torch_compile_us=compile_us,
505
  speedup_vs_eager=eager_us / flashrt_us,
 
508
  mean_abs=mean_abs,
509
  p99_abs=p99_abs,
510
  cosine=cos,
511
+ status=(
512
+ "pass"
513
+ if max_abs <= 0.5 and p99_abs <= 0.25 and cos >= 0.999 and tile_pass
514
+ else "fail"
515
+ ),
516
  )
517
 
518
 
 
529
 
530
  if not torch.cuda.is_available():
531
  raise SystemExit("CUDA is required")
532
+ capability = torch.cuda.get_device_capability(0)
533
+ if capability not in {(11, 0), (12, 0)}:
534
+ raise SystemExit("fp8-gemm benchmark requires SM110 or SM120")
535
 
536
  ops = load_source_ops() if args.backend == "source" else load_installed_ops(args.artifact)
537
+ native = load_native_reference()
538
  rows: list[Result] = []
539
  for name in MODES[args.mode]:
540
  shape = SHAPES[name]
541
  variants = [0]
542
+ if shape[0] == 1 and capability == (12, 0):
543
  variants = [0, 4, 8, 16]
544
  for variant in variants:
545
+ rows.append(bench_case(
546
+ ops, native, name, shape, variant,
547
+ args.warmup, args.iterations, args.compile_ref,
548
+ ))
549
 
550
  payload = {"rows": [asdict(row) for row in rows]}
551
  print(json.dumps(payload, indent=2, sort_keys=True))
build/torch211-cxx11-cu128-x86_64-linux/__init__.py CHANGED
@@ -37,6 +37,55 @@ def _fp8_linear_residual_bf16_fake(
37
  return None
38
 
39
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
40
  def select_fp8_linear_tile(m: int, n: int, k: int, variant: int = 0) -> str:
41
  """Return the FlashRT tile selected by the public dispatcher."""
42
 
@@ -126,3 +175,60 @@ def fp8_linear_residual_bf16(
126
 
127
  ops.fp8_linear_residual_bf16(input, weight, float(alpha), int(variant), residual)
128
  return residual
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
37
  return None
38
 
39
 
40
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_blockwise_linear_bf16"))
41
+ def _fp8_blockwise_linear_bf16_fake(
42
+ input: torch.Tensor,
43
+ weight: torch.Tensor,
44
+ input_scale: torch.Tensor,
45
+ weight_scale: torch.Tensor,
46
+ out: torch.Tensor,
47
+ ) -> None:
48
+ if input.dim() != 2 or weight.dim() != 2:
49
+ raise RuntimeError("input and weight must be rank-2 tensors")
50
+ m, k = input.shape
51
+ n = weight.shape[0]
52
+ if weight.shape[1] != k or n % 128 or k % 128:
53
+ raise RuntimeError("weight shape is invalid or N/K are not divisible by 128")
54
+ if input_scale.shape != (m, k // 128):
55
+ raise RuntimeError("input_scale must have shape (M, K / 128)")
56
+ if weight_scale.shape != (n // 128, k // 128):
57
+ raise RuntimeError("weight_scale must have shape (N / 128, K / 128)")
58
+ if out.shape != (m, n):
59
+ raise RuntimeError("out must have shape (M, N)")
60
+ return None
61
+
62
+
63
+ @torch.library.register_fake(
64
+ add_op_namespace_prefix("fp8_blockwise_swiglu_quantize_fp8")
65
+ )
66
+ def _fp8_blockwise_swiglu_quantize_fp8_fake(
67
+ input: torch.Tensor,
68
+ gate_up_weight: torch.Tensor,
69
+ input_scale: torch.Tensor,
70
+ gate_up_weight_scale: torch.Tensor,
71
+ output: torch.Tensor,
72
+ output_scale: torch.Tensor,
73
+ ) -> None:
74
+ m, k = input.shape
75
+ if gate_up_weight.dim() != 2 or gate_up_weight.shape[0] % 2:
76
+ raise RuntimeError("gate_up_weight must have shape (2*N, K)")
77
+ n = gate_up_weight.shape[0] // 2
78
+ if gate_up_weight.shape[1] != k or n % 128 or k % 128:
79
+ raise RuntimeError("gate_up_weight shape is invalid or N/K are not divisible by 128")
80
+ if input_scale.shape != (m, k // 128):
81
+ raise RuntimeError("input_scale must have shape (M, K/128)")
82
+ if gate_up_weight_scale.shape != (2 * n // 128, k // 128):
83
+ raise RuntimeError("gate_up_weight_scale must have shape (2*N/128, K/128)")
84
+ if output.shape != (m, n) or output_scale.shape != (m, n // 128):
85
+ raise RuntimeError("output buffers have invalid shapes")
86
+ return None
87
+
88
+
89
  def select_fp8_linear_tile(m: int, n: int, k: int, variant: int = 0) -> str:
90
  """Return the FlashRT tile selected by the public dispatcher."""
91
 
 
175
 
176
  ops.fp8_linear_residual_bf16(input, weight, float(alpha), int(variant), residual)
177
  return residual
178
+
179
+
180
+ def fp8_blockwise_linear_bf16(
181
+ input: torch.Tensor,
182
+ weight: torch.Tensor,
183
+ input_scale: torch.Tensor,
184
+ weight_scale: torch.Tensor,
185
+ out: torch.Tensor | None = None,
186
+ ) -> torch.Tensor:
187
+ """Block-128 scaled FP8 linear with BF16 output on SM120."""
188
+
189
+ if out is None:
190
+ out = torch.empty(
191
+ (input.shape[0], weight.shape[0]),
192
+ device=input.device,
193
+ dtype=torch.bfloat16,
194
+ )
195
+ ops.fp8_blockwise_linear_bf16(
196
+ input, weight, input_scale, weight_scale, out
197
+ )
198
+ return out
199
+
200
+
201
+ def fp8_blockwise_swiglu_quantize_fp8(
202
+ input: torch.Tensor,
203
+ gate_up_weight: torch.Tensor,
204
+ input_scale: torch.Tensor,
205
+ gate_up_weight_scale: torch.Tensor,
206
+ *,
207
+ output: torch.Tensor | None = None,
208
+ output_scale: torch.Tensor | None = None,
209
+ ) -> tuple[torch.Tensor, torch.Tensor]:
210
+ """SM89 block-128 FP8 gate/up GEMM + SiLU + FP8 requant producer."""
211
+
212
+ n = gate_up_weight.shape[0] // 2
213
+ if output is None:
214
+ output = torch.empty(
215
+ (input.shape[0], n), device=input.device, dtype=torch.float8_e4m3fn
216
+ )
217
+ if output_scale is None:
218
+ output_scale = torch.empty(
219
+ (input.shape[0], n // 128), device=input.device, dtype=torch.float32
220
+ )
221
+ ops.fp8_blockwise_swiglu_quantize_fp8(
222
+ input, gate_up_weight, input_scale, gate_up_weight_scale,
223
+ output, output_scale
224
+ )
225
+ return output, output_scale
226
+
227
+
228
+ __all__ = [
229
+ "fp8_linear_bf16",
230
+ "fp8_linear_residual_bf16",
231
+ "fp8_blockwise_linear_bf16",
232
+ "fp8_blockwise_swiglu_quantize_fp8",
233
+ "select_fp8_linear_tile",
234
+ ]
build/{torch212-cxx11-cu132-x86_64-linux/_fp8_gemm_cuda_1798e7f.abi3.so → torch211-cxx11-cu128-x86_64-linux/_fp8_gemm_cuda_bcb0782.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ae2126f11ef82012f75af2d6e20bf4d924771f8c28b2bf237d8dd2dce48d0d6d
3
- size 2622888
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:182510f121f814d7c367c58c641dc57f8992cc1b62fa25409ed9128e292fcec8
3
+ size 3942856
build/torch211-cxx11-cu128-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp8_gemm_cuda_1798e7f
3
- ops = torch.ops._fp8_gemm_cuda_1798e7f
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp8_gemm_cuda_1798e7f::{op_name}"
 
1
  import torch
2
+ from . import _fp8_gemm_cuda_bcb0782
3
+ ops = torch.ops._fp8_gemm_cuda_bcb0782
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp8_gemm_cuda_bcb0782::{op_name}"
build/torch211-cxx11-cu128-x86_64-linux/metadata.json CHANGED
@@ -1,21 +1,22 @@
1
  {
2
  "name": "fp8-gemm",
3
- "id": "_fp8_gemm_cuda_1798e7f",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
7
  "backend": {
8
  "type": "cuda",
9
  "archs": [
10
- "12.0a"
 
11
  ]
12
  },
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
- "__init__.py": "Bm2+gGxw1Jrges8cKNwvxFr7dcD5K9hQbYHhO+S60ns=",
17
- "_fp8_gemm_cuda_1798e7f.abi3.so": "nnb1/gpQoU5Y5qzjFnMgZQ4IBat5hHFe5D/wUq5rVMw=",
18
- "_ops.py": "RZGyMiLwlBDfJkYrzh6YrdosVDR9oznN9VDc/9OTrBQ=",
19
  "fp8_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
20
  }
21
  }
 
1
  {
2
  "name": "fp8-gemm",
3
+ "id": "_fp8_gemm_cuda_bcb0782",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
7
  "backend": {
8
  "type": "cuda",
9
  "archs": [
10
+ "12.0a",
11
+ "8.9"
12
  ]
13
  },
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "AhlkcnyTuk3G4aYsvMqu3QdiEZSTA6Y00poqqiX+Mps=",
18
+ "_fp8_gemm_cuda_bcb0782.abi3.so": "GCUQ8SH4FNfDZ8WMZB3Ff4mSzBti+iVAntkSjikvzsg=",
19
+ "_ops.py": "qpBYMOKsfVJtHhKzoX/hxuQPF3MGISv+MQwpPmbzUcw=",
20
  "fp8_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
  }
build/torch211-cxx11-cu130-aarch64-linux/__init__.py ADDED
@@ -0,0 +1,254 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP8 GEMM kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import torch
6
+
7
+ from ._ops import add_op_namespace_prefix, ops
8
+
9
+
10
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bf16"))
11
+ def _fp8_linear_bf16_fake(
12
+ input: torch.Tensor,
13
+ weight: torch.Tensor,
14
+ alpha: float,
15
+ variant: int,
16
+ out: torch.Tensor,
17
+ ) -> None:
18
+ if input.dim() != 2 or weight.dim() != 2:
19
+ raise RuntimeError("input and weight must be rank-2 tensors")
20
+ if out.shape != (input.shape[0], weight.shape[0]):
21
+ raise RuntimeError("out must have shape (input.shape[0], weight.shape[0])")
22
+ return None
23
+
24
+
25
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_residual_bf16"))
26
+ def _fp8_linear_residual_bf16_fake(
27
+ input: torch.Tensor,
28
+ weight: torch.Tensor,
29
+ alpha: float,
30
+ variant: int,
31
+ residual: torch.Tensor,
32
+ ) -> None:
33
+ if input.shape[0] != 1:
34
+ raise RuntimeError("residual path supports only M=1")
35
+ if residual.shape != (1, weight.shape[0]):
36
+ raise RuntimeError("residual must have shape (1, weight.shape[0])")
37
+ return None
38
+
39
+
40
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_blockwise_linear_bf16"))
41
+ def _fp8_blockwise_linear_bf16_fake(
42
+ input: torch.Tensor,
43
+ weight: torch.Tensor,
44
+ input_scale: torch.Tensor,
45
+ weight_scale: torch.Tensor,
46
+ out: torch.Tensor,
47
+ ) -> None:
48
+ if input.dim() != 2 or weight.dim() != 2:
49
+ raise RuntimeError("input and weight must be rank-2 tensors")
50
+ m, k = input.shape
51
+ n = weight.shape[0]
52
+ if weight.shape[1] != k or n % 128 or k % 128:
53
+ raise RuntimeError("weight shape is invalid or N/K are not divisible by 128")
54
+ if input_scale.shape != (m, k // 128):
55
+ raise RuntimeError("input_scale must have shape (M, K / 128)")
56
+ if weight_scale.shape != (n // 128, k // 128):
57
+ raise RuntimeError("weight_scale must have shape (N / 128, K / 128)")
58
+ if out.shape != (m, n):
59
+ raise RuntimeError("out must have shape (M, N)")
60
+ return None
61
+
62
+
63
+ @torch.library.register_fake(
64
+ add_op_namespace_prefix("fp8_blockwise_swiglu_quantize_fp8")
65
+ )
66
+ def _fp8_blockwise_swiglu_quantize_fp8_fake(
67
+ input: torch.Tensor,
68
+ gate_up_weight: torch.Tensor,
69
+ input_scale: torch.Tensor,
70
+ gate_up_weight_scale: torch.Tensor,
71
+ output: torch.Tensor,
72
+ output_scale: torch.Tensor,
73
+ ) -> None:
74
+ m, k = input.shape
75
+ if gate_up_weight.dim() != 2 or gate_up_weight.shape[0] % 2:
76
+ raise RuntimeError("gate_up_weight must have shape (2*N, K)")
77
+ n = gate_up_weight.shape[0] // 2
78
+ if gate_up_weight.shape[1] != k or n % 128 or k % 128:
79
+ raise RuntimeError("gate_up_weight shape is invalid or N/K are not divisible by 128")
80
+ if input_scale.shape != (m, k // 128):
81
+ raise RuntimeError("input_scale must have shape (M, K/128)")
82
+ if gate_up_weight_scale.shape != (2 * n // 128, k // 128):
83
+ raise RuntimeError("gate_up_weight_scale must have shape (2*N/128, K/128)")
84
+ if output.shape != (m, n) or output_scale.shape != (m, n // 128):
85
+ raise RuntimeError("output buffers have invalid shapes")
86
+ return None
87
+
88
+
89
+ def select_fp8_linear_tile(m: int, n: int, k: int, variant: int = 0) -> str:
90
+ """Return the FlashRT tile selected by the public dispatcher."""
91
+
92
+ m = int(m)
93
+ n = int(n)
94
+ k = int(k)
95
+ variant = int(variant)
96
+ if m <= 0 or n <= 0 or k <= 0:
97
+ raise RuntimeError("m, n, and k must be positive")
98
+ if k % 32 != 0:
99
+ raise RuntimeError("k must be divisible by 32")
100
+ capability = torch.cuda.get_device_capability() if torch.cuda.is_available() else None
101
+ if capability == (11, 0):
102
+ forced = {1: "sm110_sq_bf16", 2: "sm110_t1_bf16", 3: "sm110_wide_bf16"}
103
+ if variant not in {0, *forced}:
104
+ raise RuntimeError("SM110 variant must be 0 (auto), 1 (Sq), 2 (T1), or 3 (Wide)")
105
+ if n % 16 or k % 16:
106
+ raise RuntimeError("SM110 requires n and k divisible by 16")
107
+ if variant:
108
+ return forced[variant]
109
+ if n >= 8 * k:
110
+ return "sm110_wide_bf16"
111
+ if m >= 128 and k >= 4 * n:
112
+ return "sm110_sq_bf16"
113
+ if n == k and m >= 512:
114
+ return "sm110_sq_bf16" if k <= 1024 else "sm110_wide_bf16"
115
+ if n == k and m >= 128:
116
+ return "sm110_wide_bf16"
117
+ return "sm110_t1_bf16"
118
+ if m == 1:
119
+ if variant == 4:
120
+ return "gemv_fp8_m1_w4"
121
+ if variant == 8:
122
+ return "gemv_fp8_m1_w8"
123
+ if variant == 16:
124
+ return "gemv_fp8_m1_w16"
125
+ if variant != 0:
126
+ raise RuntimeError("M=1 variant must be 0, 4, 8, or 16")
127
+ if n <= 2048:
128
+ return "gemv_fp8_m1_w4"
129
+ if n <= 8192:
130
+ return "gemv_fp8_m1_w8"
131
+ return "gemv_fp8_m1_w16"
132
+ if variant != 0:
133
+ raise RuntimeError("small-M dispatcher currently supports variant=0 only")
134
+ if m <= 16:
135
+ if k % 256 == 0:
136
+ return "ld_fp8_gemm_16x128x256_w4" if n % 128 == 0 else "ld_fp8_gemm_16x64x256_w4"
137
+ if n % 256 == 0:
138
+ return "ld_fp8_gemm_16x256x128_w8"
139
+ if n % 192 == 0:
140
+ return "ld_fp8_gemm_16x192x128_w4"
141
+ if n % 128 == 0:
142
+ return "ld_fp8_gemm_16x128x128_w4"
143
+ return "ld_fp8_gemm_16x64x128_w4"
144
+ if m <= 32:
145
+ if k % 256 == 0:
146
+ return "ld_fp8_gemm_32x128x256_w4" if n % 128 == 0 else "ld_fp8_gemm_32x64x256_w4"
147
+ if n % 192 == 0:
148
+ return "ld_fp8_gemm_32x192x128_w4"
149
+ if n % 128 == 0:
150
+ return "ld_fp8_gemm_32x128x128_w4"
151
+ return "ld_fp8_gemm_32x64x128_w4"
152
+ if m <= 64:
153
+ if k % 256 == 0:
154
+ return "ld_fp8_gemm_64x128x256_w4" if n % 128 == 0 else "ld_fp8_gemm_64x64x256_w4"
155
+ if n % 128 == 0:
156
+ return "ld_fp8_gemm_64x128x128_w4"
157
+ return "ld_fp8_gemm_64x64x128_w4"
158
+ raise RuntimeError("only M=1 decode or 2 <= M <= 64 small-M rows are supported")
159
+
160
+
161
+ def fp8_linear_bf16(
162
+ input: torch.Tensor,
163
+ weight: torch.Tensor,
164
+ alpha: float = 1.0,
165
+ out: torch.Tensor | None = None,
166
+ variant: int = 0,
167
+ ) -> torch.Tensor:
168
+ """Compute ``(input @ weight.T) * alpha`` with BF16 output.
169
+
170
+ ``input`` and ``weight`` must be FP8 E4M3 CUDA tensors with shapes
171
+ ``(M, K)`` and ``(N, K)``. ``alpha`` is a host float, normally the product
172
+ of static per-tensor input and weight scales. SM110 uses the production
173
+ CUTLASS Sq/T1/Wide dispatcher over full model row counts; SM120 uses the
174
+ hand-tuned M<=64 path.
175
+ """
176
+
177
+ if out is None:
178
+ out = torch.empty(
179
+ (input.shape[0], weight.shape[0]),
180
+ device=input.device,
181
+ dtype=torch.bfloat16,
182
+ )
183
+ ops.fp8_linear_bf16(input, weight, float(alpha), int(variant), out)
184
+ return out
185
+
186
+
187
+ def fp8_linear_residual_bf16(
188
+ input: torch.Tensor,
189
+ weight: torch.Tensor,
190
+ residual: torch.Tensor,
191
+ alpha: float = 1.0,
192
+ variant: int = 0,
193
+ ) -> torch.Tensor:
194
+ """In-place ``residual += (input @ weight.T) * alpha`` for M=1 decode."""
195
+
196
+ ops.fp8_linear_residual_bf16(input, weight, float(alpha), int(variant), residual)
197
+ return residual
198
+
199
+
200
+ def fp8_blockwise_linear_bf16(
201
+ input: torch.Tensor,
202
+ weight: torch.Tensor,
203
+ input_scale: torch.Tensor,
204
+ weight_scale: torch.Tensor,
205
+ out: torch.Tensor | None = None,
206
+ ) -> torch.Tensor:
207
+ """Block-128 scaled FP8 linear with BF16 output on SM89/SM120."""
208
+
209
+ if out is None:
210
+ out = torch.empty(
211
+ (input.shape[0], weight.shape[0]),
212
+ device=input.device,
213
+ dtype=torch.bfloat16,
214
+ )
215
+ ops.fp8_blockwise_linear_bf16(
216
+ input, weight, input_scale, weight_scale, out
217
+ )
218
+ return out
219
+
220
+
221
+ def fp8_blockwise_swiglu_quantize_fp8(
222
+ input: torch.Tensor,
223
+ gate_up_weight: torch.Tensor,
224
+ input_scale: torch.Tensor,
225
+ gate_up_weight_scale: torch.Tensor,
226
+ *,
227
+ output: torch.Tensor | None = None,
228
+ output_scale: torch.Tensor | None = None,
229
+ ) -> tuple[torch.Tensor, torch.Tensor]:
230
+ """SM89 block-128 FP8 gate/up GEMM + SiLU + FP8 requant producer."""
231
+
232
+ n = gate_up_weight.shape[0] // 2
233
+ if output is None:
234
+ output = torch.empty(
235
+ (input.shape[0], n), device=input.device, dtype=torch.float8_e4m3fn
236
+ )
237
+ if output_scale is None:
238
+ output_scale = torch.empty(
239
+ (input.shape[0], n // 128), device=input.device, dtype=torch.float32
240
+ )
241
+ ops.fp8_blockwise_swiglu_quantize_fp8(
242
+ input, gate_up_weight, input_scale, gate_up_weight_scale,
243
+ output, output_scale
244
+ )
245
+ return output, output_scale
246
+
247
+
248
+ __all__ = [
249
+ "fp8_linear_bf16",
250
+ "fp8_linear_residual_bf16",
251
+ "fp8_blockwise_linear_bf16",
252
+ "fp8_blockwise_swiglu_quantize_fp8",
253
+ "select_fp8_linear_tile",
254
+ ]
build/{torch211-cxx11-cu128-x86_64-linux/_fp8_gemm_cuda_1798e7f.abi3.so → torch211-cxx11-cu130-aarch64-linux/_fp8_gemm_cuda_d31c69b1_sm110.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9e76f5fe0a50a14e58e6ace3167320650e0805ab7984715ee43ff052ae6b54cc
3
- size 2458144
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5db25f2b8481dd37a9dd8076f117e93cce85aa50779abb9febdade30f17058ac
3
+ size 1890984
build/torch211-cxx11-cu130-aarch64-linux/_ops.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _fp8_gemm_cuda_d31c69b1_sm110
3
+ ops = torch.ops._fp8_gemm_cuda_d31c69b1_sm110
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ return f"_fp8_gemm_cuda_d31c69b1_sm110::{op_name}"
build/torch211-cxx11-cu130-aarch64-linux/fp8_gemm/__init__.py ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+
6
+ p = Path(__file__).parent.parent / '__init__.py'
7
+ name = '{:x}'.format(ctypes.c_size_t(hash(p.absolute())).value)
8
+ spec = importlib.util.spec_from_file_location(name, p, submodule_search_locations=[str(p.parent)])
9
+ module = importlib.util.module_from_spec(spec)
10
+ sys.modules[name] = module
11
+ spec.loader.exec_module(module)
12
+ globals().update(vars(module))
build/torch211-cxx11-cu130-aarch64-linux/metadata.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "fp8-gemm",
3
+ "id": "_fp8_gemm_cuda_d31c69b1_sm110",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "11.0a"
11
+ ]
12
+ },
13
+ "digest": {
14
+ "algorithm": "sha256",
15
+ "files": {
16
+ "__init__.py": "hKsdFER2ZmYz36/zz9LYE7ddsK/R1dprHJFx+9yBzks=",
17
+ "_ops.py": "ql6AyUDg7Jbz6AiKZ2eN3Q6TKM+TVVeCzZauSOYe+Lk=",
18
+ "_fp8_gemm_cuda_d31c69b1_sm110.abi3.so": "XbJfK4SB3Tep3YB28RfpPM6FqlB3mruf69reMPFwWKw="
19
+ }
20
+ },
21
+ "provenance": {
22
+ "kernel": {
23
+ "sha": "d31c69b1cb97ecd703aba01e29f423097f11c86a",
24
+ "dirty": false
25
+ },
26
+ "validation": {
27
+ "torch": "2.11.0+cu130",
28
+ "cuda": "13.0"
29
+ }
30
+ }
31
+ }
build/torch211-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -37,6 +37,55 @@ def _fp8_linear_residual_bf16_fake(
37
  return None
38
 
39
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
40
  def select_fp8_linear_tile(m: int, n: int, k: int, variant: int = 0) -> str:
41
  """Return the FlashRT tile selected by the public dispatcher."""
42
 
@@ -126,3 +175,60 @@ def fp8_linear_residual_bf16(
126
 
127
  ops.fp8_linear_residual_bf16(input, weight, float(alpha), int(variant), residual)
128
  return residual
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
37
  return None
38
 
39
 
40
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_blockwise_linear_bf16"))
41
+ def _fp8_blockwise_linear_bf16_fake(
42
+ input: torch.Tensor,
43
+ weight: torch.Tensor,
44
+ input_scale: torch.Tensor,
45
+ weight_scale: torch.Tensor,
46
+ out: torch.Tensor,
47
+ ) -> None:
48
+ if input.dim() != 2 or weight.dim() != 2:
49
+ raise RuntimeError("input and weight must be rank-2 tensors")
50
+ m, k = input.shape
51
+ n = weight.shape[0]
52
+ if weight.shape[1] != k or n % 128 or k % 128:
53
+ raise RuntimeError("weight shape is invalid or N/K are not divisible by 128")
54
+ if input_scale.shape != (m, k // 128):
55
+ raise RuntimeError("input_scale must have shape (M, K / 128)")
56
+ if weight_scale.shape != (n // 128, k // 128):
57
+ raise RuntimeError("weight_scale must have shape (N / 128, K / 128)")
58
+ if out.shape != (m, n):
59
+ raise RuntimeError("out must have shape (M, N)")
60
+ return None
61
+
62
+
63
+ @torch.library.register_fake(
64
+ add_op_namespace_prefix("fp8_blockwise_swiglu_quantize_fp8")
65
+ )
66
+ def _fp8_blockwise_swiglu_quantize_fp8_fake(
67
+ input: torch.Tensor,
68
+ gate_up_weight: torch.Tensor,
69
+ input_scale: torch.Tensor,
70
+ gate_up_weight_scale: torch.Tensor,
71
+ output: torch.Tensor,
72
+ output_scale: torch.Tensor,
73
+ ) -> None:
74
+ m, k = input.shape
75
+ if gate_up_weight.dim() != 2 or gate_up_weight.shape[0] % 2:
76
+ raise RuntimeError("gate_up_weight must have shape (2*N, K)")
77
+ n = gate_up_weight.shape[0] // 2
78
+ if gate_up_weight.shape[1] != k or n % 128 or k % 128:
79
+ raise RuntimeError("gate_up_weight shape is invalid or N/K are not divisible by 128")
80
+ if input_scale.shape != (m, k // 128):
81
+ raise RuntimeError("input_scale must have shape (M, K/128)")
82
+ if gate_up_weight_scale.shape != (2 * n // 128, k // 128):
83
+ raise RuntimeError("gate_up_weight_scale must have shape (2*N/128, K/128)")
84
+ if output.shape != (m, n) or output_scale.shape != (m, n // 128):
85
+ raise RuntimeError("output buffers have invalid shapes")
86
+ return None
87
+
88
+
89
  def select_fp8_linear_tile(m: int, n: int, k: int, variant: int = 0) -> str:
90
  """Return the FlashRT tile selected by the public dispatcher."""
91
 
 
175
 
176
  ops.fp8_linear_residual_bf16(input, weight, float(alpha), int(variant), residual)
177
  return residual
178
+
179
+
180
+ def fp8_blockwise_linear_bf16(
181
+ input: torch.Tensor,
182
+ weight: torch.Tensor,
183
+ input_scale: torch.Tensor,
184
+ weight_scale: torch.Tensor,
185
+ out: torch.Tensor | None = None,
186
+ ) -> torch.Tensor:
187
+ """Block-128 scaled FP8 linear with BF16 output on SM120."""
188
+
189
+ if out is None:
190
+ out = torch.empty(
191
+ (input.shape[0], weight.shape[0]),
192
+ device=input.device,
193
+ dtype=torch.bfloat16,
194
+ )
195
+ ops.fp8_blockwise_linear_bf16(
196
+ input, weight, input_scale, weight_scale, out
197
+ )
198
+ return out
199
+
200
+
201
+ def fp8_blockwise_swiglu_quantize_fp8(
202
+ input: torch.Tensor,
203
+ gate_up_weight: torch.Tensor,
204
+ input_scale: torch.Tensor,
205
+ gate_up_weight_scale: torch.Tensor,
206
+ *,
207
+ output: torch.Tensor | None = None,
208
+ output_scale: torch.Tensor | None = None,
209
+ ) -> tuple[torch.Tensor, torch.Tensor]:
210
+ """SM89 block-128 FP8 gate/up GEMM + SiLU + FP8 requant producer."""
211
+
212
+ n = gate_up_weight.shape[0] // 2
213
+ if output is None:
214
+ output = torch.empty(
215
+ (input.shape[0], n), device=input.device, dtype=torch.float8_e4m3fn
216
+ )
217
+ if output_scale is None:
218
+ output_scale = torch.empty(
219
+ (input.shape[0], n // 128), device=input.device, dtype=torch.float32
220
+ )
221
+ ops.fp8_blockwise_swiglu_quantize_fp8(
222
+ input, gate_up_weight, input_scale, gate_up_weight_scale,
223
+ output, output_scale
224
+ )
225
+ return output, output_scale
226
+
227
+
228
+ __all__ = [
229
+ "fp8_linear_bf16",
230
+ "fp8_linear_residual_bf16",
231
+ "fp8_blockwise_linear_bf16",
232
+ "fp8_blockwise_swiglu_quantize_fp8",
233
+ "select_fp8_linear_tile",
234
+ ]
build/torch211-cxx11-cu130-x86_64-linux/{_fp8_gemm_cuda_1798e7f.abi3.so → _fp8_gemm_cuda_bcb0782.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4dc4693377b79dcf7817b22f4d0bcf9dd215c320b8945bf9b0943a068986ff4a
3
- size 2599784
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f556905c27b2b6a3e59bf294b61e699196c14a3b10aa7db3fb0f09a3b709fa4
3
+ size 4147648
build/torch211-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp8_gemm_cuda_1798e7f
3
- ops = torch.ops._fp8_gemm_cuda_1798e7f
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp8_gemm_cuda_1798e7f::{op_name}"
 
1
  import torch
2
+ from . import _fp8_gemm_cuda_bcb0782
3
+ ops = torch.ops._fp8_gemm_cuda_bcb0782
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp8_gemm_cuda_bcb0782::{op_name}"
build/torch211-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,21 +1,22 @@
1
  {
2
  "name": "fp8-gemm",
3
- "id": "_fp8_gemm_cuda_1798e7f",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
7
  "backend": {
8
  "type": "cuda",
9
  "archs": [
10
- "12.0a"
 
11
  ]
12
  },
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
- "__init__.py": "Bm2+gGxw1Jrges8cKNwvxFr7dcD5K9hQbYHhO+S60ns=",
17
- "_fp8_gemm_cuda_1798e7f.abi3.so": "TcRpM3e3nc94F7IvTQvPndIVwyC4lFv5sJQ6BomG/0o=",
18
- "_ops.py": "RZGyMiLwlBDfJkYrzh6YrdosVDR9oznN9VDc/9OTrBQ=",
19
  "fp8_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
20
  }
21
  }
 
1
  {
2
  "name": "fp8-gemm",
3
+ "id": "_fp8_gemm_cuda_bcb0782",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
7
  "backend": {
8
  "type": "cuda",
9
  "archs": [
10
+ "12.0a",
11
+ "8.9"
12
  ]
13
  },
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "AhlkcnyTuk3G4aYsvMqu3QdiEZSTA6Y00poqqiX+Mps=",
18
+ "_fp8_gemm_cuda_bcb0782.abi3.so": "T1VpBcJ7K2o+Wb8pS2HmmRlsFKOxCqfbP7Dwmjtwn6Q=",
19
+ "_ops.py": "qpBYMOKsfVJtHhKzoX/hxuQPF3MGISv+MQwpPmbzUcw=",
20
  "fp8_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
  }
build/torch212-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -37,6 +37,55 @@ def _fp8_linear_residual_bf16_fake(
37
  return None
38
 
39
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
40
  def select_fp8_linear_tile(m: int, n: int, k: int, variant: int = 0) -> str:
41
  """Return the FlashRT tile selected by the public dispatcher."""
42
 
@@ -48,6 +97,24 @@ def select_fp8_linear_tile(m: int, n: int, k: int, variant: int = 0) -> str:
48
  raise RuntimeError("m, n, and k must be positive")
49
  if k % 32 != 0:
50
  raise RuntimeError("k must be divisible by 32")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
51
  if m == 1:
52
  if variant == 4:
53
  return "gemv_fp8_m1_w4"
@@ -102,7 +169,9 @@ def fp8_linear_bf16(
102
 
103
  ``input`` and ``weight`` must be FP8 E4M3 CUDA tensors with shapes
104
  ``(M, K)`` and ``(N, K)``. ``alpha`` is a host float, normally the product
105
- of static per-tensor input and weight scales.
 
 
106
  """
107
 
108
  if out is None:
@@ -126,3 +195,60 @@ def fp8_linear_residual_bf16(
126
 
127
  ops.fp8_linear_residual_bf16(input, weight, float(alpha), int(variant), residual)
128
  return residual
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
37
  return None
38
 
39
 
40
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_blockwise_linear_bf16"))
41
+ def _fp8_blockwise_linear_bf16_fake(
42
+ input: torch.Tensor,
43
+ weight: torch.Tensor,
44
+ input_scale: torch.Tensor,
45
+ weight_scale: torch.Tensor,
46
+ out: torch.Tensor,
47
+ ) -> None:
48
+ if input.dim() != 2 or weight.dim() != 2:
49
+ raise RuntimeError("input and weight must be rank-2 tensors")
50
+ m, k = input.shape
51
+ n = weight.shape[0]
52
+ if weight.shape[1] != k or n % 128 or k % 128:
53
+ raise RuntimeError("weight shape is invalid or N/K are not divisible by 128")
54
+ if input_scale.shape != (m, k // 128):
55
+ raise RuntimeError("input_scale must have shape (M, K / 128)")
56
+ if weight_scale.shape != (n // 128, k // 128):
57
+ raise RuntimeError("weight_scale must have shape (N / 128, K / 128)")
58
+ if out.shape != (m, n):
59
+ raise RuntimeError("out must have shape (M, N)")
60
+ return None
61
+
62
+
63
+ @torch.library.register_fake(
64
+ add_op_namespace_prefix("fp8_blockwise_swiglu_quantize_fp8")
65
+ )
66
+ def _fp8_blockwise_swiglu_quantize_fp8_fake(
67
+ input: torch.Tensor,
68
+ gate_up_weight: torch.Tensor,
69
+ input_scale: torch.Tensor,
70
+ gate_up_weight_scale: torch.Tensor,
71
+ output: torch.Tensor,
72
+ output_scale: torch.Tensor,
73
+ ) -> None:
74
+ m, k = input.shape
75
+ if gate_up_weight.dim() != 2 or gate_up_weight.shape[0] % 2:
76
+ raise RuntimeError("gate_up_weight must have shape (2*N, K)")
77
+ n = gate_up_weight.shape[0] // 2
78
+ if gate_up_weight.shape[1] != k or n % 128 or k % 128:
79
+ raise RuntimeError("gate_up_weight shape is invalid or N/K are not divisible by 128")
80
+ if input_scale.shape != (m, k // 128):
81
+ raise RuntimeError("input_scale must have shape (M, K/128)")
82
+ if gate_up_weight_scale.shape != (2 * n // 128, k // 128):
83
+ raise RuntimeError("gate_up_weight_scale must have shape (2*N/128, K/128)")
84
+ if output.shape != (m, n) or output_scale.shape != (m, n // 128):
85
+ raise RuntimeError("output buffers have invalid shapes")
86
+ return None
87
+
88
+
89
  def select_fp8_linear_tile(m: int, n: int, k: int, variant: int = 0) -> str:
90
  """Return the FlashRT tile selected by the public dispatcher."""
91
 
 
97
  raise RuntimeError("m, n, and k must be positive")
98
  if k % 32 != 0:
99
  raise RuntimeError("k must be divisible by 32")
100
+ capability = torch.cuda.get_device_capability() if torch.cuda.is_available() else None
101
+ if capability == (11, 0):
102
+ forced = {1: "sm110_sq_bf16", 2: "sm110_t1_bf16", 3: "sm110_wide_bf16"}
103
+ if variant not in {0, *forced}:
104
+ raise RuntimeError("SM110 variant must be 0 (auto), 1 (Sq), 2 (T1), or 3 (Wide)")
105
+ if n % 16 or k % 16:
106
+ raise RuntimeError("SM110 requires n and k divisible by 16")
107
+ if variant:
108
+ return forced[variant]
109
+ if n >= 8 * k:
110
+ return "sm110_wide_bf16"
111
+ if m >= 128 and k >= 4 * n:
112
+ return "sm110_sq_bf16"
113
+ if n == k and m >= 512:
114
+ return "sm110_sq_bf16" if k <= 1024 else "sm110_wide_bf16"
115
+ if n == k and m >= 128:
116
+ return "sm110_wide_bf16"
117
+ return "sm110_t1_bf16"
118
  if m == 1:
119
  if variant == 4:
120
  return "gemv_fp8_m1_w4"
 
169
 
170
  ``input`` and ``weight`` must be FP8 E4M3 CUDA tensors with shapes
171
  ``(M, K)`` and ``(N, K)``. ``alpha`` is a host float, normally the product
172
+ of static per-tensor input and weight scales. SM110 uses the production
173
+ CUTLASS Sq/T1/Wide dispatcher over full model row counts; SM120 uses the
174
+ hand-tuned M<=64 path.
175
  """
176
 
177
  if out is None:
 
195
 
196
  ops.fp8_linear_residual_bf16(input, weight, float(alpha), int(variant), residual)
197
  return residual
198
+
199
+
200
+ def fp8_blockwise_linear_bf16(
201
+ input: torch.Tensor,
202
+ weight: torch.Tensor,
203
+ input_scale: torch.Tensor,
204
+ weight_scale: torch.Tensor,
205
+ out: torch.Tensor | None = None,
206
+ ) -> torch.Tensor:
207
+ """Block-128 scaled FP8 linear with BF16 output on SM89/SM120."""
208
+
209
+ if out is None:
210
+ out = torch.empty(
211
+ (input.shape[0], weight.shape[0]),
212
+ device=input.device,
213
+ dtype=torch.bfloat16,
214
+ )
215
+ ops.fp8_blockwise_linear_bf16(
216
+ input, weight, input_scale, weight_scale, out
217
+ )
218
+ return out
219
+
220
+
221
+ def fp8_blockwise_swiglu_quantize_fp8(
222
+ input: torch.Tensor,
223
+ gate_up_weight: torch.Tensor,
224
+ input_scale: torch.Tensor,
225
+ gate_up_weight_scale: torch.Tensor,
226
+ *,
227
+ output: torch.Tensor | None = None,
228
+ output_scale: torch.Tensor | None = None,
229
+ ) -> tuple[torch.Tensor, torch.Tensor]:
230
+ """SM89 block-128 FP8 gate/up GEMM + SiLU + FP8 requant producer."""
231
+
232
+ n = gate_up_weight.shape[0] // 2
233
+ if output is None:
234
+ output = torch.empty(
235
+ (input.shape[0], n), device=input.device, dtype=torch.float8_e4m3fn
236
+ )
237
+ if output_scale is None:
238
+ output_scale = torch.empty(
239
+ (input.shape[0], n // 128), device=input.device, dtype=torch.float32
240
+ )
241
+ ops.fp8_blockwise_swiglu_quantize_fp8(
242
+ input, gate_up_weight, input_scale, gate_up_weight_scale,
243
+ output, output_scale
244
+ )
245
+ return output, output_scale
246
+
247
+
248
+ __all__ = [
249
+ "fp8_linear_bf16",
250
+ "fp8_linear_residual_bf16",
251
+ "fp8_blockwise_linear_bf16",
252
+ "fp8_blockwise_swiglu_quantize_fp8",
253
+ "select_fp8_linear_tile",
254
+ ]
build/torch212-cxx11-cu130-x86_64-linux/{_fp8_gemm_cuda_1798e7f.abi3.so → _fp8_gemm_cuda_bf15968.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:2305d85c6f19a115ec72096707df0ad42741719238e6ebf486ae1727a287cf07
3
- size 2610600
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d8c90f60cb56efb79b25a6db2abe5f58336c7327902a0bcddfb7e4d3bde41dbb
3
+ size 5390512
build/torch212-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp8_gemm_cuda_1798e7f
3
- ops = torch.ops._fp8_gemm_cuda_1798e7f
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp8_gemm_cuda_1798e7f::{op_name}"
 
1
  import torch
2
+ from . import _fp8_gemm_cuda_bf15968
3
+ ops = torch.ops._fp8_gemm_cuda_bf15968
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp8_gemm_cuda_bf15968::{op_name}"
build/torch212-cxx11-cu130-x86_64-linux/fp8_gemm/__init__.py DELETED
@@ -1,26 +0,0 @@
1
- import ctypes
2
- import importlib.util
3
- import sys
4
- from pathlib import Path
5
- from types import ModuleType
6
-
7
-
8
- def _import_from_path(file_path: Path) -> ModuleType:
9
- # We cannot use the module name as-is, after adding it to `sys.modules`,
10
- # it would also be used for other imports. So, we make a module name that
11
- # depends on the path for it to be unique using the hex-encoded hash of
12
- # the path.
13
- path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
- module_name = path_hash
15
- spec = importlib.util.spec_from_file_location(module_name, file_path)
16
- if spec is None:
17
- raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
- module = importlib.util.module_from_spec(spec)
19
- if module is None:
20
- raise ImportError(f"Cannot load module {module_name} from spec")
21
- sys.modules[module_name] = module
22
- spec.loader.exec_module(module) # type: ignore
23
- return module
24
-
25
-
26
- globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
build/torch212-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,22 +1,34 @@
1
  {
2
  "name": "fp8-gemm",
3
- "id": "_fp8_gemm_cuda_1798e7f",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
7
  "backend": {
8
  "type": "cuda",
9
  "archs": [
10
- "12.0a"
 
 
11
  ]
12
  },
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
- "__init__.py": "Bm2+gGxw1Jrges8cKNwvxFr7dcD5K9hQbYHhO+S60ns=",
17
- "_fp8_gemm_cuda_1798e7f.abi3.so": "IwXYXG8ZoRXscglnB98K1CdBcZI45uv0hq4XJ6KHzwc=",
18
- "_ops.py": "RZGyMiLwlBDfJkYrzh6YrdosVDR9oznN9VDc/9OTrBQ=",
19
- "fp8_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
 
 
 
 
 
 
 
 
 
 
20
  }
21
  }
22
  }
 
1
  {
2
  "name": "fp8-gemm",
3
+ "id": "_fp8_gemm_cuda_bf15968",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
7
  "backend": {
8
  "type": "cuda",
9
  "archs": [
10
+ "11.0a",
11
+ "12.0a",
12
+ "8.9"
13
  ]
14
  },
15
  "digest": {
16
  "algorithm": "sha256",
17
  "files": {
18
+ "__init__.py": "hKsdFER2ZmYz36/zz9LYE7ddsK/R1dprHJFx+9yBzks=",
19
+ "_fp8_gemm_cuda_bf15968.abi3.so": "2MkPYMtW77ebJabbKr5fWDNscyeQKgvN37fk073kHbs=",
20
+ "_ops.py": "wDFSPD8vh2K2qjj/9mdUYv6LrXuGz6MQCb2B/e8nqWs="
21
+ }
22
+ },
23
+ "provenance": {
24
+ "kernel-builder": {
25
+ "version": "0.17.0-dev0",
26
+ "sha": "d720fa90fb9cd92d1bc60a9dc5c55bef2aafabb8",
27
+ "dirty": false
28
+ },
29
+ "kernel": {
30
+ "sha": "bf15968fd9c281a7ca107e7696300fe091f588a2",
31
+ "dirty": false
32
  }
33
  }
34
  }
build/torch212-cxx11-cu132-x86_64-linux/__init__.py CHANGED
@@ -37,6 +37,55 @@ def _fp8_linear_residual_bf16_fake(
37
  return None
38
 
39
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
40
  def select_fp8_linear_tile(m: int, n: int, k: int, variant: int = 0) -> str:
41
  """Return the FlashRT tile selected by the public dispatcher."""
42
 
@@ -48,6 +97,24 @@ def select_fp8_linear_tile(m: int, n: int, k: int, variant: int = 0) -> str:
48
  raise RuntimeError("m, n, and k must be positive")
49
  if k % 32 != 0:
50
  raise RuntimeError("k must be divisible by 32")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
51
  if m == 1:
52
  if variant == 4:
53
  return "gemv_fp8_m1_w4"
@@ -102,7 +169,9 @@ def fp8_linear_bf16(
102
 
103
  ``input`` and ``weight`` must be FP8 E4M3 CUDA tensors with shapes
104
  ``(M, K)`` and ``(N, K)``. ``alpha`` is a host float, normally the product
105
- of static per-tensor input and weight scales.
 
 
106
  """
107
 
108
  if out is None:
@@ -126,3 +195,60 @@ def fp8_linear_residual_bf16(
126
 
127
  ops.fp8_linear_residual_bf16(input, weight, float(alpha), int(variant), residual)
128
  return residual
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
37
  return None
38
 
39
 
40
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_blockwise_linear_bf16"))
41
+ def _fp8_blockwise_linear_bf16_fake(
42
+ input: torch.Tensor,
43
+ weight: torch.Tensor,
44
+ input_scale: torch.Tensor,
45
+ weight_scale: torch.Tensor,
46
+ out: torch.Tensor,
47
+ ) -> None:
48
+ if input.dim() != 2 or weight.dim() != 2:
49
+ raise RuntimeError("input and weight must be rank-2 tensors")
50
+ m, k = input.shape
51
+ n = weight.shape[0]
52
+ if weight.shape[1] != k or n % 128 or k % 128:
53
+ raise RuntimeError("weight shape is invalid or N/K are not divisible by 128")
54
+ if input_scale.shape != (m, k // 128):
55
+ raise RuntimeError("input_scale must have shape (M, K / 128)")
56
+ if weight_scale.shape != (n // 128, k // 128):
57
+ raise RuntimeError("weight_scale must have shape (N / 128, K / 128)")
58
+ if out.shape != (m, n):
59
+ raise RuntimeError("out must have shape (M, N)")
60
+ return None
61
+
62
+
63
+ @torch.library.register_fake(
64
+ add_op_namespace_prefix("fp8_blockwise_swiglu_quantize_fp8")
65
+ )
66
+ def _fp8_blockwise_swiglu_quantize_fp8_fake(
67
+ input: torch.Tensor,
68
+ gate_up_weight: torch.Tensor,
69
+ input_scale: torch.Tensor,
70
+ gate_up_weight_scale: torch.Tensor,
71
+ output: torch.Tensor,
72
+ output_scale: torch.Tensor,
73
+ ) -> None:
74
+ m, k = input.shape
75
+ if gate_up_weight.dim() != 2 or gate_up_weight.shape[0] % 2:
76
+ raise RuntimeError("gate_up_weight must have shape (2*N, K)")
77
+ n = gate_up_weight.shape[0] // 2
78
+ if gate_up_weight.shape[1] != k or n % 128 or k % 128:
79
+ raise RuntimeError("gate_up_weight shape is invalid or N/K are not divisible by 128")
80
+ if input_scale.shape != (m, k // 128):
81
+ raise RuntimeError("input_scale must have shape (M, K/128)")
82
+ if gate_up_weight_scale.shape != (2 * n // 128, k // 128):
83
+ raise RuntimeError("gate_up_weight_scale must have shape (2*N/128, K/128)")
84
+ if output.shape != (m, n) or output_scale.shape != (m, n // 128):
85
+ raise RuntimeError("output buffers have invalid shapes")
86
+ return None
87
+
88
+
89
  def select_fp8_linear_tile(m: int, n: int, k: int, variant: int = 0) -> str:
90
  """Return the FlashRT tile selected by the public dispatcher."""
91
 
 
97
  raise RuntimeError("m, n, and k must be positive")
98
  if k % 32 != 0:
99
  raise RuntimeError("k must be divisible by 32")
100
+ capability = torch.cuda.get_device_capability() if torch.cuda.is_available() else None
101
+ if capability == (11, 0):
102
+ forced = {1: "sm110_sq_bf16", 2: "sm110_t1_bf16", 3: "sm110_wide_bf16"}
103
+ if variant not in {0, *forced}:
104
+ raise RuntimeError("SM110 variant must be 0 (auto), 1 (Sq), 2 (T1), or 3 (Wide)")
105
+ if n % 16 or k % 16:
106
+ raise RuntimeError("SM110 requires n and k divisible by 16")
107
+ if variant:
108
+ return forced[variant]
109
+ if n >= 8 * k:
110
+ return "sm110_wide_bf16"
111
+ if m >= 128 and k >= 4 * n:
112
+ return "sm110_sq_bf16"
113
+ if n == k and m >= 512:
114
+ return "sm110_sq_bf16" if k <= 1024 else "sm110_wide_bf16"
115
+ if n == k and m >= 128:
116
+ return "sm110_wide_bf16"
117
+ return "sm110_t1_bf16"
118
  if m == 1:
119
  if variant == 4:
120
  return "gemv_fp8_m1_w4"
 
169
 
170
  ``input`` and ``weight`` must be FP8 E4M3 CUDA tensors with shapes
171
  ``(M, K)`` and ``(N, K)``. ``alpha`` is a host float, normally the product
172
+ of static per-tensor input and weight scales. SM110 uses the production
173
+ CUTLASS Sq/T1/Wide dispatcher over full model row counts; SM120 uses the
174
+ hand-tuned M<=64 path.
175
  """
176
 
177
  if out is None:
 
195
 
196
  ops.fp8_linear_residual_bf16(input, weight, float(alpha), int(variant), residual)
197
  return residual
198
+
199
+
200
+ def fp8_blockwise_linear_bf16(
201
+ input: torch.Tensor,
202
+ weight: torch.Tensor,
203
+ input_scale: torch.Tensor,
204
+ weight_scale: torch.Tensor,
205
+ out: torch.Tensor | None = None,
206
+ ) -> torch.Tensor:
207
+ """Block-128 scaled FP8 linear with BF16 output on SM89/SM120."""
208
+
209
+ if out is None:
210
+ out = torch.empty(
211
+ (input.shape[0], weight.shape[0]),
212
+ device=input.device,
213
+ dtype=torch.bfloat16,
214
+ )
215
+ ops.fp8_blockwise_linear_bf16(
216
+ input, weight, input_scale, weight_scale, out
217
+ )
218
+ return out
219
+
220
+
221
+ def fp8_blockwise_swiglu_quantize_fp8(
222
+ input: torch.Tensor,
223
+ gate_up_weight: torch.Tensor,
224
+ input_scale: torch.Tensor,
225
+ gate_up_weight_scale: torch.Tensor,
226
+ *,
227
+ output: torch.Tensor | None = None,
228
+ output_scale: torch.Tensor | None = None,
229
+ ) -> tuple[torch.Tensor, torch.Tensor]:
230
+ """SM89 block-128 FP8 gate/up GEMM + SiLU + FP8 requant producer."""
231
+
232
+ n = gate_up_weight.shape[0] // 2
233
+ if output is None:
234
+ output = torch.empty(
235
+ (input.shape[0], n), device=input.device, dtype=torch.float8_e4m3fn
236
+ )
237
+ if output_scale is None:
238
+ output_scale = torch.empty(
239
+ (input.shape[0], n // 128), device=input.device, dtype=torch.float32
240
+ )
241
+ ops.fp8_blockwise_swiglu_quantize_fp8(
242
+ input, gate_up_weight, input_scale, gate_up_weight_scale,
243
+ output, output_scale
244
+ )
245
+ return output, output_scale
246
+
247
+
248
+ __all__ = [
249
+ "fp8_linear_bf16",
250
+ "fp8_linear_residual_bf16",
251
+ "fp8_blockwise_linear_bf16",
252
+ "fp8_blockwise_swiglu_quantize_fp8",
253
+ "select_fp8_linear_tile",
254
+ ]
build/torch212-cxx11-cu132-x86_64-linux/_fp8_gemm_cuda_bf15968.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b025dcd2da72fc44d87e0bb8a68168e56e2972bca5417631ff1f534615c31ae
3
+ size 5431528
build/torch212-cxx11-cu132-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp8_gemm_cuda_1798e7f
3
- ops = torch.ops._fp8_gemm_cuda_1798e7f
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp8_gemm_cuda_1798e7f::{op_name}"
 
1
  import torch
2
+ from . import _fp8_gemm_cuda_bf15968
3
+ ops = torch.ops._fp8_gemm_cuda_bf15968
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp8_gemm_cuda_bf15968::{op_name}"
build/torch212-cxx11-cu132-x86_64-linux/fp8_gemm/__init__.py DELETED
@@ -1,26 +0,0 @@
1
- import ctypes
2
- import importlib.util
3
- import sys
4
- from pathlib import Path
5
- from types import ModuleType
6
-
7
-
8
- def _import_from_path(file_path: Path) -> ModuleType:
9
- # We cannot use the module name as-is, after adding it to `sys.modules`,
10
- # it would also be used for other imports. So, we make a module name that
11
- # depends on the path for it to be unique using the hex-encoded hash of
12
- # the path.
13
- path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
- module_name = path_hash
15
- spec = importlib.util.spec_from_file_location(module_name, file_path)
16
- if spec is None:
17
- raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
- module = importlib.util.module_from_spec(spec)
19
- if module is None:
20
- raise ImportError(f"Cannot load module {module_name} from spec")
21
- sys.modules[module_name] = module
22
- spec.loader.exec_module(module) # type: ignore
23
- return module
24
-
25
-
26
- globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
build/torch212-cxx11-cu132-x86_64-linux/metadata.json CHANGED
@@ -1,22 +1,34 @@
1
  {
2
  "name": "fp8-gemm",
3
- "id": "_fp8_gemm_cuda_1798e7f",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
7
  "backend": {
8
  "type": "cuda",
9
  "archs": [
10
- "12.0a"
 
 
11
  ]
12
  },
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
- "__init__.py": "Bm2+gGxw1Jrges8cKNwvxFr7dcD5K9hQbYHhO+S60ns=",
17
- "_fp8_gemm_cuda_1798e7f.abi3.so": "riEm8R74IBL3WvLW4gv02SR3H4wosr8jfY3S3OSNDW0=",
18
- "_ops.py": "RZGyMiLwlBDfJkYrzh6YrdosVDR9oznN9VDc/9OTrBQ=",
19
- "fp8_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
 
 
 
 
 
 
 
 
 
 
20
  }
21
  }
22
  }
 
1
  {
2
  "name": "fp8-gemm",
3
+ "id": "_fp8_gemm_cuda_bf15968",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
7
  "backend": {
8
  "type": "cuda",
9
  "archs": [
10
+ "11.0a",
11
+ "12.0a",
12
+ "8.9"
13
  ]
14
  },
15
  "digest": {
16
  "algorithm": "sha256",
17
  "files": {
18
+ "__init__.py": "hKsdFER2ZmYz36/zz9LYE7ddsK/R1dprHJFx+9yBzks=",
19
+ "_fp8_gemm_cuda_bf15968.abi3.so": "mwJdzS2nL8RNh+C7imgWjlbilyvKVBdjH/H1NGFcMa4=",
20
+ "_ops.py": "wDFSPD8vh2K2qjj/9mdUYv6LrXuGz6MQCb2B/e8nqWs="
21
+ }
22
+ },
23
+ "provenance": {
24
+ "kernel-builder": {
25
+ "version": "0.17.0-dev0",
26
+ "sha": "d720fa90fb9cd92d1bc60a9dc5c55bef2aafabb8",
27
+ "dirty": false
28
+ },
29
+ "kernel": {
30
+ "sha": "bf15968fd9c281a7ca107e7696300fe091f588a2",
31
+ "dirty": false
32
  }
33
  }
34
  }
build/torch213-cxx11-cu130-aarch64-linux/__init__.py ADDED
@@ -0,0 +1,254 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP8 GEMM kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import torch
6
+
7
+ from ._ops import add_op_namespace_prefix, ops
8
+
9
+
10
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bf16"))
11
+ def _fp8_linear_bf16_fake(
12
+ input: torch.Tensor,
13
+ weight: torch.Tensor,
14
+ alpha: float,
15
+ variant: int,
16
+ out: torch.Tensor,
17
+ ) -> None:
18
+ if input.dim() != 2 or weight.dim() != 2:
19
+ raise RuntimeError("input and weight must be rank-2 tensors")
20
+ if out.shape != (input.shape[0], weight.shape[0]):
21
+ raise RuntimeError("out must have shape (input.shape[0], weight.shape[0])")
22
+ return None
23
+
24
+
25
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_residual_bf16"))
26
+ def _fp8_linear_residual_bf16_fake(
27
+ input: torch.Tensor,
28
+ weight: torch.Tensor,
29
+ alpha: float,
30
+ variant: int,
31
+ residual: torch.Tensor,
32
+ ) -> None:
33
+ if input.shape[0] != 1:
34
+ raise RuntimeError("residual path supports only M=1")
35
+ if residual.shape != (1, weight.shape[0]):
36
+ raise RuntimeError("residual must have shape (1, weight.shape[0])")
37
+ return None
38
+
39
+
40
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_blockwise_linear_bf16"))
41
+ def _fp8_blockwise_linear_bf16_fake(
42
+ input: torch.Tensor,
43
+ weight: torch.Tensor,
44
+ input_scale: torch.Tensor,
45
+ weight_scale: torch.Tensor,
46
+ out: torch.Tensor,
47
+ ) -> None:
48
+ if input.dim() != 2 or weight.dim() != 2:
49
+ raise RuntimeError("input and weight must be rank-2 tensors")
50
+ m, k = input.shape
51
+ n = weight.shape[0]
52
+ if weight.shape[1] != k or n % 128 or k % 128:
53
+ raise RuntimeError("weight shape is invalid or N/K are not divisible by 128")
54
+ if input_scale.shape != (m, k // 128):
55
+ raise RuntimeError("input_scale must have shape (M, K / 128)")
56
+ if weight_scale.shape != (n // 128, k // 128):
57
+ raise RuntimeError("weight_scale must have shape (N / 128, K / 128)")
58
+ if out.shape != (m, n):
59
+ raise RuntimeError("out must have shape (M, N)")
60
+ return None
61
+
62
+
63
+ @torch.library.register_fake(
64
+ add_op_namespace_prefix("fp8_blockwise_swiglu_quantize_fp8")
65
+ )
66
+ def _fp8_blockwise_swiglu_quantize_fp8_fake(
67
+ input: torch.Tensor,
68
+ gate_up_weight: torch.Tensor,
69
+ input_scale: torch.Tensor,
70
+ gate_up_weight_scale: torch.Tensor,
71
+ output: torch.Tensor,
72
+ output_scale: torch.Tensor,
73
+ ) -> None:
74
+ m, k = input.shape
75
+ if gate_up_weight.dim() != 2 or gate_up_weight.shape[0] % 2:
76
+ raise RuntimeError("gate_up_weight must have shape (2*N, K)")
77
+ n = gate_up_weight.shape[0] // 2
78
+ if gate_up_weight.shape[1] != k or n % 128 or k % 128:
79
+ raise RuntimeError("gate_up_weight shape is invalid or N/K are not divisible by 128")
80
+ if input_scale.shape != (m, k // 128):
81
+ raise RuntimeError("input_scale must have shape (M, K/128)")
82
+ if gate_up_weight_scale.shape != (2 * n // 128, k // 128):
83
+ raise RuntimeError("gate_up_weight_scale must have shape (2*N/128, K/128)")
84
+ if output.shape != (m, n) or output_scale.shape != (m, n // 128):
85
+ raise RuntimeError("output buffers have invalid shapes")
86
+ return None
87
+
88
+
89
+ def select_fp8_linear_tile(m: int, n: int, k: int, variant: int = 0) -> str:
90
+ """Return the FlashRT tile selected by the public dispatcher."""
91
+
92
+ m = int(m)
93
+ n = int(n)
94
+ k = int(k)
95
+ variant = int(variant)
96
+ if m <= 0 or n <= 0 or k <= 0:
97
+ raise RuntimeError("m, n, and k must be positive")
98
+ if k % 32 != 0:
99
+ raise RuntimeError("k must be divisible by 32")
100
+ capability = torch.cuda.get_device_capability() if torch.cuda.is_available() else None
101
+ if capability == (11, 0):
102
+ forced = {1: "sm110_sq_bf16", 2: "sm110_t1_bf16", 3: "sm110_wide_bf16"}
103
+ if variant not in {0, *forced}:
104
+ raise RuntimeError("SM110 variant must be 0 (auto), 1 (Sq), 2 (T1), or 3 (Wide)")
105
+ if n % 16 or k % 16:
106
+ raise RuntimeError("SM110 requires n and k divisible by 16")
107
+ if variant:
108
+ return forced[variant]
109
+ if n >= 8 * k:
110
+ return "sm110_wide_bf16"
111
+ if m >= 128 and k >= 4 * n:
112
+ return "sm110_sq_bf16"
113
+ if n == k and m >= 512:
114
+ return "sm110_sq_bf16" if k <= 1024 else "sm110_wide_bf16"
115
+ if n == k and m >= 128:
116
+ return "sm110_wide_bf16"
117
+ return "sm110_t1_bf16"
118
+ if m == 1:
119
+ if variant == 4:
120
+ return "gemv_fp8_m1_w4"
121
+ if variant == 8:
122
+ return "gemv_fp8_m1_w8"
123
+ if variant == 16:
124
+ return "gemv_fp8_m1_w16"
125
+ if variant != 0:
126
+ raise RuntimeError("M=1 variant must be 0, 4, 8, or 16")
127
+ if n <= 2048:
128
+ return "gemv_fp8_m1_w4"
129
+ if n <= 8192:
130
+ return "gemv_fp8_m1_w8"
131
+ return "gemv_fp8_m1_w16"
132
+ if variant != 0:
133
+ raise RuntimeError("small-M dispatcher currently supports variant=0 only")
134
+ if m <= 16:
135
+ if k % 256 == 0:
136
+ return "ld_fp8_gemm_16x128x256_w4" if n % 128 == 0 else "ld_fp8_gemm_16x64x256_w4"
137
+ if n % 256 == 0:
138
+ return "ld_fp8_gemm_16x256x128_w8"
139
+ if n % 192 == 0:
140
+ return "ld_fp8_gemm_16x192x128_w4"
141
+ if n % 128 == 0:
142
+ return "ld_fp8_gemm_16x128x128_w4"
143
+ return "ld_fp8_gemm_16x64x128_w4"
144
+ if m <= 32:
145
+ if k % 256 == 0:
146
+ return "ld_fp8_gemm_32x128x256_w4" if n % 128 == 0 else "ld_fp8_gemm_32x64x256_w4"
147
+ if n % 192 == 0:
148
+ return "ld_fp8_gemm_32x192x128_w4"
149
+ if n % 128 == 0:
150
+ return "ld_fp8_gemm_32x128x128_w4"
151
+ return "ld_fp8_gemm_32x64x128_w4"
152
+ if m <= 64:
153
+ if k % 256 == 0:
154
+ return "ld_fp8_gemm_64x128x256_w4" if n % 128 == 0 else "ld_fp8_gemm_64x64x256_w4"
155
+ if n % 128 == 0:
156
+ return "ld_fp8_gemm_64x128x128_w4"
157
+ return "ld_fp8_gemm_64x64x128_w4"
158
+ raise RuntimeError("only M=1 decode or 2 <= M <= 64 small-M rows are supported")
159
+
160
+
161
+ def fp8_linear_bf16(
162
+ input: torch.Tensor,
163
+ weight: torch.Tensor,
164
+ alpha: float = 1.0,
165
+ out: torch.Tensor | None = None,
166
+ variant: int = 0,
167
+ ) -> torch.Tensor:
168
+ """Compute ``(input @ weight.T) * alpha`` with BF16 output.
169
+
170
+ ``input`` and ``weight`` must be FP8 E4M3 CUDA tensors with shapes
171
+ ``(M, K)`` and ``(N, K)``. ``alpha`` is a host float, normally the product
172
+ of static per-tensor input and weight scales. SM110 uses the production
173
+ CUTLASS Sq/T1/Wide dispatcher over full model row counts; SM120 uses the
174
+ hand-tuned M<=64 path.
175
+ """
176
+
177
+ if out is None:
178
+ out = torch.empty(
179
+ (input.shape[0], weight.shape[0]),
180
+ device=input.device,
181
+ dtype=torch.bfloat16,
182
+ )
183
+ ops.fp8_linear_bf16(input, weight, float(alpha), int(variant), out)
184
+ return out
185
+
186
+
187
+ def fp8_linear_residual_bf16(
188
+ input: torch.Tensor,
189
+ weight: torch.Tensor,
190
+ residual: torch.Tensor,
191
+ alpha: float = 1.0,
192
+ variant: int = 0,
193
+ ) -> torch.Tensor:
194
+ """In-place ``residual += (input @ weight.T) * alpha`` for M=1 decode."""
195
+
196
+ ops.fp8_linear_residual_bf16(input, weight, float(alpha), int(variant), residual)
197
+ return residual
198
+
199
+
200
+ def fp8_blockwise_linear_bf16(
201
+ input: torch.Tensor,
202
+ weight: torch.Tensor,
203
+ input_scale: torch.Tensor,
204
+ weight_scale: torch.Tensor,
205
+ out: torch.Tensor | None = None,
206
+ ) -> torch.Tensor:
207
+ """Block-128 scaled FP8 linear with BF16 output on SM89/SM120."""
208
+
209
+ if out is None:
210
+ out = torch.empty(
211
+ (input.shape[0], weight.shape[0]),
212
+ device=input.device,
213
+ dtype=torch.bfloat16,
214
+ )
215
+ ops.fp8_blockwise_linear_bf16(
216
+ input, weight, input_scale, weight_scale, out
217
+ )
218
+ return out
219
+
220
+
221
+ def fp8_blockwise_swiglu_quantize_fp8(
222
+ input: torch.Tensor,
223
+ gate_up_weight: torch.Tensor,
224
+ input_scale: torch.Tensor,
225
+ gate_up_weight_scale: torch.Tensor,
226
+ *,
227
+ output: torch.Tensor | None = None,
228
+ output_scale: torch.Tensor | None = None,
229
+ ) -> tuple[torch.Tensor, torch.Tensor]:
230
+ """SM89 block-128 FP8 gate/up GEMM + SiLU + FP8 requant producer."""
231
+
232
+ n = gate_up_weight.shape[0] // 2
233
+ if output is None:
234
+ output = torch.empty(
235
+ (input.shape[0], n), device=input.device, dtype=torch.float8_e4m3fn
236
+ )
237
+ if output_scale is None:
238
+ output_scale = torch.empty(
239
+ (input.shape[0], n // 128), device=input.device, dtype=torch.float32
240
+ )
241
+ ops.fp8_blockwise_swiglu_quantize_fp8(
242
+ input, gate_up_weight, input_scale, gate_up_weight_scale,
243
+ output, output_scale
244
+ )
245
+ return output, output_scale
246
+
247
+
248
+ __all__ = [
249
+ "fp8_linear_bf16",
250
+ "fp8_linear_residual_bf16",
251
+ "fp8_blockwise_linear_bf16",
252
+ "fp8_blockwise_swiglu_quantize_fp8",
253
+ "select_fp8_linear_tile",
254
+ ]
build/torch213-cxx11-cu130-aarch64-linux/_ops.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import fp8_gemm_source_test
3
+ ops = torch.ops.fp8_gemm_source_test
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ return f"fp8_gemm_source_test::{op_name}"
build/torch213-cxx11-cu130-aarch64-linux/fp8_gemm/__init__.py ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+
6
+ def _import_from_path(file_path: Path):
7
+ path_hash = '{:x}'.format(ctypes.c_size_t(hash(file_path.absolute())).value)
8
+ spec = importlib.util.spec_from_file_location(path_hash, file_path)
9
+ module = importlib.util.module_from_spec(spec)
10
+ sys.modules[path_hash] = module
11
+ spec.loader.exec_module(module)
12
+ return module
13
+
14
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / '__init__.py')))
build/torch213-cxx11-cu130-aarch64-linux/fp8_gemm_source_test.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6da6757d21dbf032ac9c79d54dceffafed47cbf28b392de481d03e6078f198a8
3
+ size 1893488
build/torch213-cxx11-cu130-aarch64-linux/metadata.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "fp8-gemm",
3
+ "id": "fp8_gemm_source_test",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "11.0a"
11
+ ]
12
+ },
13
+ "digest": {
14
+ "algorithm": "sha256",
15
+ "files": {
16
+ "__init__.py": "hKsdFER2ZmYz36/zz9LYE7ddsK/R1dprHJFx+9yBzks=",
17
+ "fp8_gemm_source_test.abi3.so": "baZ1fSHb8DKsnHnVTc7/r+1Hy/KLOS3kgdA+YHjxmKg=",
18
+ "_ops.py": "zi4cl9u1hlmXMSZ8ZPvvrFIj8y1rvp8y3OTUWOXsp84=",
19
+ "fp8_gemm/__init__.py": "v6p5XMfQzddhi1fLSAw4HX9CyS0rQsidvu9VsT01xi4="
20
+ }
21
+ },
22
+ "provenance": {
23
+ "kernel": {
24
+ "sha": "456d297",
25
+ "dirty": false
26
+ },
27
+ "validation": {
28
+ "torch": "2.13.0+cu130",
29
+ "cuda": "13.0"
30
+ }
31
+ }
32
+ }
build/torch213-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,254 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP8 GEMM kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import torch
6
+
7
+ from ._ops import add_op_namespace_prefix, ops
8
+
9
+
10
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bf16"))
11
+ def _fp8_linear_bf16_fake(
12
+ input: torch.Tensor,
13
+ weight: torch.Tensor,
14
+ alpha: float,
15
+ variant: int,
16
+ out: torch.Tensor,
17
+ ) -> None:
18
+ if input.dim() != 2 or weight.dim() != 2:
19
+ raise RuntimeError("input and weight must be rank-2 tensors")
20
+ if out.shape != (input.shape[0], weight.shape[0]):
21
+ raise RuntimeError("out must have shape (input.shape[0], weight.shape[0])")
22
+ return None
23
+
24
+
25
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_residual_bf16"))
26
+ def _fp8_linear_residual_bf16_fake(
27
+ input: torch.Tensor,
28
+ weight: torch.Tensor,
29
+ alpha: float,
30
+ variant: int,
31
+ residual: torch.Tensor,
32
+ ) -> None:
33
+ if input.shape[0] != 1:
34
+ raise RuntimeError("residual path supports only M=1")
35
+ if residual.shape != (1, weight.shape[0]):
36
+ raise RuntimeError("residual must have shape (1, weight.shape[0])")
37
+ return None
38
+
39
+
40
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_blockwise_linear_bf16"))
41
+ def _fp8_blockwise_linear_bf16_fake(
42
+ input: torch.Tensor,
43
+ weight: torch.Tensor,
44
+ input_scale: torch.Tensor,
45
+ weight_scale: torch.Tensor,
46
+ out: torch.Tensor,
47
+ ) -> None:
48
+ if input.dim() != 2 or weight.dim() != 2:
49
+ raise RuntimeError("input and weight must be rank-2 tensors")
50
+ m, k = input.shape
51
+ n = weight.shape[0]
52
+ if weight.shape[1] != k or n % 128 or k % 128:
53
+ raise RuntimeError("weight shape is invalid or N/K are not divisible by 128")
54
+ if input_scale.shape != (m, k // 128):
55
+ raise RuntimeError("input_scale must have shape (M, K / 128)")
56
+ if weight_scale.shape != (n // 128, k // 128):
57
+ raise RuntimeError("weight_scale must have shape (N / 128, K / 128)")
58
+ if out.shape != (m, n):
59
+ raise RuntimeError("out must have shape (M, N)")
60
+ return None
61
+
62
+
63
+ @torch.library.register_fake(
64
+ add_op_namespace_prefix("fp8_blockwise_swiglu_quantize_fp8")
65
+ )
66
+ def _fp8_blockwise_swiglu_quantize_fp8_fake(
67
+ input: torch.Tensor,
68
+ gate_up_weight: torch.Tensor,
69
+ input_scale: torch.Tensor,
70
+ gate_up_weight_scale: torch.Tensor,
71
+ output: torch.Tensor,
72
+ output_scale: torch.Tensor,
73
+ ) -> None:
74
+ m, k = input.shape
75
+ if gate_up_weight.dim() != 2 or gate_up_weight.shape[0] % 2:
76
+ raise RuntimeError("gate_up_weight must have shape (2*N, K)")
77
+ n = gate_up_weight.shape[0] // 2
78
+ if gate_up_weight.shape[1] != k or n % 128 or k % 128:
79
+ raise RuntimeError("gate_up_weight shape is invalid or N/K are not divisible by 128")
80
+ if input_scale.shape != (m, k // 128):
81
+ raise RuntimeError("input_scale must have shape (M, K/128)")
82
+ if gate_up_weight_scale.shape != (2 * n // 128, k // 128):
83
+ raise RuntimeError("gate_up_weight_scale must have shape (2*N/128, K/128)")
84
+ if output.shape != (m, n) or output_scale.shape != (m, n // 128):
85
+ raise RuntimeError("output buffers have invalid shapes")
86
+ return None
87
+
88
+
89
+ def select_fp8_linear_tile(m: int, n: int, k: int, variant: int = 0) -> str:
90
+ """Return the FlashRT tile selected by the public dispatcher."""
91
+
92
+ m = int(m)
93
+ n = int(n)
94
+ k = int(k)
95
+ variant = int(variant)
96
+ if m <= 0 or n <= 0 or k <= 0:
97
+ raise RuntimeError("m, n, and k must be positive")
98
+ if k % 32 != 0:
99
+ raise RuntimeError("k must be divisible by 32")
100
+ capability = torch.cuda.get_device_capability() if torch.cuda.is_available() else None
101
+ if capability == (11, 0):
102
+ forced = {1: "sm110_sq_bf16", 2: "sm110_t1_bf16", 3: "sm110_wide_bf16"}
103
+ if variant not in {0, *forced}:
104
+ raise RuntimeError("SM110 variant must be 0 (auto), 1 (Sq), 2 (T1), or 3 (Wide)")
105
+ if n % 16 or k % 16:
106
+ raise RuntimeError("SM110 requires n and k divisible by 16")
107
+ if variant:
108
+ return forced[variant]
109
+ if n >= 8 * k:
110
+ return "sm110_wide_bf16"
111
+ if m >= 128 and k >= 4 * n:
112
+ return "sm110_sq_bf16"
113
+ if n == k and m >= 512:
114
+ return "sm110_sq_bf16" if k <= 1024 else "sm110_wide_bf16"
115
+ if n == k and m >= 128:
116
+ return "sm110_wide_bf16"
117
+ return "sm110_t1_bf16"
118
+ if m == 1:
119
+ if variant == 4:
120
+ return "gemv_fp8_m1_w4"
121
+ if variant == 8:
122
+ return "gemv_fp8_m1_w8"
123
+ if variant == 16:
124
+ return "gemv_fp8_m1_w16"
125
+ if variant != 0:
126
+ raise RuntimeError("M=1 variant must be 0, 4, 8, or 16")
127
+ if n <= 2048:
128
+ return "gemv_fp8_m1_w4"
129
+ if n <= 8192:
130
+ return "gemv_fp8_m1_w8"
131
+ return "gemv_fp8_m1_w16"
132
+ if variant != 0:
133
+ raise RuntimeError("small-M dispatcher currently supports variant=0 only")
134
+ if m <= 16:
135
+ if k % 256 == 0:
136
+ return "ld_fp8_gemm_16x128x256_w4" if n % 128 == 0 else "ld_fp8_gemm_16x64x256_w4"
137
+ if n % 256 == 0:
138
+ return "ld_fp8_gemm_16x256x128_w8"
139
+ if n % 192 == 0:
140
+ return "ld_fp8_gemm_16x192x128_w4"
141
+ if n % 128 == 0:
142
+ return "ld_fp8_gemm_16x128x128_w4"
143
+ return "ld_fp8_gemm_16x64x128_w4"
144
+ if m <= 32:
145
+ if k % 256 == 0:
146
+ return "ld_fp8_gemm_32x128x256_w4" if n % 128 == 0 else "ld_fp8_gemm_32x64x256_w4"
147
+ if n % 192 == 0:
148
+ return "ld_fp8_gemm_32x192x128_w4"
149
+ if n % 128 == 0:
150
+ return "ld_fp8_gemm_32x128x128_w4"
151
+ return "ld_fp8_gemm_32x64x128_w4"
152
+ if m <= 64:
153
+ if k % 256 == 0:
154
+ return "ld_fp8_gemm_64x128x256_w4" if n % 128 == 0 else "ld_fp8_gemm_64x64x256_w4"
155
+ if n % 128 == 0:
156
+ return "ld_fp8_gemm_64x128x128_w4"
157
+ return "ld_fp8_gemm_64x64x128_w4"
158
+ raise RuntimeError("only M=1 decode or 2 <= M <= 64 small-M rows are supported")
159
+
160
+
161
+ def fp8_linear_bf16(
162
+ input: torch.Tensor,
163
+ weight: torch.Tensor,
164
+ alpha: float = 1.0,
165
+ out: torch.Tensor | None = None,
166
+ variant: int = 0,
167
+ ) -> torch.Tensor:
168
+ """Compute ``(input @ weight.T) * alpha`` with BF16 output.
169
+
170
+ ``input`` and ``weight`` must be FP8 E4M3 CUDA tensors with shapes
171
+ ``(M, K)`` and ``(N, K)``. ``alpha`` is a host float, normally the product
172
+ of static per-tensor input and weight scales. SM110 uses the production
173
+ CUTLASS Sq/T1/Wide dispatcher over full model row counts; SM120 uses the
174
+ hand-tuned M<=64 path.
175
+ """
176
+
177
+ if out is None:
178
+ out = torch.empty(
179
+ (input.shape[0], weight.shape[0]),
180
+ device=input.device,
181
+ dtype=torch.bfloat16,
182
+ )
183
+ ops.fp8_linear_bf16(input, weight, float(alpha), int(variant), out)
184
+ return out
185
+
186
+
187
+ def fp8_linear_residual_bf16(
188
+ input: torch.Tensor,
189
+ weight: torch.Tensor,
190
+ residual: torch.Tensor,
191
+ alpha: float = 1.0,
192
+ variant: int = 0,
193
+ ) -> torch.Tensor:
194
+ """In-place ``residual += (input @ weight.T) * alpha`` for M=1 decode."""
195
+
196
+ ops.fp8_linear_residual_bf16(input, weight, float(alpha), int(variant), residual)
197
+ return residual
198
+
199
+
200
+ def fp8_blockwise_linear_bf16(
201
+ input: torch.Tensor,
202
+ weight: torch.Tensor,
203
+ input_scale: torch.Tensor,
204
+ weight_scale: torch.Tensor,
205
+ out: torch.Tensor | None = None,
206
+ ) -> torch.Tensor:
207
+ """Block-128 scaled FP8 linear with BF16 output on SM89/SM120."""
208
+
209
+ if out is None:
210
+ out = torch.empty(
211
+ (input.shape[0], weight.shape[0]),
212
+ device=input.device,
213
+ dtype=torch.bfloat16,
214
+ )
215
+ ops.fp8_blockwise_linear_bf16(
216
+ input, weight, input_scale, weight_scale, out
217
+ )
218
+ return out
219
+
220
+
221
+ def fp8_blockwise_swiglu_quantize_fp8(
222
+ input: torch.Tensor,
223
+ gate_up_weight: torch.Tensor,
224
+ input_scale: torch.Tensor,
225
+ gate_up_weight_scale: torch.Tensor,
226
+ *,
227
+ output: torch.Tensor | None = None,
228
+ output_scale: torch.Tensor | None = None,
229
+ ) -> tuple[torch.Tensor, torch.Tensor]:
230
+ """SM89 block-128 FP8 gate/up GEMM + SiLU + FP8 requant producer."""
231
+
232
+ n = gate_up_weight.shape[0] // 2
233
+ if output is None:
234
+ output = torch.empty(
235
+ (input.shape[0], n), device=input.device, dtype=torch.float8_e4m3fn
236
+ )
237
+ if output_scale is None:
238
+ output_scale = torch.empty(
239
+ (input.shape[0], n // 128), device=input.device, dtype=torch.float32
240
+ )
241
+ ops.fp8_blockwise_swiglu_quantize_fp8(
242
+ input, gate_up_weight, input_scale, gate_up_weight_scale,
243
+ output, output_scale
244
+ )
245
+ return output, output_scale
246
+
247
+
248
+ __all__ = [
249
+ "fp8_linear_bf16",
250
+ "fp8_linear_residual_bf16",
251
+ "fp8_blockwise_linear_bf16",
252
+ "fp8_blockwise_swiglu_quantize_fp8",
253
+ "select_fp8_linear_tile",
254
+ ]
build/torch213-cxx11-cu130-x86_64-linux/_fp8_gemm_cuda_bf15968.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:90adef3d22bb98b7dc1b2b48d860f0fa2fe682125acb31b1c554f889e0f171ce
3
+ size 5390352
build/torch213-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _fp8_gemm_cuda_bf15968
3
+ ops = torch.ops._fp8_gemm_cuda_bf15968
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_fp8_gemm_cuda_bf15968::{op_name}"
build/torch213-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "fp8-gemm",
3
+ "id": "_fp8_gemm_cuda_bf15968",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "11.0a",
11
+ "12.0a",
12
+ "8.9"
13
+ ]
14
+ },
15
+ "digest": {
16
+ "algorithm": "sha256",
17
+ "files": {
18
+ "__init__.py": "hKsdFER2ZmYz36/zz9LYE7ddsK/R1dprHJFx+9yBzks=",
19
+ "_fp8_gemm_cuda_bf15968.abi3.so": "kK3vPSK7mLfcGytI2GDw+i/mghJayzGxxVT4ieDxcc4=",
20
+ "_ops.py": "wDFSPD8vh2K2qjj/9mdUYv6LrXuGz6MQCb2B/e8nqWs="
21
+ }
22
+ },
23
+ "provenance": {
24
+ "kernel-builder": {
25
+ "version": "0.17.0-dev0",
26
+ "sha": "d720fa90fb9cd92d1bc60a9dc5c55bef2aafabb8",
27
+ "dirty": false
28
+ },
29
+ "kernel": {
30
+ "sha": "bf15968fd9c281a7ca107e7696300fe091f588a2",
31
+ "dirty": false
32
+ }
33
+ }
34
+ }
build/torch213-cxx11-cu132-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,254 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP8 GEMM kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import torch
6
+
7
+ from ._ops import add_op_namespace_prefix, ops
8
+
9
+
10
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bf16"))
11
+ def _fp8_linear_bf16_fake(
12
+ input: torch.Tensor,
13
+ weight: torch.Tensor,
14
+ alpha: float,
15
+ variant: int,
16
+ out: torch.Tensor,
17
+ ) -> None:
18
+ if input.dim() != 2 or weight.dim() != 2:
19
+ raise RuntimeError("input and weight must be rank-2 tensors")
20
+ if out.shape != (input.shape[0], weight.shape[0]):
21
+ raise RuntimeError("out must have shape (input.shape[0], weight.shape[0])")
22
+ return None
23
+
24
+
25
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_residual_bf16"))
26
+ def _fp8_linear_residual_bf16_fake(
27
+ input: torch.Tensor,
28
+ weight: torch.Tensor,
29
+ alpha: float,
30
+ variant: int,
31
+ residual: torch.Tensor,
32
+ ) -> None:
33
+ if input.shape[0] != 1:
34
+ raise RuntimeError("residual path supports only M=1")
35
+ if residual.shape != (1, weight.shape[0]):
36
+ raise RuntimeError("residual must have shape (1, weight.shape[0])")
37
+ return None
38
+
39
+
40
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_blockwise_linear_bf16"))
41
+ def _fp8_blockwise_linear_bf16_fake(
42
+ input: torch.Tensor,
43
+ weight: torch.Tensor,
44
+ input_scale: torch.Tensor,
45
+ weight_scale: torch.Tensor,
46
+ out: torch.Tensor,
47
+ ) -> None:
48
+ if input.dim() != 2 or weight.dim() != 2:
49
+ raise RuntimeError("input and weight must be rank-2 tensors")
50
+ m, k = input.shape
51
+ n = weight.shape[0]
52
+ if weight.shape[1] != k or n % 128 or k % 128:
53
+ raise RuntimeError("weight shape is invalid or N/K are not divisible by 128")
54
+ if input_scale.shape != (m, k // 128):
55
+ raise RuntimeError("input_scale must have shape (M, K / 128)")
56
+ if weight_scale.shape != (n // 128, k // 128):
57
+ raise RuntimeError("weight_scale must have shape (N / 128, K / 128)")
58
+ if out.shape != (m, n):
59
+ raise RuntimeError("out must have shape (M, N)")
60
+ return None
61
+
62
+
63
+ @torch.library.register_fake(
64
+ add_op_namespace_prefix("fp8_blockwise_swiglu_quantize_fp8")
65
+ )
66
+ def _fp8_blockwise_swiglu_quantize_fp8_fake(
67
+ input: torch.Tensor,
68
+ gate_up_weight: torch.Tensor,
69
+ input_scale: torch.Tensor,
70
+ gate_up_weight_scale: torch.Tensor,
71
+ output: torch.Tensor,
72
+ output_scale: torch.Tensor,
73
+ ) -> None:
74
+ m, k = input.shape
75
+ if gate_up_weight.dim() != 2 or gate_up_weight.shape[0] % 2:
76
+ raise RuntimeError("gate_up_weight must have shape (2*N, K)")
77
+ n = gate_up_weight.shape[0] // 2
78
+ if gate_up_weight.shape[1] != k or n % 128 or k % 128:
79
+ raise RuntimeError("gate_up_weight shape is invalid or N/K are not divisible by 128")
80
+ if input_scale.shape != (m, k // 128):
81
+ raise RuntimeError("input_scale must have shape (M, K/128)")
82
+ if gate_up_weight_scale.shape != (2 * n // 128, k // 128):
83
+ raise RuntimeError("gate_up_weight_scale must have shape (2*N/128, K/128)")
84
+ if output.shape != (m, n) or output_scale.shape != (m, n // 128):
85
+ raise RuntimeError("output buffers have invalid shapes")
86
+ return None
87
+
88
+
89
+ def select_fp8_linear_tile(m: int, n: int, k: int, variant: int = 0) -> str:
90
+ """Return the FlashRT tile selected by the public dispatcher."""
91
+
92
+ m = int(m)
93
+ n = int(n)
94
+ k = int(k)
95
+ variant = int(variant)
96
+ if m <= 0 or n <= 0 or k <= 0:
97
+ raise RuntimeError("m, n, and k must be positive")
98
+ if k % 32 != 0:
99
+ raise RuntimeError("k must be divisible by 32")
100
+ capability = torch.cuda.get_device_capability() if torch.cuda.is_available() else None
101
+ if capability == (11, 0):
102
+ forced = {1: "sm110_sq_bf16", 2: "sm110_t1_bf16", 3: "sm110_wide_bf16"}
103
+ if variant not in {0, *forced}:
104
+ raise RuntimeError("SM110 variant must be 0 (auto), 1 (Sq), 2 (T1), or 3 (Wide)")
105
+ if n % 16 or k % 16:
106
+ raise RuntimeError("SM110 requires n and k divisible by 16")
107
+ if variant:
108
+ return forced[variant]
109
+ if n >= 8 * k:
110
+ return "sm110_wide_bf16"
111
+ if m >= 128 and k >= 4 * n:
112
+ return "sm110_sq_bf16"
113
+ if n == k and m >= 512:
114
+ return "sm110_sq_bf16" if k <= 1024 else "sm110_wide_bf16"
115
+ if n == k and m >= 128:
116
+ return "sm110_wide_bf16"
117
+ return "sm110_t1_bf16"
118
+ if m == 1:
119
+ if variant == 4:
120
+ return "gemv_fp8_m1_w4"
121
+ if variant == 8:
122
+ return "gemv_fp8_m1_w8"
123
+ if variant == 16:
124
+ return "gemv_fp8_m1_w16"
125
+ if variant != 0:
126
+ raise RuntimeError("M=1 variant must be 0, 4, 8, or 16")
127
+ if n <= 2048:
128
+ return "gemv_fp8_m1_w4"
129
+ if n <= 8192:
130
+ return "gemv_fp8_m1_w8"
131
+ return "gemv_fp8_m1_w16"
132
+ if variant != 0:
133
+ raise RuntimeError("small-M dispatcher currently supports variant=0 only")
134
+ if m <= 16:
135
+ if k % 256 == 0:
136
+ return "ld_fp8_gemm_16x128x256_w4" if n % 128 == 0 else "ld_fp8_gemm_16x64x256_w4"
137
+ if n % 256 == 0:
138
+ return "ld_fp8_gemm_16x256x128_w8"
139
+ if n % 192 == 0:
140
+ return "ld_fp8_gemm_16x192x128_w4"
141
+ if n % 128 == 0:
142
+ return "ld_fp8_gemm_16x128x128_w4"
143
+ return "ld_fp8_gemm_16x64x128_w4"
144
+ if m <= 32:
145
+ if k % 256 == 0:
146
+ return "ld_fp8_gemm_32x128x256_w4" if n % 128 == 0 else "ld_fp8_gemm_32x64x256_w4"
147
+ if n % 192 == 0:
148
+ return "ld_fp8_gemm_32x192x128_w4"
149
+ if n % 128 == 0:
150
+ return "ld_fp8_gemm_32x128x128_w4"
151
+ return "ld_fp8_gemm_32x64x128_w4"
152
+ if m <= 64:
153
+ if k % 256 == 0:
154
+ return "ld_fp8_gemm_64x128x256_w4" if n % 128 == 0 else "ld_fp8_gemm_64x64x256_w4"
155
+ if n % 128 == 0:
156
+ return "ld_fp8_gemm_64x128x128_w4"
157
+ return "ld_fp8_gemm_64x64x128_w4"
158
+ raise RuntimeError("only M=1 decode or 2 <= M <= 64 small-M rows are supported")
159
+
160
+
161
+ def fp8_linear_bf16(
162
+ input: torch.Tensor,
163
+ weight: torch.Tensor,
164
+ alpha: float = 1.0,
165
+ out: torch.Tensor | None = None,
166
+ variant: int = 0,
167
+ ) -> torch.Tensor:
168
+ """Compute ``(input @ weight.T) * alpha`` with BF16 output.
169
+
170
+ ``input`` and ``weight`` must be FP8 E4M3 CUDA tensors with shapes
171
+ ``(M, K)`` and ``(N, K)``. ``alpha`` is a host float, normally the product
172
+ of static per-tensor input and weight scales. SM110 uses the production
173
+ CUTLASS Sq/T1/Wide dispatcher over full model row counts; SM120 uses the
174
+ hand-tuned M<=64 path.
175
+ """
176
+
177
+ if out is None:
178
+ out = torch.empty(
179
+ (input.shape[0], weight.shape[0]),
180
+ device=input.device,
181
+ dtype=torch.bfloat16,
182
+ )
183
+ ops.fp8_linear_bf16(input, weight, float(alpha), int(variant), out)
184
+ return out
185
+
186
+
187
+ def fp8_linear_residual_bf16(
188
+ input: torch.Tensor,
189
+ weight: torch.Tensor,
190
+ residual: torch.Tensor,
191
+ alpha: float = 1.0,
192
+ variant: int = 0,
193
+ ) -> torch.Tensor:
194
+ """In-place ``residual += (input @ weight.T) * alpha`` for M=1 decode."""
195
+
196
+ ops.fp8_linear_residual_bf16(input, weight, float(alpha), int(variant), residual)
197
+ return residual
198
+
199
+
200
+ def fp8_blockwise_linear_bf16(
201
+ input: torch.Tensor,
202
+ weight: torch.Tensor,
203
+ input_scale: torch.Tensor,
204
+ weight_scale: torch.Tensor,
205
+ out: torch.Tensor | None = None,
206
+ ) -> torch.Tensor:
207
+ """Block-128 scaled FP8 linear with BF16 output on SM89/SM120."""
208
+
209
+ if out is None:
210
+ out = torch.empty(
211
+ (input.shape[0], weight.shape[0]),
212
+ device=input.device,
213
+ dtype=torch.bfloat16,
214
+ )
215
+ ops.fp8_blockwise_linear_bf16(
216
+ input, weight, input_scale, weight_scale, out
217
+ )
218
+ return out
219
+
220
+
221
+ def fp8_blockwise_swiglu_quantize_fp8(
222
+ input: torch.Tensor,
223
+ gate_up_weight: torch.Tensor,
224
+ input_scale: torch.Tensor,
225
+ gate_up_weight_scale: torch.Tensor,
226
+ *,
227
+ output: torch.Tensor | None = None,
228
+ output_scale: torch.Tensor | None = None,
229
+ ) -> tuple[torch.Tensor, torch.Tensor]:
230
+ """SM89 block-128 FP8 gate/up GEMM + SiLU + FP8 requant producer."""
231
+
232
+ n = gate_up_weight.shape[0] // 2
233
+ if output is None:
234
+ output = torch.empty(
235
+ (input.shape[0], n), device=input.device, dtype=torch.float8_e4m3fn
236
+ )
237
+ if output_scale is None:
238
+ output_scale = torch.empty(
239
+ (input.shape[0], n // 128), device=input.device, dtype=torch.float32
240
+ )
241
+ ops.fp8_blockwise_swiglu_quantize_fp8(
242
+ input, gate_up_weight, input_scale, gate_up_weight_scale,
243
+ output, output_scale
244
+ )
245
+ return output, output_scale
246
+
247
+
248
+ __all__ = [
249
+ "fp8_linear_bf16",
250
+ "fp8_linear_residual_bf16",
251
+ "fp8_blockwise_linear_bf16",
252
+ "fp8_blockwise_swiglu_quantize_fp8",
253
+ "select_fp8_linear_tile",
254
+ ]
build/torch213-cxx11-cu132-x86_64-linux/_fp8_gemm_cuda_bf15968.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5395ce05668ccf4bec5f12109f073fb6ce743810f486ea98d67de77574a17b7
3
+ size 5431368
build/torch213-cxx11-cu132-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _fp8_gemm_cuda_bf15968
3
+ ops = torch.ops._fp8_gemm_cuda_bf15968
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_fp8_gemm_cuda_bf15968::{op_name}"
build/torch213-cxx11-cu132-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "fp8-gemm",
3
+ "id": "_fp8_gemm_cuda_bf15968",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "11.0a",
11
+ "12.0a",
12
+ "8.9"
13
+ ]
14
+ },
15
+ "digest": {
16
+ "algorithm": "sha256",
17
+ "files": {
18
+ "__init__.py": "hKsdFER2ZmYz36/zz9LYE7ddsK/R1dprHJFx+9yBzks=",
19
+ "_fp8_gemm_cuda_bf15968.abi3.so": "5Tlc4FZozPS+xfEhCfBz+2znQ4EPSG6pjWfed1dKF7c=",
20
+ "_ops.py": "wDFSPD8vh2K2qjj/9mdUYv6LrXuGz6MQCb2B/e8nqWs="
21
+ }
22
+ },
23
+ "provenance": {
24
+ "kernel-builder": {
25
+ "version": "0.17.0-dev0",
26
+ "sha": "d720fa90fb9cd92d1bc60a9dc5c55bef2aafabb8",
27
+ "dirty": false
28
+ },
29
+ "kernel": {
30
+ "sha": "bf15968fd9c281a7ca107e7696300fe091f588a2",
31
+ "dirty": false
32
+ }
33
+ }
34
+ }