liangsu9988 commited on
Commit
a47a242
·
verified ·
1 Parent(s): 4dd8b8b

Update torch211-cxx11-cu130-x86_64-linux from source 92c4b99

Browse files
build/torch211-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -19,7 +19,8 @@ def sfa_size_bytes(rows: int, dim: int) -> int:
19
  def _alloc_fp4(rows: int, dim: int, device: torch.device | str):
20
  return (
21
  torch.empty((rows, dim // 2), device=device, dtype=torch.uint8),
22
- torch.empty((sfa_size_bytes(rows, dim),), device=device, dtype=torch.uint8),
 
23
  )
24
 
25
 
@@ -36,6 +37,36 @@ def _linear_fake(
36
  return None
37
 
38
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
39
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
40
  def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
41
  if a_packed.shape[0] != 1:
@@ -58,11 +89,46 @@ def _legacy_linear_fake(
58
  return None
59
 
60
 
 
 
 
 
 
 
 
 
 
 
61
  @torch.library.register_fake(add_op_namespace_prefix("quantize_fp4_sfa_fp16"))
62
  def _quant_fake(x: torch.Tensor, packed: torch.Tensor, sfa: torch.Tensor, is_sfb: bool = False) -> None:
63
  return None
64
 
65
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
66
  @torch.library.register_fake(add_op_namespace_prefix("quantize_fp4_sfa_bf16"))
67
  def _quant_bf16_fake(x: torch.Tensor, packed: torch.Tensor, sfa: torch.Tensor, is_sfb: bool = False) -> None:
68
  return None
@@ -112,6 +178,160 @@ def quantize_fp4_sfa_fp16(
112
  return packed, sfa
113
 
114
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
115
  def quantize_fp4_sfa_bf16(
116
  x: torch.Tensor,
117
  packed: torch.Tensor | None = None,
@@ -152,6 +372,169 @@ def nvfp4_gemm_bf16(
152
  return out
153
 
154
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
155
  def fp4_w4a4_gemv_warpsplit_bf16(
156
  a_packed: torch.Tensor,
157
  b_packed: torch.Tensor,
@@ -191,6 +574,45 @@ def fp4_w4a16_linear_bf16(
191
  )
192
 
193
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
194
  def nvfp4_gemm_residual_bf16(
195
  a_packed: torch.Tensor,
196
  b_packed: torch.Tensor,
@@ -243,7 +665,7 @@ def nvfp4_gemm_bias_gelu_nvfp4(
243
  if out_packed is None:
244
  out_packed = torch.empty((m, n // 2), device=a_packed.device, dtype=torch.uint8)
245
  if out_sfa is None:
246
- out_sfa = torch.empty((sfa_size_bytes(m, n),), device=a_packed.device, dtype=torch.uint8)
247
  ops.nvfp4_gemm_bias_gelu_nvfp4(
248
  a_packed, b_packed, sfa, sfb, bias, out_packed, out_sfa, float(alpha)
249
  )
@@ -292,16 +714,33 @@ def nvfp4_gemm_streamk_bias_bf16(
292
 
293
 
294
  __all__ = [
 
 
295
  "dequantize_fp4_sfa_fp16",
 
296
  "fp4_w4a16_linear_bf16",
297
  "fp4_w4a4_gemv_warpsplit_bf16",
298
  "nvfp4_gemm_bf16",
 
 
 
 
 
 
 
299
  "nvfp4_gemm_bias_gelu_bf16",
300
  "nvfp4_gemm_bias_gelu_nvfp4",
 
301
  "nvfp4_gemm_residual_bf16",
 
302
  "nvfp4_gemm_streamk_bf16",
303
  "nvfp4_gemm_streamk_bias_bf16",
304
  "quantize_fp4_sfa_fp16",
 
 
 
 
 
305
  "quantize_fp4_sfa_bf16",
306
  "sfa_size_bytes",
307
  ]
 
19
  def _alloc_fp4(rows: int, dim: int, device: torch.device | str):
20
  return (
21
  torch.empty((rows, dim // 2), device=device, dtype=torch.uint8),
22
+ # Tile-layout padding entries are not written by every quantizer.
23
+ torch.zeros((sfa_size_bytes(rows, dim),), device=device, dtype=torch.uint8),
24
  )
25
 
26
 
 
37
  return None
38
 
39
 
40
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_fp16"))
41
+ def _linear_fp16_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, variant: int = -1) -> None:
42
+ return None
43
+
44
+
45
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_variant_bf16"))
46
+ def _linear_variant_bf16_fake(a, b, sfa, sfb, out, alpha: float = 1.0, variant: int = 7) -> None:
47
+ return None
48
+
49
+
50
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_nvfp4"))
51
+ def _linear_fp4out_fake(a, b, sfa, sfb, out_packed, out_sfa) -> None:
52
+ return None
53
+
54
+
55
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_geglu_nvfp4_fp16"))
56
+ def _geglu_fp4_fake(a, b, sfa, sfb, scratch, out_packed, out_sfa, skinny: bool = False) -> None:
57
+ return None
58
+
59
+
60
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bias_gelu_nvfp4_fp16"))
61
+ def _bias_gelu_fp4_fp16_fake(a, b, sfa, sfb, bias, out_packed, out_sfa) -> None:
62
+ return None
63
+
64
+
65
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bias_residual_fp16"))
66
+ def _bias_residual_fp16_fake(a, b, sfa, sfb, bias, residual, out) -> None:
67
+ return None
68
+
69
+
70
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
71
  def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
72
  if a_packed.shape[0] != 1:
 
89
  return None
90
 
91
 
92
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bias_bf16"))
93
+ def _bias_fake(a, b, sfa, sfb, bias, out) -> None:
94
+ return None
95
+
96
+
97
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bias_residual_bf16"))
98
+ def _bias_residual_fake(a, b, sfa, sfb, bias, residual, out) -> None:
99
+ return None
100
+
101
+
102
  @torch.library.register_fake(add_op_namespace_prefix("quantize_fp4_sfa_fp16"))
103
  def _quant_fake(x: torch.Tensor, packed: torch.Tensor, sfa: torch.Tensor, is_sfb: bool = False) -> None:
104
  return None
105
 
106
 
107
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_fp4_sfa_mse_fp16"))
108
+ def _quant_mse_fake(x, packed, sfa, is_sfb: bool = False) -> None:
109
+ return None
110
+
111
+
112
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_fp4_sfa_mse_bf16"))
113
+ def _quant_mse_bf16_fake(x, packed, sfa, is_sfb: bool = False) -> None:
114
+ return None
115
+
116
+
117
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_e0m3_sfa_fp16"))
118
+ def _quant_e0m3_fake(x, packed, sfa, is_sfb: bool = False) -> None:
119
+ return None
120
+
121
+
122
+ @torch.library.register_fake(add_op_namespace_prefix("e0m3_weight_gemm_fp16"))
123
+ def _e0m3_gemm_fake(a, b, sfa, sfb, out, alpha: float = 1.0, a_format: int = 1) -> None:
124
+ return None
125
+
126
+
127
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_relu2_nvfp4"))
128
+ def _relu2_gemm_fake(a, b, sfa, sfb, out_packed, out_sfa) -> None:
129
+ return None
130
+
131
+
132
  @torch.library.register_fake(add_op_namespace_prefix("quantize_fp4_sfa_bf16"))
133
  def _quant_bf16_fake(x: torch.Tensor, packed: torch.Tensor, sfa: torch.Tensor, is_sfb: bool = False) -> None:
134
  return None
 
178
  return packed, sfa
179
 
180
 
181
+ def quantize_fp4_sfa_mse_fp16(
182
+ x: torch.Tensor,
183
+ packed: torch.Tensor | None = None,
184
+ sfa: torch.Tensor | None = None,
185
+ is_sfb: bool = False,
186
+ ):
187
+ """Pack FP16 weights using per-block reconstruction-MSE scale search."""
188
+ if packed is None or sfa is None:
189
+ packed, sfa = _alloc_fp4(x.shape[0], x.shape[1], x.device)
190
+ ops.quantize_fp4_sfa_mse_fp16(x, packed, sfa, bool(is_sfb))
191
+ return packed, sfa
192
+
193
+
194
+ def quantize_fp4_sfa_mse_bf16(
195
+ x: torch.Tensor,
196
+ packed: torch.Tensor | None = None,
197
+ sfa: torch.Tensor | None = None,
198
+ is_sfb: bool = False,
199
+ ):
200
+ """Pack BF16 weights using per-block reconstruction-MSE scale search."""
201
+ if packed is None or sfa is None:
202
+ packed, sfa = _alloc_fp4(x.shape[0], x.shape[1], x.device)
203
+ ops.quantize_fp4_sfa_mse_bf16(x, packed, sfa, bool(is_sfb))
204
+ return packed, sfa
205
+
206
+
207
+ def aligned_fp4_dim(dim: int, alignment: int = 32) -> int:
208
+ """Return the physical FP4 GEMM dimension for a logical dimension."""
209
+ if dim <= 0 or alignment <= 0 or alignment % 16 != 0:
210
+ raise ValueError("dim must be positive and alignment divisible by 16")
211
+ return ((dim + alignment - 1) // alignment) * alignment
212
+
213
+
214
+ def quantize_fp4_sfa_padded_bf16(
215
+ x: torch.Tensor,
216
+ *,
217
+ alignment: int = 32,
218
+ is_sfb: bool = False,
219
+ ):
220
+ """Bind-time BF16 pack with zero padding on the final dimension.
221
+
222
+ The returned packed tensor uses the physical aligned width. Keep the
223
+ logical width returned as the third item for slicing final model outputs.
224
+ This helper allocates and is intentionally not a runtime hot-path op.
225
+ """
226
+ if x.ndim != 2 or x.dtype != torch.bfloat16:
227
+ raise ValueError("x must be a 2-D BF16 tensor")
228
+ logical_dim = int(x.shape[1])
229
+ physical_dim = aligned_fp4_dim(logical_dim, alignment)
230
+ if physical_dim == logical_dim:
231
+ padded = x.contiguous()
232
+ else:
233
+ padded = torch.zeros(
234
+ (x.shape[0], physical_dim), device=x.device, dtype=x.dtype
235
+ )
236
+ padded[:, :logical_dim].copy_(x)
237
+ packed, sfa = quantize_fp4_sfa_bf16(padded, is_sfb=is_sfb)
238
+ return packed, sfa, logical_dim
239
+
240
+
241
+ def pack_nvfp4_weight_bf16(
242
+ weight: torch.Tensor,
243
+ bias: torch.Tensor | None = None,
244
+ *,
245
+ alignment: int = 32,
246
+ mse: bool = False,
247
+ ):
248
+ """Zero-pad and pack a logical ``(N, K)`` BF16 weight at bind time.
249
+
250
+ Both physical ``N`` and ``K`` are aligned. This is the model-neutral
251
+ contract used by widths such as SigLIP's logical 4304, which becomes 4320
252
+ for Blackwell NVFP4 TMA. The hot path consumes the returned static tensors
253
+ directly and performs no padding or allocation.
254
+ """
255
+ if weight.ndim != 2 or weight.dtype != torch.bfloat16:
256
+ raise ValueError("weight must be a 2-D BF16 tensor")
257
+ logical_n, logical_k = map(int, weight.shape)
258
+ physical_n = aligned_fp4_dim(logical_n, alignment)
259
+ physical_k = aligned_fp4_dim(logical_k, alignment)
260
+ padded_weight = torch.zeros(
261
+ (physical_n, physical_k), device=weight.device, dtype=weight.dtype
262
+ )
263
+ padded_weight[:logical_n, :logical_k].copy_(weight)
264
+ quantize = quantize_fp4_sfa_mse_bf16 if mse else quantize_fp4_sfa_bf16
265
+ packed, sfb = quantize(padded_weight, is_sfb=True)
266
+ padded_bias = None
267
+ if bias is not None:
268
+ if bias.ndim != 1 or bias.numel() != logical_n:
269
+ raise ValueError("bias must have shape (N,)")
270
+ if bias.dtype != torch.bfloat16 or bias.device != weight.device:
271
+ raise ValueError("bias must match the weight dtype and device")
272
+ padded_bias = torch.zeros(
273
+ (physical_n,), device=bias.device, dtype=bias.dtype
274
+ )
275
+ padded_bias[:logical_n].copy_(bias)
276
+ return packed, sfb, padded_bias, (logical_n, logical_k)
277
+
278
+
279
+ def quantize_e0m3_sfa_fp16(
280
+ x: torch.Tensor,
281
+ packed: torch.Tensor | None = None,
282
+ sfa: torch.Tensor | None = None,
283
+ is_sfb: bool = False,
284
+ ):
285
+ """Quantize FP16 to packed E0M3 with CUTLASS SFA/SFB layout."""
286
+ if packed is None or sfa is None:
287
+ packed, sfa = _alloc_fp4(x.shape[0], x.shape[1], x.device)
288
+ ops.quantize_e0m3_sfa_fp16(x, packed, sfa, bool(is_sfb))
289
+ return packed, sfa
290
+
291
+
292
+ def e0m3_weight_gemm_fp16(
293
+ a_packed: torch.Tensor,
294
+ b_packed: torch.Tensor,
295
+ sfa: torch.Tensor,
296
+ sfb: torch.Tensor,
297
+ *,
298
+ alpha: float = 1.0,
299
+ a_format: int = 1,
300
+ out: torch.Tensor | None = None,
301
+ ):
302
+ """SM110 GEMM with E0M3 weights and E2M1 or E0M3 activations."""
303
+ if out is None:
304
+ out = torch.empty(
305
+ (a_packed.shape[0], b_packed.shape[0]),
306
+ device=a_packed.device,
307
+ dtype=torch.float16,
308
+ )
309
+ ops.e0m3_weight_gemm_fp16(
310
+ a_packed, b_packed, sfa, sfb, out, float(alpha), int(a_format)
311
+ )
312
+ return out
313
+
314
+
315
+ def nvfp4_gemm_relu2_nvfp4(
316
+ a_packed: torch.Tensor,
317
+ b_packed: torch.Tensor,
318
+ sfa: torch.Tensor,
319
+ sfb: torch.Tensor,
320
+ *,
321
+ out_packed: torch.Tensor | None = None,
322
+ out_sfa: torch.Tensor | None = None,
323
+ ):
324
+ """SM110 NVFP4 GEMM with fused ReLU-squared and NVFP4 output."""
325
+ if out_packed is None or out_sfa is None:
326
+ out_packed, out_sfa = _alloc_fp4(
327
+ a_packed.shape[0], b_packed.shape[0], a_packed.device
328
+ )
329
+ ops.nvfp4_gemm_relu2_nvfp4(
330
+ a_packed, b_packed, sfa, sfb, out_packed, out_sfa
331
+ )
332
+ return out_packed, out_sfa
333
+
334
+
335
  def quantize_fp4_sfa_bf16(
336
  x: torch.Tensor,
337
  packed: torch.Tensor | None = None,
 
372
  return out
373
 
374
 
375
+ def nvfp4_gemm_fp16(
376
+ a_packed: torch.Tensor,
377
+ b_packed: torch.Tensor,
378
+ sfa: torch.Tensor,
379
+ sfb: torch.Tensor,
380
+ alpha: float = 1.0,
381
+ out: torch.Tensor | None = None,
382
+ variant: int = -1,
383
+ ) -> torch.Tensor:
384
+ """SM110 native NVFP4 GEMM with FP16 output."""
385
+ if out is None:
386
+ out = torch.empty(
387
+ (a_packed.shape[0], b_packed.shape[0]),
388
+ device=a_packed.device,
389
+ dtype=torch.float16,
390
+ )
391
+ ops.nvfp4_gemm_fp16(
392
+ a_packed, b_packed, sfa, sfb, out, float(alpha), int(variant)
393
+ )
394
+ return out
395
+
396
+
397
+ def nvfp4_gemm_variant_bf16(
398
+ a_packed: torch.Tensor,
399
+ b_packed: torch.Tensor,
400
+ sfa: torch.Tensor,
401
+ sfb: torch.Tensor,
402
+ alpha: float = 1.0,
403
+ out: torch.Tensor | None = None,
404
+ variant: int = 7,
405
+ ) -> torch.Tensor:
406
+ """SM110 native NVFP4 GEMM with BF16 output.
407
+
408
+ ``variant=7`` serves large-M encoder/SigLIP shapes; ``variant=10`` is
409
+ the narrow-N PI0.5 decoder tile. The indices match FlashRT native.
410
+ """
411
+ if out is None:
412
+ out = torch.empty(
413
+ (a_packed.shape[0], b_packed.shape[0]),
414
+ device=a_packed.device,
415
+ dtype=torch.bfloat16,
416
+ )
417
+ ops.nvfp4_gemm_variant_bf16(
418
+ a_packed, b_packed, sfa, sfb, out, float(alpha), int(variant)
419
+ )
420
+ return out
421
+
422
+
423
+ def nvfp4_gemm_nvfp4(
424
+ a_packed: torch.Tensor,
425
+ b_packed: torch.Tensor,
426
+ sfa: torch.Tensor,
427
+ sfb: torch.Tensor,
428
+ *,
429
+ out_packed: torch.Tensor | None = None,
430
+ out_sfa: torch.Tensor | None = None,
431
+ ) -> tuple[torch.Tensor, torch.Tensor]:
432
+ """SM110 NVFP4 GEMM with an NVFP4 output and CUTLASS SFA layout."""
433
+ if out_packed is None or out_sfa is None:
434
+ out_packed, out_sfa = _alloc_fp4(
435
+ a_packed.shape[0], b_packed.shape[0], a_packed.device
436
+ )
437
+ ops.nvfp4_gemm_nvfp4(
438
+ a_packed, b_packed, sfa, sfb, out_packed, out_sfa
439
+ )
440
+ return out_packed, out_sfa
441
+
442
+
443
+ def nvfp4_gemm_geglu_nvfp4_fp16(
444
+ a_packed: torch.Tensor,
445
+ b_interleaved_packed: torch.Tensor,
446
+ sfa: torch.Tensor,
447
+ sfb: torch.Tensor,
448
+ *,
449
+ skinny: bool = False,
450
+ scratch: torch.Tensor | None = None,
451
+ out_packed: torch.Tensor | None = None,
452
+ out_sfa: torch.Tensor | None = None,
453
+ ) -> tuple[torch.Tensor, torch.Tensor]:
454
+ """GEMM with fused GeGLU and compact NVFP4 output on SM110.
455
+
456
+ ``b_interleaved_packed`` stores gate/up rows pairwise, so its first
457
+ dimension is twice the logical hidden width.
458
+ """
459
+ m, n_twice = a_packed.shape[0], b_interleaved_packed.shape[0]
460
+ hidden = n_twice // 2
461
+ if scratch is None:
462
+ scratch = torch.empty((m, hidden), device=a_packed.device, dtype=torch.uint8)
463
+ if out_packed is None:
464
+ out_packed = torch.empty((m, hidden // 2), device=a_packed.device, dtype=torch.uint8)
465
+ if out_sfa is None:
466
+ out_sfa = torch.zeros((sfa_size_bytes(m, hidden),), device=a_packed.device, dtype=torch.uint8)
467
+ ops.nvfp4_gemm_geglu_nvfp4_fp16(
468
+ a_packed, b_interleaved_packed, sfa, sfb, scratch,
469
+ out_packed, out_sfa, bool(skinny)
470
+ )
471
+ return out_packed, out_sfa
472
+
473
+
474
+ def cutlass_fp4_gemm_geglu_il_hw_v10(
475
+ a_packed: torch.Tensor,
476
+ b_interleaved_packed: torch.Tensor,
477
+ sfa: torch.Tensor,
478
+ sfb: torch.Tensor,
479
+ *,
480
+ scratch: torch.Tensor | None = None,
481
+ out_packed: torch.Tensor | None = None,
482
+ out_sfa: torch.Tensor | None = None,
483
+ ) -> tuple[torch.Tensor, torch.Tensor]:
484
+ """Explicit public alias for the native SM110 v10 GeGLU schedule."""
485
+ return nvfp4_gemm_geglu_nvfp4_fp16(
486
+ a_packed,
487
+ b_interleaved_packed,
488
+ sfa,
489
+ sfb,
490
+ skinny=True,
491
+ scratch=scratch,
492
+ out_packed=out_packed,
493
+ out_sfa=out_sfa,
494
+ )
495
+
496
+
497
+ def nvfp4_gemm_bias_gelu_nvfp4_fp16(
498
+ a_packed: torch.Tensor,
499
+ b_packed: torch.Tensor,
500
+ sfa: torch.Tensor,
501
+ sfb: torch.Tensor,
502
+ bias: torch.Tensor,
503
+ *,
504
+ out_packed: torch.Tensor | None = None,
505
+ out_sfa: torch.Tensor | None = None,
506
+ ) -> tuple[torch.Tensor, torch.Tensor]:
507
+ """FP16-bias GEMM with fused GELU and NVFP4 output on SM110."""
508
+ m, n = a_packed.shape[0], b_packed.shape[0]
509
+ if out_packed is None:
510
+ out_packed = torch.empty((m, n // 2), device=a_packed.device, dtype=torch.uint8)
511
+ if out_sfa is None:
512
+ out_sfa = torch.zeros((sfa_size_bytes(m, n),), device=a_packed.device, dtype=torch.uint8)
513
+ ops.nvfp4_gemm_bias_gelu_nvfp4_fp16(
514
+ a_packed, b_packed, sfa, sfb, bias, out_packed, out_sfa
515
+ )
516
+ return out_packed, out_sfa
517
+
518
+
519
+ def nvfp4_gemm_bias_residual_fp16(
520
+ a_packed: torch.Tensor,
521
+ b_packed: torch.Tensor,
522
+ sfa: torch.Tensor,
523
+ sfb: torch.Tensor,
524
+ bias: torch.Tensor,
525
+ residual: torch.Tensor,
526
+ *,
527
+ out: torch.Tensor | None = None,
528
+ ) -> torch.Tensor:
529
+ """FP16-output GEMM with fused FP16 bias and residual on SM110."""
530
+ if out is None:
531
+ out = torch.empty_like(residual)
532
+ ops.nvfp4_gemm_bias_residual_fp16(
533
+ a_packed, b_packed, sfa, sfb, bias, residual, out
534
+ )
535
+ return out
536
+
537
+
538
  def fp4_w4a4_gemv_warpsplit_bf16(
539
  a_packed: torch.Tensor,
540
  b_packed: torch.Tensor,
 
574
  )
575
 
576
 
577
+ def nvfp4_gemm_bias_bf16(
578
+ a_packed: torch.Tensor,
579
+ b_packed: torch.Tensor,
580
+ sfa: torch.Tensor,
581
+ sfb: torch.Tensor,
582
+ bias: torch.Tensor,
583
+ *,
584
+ out: torch.Tensor | None = None,
585
+ ) -> torch.Tensor:
586
+ """SM110 NVFP4 GEMM with a fused per-column BF16 bias."""
587
+ if out is None:
588
+ out = torch.empty(
589
+ (a_packed.shape[0], b_packed.shape[0]),
590
+ device=a_packed.device,
591
+ dtype=torch.bfloat16,
592
+ )
593
+ ops.nvfp4_gemm_bias_bf16(a_packed, b_packed, sfa, sfb, bias, out)
594
+ return out
595
+
596
+
597
+ def nvfp4_gemm_bias_residual_bf16(
598
+ a_packed: torch.Tensor,
599
+ b_packed: torch.Tensor,
600
+ sfa: torch.Tensor,
601
+ sfb: torch.Tensor,
602
+ bias: torch.Tensor,
603
+ residual: torch.Tensor,
604
+ *,
605
+ out: torch.Tensor | None = None,
606
+ ) -> torch.Tensor:
607
+ """SM110 NVFP4 GEMM with fused BF16 bias and residual add."""
608
+ if out is None:
609
+ out = torch.empty_like(residual)
610
+ ops.nvfp4_gemm_bias_residual_bf16(
611
+ a_packed, b_packed, sfa, sfb, bias, residual, out
612
+ )
613
+ return out
614
+
615
+
616
  def nvfp4_gemm_residual_bf16(
617
  a_packed: torch.Tensor,
618
  b_packed: torch.Tensor,
 
665
  if out_packed is None:
666
  out_packed = torch.empty((m, n // 2), device=a_packed.device, dtype=torch.uint8)
667
  if out_sfa is None:
668
+ out_sfa = torch.zeros((sfa_size_bytes(m, n),), device=a_packed.device, dtype=torch.uint8)
669
  ops.nvfp4_gemm_bias_gelu_nvfp4(
670
  a_packed, b_packed, sfa, sfb, bias, out_packed, out_sfa, float(alpha)
671
  )
 
714
 
715
 
716
  __all__ = [
717
+ "aligned_fp4_dim",
718
+ "cutlass_fp4_gemm_geglu_il_hw_v10",
719
  "dequantize_fp4_sfa_fp16",
720
+ "e0m3_weight_gemm_fp16",
721
  "fp4_w4a16_linear_bf16",
722
  "fp4_w4a4_gemv_warpsplit_bf16",
723
  "nvfp4_gemm_bf16",
724
+ "nvfp4_gemm_fp16",
725
+ "nvfp4_gemm_variant_bf16",
726
+ "nvfp4_gemm_nvfp4",
727
+ "nvfp4_gemm_geglu_nvfp4_fp16",
728
+ "nvfp4_gemm_bias_gelu_nvfp4_fp16",
729
+ "nvfp4_gemm_bias_residual_fp16",
730
+ "nvfp4_gemm_bias_bf16",
731
  "nvfp4_gemm_bias_gelu_bf16",
732
  "nvfp4_gemm_bias_gelu_nvfp4",
733
+ "nvfp4_gemm_bias_residual_bf16",
734
  "nvfp4_gemm_residual_bf16",
735
+ "nvfp4_gemm_relu2_nvfp4",
736
  "nvfp4_gemm_streamk_bf16",
737
  "nvfp4_gemm_streamk_bias_bf16",
738
  "quantize_fp4_sfa_fp16",
739
+ "quantize_fp4_sfa_mse_fp16",
740
+ "quantize_fp4_sfa_mse_bf16",
741
+ "quantize_fp4_sfa_padded_bf16",
742
+ "pack_nvfp4_weight_bf16",
743
+ "quantize_e0m3_sfa_fp16",
744
  "quantize_fp4_sfa_bf16",
745
  "sfa_size_bytes",
746
  ]
build/torch211-cxx11-cu130-x86_64-linux/_fp4_gemm_cuda_92c4b99.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:199bf055035b1b7d840c3b83094f0b95fef7be344a8e4ee5e180798a26b74b10
3
+ size 2330536
build/torch211-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,6 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_83a7cd0
3
- ops = torch.ops._fp4_gemm_cuda_83a7cd0
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
- """
7
- Prefix op by namespace.
8
- """
9
- return f"_fp4_gemm_cuda_83a7cd0::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_92c4b99
3
+ ops = torch.ops._fp4_gemm_cuda_92c4b99
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
+ return f"_fp4_gemm_cuda_92c4b99::{op_name}"
 
 
 
build/torch211-cxx11-cu130-x86_64-linux/fp4_gemm/__init__.py CHANGED
@@ -2,25 +2,13 @@ import ctypes
2
  import importlib.util
3
  import sys
4
  from pathlib import Path
5
- from types import ModuleType
6
 
7
-
8
- def _import_from_path(file_path: Path) -> ModuleType:
9
- # We cannot use the module name as-is, after adding it to `sys.modules`,
10
- # it would also be used for other imports. So, we make a module name that
11
- # depends on the path for it to be unique using the hex-encoded hash of
12
- # the path.
13
- path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
- module_name = path_hash
15
- spec = importlib.util.spec_from_file_location(module_name, file_path)
16
- if spec is None:
17
- raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
  module = importlib.util.module_from_spec(spec)
19
- if module is None:
20
- raise ImportError(f"Cannot load module {module_name} from spec")
21
- sys.modules[module_name] = module
22
- spec.loader.exec_module(module) # type: ignore
23
  return module
24
 
25
-
26
- globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
 
2
  import importlib.util
3
  import sys
4
  from pathlib import Path
 
5
 
6
+ def _import_from_path(file_path: Path):
7
+ path_hash = '{:x}'.format(ctypes.c_size_t(hash(file_path.absolute())).value)
8
+ spec = importlib.util.spec_from_file_location(path_hash, file_path)
 
 
 
 
 
 
 
 
9
  module = importlib.util.module_from_spec(spec)
10
+ sys.modules[path_hash] = module
11
+ spec.loader.exec_module(module)
 
 
12
  return module
13
 
14
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / '__init__.py')))
 
build/torch211-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,14 +1,32 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_83a7cd0",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
7
  "backend": {
8
  "type": "cuda",
9
  "archs": [
10
- "11.0a",
11
  "12.0a"
12
  ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13
  }
14
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_92c4b99",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
7
  "backend": {
8
  "type": "cuda",
9
  "archs": [
 
10
  "12.0a"
11
  ]
12
+ },
13
+ "digest": {
14
+ "algorithm": "sha256",
15
+ "files": {
16
+ "__init__.py": "Mr1/bAUzs+af68D+bRXX6iONDof//dzKgn6QU5pKdu8=",
17
+ "_fp4_gemm_cuda_92c4b99.abi3.so": "GZvwVQNbG32EDDuDCU8Llf73vjRKjk7l4YB5iia3SxA=",
18
+ "_ops.py": "twu0zeqTyaITBKAdr9FudRbkbXpvBQdmAHcVf/OXw2I=",
19
+ "fp4_gemm/__init__.py": "v6p5XMfQzddhi1fLSAw4HX9CyS0rQsidvu9VsT01xi4="
20
+ }
21
+ },
22
+ "provenance": {
23
+ "kernel": {
24
+ "sha": "92c4b991b6f1fbd89744aa6f46f62b1bfac7ca7e",
25
+ "dirty": false
26
+ },
27
+ "validation": {
28
+ "torch": "2.11.0+cu130",
29
+ "cuda": "13.0"
30
+ }
31
  }
32
  }