liangsu9988 commited on
Commit
96216b3
·
verified ·
1 Parent(s): 5c00374

Uploaded using `kernel-builder`.

Browse files
build/torch213-cxx11-cu132-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,543 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT adaptive LayerNorm producer kernels for DiT/Wan-style blocks."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import torch
6
+
7
+ from ._ops import add_op_namespace_prefix, ops
8
+
9
+
10
+ def _check_x(x: torch.Tensor) -> None:
11
+ if x.dim() != 2:
12
+ raise RuntimeError("x must have shape (rows, dim)")
13
+ if x.shape[1] % 2 != 0:
14
+ raise RuntimeError("x.shape[1] must be even")
15
+
16
+
17
+ def _check_mod(x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor) -> None:
18
+ _check_x(x)
19
+ dim = x.shape[1]
20
+ if scale.shape != (dim,) or shift.shape != (dim,):
21
+ raise RuntimeError("scale and shift must have shape (dim,)")
22
+
23
+
24
+ def _check_scalar(scale: torch.Tensor, name: str) -> None:
25
+ if scale.numel() != 1:
26
+ raise RuntimeError(f"{name} must be a scalar tensor")
27
+
28
+
29
+ def swizzled_sf_size(rows: int, dim: int) -> int:
30
+ """Return bytes required by the FlashRT/CUTLASS NVFP4 swizzled scale layout."""
31
+
32
+ if dim % 16 != 0:
33
+ raise RuntimeError("dim must be divisible by 16 for NVFP4 swizzled output")
34
+ n_blocks = dim // 16
35
+ n_row_super = (rows + 127) // 128
36
+ n_col_super = (n_blocks + 3) // 4
37
+ return n_row_super * n_col_super * 128 * 64
38
+
39
+
40
+ def _alloc_fp8(x: torch.Tensor) -> torch.Tensor:
41
+ return torch.empty_like(x, dtype=torch.float8_e4m3fn)
42
+
43
+
44
+ def _alloc_nvfp4(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
45
+ rows, dim = x.shape
46
+ packed = torch.empty((rows, dim // 2), device=x.device, dtype=torch.uint8)
47
+ sf = torch.zeros((swizzled_sf_size(rows, dim),), device=x.device, dtype=torch.uint8)
48
+ return packed, sf
49
+
50
+
51
+ @torch.library.register_fake(add_op_namespace_prefix("ada_layer_norm_quant_fp8_bf16"))
52
+ def _ada_layer_norm_quant_fp8_bf16_fake(
53
+ x: torch.Tensor,
54
+ scale: torch.Tensor,
55
+ shift: torch.Tensor,
56
+ act_scale: torch.Tensor,
57
+ eps: float,
58
+ out: torch.Tensor,
59
+ ) -> None:
60
+ _check_mod(x, scale, shift)
61
+ _check_scalar(act_scale, "act_scale")
62
+ if out.shape != x.shape:
63
+ raise RuntimeError("out must have the same shape as x")
64
+ return None
65
+
66
+
67
+ @torch.library.register_fake(add_op_namespace_prefix("ada_layer_norm_quant_fp8_ptok_bf16"))
68
+ def _ada_layer_norm_quant_fp8_ptok_bf16_fake(
69
+ x: torch.Tensor,
70
+ scale: torch.Tensor,
71
+ shift: torch.Tensor,
72
+ act_scale: torch.Tensor,
73
+ eps: float,
74
+ out: torch.Tensor,
75
+ ) -> None:
76
+ _check_x(x)
77
+ if scale.shape != x.shape or shift.shape != x.shape:
78
+ raise RuntimeError(
79
+ "per-token scale and shift must have shape (seq_len, dim)")
80
+ _check_scalar(act_scale, "act_scale")
81
+ if out.shape != x.shape:
82
+ raise RuntimeError("out must have the same shape as x")
83
+ return None
84
+
85
+
86
+ @torch.library.register_fake(add_op_namespace_prefix("ada_layer_norm_quant_fp8_ptok_table_bf16"))
87
+ def _ada_layer_norm_quant_fp8_ptok_table_bf16_fake(
88
+ x: torch.Tensor,
89
+ temb: torch.Tensor,
90
+ table: torch.Tensor,
91
+ act_scale: torch.Tensor,
92
+ shift_idx: int,
93
+ scale_idx: int,
94
+ eps: float,
95
+ out: torch.Tensor,
96
+ ) -> None:
97
+ _check_x(x)
98
+ if temb.dim() != 3 or temb.shape[0] != x.shape[0] or temb.shape[2] != x.shape[1]:
99
+ raise RuntimeError("temb must have shape (seq_len, n_chunks, dim)")
100
+ if table.dim() != 2 or table.shape[0] != temb.shape[1] or table.shape[1] != x.shape[1]:
101
+ raise RuntimeError("table must have shape (n_chunks, dim)")
102
+ _check_scalar(act_scale, "act_scale")
103
+ if out.shape != x.shape:
104
+ raise RuntimeError("out must have the same shape as x")
105
+ return None
106
+
107
+
108
+ @torch.library.register_fake(add_op_namespace_prefix("ada_layer_norm_ptok_table_bf16"))
109
+ def _ada_layer_norm_ptok_table_bf16_fake(
110
+ x: torch.Tensor,
111
+ temb: torch.Tensor,
112
+ table: torch.Tensor,
113
+ shift_idx: int,
114
+ scale_idx: int,
115
+ eps: float,
116
+ out: torch.Tensor,
117
+ ) -> None:
118
+ _check_x(x)
119
+ if temb.dim() != 3 or temb.shape[0] != x.shape[0] or temb.shape[2] != x.shape[1]:
120
+ raise RuntimeError("temb must have shape (seq_len, n_chunks, dim)")
121
+ if table.shape != (temb.shape[1], x.shape[1]):
122
+ raise RuntimeError("table must have shape (n_chunks, dim)")
123
+ if out.shape != x.shape:
124
+ raise RuntimeError("out must have the same shape as x")
125
+ return None
126
+
127
+
128
+ @torch.library.register_fake(add_op_namespace_prefix("ada_layer_norm_quant_fp8_modfp8_bf16"))
129
+ def _ada_layer_norm_quant_fp8_modfp8_bf16_fake(
130
+ x: torch.Tensor,
131
+ scale_fp8: torch.Tensor,
132
+ shift_fp8: torch.Tensor,
133
+ scale_deq: torch.Tensor,
134
+ shift_deq: torch.Tensor,
135
+ act_scale: torch.Tensor,
136
+ eps: float,
137
+ out: torch.Tensor,
138
+ ) -> None:
139
+ _check_mod(x, scale_fp8, shift_fp8)
140
+ _check_scalar(scale_deq, "scale_deq")
141
+ _check_scalar(shift_deq, "shift_deq")
142
+ _check_scalar(act_scale, "act_scale")
143
+ if out.shape != x.shape:
144
+ raise RuntimeError("out must have the same shape as x")
145
+ return None
146
+
147
+
148
+ @torch.library.register_fake(add_op_namespace_prefix("awq_ada_layer_norm_quant_fp8_bf16"))
149
+ def _awq_ada_layer_norm_quant_fp8_bf16_fake(
150
+ x: torch.Tensor,
151
+ scale: torch.Tensor,
152
+ shift: torch.Tensor,
153
+ inv_s: torch.Tensor,
154
+ act_scale: torch.Tensor,
155
+ eps: float,
156
+ out: torch.Tensor,
157
+ ) -> None:
158
+ _check_mod(x, scale, shift)
159
+ if inv_s.shape != (x.shape[1],):
160
+ raise RuntimeError("inv_s must have shape (dim,)")
161
+ _check_scalar(act_scale, "act_scale")
162
+ if out.shape != x.shape:
163
+ raise RuntimeError("out must have the same shape as x")
164
+ return None
165
+
166
+
167
+ @torch.library.register_fake(add_op_namespace_prefix("ada_layer_norm_quant_nvfp4_swizzled_bf16"))
168
+ def _ada_layer_norm_quant_nvfp4_swizzled_bf16_fake(
169
+ x: torch.Tensor,
170
+ scale: torch.Tensor,
171
+ shift: torch.Tensor,
172
+ eps: float,
173
+ packed: torch.Tensor,
174
+ sf_swizzled: torch.Tensor,
175
+ ) -> None:
176
+ _check_mod(x, scale, shift)
177
+ rows, dim = x.shape
178
+ if packed.shape != (rows, dim // 2):
179
+ raise RuntimeError("packed must have shape (rows, dim // 2)")
180
+ if sf_swizzled.numel() < swizzled_sf_size(rows, dim):
181
+ raise RuntimeError("sf_swizzled is too small")
182
+ return None
183
+
184
+
185
+ @torch.library.register_fake(
186
+ add_op_namespace_prefix("ada_layer_norm_quant_nvfp4_swizzled_ptok_table_bf16")
187
+ )
188
+ def _ada_layer_norm_quant_nvfp4_swizzled_ptok_table_bf16_fake(
189
+ x: torch.Tensor,
190
+ temb: torch.Tensor,
191
+ table: torch.Tensor,
192
+ shift_idx: int,
193
+ scale_idx: int,
194
+ eps: float,
195
+ packed: torch.Tensor,
196
+ sf_swizzled: torch.Tensor,
197
+ ) -> None:
198
+ _check_x(x)
199
+ if temb.dim() != 3 or temb.shape[0] != x.shape[0] or temb.shape[2] != x.shape[1]:
200
+ raise RuntimeError("temb must have shape (seq_len, n_chunks, dim)")
201
+ if table.shape != (temb.shape[1], x.shape[1]):
202
+ raise RuntimeError("table must have shape (n_chunks, dim)")
203
+ rows, dim = x.shape
204
+ if packed.shape != (rows, dim // 2):
205
+ raise RuntimeError("packed must have shape (rows, dim // 2)")
206
+ if sf_swizzled.numel() < swizzled_sf_size(rows, dim):
207
+ raise RuntimeError("sf_swizzled is too small")
208
+ return None
209
+
210
+
211
+ @torch.library.register_fake(add_op_namespace_prefix("ada_layer_norm_quant_nvfp4_swizzled_modfp8_bf16"))
212
+ def _ada_layer_norm_quant_nvfp4_swizzled_modfp8_bf16_fake(
213
+ x: torch.Tensor,
214
+ scale_fp8: torch.Tensor,
215
+ shift_fp8: torch.Tensor,
216
+ scale_deq: torch.Tensor,
217
+ shift_deq: torch.Tensor,
218
+ eps: float,
219
+ packed: torch.Tensor,
220
+ sf_swizzled: torch.Tensor,
221
+ ) -> None:
222
+ _check_mod(x, scale_fp8, shift_fp8)
223
+ _check_scalar(scale_deq, "scale_deq")
224
+ _check_scalar(shift_deq, "shift_deq")
225
+ rows, dim = x.shape
226
+ if packed.shape != (rows, dim // 2):
227
+ raise RuntimeError("packed must have shape (rows, dim // 2)")
228
+ if sf_swizzled.numel() < swizzled_sf_size(rows, dim):
229
+ raise RuntimeError("sf_swizzled is too small")
230
+ return None
231
+
232
+
233
+ @torch.library.register_fake(add_op_namespace_prefix("layer_norm_no_affine_quant_fp8_static_bf16"))
234
+ def _layer_norm_no_affine_quant_fp8_static_bf16_fake(
235
+ x: torch.Tensor,
236
+ act_scale: torch.Tensor,
237
+ eps: float,
238
+ out: torch.Tensor,
239
+ ) -> None:
240
+ _check_x(x)
241
+ _check_scalar(act_scale, "act_scale")
242
+ if out.shape != x.shape:
243
+ raise RuntimeError("out must have the same shape as x")
244
+ return None
245
+
246
+
247
+ @torch.library.register_fake(add_op_namespace_prefix("layer_norm_no_affine_quant_nvfp4_swizzled_bf16"))
248
+ def _layer_norm_no_affine_quant_nvfp4_swizzled_bf16_fake(
249
+ x: torch.Tensor,
250
+ eps: float,
251
+ packed: torch.Tensor,
252
+ sf_swizzled: torch.Tensor,
253
+ ) -> None:
254
+ _check_x(x)
255
+ rows, dim = x.shape
256
+ if packed.shape != (rows, dim // 2):
257
+ raise RuntimeError("packed must have shape (rows, dim // 2)")
258
+ if sf_swizzled.numel() < swizzled_sf_size(rows, dim):
259
+ raise RuntimeError("sf_swizzled is too small")
260
+ return None
261
+
262
+
263
+ def ada_layer_norm_quant_fp8_bf16(
264
+ x: torch.Tensor,
265
+ scale: torch.Tensor,
266
+ shift: torch.Tensor,
267
+ act_scale: torch.Tensor,
268
+ eps: float = 1e-5,
269
+ out: torch.Tensor | None = None,
270
+ ) -> torch.Tensor:
271
+ """Fused no-affine LayerNorm + AdaLN scale/shift + static FP8 quantize."""
272
+
273
+ if out is None:
274
+ out = _alloc_fp8(x)
275
+ ops.ada_layer_norm_quant_fp8_bf16(x, scale, shift, act_scale, float(eps), out)
276
+ return out
277
+
278
+
279
+ def ada_layer_norm_quant_fp8_ptok_bf16(
280
+ x: torch.Tensor,
281
+ scale: torch.Tensor,
282
+ shift: torch.Tensor,
283
+ act_scale: torch.Tensor,
284
+ eps: float = 1e-5,
285
+ out: torch.Tensor | None = None,
286
+ ) -> torch.Tensor:
287
+ """Per-token AdaLN producer: no-affine LayerNorm + [M, D] scale/shift
288
+ modulation + static FP8 quantize, one pass.
289
+
290
+ The per-token form serves hosts whose timestep embedding carries one
291
+ modulation vector per activation row (video DiTs with [B, S, 6, D]
292
+ tables); the broadcast form above cannot express that seam.
293
+ """
294
+
295
+ if out is None:
296
+ out = _alloc_fp8(x)
297
+ ops.ada_layer_norm_quant_fp8_ptok_bf16(
298
+ x, scale, shift, act_scale, float(eps), out)
299
+ return out
300
+
301
+
302
+ def ada_layer_norm_quant_fp8_ptok_table_bf16(
303
+ x: torch.Tensor,
304
+ temb: torch.Tensor,
305
+ table: torch.Tensor,
306
+ act_scale: torch.Tensor,
307
+ shift_idx: int,
308
+ scale_idx: int,
309
+ eps: float = 1e-5,
310
+ out: torch.Tensor | None = None,
311
+ ) -> torch.Tensor:
312
+ """Per-token AdaLN producer, table form.
313
+
314
+ ``temb`` is the host's per-token modulation embedding
315
+ ``[seq_len, n_chunks, dim]`` (cast to BF16 once per call) and
316
+ ``table`` the block's own ``[n_chunks, dim]`` FP32 modulation
317
+ parameter. The table add, chunk selection, no-affine LayerNorm,
318
+ modulation and static FP8 quantize all happen in one pass — the
319
+ per-block six-chunk materialization the host would otherwise pay
320
+ never exists.
321
+ """
322
+
323
+ if out is None:
324
+ out = _alloc_fp8(x)
325
+ ops.ada_layer_norm_quant_fp8_ptok_table_bf16(
326
+ x, temb, table, act_scale, int(shift_idx), int(scale_idx),
327
+ float(eps), out)
328
+ return out
329
+
330
+
331
+ def ada_layer_norm_ptok_table_bf16(
332
+ x: torch.Tensor,
333
+ temb: torch.Tensor,
334
+ table: torch.Tensor,
335
+ shift_idx: int,
336
+ scale_idx: int,
337
+ eps: float = 1e-5,
338
+ out: torch.Tensor | None = None,
339
+ ) -> torch.Tensor:
340
+ """Fuse table add, chunk selection and AdaLN into a BF16 output."""
341
+
342
+ if out is None:
343
+ out = torch.empty_like(x)
344
+ ops.ada_layer_norm_ptok_table_bf16(
345
+ x, temb, table, int(shift_idx), int(scale_idx), float(eps), out
346
+ )
347
+ return out
348
+
349
+
350
+ def ada_layer_norm_quant_fp8_modfp8_bf16(
351
+ x: torch.Tensor,
352
+ scale_fp8: torch.Tensor,
353
+ shift_fp8: torch.Tensor,
354
+ scale_deq: torch.Tensor,
355
+ shift_deq: torch.Tensor,
356
+ act_scale: torch.Tensor,
357
+ eps: float = 1e-5,
358
+ out: torch.Tensor | None = None,
359
+ ) -> torch.Tensor:
360
+ """Fused AdaLN producer when modulation vectors are stored in FP8."""
361
+
362
+ if out is None:
363
+ out = _alloc_fp8(x)
364
+ ops.ada_layer_norm_quant_fp8_modfp8_bf16(
365
+ x, scale_fp8, shift_fp8, scale_deq, shift_deq, act_scale, float(eps), out
366
+ )
367
+ return out
368
+
369
+
370
+ def awq_ada_layer_norm_quant_fp8_bf16(
371
+ x: torch.Tensor,
372
+ scale: torch.Tensor,
373
+ shift: torch.Tensor,
374
+ inv_s: torch.Tensor,
375
+ act_scale: torch.Tensor,
376
+ eps: float = 1e-5,
377
+ out: torch.Tensor | None = None,
378
+ ) -> torch.Tensor:
379
+ """Fused AdaLN producer with AWQ/SmoothQuant per-channel activation scale."""
380
+
381
+ if out is None:
382
+ out = _alloc_fp8(x)
383
+ ops.awq_ada_layer_norm_quant_fp8_bf16(x, scale, shift, inv_s, act_scale, float(eps), out)
384
+ return out
385
+
386
+
387
+ def ada_layer_norm_quant_nvfp4_swizzled_bf16(
388
+ x: torch.Tensor,
389
+ scale: torch.Tensor,
390
+ shift: torch.Tensor,
391
+ eps: float = 1e-5,
392
+ packed: torch.Tensor | None = None,
393
+ sf_swizzled: torch.Tensor | None = None,
394
+ ) -> tuple[torch.Tensor, torch.Tensor]:
395
+ """Fused AdaLN producer to NVFP4 packed activations and swizzled scale factors."""
396
+
397
+ if packed is None or sf_swizzled is None:
398
+ packed, sf_swizzled = _alloc_nvfp4(x)
399
+ ops.ada_layer_norm_quant_nvfp4_swizzled_bf16(x, scale, shift, float(eps), packed, sf_swizzled)
400
+ return packed, sf_swizzled
401
+
402
+
403
+ def ada_layer_norm_quant_nvfp4_swizzled_ptok_table_bf16(
404
+ x: torch.Tensor,
405
+ temb: torch.Tensor,
406
+ table: torch.Tensor,
407
+ shift_idx: int,
408
+ scale_idx: int,
409
+ eps: float = 1e-5,
410
+ packed: torch.Tensor | None = None,
411
+ sf_swizzled: torch.Tensor | None = None,
412
+ ) -> tuple[torch.Tensor, torch.Tensor]:
413
+ """Fuse per-token table AdaLN into NVFP4 and CUTLASS scale factors."""
414
+
415
+ if packed is None or sf_swizzled is None:
416
+ packed, sf_swizzled = _alloc_nvfp4(x)
417
+ ops.ada_layer_norm_quant_nvfp4_swizzled_ptok_table_bf16(
418
+ x, temb, table, int(shift_idx), int(scale_idx), float(eps),
419
+ packed, sf_swizzled
420
+ )
421
+ return packed, sf_swizzled
422
+
423
+
424
+ def ada_layer_norm_quant_nvfp4_swizzled_modfp8_bf16(
425
+ x: torch.Tensor,
426
+ scale_fp8: torch.Tensor,
427
+ shift_fp8: torch.Tensor,
428
+ scale_deq: torch.Tensor,
429
+ shift_deq: torch.Tensor,
430
+ eps: float = 1e-5,
431
+ packed: torch.Tensor | None = None,
432
+ sf_swizzled: torch.Tensor | None = None,
433
+ ) -> tuple[torch.Tensor, torch.Tensor]:
434
+ """Fused FP8-modulated AdaLN producer to NVFP4 packed activations."""
435
+
436
+ if packed is None or sf_swizzled is None:
437
+ packed, sf_swizzled = _alloc_nvfp4(x)
438
+ ops.ada_layer_norm_quant_nvfp4_swizzled_modfp8_bf16(
439
+ x, scale_fp8, shift_fp8, scale_deq, shift_deq, float(eps), packed, sf_swizzled
440
+ )
441
+ return packed, sf_swizzled
442
+
443
+
444
+ def layer_norm_no_affine_quant_fp8_static_bf16(
445
+ x: torch.Tensor,
446
+ act_scale: torch.Tensor,
447
+ eps: float = 1e-5,
448
+ out: torch.Tensor | None = None,
449
+ ) -> torch.Tensor:
450
+ """Fused no-affine LayerNorm + static FP8 quantize."""
451
+
452
+ if out is None:
453
+ out = _alloc_fp8(x)
454
+ ops.layer_norm_no_affine_quant_fp8_static_bf16(x, act_scale, float(eps), out)
455
+ return out
456
+
457
+
458
+ def layer_norm_no_affine_quant_nvfp4_swizzled_bf16(
459
+ x: torch.Tensor,
460
+ eps: float = 1e-5,
461
+ packed: torch.Tensor | None = None,
462
+ sf_swizzled: torch.Tensor | None = None,
463
+ ) -> tuple[torch.Tensor, torch.Tensor]:
464
+ """SM110 no-affine LayerNorm producing NVFP4 and CUTLASS SFA."""
465
+ if packed is None or sf_swizzled is None:
466
+ packed, sf_swizzled = _alloc_nvfp4(x)
467
+ ops.layer_norm_no_affine_quant_nvfp4_swizzled_bf16(
468
+ x, float(eps), packed, sf_swizzled
469
+ )
470
+ return packed, sf_swizzled
471
+
472
+
473
+ @torch.library.register_fake(add_op_namespace_prefix("adaln_modulation6_bf16"))
474
+ def _adaln_modulation6_bf16_fake(
475
+ adaln_params: torch.Tensor,
476
+ layer_modulation: torch.Tensor,
477
+ out0: torch.Tensor,
478
+ out1: torch.Tensor,
479
+ out2: torch.Tensor,
480
+ out3: torch.Tensor,
481
+ out4: torch.Tensor,
482
+ out5: torch.Tensor,
483
+ ) -> None:
484
+ if adaln_params.dim() != 4 or adaln_params.shape[2] != 6:
485
+ raise RuntimeError(
486
+ "adaln_params must have shape (batch, sequence, 6, dim)"
487
+ )
488
+ expected = (
489
+ adaln_params.shape[0],
490
+ adaln_params.shape[1],
491
+ adaln_params.shape[3],
492
+ )
493
+ if layer_modulation.shape != (6, expected[2]):
494
+ raise RuntimeError("layer_modulation must have shape (6, dim)")
495
+ for output in (out0, out1, out2, out3, out4, out5):
496
+ if output.shape != expected:
497
+ raise RuntimeError(
498
+ "each output must have shape (batch, sequence, dim)"
499
+ )
500
+ return None
501
+
502
+
503
+ def adaln_modulation6_bf16(
504
+ adaln_params: torch.Tensor,
505
+ layer_modulation: torch.Tensor,
506
+ *,
507
+ out: tuple[torch.Tensor, ...] | None = None,
508
+ ) -> tuple[torch.Tensor, ...]:
509
+ """Add six FP32 AdaLN modulation vectors and return six BF16 tensors."""
510
+
511
+ batch, sequence, _, dim = adaln_params.shape
512
+ if out is None:
513
+ out = tuple(
514
+ torch.empty(
515
+ (batch, sequence, dim),
516
+ device=adaln_params.device,
517
+ dtype=torch.bfloat16,
518
+ )
519
+ for _ in range(6)
520
+ )
521
+ if len(out) != 6:
522
+ raise RuntimeError("out must contain six tensors")
523
+ ops.adaln_modulation6_bf16(
524
+ adaln_params, layer_modulation, *out
525
+ )
526
+ return out
527
+
528
+
529
+ __all__ = [
530
+ "ada_layer_norm_quant_fp8_bf16",
531
+ "ada_layer_norm_quant_fp8_ptok_bf16",
532
+ "ada_layer_norm_quant_fp8_ptok_table_bf16",
533
+ "ada_layer_norm_ptok_table_bf16",
534
+ "ada_layer_norm_quant_fp8_modfp8_bf16",
535
+ "awq_ada_layer_norm_quant_fp8_bf16",
536
+ "ada_layer_norm_quant_nvfp4_swizzled_bf16",
537
+ "ada_layer_norm_quant_nvfp4_swizzled_ptok_table_bf16",
538
+ "ada_layer_norm_quant_nvfp4_swizzled_modfp8_bf16",
539
+ "layer_norm_no_affine_quant_fp8_static_bf16",
540
+ "layer_norm_no_affine_quant_nvfp4_swizzled_bf16",
541
+ "adaln_modulation6_bf16",
542
+ "swizzled_sf_size",
543
+ ]
build/torch213-cxx11-cu132-x86_64-linux/_adaptive_layernorm_producers_cuda_559acf6.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:53e79cefbdf04488df4d2d76111fa8bd0a20e243eac99e57bb54343336e44507
3
+ size 2000456
build/torch213-cxx11-cu132-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _adaptive_layernorm_producers_cuda_559acf6
3
+ ops = torch.ops._adaptive_layernorm_producers_cuda_559acf6
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_adaptive_layernorm_producers_cuda_559acf6::{op_name}"
build/torch213-cxx11-cu132-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "adaptive-layernorm-producers",
3
+ "id": "_adaptive_layernorm_producers_cuda_559acf6",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "kernel-depends": [],
8
+ "backend": {
9
+ "type": "cuda",
10
+ "archs": [
11
+ "10.0",
12
+ "11.0",
13
+ "11.0a",
14
+ "12.0",
15
+ "12.1+PTX",
16
+ "7.5",
17
+ "8.0",
18
+ "8.6",
19
+ "8.7",
20
+ "8.9",
21
+ "9.0"
22
+ ]
23
+ },
24
+ "digest": {
25
+ "algorithm": "sha256",
26
+ "files": {
27
+ "__init__.py": "+ZwPYHOKPDe9jTo0jlUuIRwuU4XT1gxfK/Bxk+ZLkNI=",
28
+ "_adaptive_layernorm_producers_cuda_559acf6.abi3.so": "U+ec773wRIjfTS12ER+ovQog4kPqyZ5Xu1Q0MzbkRQc=",
29
+ "_ops.py": "y8e0DCkDQ0iu8HnSb+bgXwPrRQARnnr1IrA1B/CWs9M="
30
+ }
31
+ },
32
+ "provenance": {
33
+ "kernel-builder": {
34
+ "version": "0.17.0-dev0",
35
+ "sha": "3a5c09d941a19f582fe551d038dc2b3bc6b0a03d",
36
+ "dirty": false
37
+ },
38
+ "kernel": {
39
+ "sha": "559acf64fc1eacf17cb382dc35bebff41deb9110",
40
+ "dirty": false
41
+ }
42
+ }
43
+ }