Spaces:
Running on Zero
Running on Zero
Download src/modeling/kernels/whitening.py from KeisukeMiyamoto/lambda-audio-v6: direct link, hf CLI and curl.
- Browser
- Download file 6.31 kB
-
https://huggingface.co/spaces/KeisukeMiyamoto/lambda-audio-v6/resolve/main/src/modeling/kernels/whitening.py
- Command line
-
hf download hf://spaces/KeisukeMiyamoto/lambda-audio-v6/src/modeling/kernels/whitening.py
-
curl -L -o whitening.py https://huggingface.co/spaces/KeisukeMiyamoto/lambda-audio-v6/resolve/main/src/modeling/kernels/whitening.py
6.31 kB
| from typing import Any | |
| import torch | |
| import triton | |
| import triton.language as tl | |
| def _gated_mm(A: tl.tensor, B: tl.tensor, OUT: tl.tensor, ACTIVE: tl.tensor, M: tl.tensor, K: tl.tensor, N: tl.tensor, TRANSPOSE: tl.constexpr, BLOCK: tl.constexpr) -> None: | |
| group = tl.program_id(0) | |
| rows = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK) | |
| cols = tl.program_id(2) * BLOCK + tl.arange(0, BLOCK) | |
| inner = tl.arange(0, BLOCK) | |
| acc = tl.full((BLOCK, BLOCK), 0, tl.float32) | |
| if tl.load(ACTIVE): | |
| for start in range(tl.cdiv(K, BLOCK)): | |
| k = start * BLOCK + inner | |
| if TRANSPOSE: | |
| a = tl.load(A + group * M * K + k[None, :] * M + rows[:, None], (rows[:, None] < M) & (k[None, :] < K), 0) | |
| else: | |
| a = tl.load(A + group * M * K + rows[:, None] * K + k[None, :], (rows[:, None] < M) & (k[None, :] < K), 0) | |
| b = tl.load(B + group * K * N + k[:, None] * N + cols[None, :], (k[:, None] < K) & (cols[None, :] < N), 0) | |
| acc += tl.dot(a, b, input_precision="tf32x3") | |
| tl.store(OUT + group * M * N + rows[:, None] * N + cols[None, :], acc, (rows[:, None] < M) & (cols[None, :] < N)) | |
| def _matrix( | |
| COV: tl.tensor, | |
| DMEAN: tl.tensor, | |
| QMEAN: tl.tensor, | |
| SELECTED: tl.tensor, | |
| PROB: tl.tensor, | |
| ACTIVE: tl.tensor, | |
| OUT: tl.tensor, | |
| D: tl.tensor, | |
| TOTAL: tl.tensor, | |
| LIMIT: tl.tensor, | |
| PMIN: tl.tensor, | |
| PMAX: tl.tensor, | |
| BLOCK: tl.constexpr, | |
| ) -> None: | |
| i = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) | |
| d = tl.load(DMEAN) | |
| q = tl.load(QMEAN) | |
| denom = d * d + 1.0e-20 | |
| selected = tl.load(SELECTED) | |
| active = selected & (q / denom >= LIMIT) | |
| if tl.program_id(0) == 0: | |
| tl.store(ACTIVE, active) | |
| if selected: | |
| tl.store(PROB, tl.where(active, PMAX, PMIN)) | |
| cov = tl.load(COV + i, i < TOTAL * D, 0) | |
| diagonal = (i % D) == ((i // D) % D) | |
| value = (4.0 / (TOTAL * denom)) * (cov - tl.where(diagonal, (q / denom) * d, 0.0)) | |
| tl.store(OUT + i, value, i < TOTAL * D) | |
| def _norms(P: tl.tensor, G: tl.tensor, ACTIVE: tl.tensor, SUMS: tl.tensor, SIZE: tl.tensor, BLOCK: tl.constexpr) -> None: | |
| block = tl.program_id(0) | |
| i = block * BLOCK + tl.arange(0, BLOCK) | |
| ps = tl.full((), 0, tl.float32) | |
| gs = tl.full((), 0, tl.float32) | |
| if tl.load(ACTIVE): | |
| p = tl.load(P + i, i < SIZE, 0) | |
| g = tl.load(G + i, i < SIZE, 0).to(tl.float32) | |
| ps = tl.sum(p * p, 0) | |
| gs = tl.sum(g * g, 0) | |
| tl.store(SUMS + block * 2, ps) | |
| tl.store(SUMS + block * 2 + 1, gs) | |
| def _scale(SUMS: tl.tensor, OUT: tl.tensor, PARTS: tl.tensor, SCALE: tl.tensor, BLOCK: tl.constexpr) -> None: | |
| i = tl.arange(0, BLOCK) | |
| p = tl.sum(tl.load(SUMS + i * 2, i < PARTS, 0), 0) | |
| g = tl.sum(tl.load(SUMS + i * 2 + 1, i < PARTS, 0), 0) | |
| tl.store(OUT, SCALE * (tl.sqrt(g) / (tl.sqrt(p) + 1.0e-20))) | |
| def _apply( | |
| P: tl.tensor, G: tl.tensor, SCALE: tl.tensor, ACTIVE: tl.tensor, OUT: tl.tensor, N: tl.tensor, D: tl.tensor, GROUPS: tl.tensor, SIZE: tl.tensor, BLOCK: tl.constexpr | |
| ) -> None: | |
| i = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) | |
| g = tl.load(G + i, i < SIZE, 0) | |
| y = g.to(tl.float32) | |
| if tl.load(ACTIVE): | |
| offset = ((i // D) % GROUPS) * N * D + (i // (D * GROUPS)) * D + i % D | |
| p = tl.load(P + offset, i < SIZE, 0) | |
| correction = (p * tl.load(SCALE)).to(g.dtype) | |
| y += correction.to(tl.float32) | |
| tl.store(OUT + i, y, i < SIZE) | |
| def whitening_backward( | |
| x: torch.Tensor, grad: torch.Tensor, probability: torch.Tensor, selected: torch.Tensor, groups: int, limit: float, min_prob: float, max_prob: float, scale: float | |
| ) -> torch.Tensor: | |
| channels = x.shape[-1] | |
| n = x.numel() // channels | |
| d = channels // groups | |
| with torch.autocast("cuda", enabled=False): | |
| centered = x.float().reshape(n, groups, d).permute(1, 0, 2).contiguous() | |
| centered = centered - centered.mean(1, keepdim=True) | |
| covariance = torch.empty((groups, d, d), device=x.device) | |
| _gated_mm[(groups, triton.cdiv(d, 32), triton.cdiv(d, 32))](centered, centered, covariance, selected, d, n, d, True, 32) | |
| diagonal_mean = covariance.diagonal(dim1=-2, dim2=-1).mean() | |
| square_mean = covariance.square().sum() / channels | |
| matrix = torch.empty_like(covariance) | |
| active = torch.empty((), device=x.device, dtype=torch.bool) | |
| _matrix[(triton.cdiv(covariance.numel(), 256),)]( | |
| covariance, diagonal_mean, square_mean, selected, probability, active, matrix, d, channels, limit, min_prob, max_prob, 256, enable_fp_fusion=False | |
| ) | |
| penalty = torch.empty_like(centered) | |
| _gated_mm[(groups, triton.cdiv(n, 32), triton.cdiv(d, 32))](centered, matrix, penalty, active, n, d, d, False, 32) | |
| penalty = penalty - penalty.mean(1, keepdim=True) | |
| g = grad.contiguous() | |
| parts = triton.cdiv(g.numel(), 1024) | |
| sums = torch.empty((parts, 2), device=x.device) | |
| factor = torch.empty((), device=x.device) | |
| result = torch.empty_like(g) | |
| _norms[(parts,)](penalty, g, active, sums, g.numel(), 1024) | |
| _scale[(1,)](sums, factor, parts, scale, triton.next_power_of_2(parts)) | |
| _apply[(parts,)](penalty, g, factor, active, result, n, d, groups, g.numel(), 1024) | |
| return result | |
| class _Whitening(torch.autograd.Function): | |
| def forward(ctx: Any, x: torch.Tensor, module: Any, draw: float) -> torch.Tensor: | |
| ctx.save_for_backward(x, module.kernel_probability >= draw) | |
| ctx.module = module | |
| return x | |
| def backward(ctx: Any, grad: torch.Tensor) -> tuple[torch.Tensor, None, None]: | |
| x, selected = ctx.saved_tensors | |
| w = ctx.module | |
| result = whitening_backward(x, grad, w.kernel_probability, selected, w.num_groups, float(w.whitening_limit), w.min_prob, w.max_prob, float(w.grad_scale)) | |
| return result, None, None | |
| def whiten(x: torch.Tensor, module: Any, draw: float) -> torch.Tensor: | |
| return _Whitening.apply(x, module, draw) | |