PRIYANSHUDHAKED commited on
Commit
7a2cbfa
·
verified ·
1 Parent(s): 083bbc8

Delete src/miocodec/module/fsq.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. src/miocodec/module/fsq.py +0 -142
src/miocodec/module/fsq.py DELETED
@@ -1,142 +0,0 @@
1
- # Finite Scalar Quantization: https://arxiv.org/abs/2309.15505
2
-
3
- import torch
4
- from torch import nn
5
-
6
- from ..util import get_logger
7
-
8
- logger = get_logger()
9
-
10
-
11
- def round_ste(z: torch.Tensor) -> torch.Tensor:
12
- """Round with straight through gradients."""
13
- zhat = z.round()
14
- return z + (zhat - z).detach()
15
-
16
-
17
- def get_entropy(prob: torch.Tensor, eps: float = 1e-10) -> torch.Tensor:
18
- return -torch.sum(prob * torch.log(prob + eps), dim=-1)
19
-
20
-
21
- class FSQ(nn.Module):
22
- def __init__(self, levels: list[int]):
23
- super().__init__()
24
- self.levels = levels
25
- self.dim = len(levels)
26
-
27
- _levels = torch.tensor(levels, dtype=torch.long)
28
- self.register_buffer("_levels", _levels, persistent=False)
29
- _basis = torch.cumprod(torch.tensor([1] + levels[:-1]), dim=0, dtype=torch.long)
30
- self.register_buffer("_basis", _basis, persistent=False)
31
-
32
- def bound(self, z: torch.Tensor, eps: float = 1e-3) -> torch.Tensor:
33
- """Bound `z`, an array of shape (..., d)."""
34
- half_l = (self._levels - 1) * (1 - eps) / 2
35
- offset = torch.where(self._levels % 2 == 0, 0.5, 0.0)
36
- shift = (offset / half_l).tan()
37
- return (z + shift).tanh() * half_l - offset
38
-
39
- def quantize(self, z: torch.Tensor) -> torch.Tensor:
40
- """Quantizes z, returns quantized zhat, same shape as z."""
41
- quantized = round_ste(self.bound(z))
42
- half_width = self._levels // 2 # Renormalize to [-1, 1].
43
- return quantized / half_width
44
-
45
- def _scale_and_shift(self, zhat_normalized: torch.Tensor) -> torch.Tensor:
46
- half_width = self._levels // 2
47
- return (zhat_normalized * half_width) + half_width
48
-
49
- def _scale_and_shift_inverse(self, zhat: torch.Tensor) -> torch.Tensor:
50
- half_width = self._levels // 2
51
- return (zhat - half_width) / half_width
52
-
53
- def codes_to_indices(self, zhat: torch.Tensor) -> torch.Tensor:
54
- """Converts a `code` to an index in the codebook."""
55
- # (B, T, C) -> (B, T)
56
- assert zhat.shape[-1] == len(self.levels)
57
- zhat = self._scale_and_shift(zhat)
58
- # MPS doesn't support float64, fall back to float32
59
- dtype = torch.float32 if zhat.device.type == "mps" else torch.float64
60
- return (zhat * self._basis.to(dtype)).to(torch.long).sum(dim=-1)
61
-
62
- def indices_to_codes(self, indices: torch.Tensor) -> torch.Tensor:
63
- """Inverse of `codes_to_indices`."""
64
- # (B, T) -> (B, T, C)
65
- indices = indices.unsqueeze(-1)
66
- codes_non_centered = (indices // self._basis) % self._levels
67
- return self._scale_and_shift_inverse(codes_non_centered)
68
-
69
- def encode(self, z: torch.Tensor) -> torch.Tensor:
70
- z_q = self.quantize(z)
71
- indices = self.codes_to_indices(z_q) # (B, T)
72
- return z_q, indices
73
-
74
- def decode(self, indices: torch.Tensor) -> torch.Tensor:
75
- z_q = self.indices_to_codes(indices) # (B, T, C)
76
- return z_q
77
-
78
- def forward(self, z: torch.Tensor):
79
- z_q = self.quantize(z)
80
- indices = self.codes_to_indices(z_q) # (B, T)
81
- return z_q, indices
82
-
83
-
84
- class FiniteScalarQuantizer(nn.Module):
85
- def __init__(self, input_dim: int, output_dim: int, levels: list[int]) -> None:
86
- super().__init__()
87
- self.input_dim_ = input_dim
88
- self.output_dim_ = output_dim
89
-
90
- self.fsq = FSQ(levels)
91
- logger.debug(
92
- f"Finite Scalar Quantizer with levels: {levels}, input_dim: {input_dim}, output_dim: {output_dim}, codebook_size: {self.all_codebook_size}"
93
- )
94
-
95
- self.proj_in = nn.Linear(input_dim, len(levels)) if len(levels) != input_dim else nn.Identity()
96
- self.proj_out = nn.Linear(len(levels), output_dim) if len(levels) != output_dim else nn.Identity()
97
-
98
- def build_codebook(self) -> None:
99
- pass
100
-
101
- @property
102
- def output_dim(self) -> int:
103
- return self.output_dim_
104
-
105
- @property
106
- def all_codebook_size(self) -> int:
107
- size = 1
108
- for level in self.fsq.levels:
109
- size *= level
110
- return size
111
-
112
- def forward(self, z: torch.Tensor) -> tuple[torch.Tensor, dict]:
113
- latent = self.proj_in(z) # Latent projected by proj_in
114
- quantized_latent, indices = self.fsq(latent) # Quantized latent before proj_out
115
- z_q = self.proj_out(quantized_latent)
116
-
117
- # Compute perplexity from used indices distribution
118
- flat_indices = indices.view(-1)
119
- unique_indices, counts = torch.unique(flat_indices, return_counts=True)
120
- used_indices_probs = counts.float() / flat_indices.numel()
121
- entropy = get_entropy(used_indices_probs)
122
- perplexity = torch.exp(entropy)
123
-
124
- info_dict = {
125
- "latent": latent,
126
- "quantized_latent": quantized_latent,
127
- "indices": indices,
128
- "perplexity": perplexity,
129
- }
130
- return z_q, info_dict
131
-
132
- def encode(self, z: torch.Tensor, skip_proj: bool = False) -> tuple[torch.Tensor, torch.Tensor]:
133
- z = self.proj_in(z)
134
- z_q, indices = self.fsq.encode(z)
135
- if not skip_proj:
136
- z_q = self.proj_out(z_q)
137
- return z_q, indices
138
-
139
- def decode(self, indices: torch.Tensor) -> torch.Tensor:
140
- z_q = self.fsq.decode(indices)
141
- z_q = self.proj_out(z_q)
142
- return z_q