taoyy commited on
Commit
100cd4f
·
verified ·
1 Parent(s): f4915cf

Add 14 files (batch 3)

Browse files
third_party/Matcha-TTS/matcha/models/components/decoder.py ADDED
@@ -0,0 +1,443 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ from typing import Optional
3
+
4
+ import torch
5
+ import torch.nn as nn # pylint: disable=consider-using-from-import
6
+ import torch.nn.functional as F
7
+ from conformer import ConformerBlock
8
+ from diffusers.models.activations import get_activation
9
+ from einops import pack, rearrange, repeat
10
+
11
+ from matcha.models.components.transformer import BasicTransformerBlock
12
+
13
+
14
+ class SinusoidalPosEmb(torch.nn.Module):
15
+ def __init__(self, dim):
16
+ super().__init__()
17
+ self.dim = dim
18
+ assert self.dim % 2 == 0, "SinusoidalPosEmb requires dim to be even"
19
+
20
+ def forward(self, x, scale=1000):
21
+ if x.ndim < 1:
22
+ x = x.unsqueeze(0)
23
+ device = x.device
24
+ half_dim = self.dim // 2
25
+ emb = math.log(10000) / (half_dim - 1)
26
+ emb = torch.exp(torch.arange(half_dim, device=device).float() * -emb)
27
+ emb = scale * x.unsqueeze(1) * emb.unsqueeze(0)
28
+ emb = torch.cat((emb.sin(), emb.cos()), dim=-1)
29
+ return emb
30
+
31
+
32
+ class Block1D(torch.nn.Module):
33
+ def __init__(self, dim, dim_out, groups=8):
34
+ super().__init__()
35
+ self.block = torch.nn.Sequential(
36
+ torch.nn.Conv1d(dim, dim_out, 3, padding=1),
37
+ torch.nn.GroupNorm(groups, dim_out),
38
+ nn.Mish(),
39
+ )
40
+
41
+ def forward(self, x, mask):
42
+ output = self.block(x * mask)
43
+ return output * mask
44
+
45
+
46
+ class ResnetBlock1D(torch.nn.Module):
47
+ def __init__(self, dim, dim_out, time_emb_dim, groups=8):
48
+ super().__init__()
49
+ self.mlp = torch.nn.Sequential(nn.Mish(), torch.nn.Linear(time_emb_dim, dim_out))
50
+
51
+ self.block1 = Block1D(dim, dim_out, groups=groups)
52
+ self.block2 = Block1D(dim_out, dim_out, groups=groups)
53
+
54
+ self.res_conv = torch.nn.Conv1d(dim, dim_out, 1)
55
+
56
+ def forward(self, x, mask, time_emb):
57
+ h = self.block1(x, mask)
58
+ h += self.mlp(time_emb).unsqueeze(-1)
59
+ h = self.block2(h, mask)
60
+ output = h + self.res_conv(x * mask)
61
+ return output
62
+
63
+
64
+ class Downsample1D(nn.Module):
65
+ def __init__(self, dim):
66
+ super().__init__()
67
+ self.conv = torch.nn.Conv1d(dim, dim, 3, 2, 1)
68
+
69
+ def forward(self, x):
70
+ return self.conv(x)
71
+
72
+
73
+ class TimestepEmbedding(nn.Module):
74
+ def __init__(
75
+ self,
76
+ in_channels: int,
77
+ time_embed_dim: int,
78
+ act_fn: str = "silu",
79
+ out_dim: int = None,
80
+ post_act_fn: Optional[str] = None,
81
+ cond_proj_dim=None,
82
+ ):
83
+ super().__init__()
84
+
85
+ self.linear_1 = nn.Linear(in_channels, time_embed_dim)
86
+
87
+ if cond_proj_dim is not None:
88
+ self.cond_proj = nn.Linear(cond_proj_dim, in_channels, bias=False)
89
+ else:
90
+ self.cond_proj = None
91
+
92
+ self.act = get_activation(act_fn)
93
+
94
+ if out_dim is not None:
95
+ time_embed_dim_out = out_dim
96
+ else:
97
+ time_embed_dim_out = time_embed_dim
98
+ self.linear_2 = nn.Linear(time_embed_dim, time_embed_dim_out)
99
+
100
+ if post_act_fn is None:
101
+ self.post_act = None
102
+ else:
103
+ self.post_act = get_activation(post_act_fn)
104
+
105
+ def forward(self, sample, condition=None):
106
+ if condition is not None:
107
+ sample = sample + self.cond_proj(condition)
108
+ sample = self.linear_1(sample)
109
+
110
+ if self.act is not None:
111
+ sample = self.act(sample)
112
+
113
+ sample = self.linear_2(sample)
114
+
115
+ if self.post_act is not None:
116
+ sample = self.post_act(sample)
117
+ return sample
118
+
119
+
120
+ class Upsample1D(nn.Module):
121
+ """A 1D upsampling layer with an optional convolution.
122
+
123
+ Parameters:
124
+ channels (`int`):
125
+ number of channels in the inputs and outputs.
126
+ use_conv (`bool`, default `False`):
127
+ option to use a convolution.
128
+ use_conv_transpose (`bool`, default `False`):
129
+ option to use a convolution transpose.
130
+ out_channels (`int`, optional):
131
+ number of output channels. Defaults to `channels`.
132
+ """
133
+
134
+ def __init__(self, channels, use_conv=False, use_conv_transpose=True, out_channels=None, name="conv"):
135
+ super().__init__()
136
+ self.channels = channels
137
+ self.out_channels = out_channels or channels
138
+ self.use_conv = use_conv
139
+ self.use_conv_transpose = use_conv_transpose
140
+ self.name = name
141
+
142
+ self.conv = None
143
+ if use_conv_transpose:
144
+ self.conv = nn.ConvTranspose1d(channels, self.out_channels, 4, 2, 1)
145
+ elif use_conv:
146
+ self.conv = nn.Conv1d(self.channels, self.out_channels, 3, padding=1)
147
+
148
+ def forward(self, inputs):
149
+ assert inputs.shape[1] == self.channels
150
+ if self.use_conv_transpose:
151
+ return self.conv(inputs)
152
+
153
+ outputs = F.interpolate(inputs, scale_factor=2.0, mode="nearest")
154
+
155
+ if self.use_conv:
156
+ outputs = self.conv(outputs)
157
+
158
+ return outputs
159
+
160
+
161
+ class ConformerWrapper(ConformerBlock):
162
+ def __init__( # pylint: disable=useless-super-delegation
163
+ self,
164
+ *,
165
+ dim,
166
+ dim_head=64,
167
+ heads=8,
168
+ ff_mult=4,
169
+ conv_expansion_factor=2,
170
+ conv_kernel_size=31,
171
+ attn_dropout=0,
172
+ ff_dropout=0,
173
+ conv_dropout=0,
174
+ conv_causal=False,
175
+ ):
176
+ super().__init__(
177
+ dim=dim,
178
+ dim_head=dim_head,
179
+ heads=heads,
180
+ ff_mult=ff_mult,
181
+ conv_expansion_factor=conv_expansion_factor,
182
+ conv_kernel_size=conv_kernel_size,
183
+ attn_dropout=attn_dropout,
184
+ ff_dropout=ff_dropout,
185
+ conv_dropout=conv_dropout,
186
+ conv_causal=conv_causal,
187
+ )
188
+
189
+ def forward(
190
+ self,
191
+ hidden_states,
192
+ attention_mask,
193
+ encoder_hidden_states=None,
194
+ encoder_attention_mask=None,
195
+ timestep=None,
196
+ ):
197
+ return super().forward(x=hidden_states, mask=attention_mask.bool())
198
+
199
+
200
+ class Decoder(nn.Module):
201
+ def __init__(
202
+ self,
203
+ in_channels,
204
+ out_channels,
205
+ channels=(256, 256),
206
+ dropout=0.05,
207
+ attention_head_dim=64,
208
+ n_blocks=1,
209
+ num_mid_blocks=2,
210
+ num_heads=4,
211
+ act_fn="snake",
212
+ down_block_type="transformer",
213
+ mid_block_type="transformer",
214
+ up_block_type="transformer",
215
+ ):
216
+ super().__init__()
217
+ channels = tuple(channels)
218
+ self.in_channels = in_channels
219
+ self.out_channels = out_channels
220
+
221
+ self.time_embeddings = SinusoidalPosEmb(in_channels)
222
+ time_embed_dim = channels[0] * 4
223
+ self.time_mlp = TimestepEmbedding(
224
+ in_channels=in_channels,
225
+ time_embed_dim=time_embed_dim,
226
+ act_fn="silu",
227
+ )
228
+
229
+ self.down_blocks = nn.ModuleList([])
230
+ self.mid_blocks = nn.ModuleList([])
231
+ self.up_blocks = nn.ModuleList([])
232
+
233
+ output_channel = in_channels
234
+ for i in range(len(channels)): # pylint: disable=consider-using-enumerate
235
+ input_channel = output_channel
236
+ output_channel = channels[i]
237
+ is_last = i == len(channels) - 1
238
+ resnet = ResnetBlock1D(dim=input_channel, dim_out=output_channel, time_emb_dim=time_embed_dim)
239
+ transformer_blocks = nn.ModuleList(
240
+ [
241
+ self.get_block(
242
+ down_block_type,
243
+ output_channel,
244
+ attention_head_dim,
245
+ num_heads,
246
+ dropout,
247
+ act_fn,
248
+ )
249
+ for _ in range(n_blocks)
250
+ ]
251
+ )
252
+ downsample = (
253
+ Downsample1D(output_channel) if not is_last else nn.Conv1d(output_channel, output_channel, 3, padding=1)
254
+ )
255
+
256
+ self.down_blocks.append(nn.ModuleList([resnet, transformer_blocks, downsample]))
257
+
258
+ for i in range(num_mid_blocks):
259
+ input_channel = channels[-1]
260
+ out_channels = channels[-1]
261
+
262
+ resnet = ResnetBlock1D(dim=input_channel, dim_out=output_channel, time_emb_dim=time_embed_dim)
263
+
264
+ transformer_blocks = nn.ModuleList(
265
+ [
266
+ self.get_block(
267
+ mid_block_type,
268
+ output_channel,
269
+ attention_head_dim,
270
+ num_heads,
271
+ dropout,
272
+ act_fn,
273
+ )
274
+ for _ in range(n_blocks)
275
+ ]
276
+ )
277
+
278
+ self.mid_blocks.append(nn.ModuleList([resnet, transformer_blocks]))
279
+
280
+ channels = channels[::-1] + (channels[0],)
281
+ for i in range(len(channels) - 1):
282
+ input_channel = channels[i]
283
+ output_channel = channels[i + 1]
284
+ is_last = i == len(channels) - 2
285
+
286
+ resnet = ResnetBlock1D(
287
+ dim=2 * input_channel,
288
+ dim_out=output_channel,
289
+ time_emb_dim=time_embed_dim,
290
+ )
291
+ transformer_blocks = nn.ModuleList(
292
+ [
293
+ self.get_block(
294
+ up_block_type,
295
+ output_channel,
296
+ attention_head_dim,
297
+ num_heads,
298
+ dropout,
299
+ act_fn,
300
+ )
301
+ for _ in range(n_blocks)
302
+ ]
303
+ )
304
+ upsample = (
305
+ Upsample1D(output_channel, use_conv_transpose=True)
306
+ if not is_last
307
+ else nn.Conv1d(output_channel, output_channel, 3, padding=1)
308
+ )
309
+
310
+ self.up_blocks.append(nn.ModuleList([resnet, transformer_blocks, upsample]))
311
+
312
+ self.final_block = Block1D(channels[-1], channels[-1])
313
+ self.final_proj = nn.Conv1d(channels[-1], self.out_channels, 1)
314
+
315
+ self.initialize_weights()
316
+ # nn.init.normal_(self.final_proj.weight)
317
+
318
+ @staticmethod
319
+ def get_block(block_type, dim, attention_head_dim, num_heads, dropout, act_fn):
320
+ if block_type == "conformer":
321
+ block = ConformerWrapper(
322
+ dim=dim,
323
+ dim_head=attention_head_dim,
324
+ heads=num_heads,
325
+ ff_mult=1,
326
+ conv_expansion_factor=2,
327
+ ff_dropout=dropout,
328
+ attn_dropout=dropout,
329
+ conv_dropout=dropout,
330
+ conv_kernel_size=31,
331
+ )
332
+ elif block_type == "transformer":
333
+ block = BasicTransformerBlock(
334
+ dim=dim,
335
+ num_attention_heads=num_heads,
336
+ attention_head_dim=attention_head_dim,
337
+ dropout=dropout,
338
+ activation_fn=act_fn,
339
+ )
340
+ else:
341
+ raise ValueError(f"Unknown block type {block_type}")
342
+
343
+ return block
344
+
345
+ def initialize_weights(self):
346
+ for m in self.modules():
347
+ if isinstance(m, nn.Conv1d):
348
+ nn.init.kaiming_normal_(m.weight, nonlinearity="relu")
349
+
350
+ if m.bias is not None:
351
+ nn.init.constant_(m.bias, 0)
352
+
353
+ elif isinstance(m, nn.GroupNorm):
354
+ nn.init.constant_(m.weight, 1)
355
+ nn.init.constant_(m.bias, 0)
356
+
357
+ elif isinstance(m, nn.Linear):
358
+ nn.init.kaiming_normal_(m.weight, nonlinearity="relu")
359
+
360
+ if m.bias is not None:
361
+ nn.init.constant_(m.bias, 0)
362
+
363
+ def forward(self, x, mask, mu, t, spks=None, cond=None):
364
+ """Forward pass of the UNet1DConditional model.
365
+
366
+ Args:
367
+ x (torch.Tensor): shape (batch_size, in_channels, time)
368
+ mask (_type_): shape (batch_size, 1, time)
369
+ t (_type_): shape (batch_size)
370
+ spks (_type_, optional): shape: (batch_size, condition_channels). Defaults to None.
371
+ cond (_type_, optional): placeholder for future use. Defaults to None.
372
+
373
+ Raises:
374
+ ValueError: _description_
375
+ ValueError: _description_
376
+
377
+ Returns:
378
+ _type_: _description_
379
+ """
380
+
381
+ t = self.time_embeddings(t)
382
+ t = self.time_mlp(t)
383
+
384
+ x = pack([x, mu], "b * t")[0]
385
+
386
+ if spks is not None:
387
+ spks = repeat(spks, "b c -> b c t", t=x.shape[-1])
388
+ x = pack([x, spks], "b * t")[0]
389
+
390
+ hiddens = []
391
+ masks = [mask]
392
+ for resnet, transformer_blocks, downsample in self.down_blocks:
393
+ mask_down = masks[-1]
394
+ x = resnet(x, mask_down, t)
395
+ x = rearrange(x, "b c t -> b t c")
396
+ mask_down = rearrange(mask_down, "b 1 t -> b t")
397
+ for transformer_block in transformer_blocks:
398
+ x = transformer_block(
399
+ hidden_states=x,
400
+ attention_mask=mask_down,
401
+ timestep=t,
402
+ )
403
+ x = rearrange(x, "b t c -> b c t")
404
+ mask_down = rearrange(mask_down, "b t -> b 1 t")
405
+ hiddens.append(x) # Save hidden states for skip connections
406
+ x = downsample(x * mask_down)
407
+ masks.append(mask_down[:, :, ::2])
408
+
409
+ masks = masks[:-1]
410
+ mask_mid = masks[-1]
411
+
412
+ for resnet, transformer_blocks in self.mid_blocks:
413
+ x = resnet(x, mask_mid, t)
414
+ x = rearrange(x, "b c t -> b t c")
415
+ mask_mid = rearrange(mask_mid, "b 1 t -> b t")
416
+ for transformer_block in transformer_blocks:
417
+ x = transformer_block(
418
+ hidden_states=x,
419
+ attention_mask=mask_mid,
420
+ timestep=t,
421
+ )
422
+ x = rearrange(x, "b t c -> b c t")
423
+ mask_mid = rearrange(mask_mid, "b t -> b 1 t")
424
+
425
+ for resnet, transformer_blocks, upsample in self.up_blocks:
426
+ mask_up = masks.pop()
427
+ x = resnet(pack([x, hiddens.pop()], "b * t")[0], mask_up, t)
428
+ x = rearrange(x, "b c t -> b t c")
429
+ mask_up = rearrange(mask_up, "b 1 t -> b t")
430
+ for transformer_block in transformer_blocks:
431
+ x = transformer_block(
432
+ hidden_states=x,
433
+ attention_mask=mask_up,
434
+ timestep=t,
435
+ )
436
+ x = rearrange(x, "b t c -> b c t")
437
+ mask_up = rearrange(mask_up, "b t -> b 1 t")
438
+ x = upsample(x * mask_up)
439
+
440
+ x = self.final_block(x, mask_up)
441
+ output = self.final_proj(x * mask_up)
442
+
443
+ return output * mask
third_party/Matcha-TTS/matcha/models/components/text_encoder.py ADDED
@@ -0,0 +1,410 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ from https://github.com/jaywalnut310/glow-tts """
2
+
3
+ import math
4
+
5
+ import torch
6
+ import torch.nn as nn # pylint: disable=consider-using-from-import
7
+ from einops import rearrange
8
+
9
+ import matcha.utils as utils # pylint: disable=consider-using-from-import
10
+ from matcha.utils.model import sequence_mask
11
+
12
+ log = utils.get_pylogger(__name__)
13
+
14
+
15
+ class LayerNorm(nn.Module):
16
+ def __init__(self, channels, eps=1e-4):
17
+ super().__init__()
18
+ self.channels = channels
19
+ self.eps = eps
20
+
21
+ self.gamma = torch.nn.Parameter(torch.ones(channels))
22
+ self.beta = torch.nn.Parameter(torch.zeros(channels))
23
+
24
+ def forward(self, x):
25
+ n_dims = len(x.shape)
26
+ mean = torch.mean(x, 1, keepdim=True)
27
+ variance = torch.mean((x - mean) ** 2, 1, keepdim=True)
28
+
29
+ x = (x - mean) * torch.rsqrt(variance + self.eps)
30
+
31
+ shape = [1, -1] + [1] * (n_dims - 2)
32
+ x = x * self.gamma.view(*shape) + self.beta.view(*shape)
33
+ return x
34
+
35
+
36
+ class ConvReluNorm(nn.Module):
37
+ def __init__(self, in_channels, hidden_channels, out_channels, kernel_size, n_layers, p_dropout):
38
+ super().__init__()
39
+ self.in_channels = in_channels
40
+ self.hidden_channels = hidden_channels
41
+ self.out_channels = out_channels
42
+ self.kernel_size = kernel_size
43
+ self.n_layers = n_layers
44
+ self.p_dropout = p_dropout
45
+
46
+ self.conv_layers = torch.nn.ModuleList()
47
+ self.norm_layers = torch.nn.ModuleList()
48
+ self.conv_layers.append(torch.nn.Conv1d(in_channels, hidden_channels, kernel_size, padding=kernel_size // 2))
49
+ self.norm_layers.append(LayerNorm(hidden_channels))
50
+ self.relu_drop = torch.nn.Sequential(torch.nn.ReLU(), torch.nn.Dropout(p_dropout))
51
+ for _ in range(n_layers - 1):
52
+ self.conv_layers.append(
53
+ torch.nn.Conv1d(hidden_channels, hidden_channels, kernel_size, padding=kernel_size // 2)
54
+ )
55
+ self.norm_layers.append(LayerNorm(hidden_channels))
56
+ self.proj = torch.nn.Conv1d(hidden_channels, out_channels, 1)
57
+ self.proj.weight.data.zero_()
58
+ self.proj.bias.data.zero_()
59
+
60
+ def forward(self, x, x_mask):
61
+ x_org = x
62
+ for i in range(self.n_layers):
63
+ x = self.conv_layers[i](x * x_mask)
64
+ x = self.norm_layers[i](x)
65
+ x = self.relu_drop(x)
66
+ x = x_org + self.proj(x)
67
+ return x * x_mask
68
+
69
+
70
+ class DurationPredictor(nn.Module):
71
+ def __init__(self, in_channels, filter_channels, kernel_size, p_dropout):
72
+ super().__init__()
73
+ self.in_channels = in_channels
74
+ self.filter_channels = filter_channels
75
+ self.p_dropout = p_dropout
76
+
77
+ self.drop = torch.nn.Dropout(p_dropout)
78
+ self.conv_1 = torch.nn.Conv1d(in_channels, filter_channels, kernel_size, padding=kernel_size // 2)
79
+ self.norm_1 = LayerNorm(filter_channels)
80
+ self.conv_2 = torch.nn.Conv1d(filter_channels, filter_channels, kernel_size, padding=kernel_size // 2)
81
+ self.norm_2 = LayerNorm(filter_channels)
82
+ self.proj = torch.nn.Conv1d(filter_channels, 1, 1)
83
+
84
+ def forward(self, x, x_mask):
85
+ x = self.conv_1(x * x_mask)
86
+ x = torch.relu(x)
87
+ x = self.norm_1(x)
88
+ x = self.drop(x)
89
+ x = self.conv_2(x * x_mask)
90
+ x = torch.relu(x)
91
+ x = self.norm_2(x)
92
+ x = self.drop(x)
93
+ x = self.proj(x * x_mask)
94
+ return x * x_mask
95
+
96
+
97
+ class RotaryPositionalEmbeddings(nn.Module):
98
+ """
99
+ ## RoPE module
100
+
101
+ Rotary encoding transforms pairs of features by rotating in the 2D plane.
102
+ That is, it organizes the $d$ features as $\frac{d}{2}$ pairs.
103
+ Each pair can be considered a coordinate in a 2D plane, and the encoding will rotate it
104
+ by an angle depending on the position of the token.
105
+ """
106
+
107
+ def __init__(self, d: int, base: int = 10_000):
108
+ r"""
109
+ * `d` is the number of features $d$
110
+ * `base` is the constant used for calculating $\Theta$
111
+ """
112
+ super().__init__()
113
+
114
+ self.base = base
115
+ self.d = int(d)
116
+ self.cos_cached = None
117
+ self.sin_cached = None
118
+
119
+ def _build_cache(self, x: torch.Tensor):
120
+ r"""
121
+ Cache $\cos$ and $\sin$ values
122
+ """
123
+ # Return if cache is already built
124
+ if self.cos_cached is not None and x.shape[0] <= self.cos_cached.shape[0]:
125
+ return
126
+
127
+ # Get sequence length
128
+ seq_len = x.shape[0]
129
+
130
+ # $\Theta = {\theta_i = 10000^{-\frac{2(i-1)}{d}}, i \in [1, 2, ..., \frac{d}{2}]}$
131
+ theta = 1.0 / (self.base ** (torch.arange(0, self.d, 2).float() / self.d)).to(x.device)
132
+
133
+ # Create position indexes `[0, 1, ..., seq_len - 1]`
134
+ seq_idx = torch.arange(seq_len, device=x.device).float().to(x.device)
135
+
136
+ # Calculate the product of position index and $\theta_i$
137
+ idx_theta = torch.einsum("n,d->nd", seq_idx, theta)
138
+
139
+ # Concatenate so that for row $m$ we have
140
+ # $[m \theta_0, m \theta_1, ..., m \theta_{\frac{d}{2}}, m \theta_0, m \theta_1, ..., m \theta_{\frac{d}{2}}]$
141
+ idx_theta2 = torch.cat([idx_theta, idx_theta], dim=1)
142
+
143
+ # Cache them
144
+ self.cos_cached = idx_theta2.cos()[:, None, None, :]
145
+ self.sin_cached = idx_theta2.sin()[:, None, None, :]
146
+
147
+ def _neg_half(self, x: torch.Tensor):
148
+ # $\frac{d}{2}$
149
+ d_2 = self.d // 2
150
+
151
+ # Calculate $[-x^{(\frac{d}{2} + 1)}, -x^{(\frac{d}{2} + 2)}, ..., -x^{(d)}, x^{(1)}, x^{(2)}, ..., x^{(\frac{d}{2})}]$
152
+ return torch.cat([-x[:, :, :, d_2:], x[:, :, :, :d_2]], dim=-1)
153
+
154
+ def forward(self, x: torch.Tensor):
155
+ """
156
+ * `x` is the Tensor at the head of a key or a query with shape `[seq_len, batch_size, n_heads, d]`
157
+ """
158
+ # Cache $\cos$ and $\sin$ values
159
+ x = rearrange(x, "b h t d -> t b h d")
160
+
161
+ self._build_cache(x)
162
+
163
+ # Split the features, we can choose to apply rotary embeddings only to a partial set of features.
164
+ x_rope, x_pass = x[..., : self.d], x[..., self.d :]
165
+
166
+ # Calculate
167
+ # $[-x^{(\frac{d}{2} + 1)}, -x^{(\frac{d}{2} + 2)}, ..., -x^{(d)}, x^{(1)}, x^{(2)}, ..., x^{(\frac{d}{2})}]$
168
+ neg_half_x = self._neg_half(x_rope)
169
+
170
+ x_rope = (x_rope * self.cos_cached[: x.shape[0]]) + (neg_half_x * self.sin_cached[: x.shape[0]])
171
+
172
+ return rearrange(torch.cat((x_rope, x_pass), dim=-1), "t b h d -> b h t d")
173
+
174
+
175
+ class MultiHeadAttention(nn.Module):
176
+ def __init__(
177
+ self,
178
+ channels,
179
+ out_channels,
180
+ n_heads,
181
+ heads_share=True,
182
+ p_dropout=0.0,
183
+ proximal_bias=False,
184
+ proximal_init=False,
185
+ ):
186
+ super().__init__()
187
+ assert channels % n_heads == 0
188
+
189
+ self.channels = channels
190
+ self.out_channels = out_channels
191
+ self.n_heads = n_heads
192
+ self.heads_share = heads_share
193
+ self.proximal_bias = proximal_bias
194
+ self.p_dropout = p_dropout
195
+ self.attn = None
196
+
197
+ self.k_channels = channels // n_heads
198
+ self.conv_q = torch.nn.Conv1d(channels, channels, 1)
199
+ self.conv_k = torch.nn.Conv1d(channels, channels, 1)
200
+ self.conv_v = torch.nn.Conv1d(channels, channels, 1)
201
+
202
+ # from https://nn.labml.ai/transformers/rope/index.html
203
+ self.query_rotary_pe = RotaryPositionalEmbeddings(self.k_channels * 0.5)
204
+ self.key_rotary_pe = RotaryPositionalEmbeddings(self.k_channels * 0.5)
205
+
206
+ self.conv_o = torch.nn.Conv1d(channels, out_channels, 1)
207
+ self.drop = torch.nn.Dropout(p_dropout)
208
+
209
+ torch.nn.init.xavier_uniform_(self.conv_q.weight)
210
+ torch.nn.init.xavier_uniform_(self.conv_k.weight)
211
+ if proximal_init:
212
+ self.conv_k.weight.data.copy_(self.conv_q.weight.data)
213
+ self.conv_k.bias.data.copy_(self.conv_q.bias.data)
214
+ torch.nn.init.xavier_uniform_(self.conv_v.weight)
215
+
216
+ def forward(self, x, c, attn_mask=None):
217
+ q = self.conv_q(x)
218
+ k = self.conv_k(c)
219
+ v = self.conv_v(c)
220
+
221
+ x, self.attn = self.attention(q, k, v, mask=attn_mask)
222
+
223
+ x = self.conv_o(x)
224
+ return x
225
+
226
+ def attention(self, query, key, value, mask=None):
227
+ b, d, t_s, t_t = (*key.size(), query.size(2))
228
+ query = rearrange(query, "b (h c) t-> b h t c", h=self.n_heads)
229
+ key = rearrange(key, "b (h c) t-> b h t c", h=self.n_heads)
230
+ value = rearrange(value, "b (h c) t-> b h t c", h=self.n_heads)
231
+
232
+ query = self.query_rotary_pe(query)
233
+ key = self.key_rotary_pe(key)
234
+
235
+ scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(self.k_channels)
236
+
237
+ if self.proximal_bias:
238
+ assert t_s == t_t, "Proximal bias is only available for self-attention."
239
+ scores = scores + self._attention_bias_proximal(t_s).to(device=scores.device, dtype=scores.dtype)
240
+ if mask is not None:
241
+ scores = scores.masked_fill(mask == 0, -1e4)
242
+ p_attn = torch.nn.functional.softmax(scores, dim=-1)
243
+ p_attn = self.drop(p_attn)
244
+ output = torch.matmul(p_attn, value)
245
+ output = output.transpose(2, 3).contiguous().view(b, d, t_t)
246
+ return output, p_attn
247
+
248
+ @staticmethod
249
+ def _attention_bias_proximal(length):
250
+ r = torch.arange(length, dtype=torch.float32)
251
+ diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1)
252
+ return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0)
253
+
254
+
255
+ class FFN(nn.Module):
256
+ def __init__(self, in_channels, out_channels, filter_channels, kernel_size, p_dropout=0.0):
257
+ super().__init__()
258
+ self.in_channels = in_channels
259
+ self.out_channels = out_channels
260
+ self.filter_channels = filter_channels
261
+ self.kernel_size = kernel_size
262
+ self.p_dropout = p_dropout
263
+
264
+ self.conv_1 = torch.nn.Conv1d(in_channels, filter_channels, kernel_size, padding=kernel_size // 2)
265
+ self.conv_2 = torch.nn.Conv1d(filter_channels, out_channels, kernel_size, padding=kernel_size // 2)
266
+ self.drop = torch.nn.Dropout(p_dropout)
267
+
268
+ def forward(self, x, x_mask):
269
+ x = self.conv_1(x * x_mask)
270
+ x = torch.relu(x)
271
+ x = self.drop(x)
272
+ x = self.conv_2(x * x_mask)
273
+ return x * x_mask
274
+
275
+
276
+ class Encoder(nn.Module):
277
+ def __init__(
278
+ self,
279
+ hidden_channels,
280
+ filter_channels,
281
+ n_heads,
282
+ n_layers,
283
+ kernel_size=1,
284
+ p_dropout=0.0,
285
+ **kwargs,
286
+ ):
287
+ super().__init__()
288
+ self.hidden_channels = hidden_channels
289
+ self.filter_channels = filter_channels
290
+ self.n_heads = n_heads
291
+ self.n_layers = n_layers
292
+ self.kernel_size = kernel_size
293
+ self.p_dropout = p_dropout
294
+
295
+ self.drop = torch.nn.Dropout(p_dropout)
296
+ self.attn_layers = torch.nn.ModuleList()
297
+ self.norm_layers_1 = torch.nn.ModuleList()
298
+ self.ffn_layers = torch.nn.ModuleList()
299
+ self.norm_layers_2 = torch.nn.ModuleList()
300
+ for _ in range(self.n_layers):
301
+ self.attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout))
302
+ self.norm_layers_1.append(LayerNorm(hidden_channels))
303
+ self.ffn_layers.append(
304
+ FFN(
305
+ hidden_channels,
306
+ hidden_channels,
307
+ filter_channels,
308
+ kernel_size,
309
+ p_dropout=p_dropout,
310
+ )
311
+ )
312
+ self.norm_layers_2.append(LayerNorm(hidden_channels))
313
+
314
+ def forward(self, x, x_mask):
315
+ attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
316
+ for i in range(self.n_layers):
317
+ x = x * x_mask
318
+ y = self.attn_layers[i](x, x, attn_mask)
319
+ y = self.drop(y)
320
+ x = self.norm_layers_1[i](x + y)
321
+ y = self.ffn_layers[i](x, x_mask)
322
+ y = self.drop(y)
323
+ x = self.norm_layers_2[i](x + y)
324
+ x = x * x_mask
325
+ return x
326
+
327
+
328
+ class TextEncoder(nn.Module):
329
+ def __init__(
330
+ self,
331
+ encoder_type,
332
+ encoder_params,
333
+ duration_predictor_params,
334
+ n_vocab,
335
+ n_spks=1,
336
+ spk_emb_dim=128,
337
+ ):
338
+ super().__init__()
339
+ self.encoder_type = encoder_type
340
+ self.n_vocab = n_vocab
341
+ self.n_feats = encoder_params.n_feats
342
+ self.n_channels = encoder_params.n_channels
343
+ self.spk_emb_dim = spk_emb_dim
344
+ self.n_spks = n_spks
345
+
346
+ self.emb = torch.nn.Embedding(n_vocab, self.n_channels)
347
+ torch.nn.init.normal_(self.emb.weight, 0.0, self.n_channels**-0.5)
348
+
349
+ if encoder_params.prenet:
350
+ self.prenet = ConvReluNorm(
351
+ self.n_channels,
352
+ self.n_channels,
353
+ self.n_channels,
354
+ kernel_size=5,
355
+ n_layers=3,
356
+ p_dropout=0.5,
357
+ )
358
+ else:
359
+ self.prenet = lambda x, x_mask: x
360
+
361
+ self.encoder = Encoder(
362
+ encoder_params.n_channels + (spk_emb_dim if n_spks > 1 else 0),
363
+ encoder_params.filter_channels,
364
+ encoder_params.n_heads,
365
+ encoder_params.n_layers,
366
+ encoder_params.kernel_size,
367
+ encoder_params.p_dropout,
368
+ )
369
+
370
+ self.proj_m = torch.nn.Conv1d(self.n_channels + (spk_emb_dim if n_spks > 1 else 0), self.n_feats, 1)
371
+ self.proj_w = DurationPredictor(
372
+ self.n_channels + (spk_emb_dim if n_spks > 1 else 0),
373
+ duration_predictor_params.filter_channels_dp,
374
+ duration_predictor_params.kernel_size,
375
+ duration_predictor_params.p_dropout,
376
+ )
377
+
378
+ def forward(self, x, x_lengths, spks=None):
379
+ """Run forward pass to the transformer based encoder and duration predictor
380
+
381
+ Args:
382
+ x (torch.Tensor): text input
383
+ shape: (batch_size, max_text_length)
384
+ x_lengths (torch.Tensor): text input lengths
385
+ shape: (batch_size,)
386
+ spks (torch.Tensor, optional): speaker ids. Defaults to None.
387
+ shape: (batch_size,)
388
+
389
+ Returns:
390
+ mu (torch.Tensor): average output of the encoder
391
+ shape: (batch_size, n_feats, max_text_length)
392
+ logw (torch.Tensor): log duration predicted by the duration predictor
393
+ shape: (batch_size, 1, max_text_length)
394
+ x_mask (torch.Tensor): mask for the text input
395
+ shape: (batch_size, 1, max_text_length)
396
+ """
397
+ x = self.emb(x) * math.sqrt(self.n_channels)
398
+ x = torch.transpose(x, 1, -1)
399
+ x_mask = torch.unsqueeze(sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype)
400
+
401
+ x = self.prenet(x, x_mask)
402
+ if self.n_spks > 1:
403
+ x = torch.cat([x, spks.unsqueeze(-1).repeat(1, 1, x.shape[-1])], dim=1)
404
+ x = self.encoder(x, x_mask)
405
+ mu = self.proj_m(x) * x_mask
406
+
407
+ x_dp = torch.detach(x)
408
+ logw = self.proj_w(x_dp, x_mask)
409
+
410
+ return mu, logw, x_mask
third_party/Matcha-TTS/matcha/models/components/transformer.py ADDED
@@ -0,0 +1,316 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any, Dict, Optional
2
+
3
+ import torch
4
+ import torch.nn as nn # pylint: disable=consider-using-from-import
5
+ from diffusers.models.attention import (
6
+ GEGLU,
7
+ GELU,
8
+ AdaLayerNorm,
9
+ AdaLayerNormZero,
10
+ ApproximateGELU,
11
+ )
12
+ from diffusers.models.attention_processor import Attention
13
+ from diffusers.models.lora import LoRACompatibleLinear
14
+ from diffusers.utils.torch_utils import maybe_allow_in_graph
15
+
16
+
17
+ class SnakeBeta(nn.Module):
18
+ """
19
+ A modified Snake function which uses separate parameters for the magnitude of the periodic components
20
+ Shape:
21
+ - Input: (B, C, T)
22
+ - Output: (B, C, T), same shape as the input
23
+ Parameters:
24
+ - alpha - trainable parameter that controls frequency
25
+ - beta - trainable parameter that controls magnitude
26
+ References:
27
+ - This activation function is a modified version based on this paper by Liu Ziyin, Tilman Hartwig, Masahito Ueda:
28
+ https://arxiv.org/abs/2006.08195
29
+ Examples:
30
+ >>> a1 = snakebeta(256)
31
+ >>> x = torch.randn(256)
32
+ >>> x = a1(x)
33
+ """
34
+
35
+ def __init__(self, in_features, out_features, alpha=1.0, alpha_trainable=True, alpha_logscale=True):
36
+ """
37
+ Initialization.
38
+ INPUT:
39
+ - in_features: shape of the input
40
+ - alpha - trainable parameter that controls frequency
41
+ - beta - trainable parameter that controls magnitude
42
+ alpha is initialized to 1 by default, higher values = higher-frequency.
43
+ beta is initialized to 1 by default, higher values = higher-magnitude.
44
+ alpha will be trained along with the rest of your model.
45
+ """
46
+ super().__init__()
47
+ self.in_features = out_features if isinstance(out_features, list) else [out_features]
48
+ self.proj = LoRACompatibleLinear(in_features, out_features)
49
+
50
+ # initialize alpha
51
+ self.alpha_logscale = alpha_logscale
52
+ if self.alpha_logscale: # log scale alphas initialized to zeros
53
+ self.alpha = nn.Parameter(torch.zeros(self.in_features) * alpha)
54
+ self.beta = nn.Parameter(torch.zeros(self.in_features) * alpha)
55
+ else: # linear scale alphas initialized to ones
56
+ self.alpha = nn.Parameter(torch.ones(self.in_features) * alpha)
57
+ self.beta = nn.Parameter(torch.ones(self.in_features) * alpha)
58
+
59
+ self.alpha.requires_grad = alpha_trainable
60
+ self.beta.requires_grad = alpha_trainable
61
+
62
+ self.no_div_by_zero = 0.000000001
63
+
64
+ def forward(self, x):
65
+ """
66
+ Forward pass of the function.
67
+ Applies the function to the input elementwise.
68
+ SnakeBeta ∶= x + 1/b * sin^2 (xa)
69
+ """
70
+ x = self.proj(x)
71
+ if self.alpha_logscale:
72
+ alpha = torch.exp(self.alpha)
73
+ beta = torch.exp(self.beta)
74
+ else:
75
+ alpha = self.alpha
76
+ beta = self.beta
77
+
78
+ x = x + (1.0 / (beta + self.no_div_by_zero)) * torch.pow(torch.sin(x * alpha), 2)
79
+
80
+ return x
81
+
82
+
83
+ class FeedForward(nn.Module):
84
+ r"""
85
+ A feed-forward layer.
86
+
87
+ Parameters:
88
+ dim (`int`): The number of channels in the input.
89
+ dim_out (`int`, *optional*): The number of channels in the output. If not given, defaults to `dim`.
90
+ mult (`int`, *optional*, defaults to 4): The multiplier to use for the hidden dimension.
91
+ dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.
92
+ activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward.
93
+ final_dropout (`bool` *optional*, defaults to False): Apply a final dropout.
94
+ """
95
+
96
+ def __init__(
97
+ self,
98
+ dim: int,
99
+ dim_out: Optional[int] = None,
100
+ mult: int = 4,
101
+ dropout: float = 0.0,
102
+ activation_fn: str = "geglu",
103
+ final_dropout: bool = False,
104
+ ):
105
+ super().__init__()
106
+ inner_dim = int(dim * mult)
107
+ dim_out = dim_out if dim_out is not None else dim
108
+
109
+ if activation_fn == "gelu":
110
+ act_fn = GELU(dim, inner_dim)
111
+ if activation_fn == "gelu-approximate":
112
+ act_fn = GELU(dim, inner_dim, approximate="tanh")
113
+ elif activation_fn == "geglu":
114
+ act_fn = GEGLU(dim, inner_dim)
115
+ elif activation_fn == "geglu-approximate":
116
+ act_fn = ApproximateGELU(dim, inner_dim)
117
+ elif activation_fn == "snakebeta":
118
+ act_fn = SnakeBeta(dim, inner_dim)
119
+
120
+ self.net = nn.ModuleList([])
121
+ # project in
122
+ self.net.append(act_fn)
123
+ # project dropout
124
+ self.net.append(nn.Dropout(dropout))
125
+ # project out
126
+ self.net.append(LoRACompatibleLinear(inner_dim, dim_out))
127
+ # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
128
+ if final_dropout:
129
+ self.net.append(nn.Dropout(dropout))
130
+
131
+ def forward(self, hidden_states):
132
+ for module in self.net:
133
+ hidden_states = module(hidden_states)
134
+ return hidden_states
135
+
136
+
137
+ @maybe_allow_in_graph
138
+ class BasicTransformerBlock(nn.Module):
139
+ r"""
140
+ A basic Transformer block.
141
+
142
+ Parameters:
143
+ dim (`int`): The number of channels in the input and output.
144
+ num_attention_heads (`int`): The number of heads to use for multi-head attention.
145
+ attention_head_dim (`int`): The number of channels in each head.
146
+ dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.
147
+ cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention.
148
+ only_cross_attention (`bool`, *optional*):
149
+ Whether to use only cross-attention layers. In this case two cross attention layers are used.
150
+ double_self_attention (`bool`, *optional*):
151
+ Whether to use two self-attention layers. In this case no cross attention layers are used.
152
+ activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward.
153
+ num_embeds_ada_norm (:
154
+ obj: `int`, *optional*): The number of diffusion steps used during training. See `Transformer2DModel`.
155
+ attention_bias (:
156
+ obj: `bool`, *optional*, defaults to `False`): Configure if the attentions should contain a bias parameter.
157
+ """
158
+
159
+ def __init__(
160
+ self,
161
+ dim: int,
162
+ num_attention_heads: int,
163
+ attention_head_dim: int,
164
+ dropout=0.0,
165
+ cross_attention_dim: Optional[int] = None,
166
+ activation_fn: str = "geglu",
167
+ num_embeds_ada_norm: Optional[int] = None,
168
+ attention_bias: bool = False,
169
+ only_cross_attention: bool = False,
170
+ double_self_attention: bool = False,
171
+ upcast_attention: bool = False,
172
+ norm_elementwise_affine: bool = True,
173
+ norm_type: str = "layer_norm",
174
+ final_dropout: bool = False,
175
+ ):
176
+ super().__init__()
177
+ self.only_cross_attention = only_cross_attention
178
+
179
+ self.use_ada_layer_norm_zero = (num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero"
180
+ self.use_ada_layer_norm = (num_embeds_ada_norm is not None) and norm_type == "ada_norm"
181
+
182
+ if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
183
+ raise ValueError(
184
+ f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"
185
+ f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."
186
+ )
187
+
188
+ # Define 3 blocks. Each block has its own normalization layer.
189
+ # 1. Self-Attn
190
+ if self.use_ada_layer_norm:
191
+ self.norm1 = AdaLayerNorm(dim, num_embeds_ada_norm)
192
+ elif self.use_ada_layer_norm_zero:
193
+ self.norm1 = AdaLayerNormZero(dim, num_embeds_ada_norm)
194
+ else:
195
+ self.norm1 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine)
196
+ self.attn1 = Attention(
197
+ query_dim=dim,
198
+ heads=num_attention_heads,
199
+ dim_head=attention_head_dim,
200
+ dropout=dropout,
201
+ bias=attention_bias,
202
+ cross_attention_dim=cross_attention_dim if only_cross_attention else None,
203
+ upcast_attention=upcast_attention,
204
+ )
205
+
206
+ # 2. Cross-Attn
207
+ if cross_attention_dim is not None or double_self_attention:
208
+ # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
209
+ # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
210
+ # the second cross attention block.
211
+ self.norm2 = (
212
+ AdaLayerNorm(dim, num_embeds_ada_norm)
213
+ if self.use_ada_layer_norm
214
+ else nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine)
215
+ )
216
+ self.attn2 = Attention(
217
+ query_dim=dim,
218
+ cross_attention_dim=cross_attention_dim if not double_self_attention else None,
219
+ heads=num_attention_heads,
220
+ dim_head=attention_head_dim,
221
+ dropout=dropout,
222
+ bias=attention_bias,
223
+ upcast_attention=upcast_attention,
224
+ # scale_qk=False, # uncomment this to not to use flash attention
225
+ ) # is self-attn if encoder_hidden_states is none
226
+ else:
227
+ self.norm2 = None
228
+ self.attn2 = None
229
+
230
+ # 3. Feed-forward
231
+ self.norm3 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine)
232
+ self.ff = FeedForward(dim, dropout=dropout, activation_fn=activation_fn, final_dropout=final_dropout)
233
+
234
+ # let chunk size default to None
235
+ self._chunk_size = None
236
+ self._chunk_dim = 0
237
+
238
+ def set_chunk_feed_forward(self, chunk_size: Optional[int], dim: int):
239
+ # Sets chunk feed-forward
240
+ self._chunk_size = chunk_size
241
+ self._chunk_dim = dim
242
+
243
+ def forward(
244
+ self,
245
+ hidden_states: torch.FloatTensor,
246
+ attention_mask: Optional[torch.FloatTensor] = None,
247
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
248
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
249
+ timestep: Optional[torch.LongTensor] = None,
250
+ cross_attention_kwargs: Dict[str, Any] = None,
251
+ class_labels: Optional[torch.LongTensor] = None,
252
+ ):
253
+ # Notice that normalization is always applied before the real computation in the following blocks.
254
+ # 1. Self-Attention
255
+ if self.use_ada_layer_norm:
256
+ norm_hidden_states = self.norm1(hidden_states, timestep)
257
+ elif self.use_ada_layer_norm_zero:
258
+ norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(
259
+ hidden_states, timestep, class_labels, hidden_dtype=hidden_states.dtype
260
+ )
261
+ else:
262
+ norm_hidden_states = self.norm1(hidden_states)
263
+
264
+ cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {}
265
+
266
+ attn_output = self.attn1(
267
+ norm_hidden_states,
268
+ encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None,
269
+ attention_mask=encoder_attention_mask if self.only_cross_attention else attention_mask,
270
+ **cross_attention_kwargs,
271
+ )
272
+ if self.use_ada_layer_norm_zero:
273
+ attn_output = gate_msa.unsqueeze(1) * attn_output
274
+ hidden_states = attn_output + hidden_states
275
+
276
+ # 2. Cross-Attention
277
+ if self.attn2 is not None:
278
+ norm_hidden_states = (
279
+ self.norm2(hidden_states, timestep) if self.use_ada_layer_norm else self.norm2(hidden_states)
280
+ )
281
+
282
+ attn_output = self.attn2(
283
+ norm_hidden_states,
284
+ encoder_hidden_states=encoder_hidden_states,
285
+ attention_mask=encoder_attention_mask,
286
+ **cross_attention_kwargs,
287
+ )
288
+ hidden_states = attn_output + hidden_states
289
+
290
+ # 3. Feed-forward
291
+ norm_hidden_states = self.norm3(hidden_states)
292
+
293
+ if self.use_ada_layer_norm_zero:
294
+ norm_hidden_states = norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]
295
+
296
+ if self._chunk_size is not None:
297
+ # "feed_forward_chunk_size" can be used to save memory
298
+ if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
299
+ raise ValueError(
300
+ f"`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`."
301
+ )
302
+
303
+ num_chunks = norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
304
+ ff_output = torch.cat(
305
+ [self.ff(hid_slice) for hid_slice in norm_hidden_states.chunk(num_chunks, dim=self._chunk_dim)],
306
+ dim=self._chunk_dim,
307
+ )
308
+ else:
309
+ ff_output = self.ff(norm_hidden_states)
310
+
311
+ if self.use_ada_layer_norm_zero:
312
+ ff_output = gate_mlp.unsqueeze(1) * ff_output
313
+
314
+ hidden_states = ff_output + hidden_states
315
+
316
+ return hidden_states
third_party/Matcha-TTS/matcha/models/matcha_tts.py ADDED
@@ -0,0 +1,245 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import datetime as dt
2
+ import math
3
+ import random
4
+
5
+ import torch
6
+
7
+ import matcha.utils.monotonic_align as monotonic_align # pylint: disable=consider-using-from-import
8
+ from matcha import utils
9
+ from matcha.models.baselightningmodule import BaseLightningClass
10
+ from matcha.models.components.flow_matching import CFM
11
+ from matcha.models.components.text_encoder import TextEncoder
12
+ from matcha.utils.model import (
13
+ denormalize,
14
+ duration_loss,
15
+ fix_len_compatibility,
16
+ generate_path,
17
+ sequence_mask,
18
+ )
19
+
20
+ log = utils.get_pylogger(__name__)
21
+
22
+
23
+ class MatchaTTS(BaseLightningClass): # 🍵
24
+ def __init__(
25
+ self,
26
+ n_vocab,
27
+ n_spks,
28
+ spk_emb_dim,
29
+ n_feats,
30
+ encoder,
31
+ decoder,
32
+ cfm,
33
+ data_statistics,
34
+ out_size,
35
+ optimizer=None,
36
+ scheduler=None,
37
+ prior_loss=True,
38
+ use_precomputed_durations=False,
39
+ ):
40
+ super().__init__()
41
+
42
+ self.save_hyperparameters(logger=False)
43
+
44
+ self.n_vocab = n_vocab
45
+ self.n_spks = n_spks
46
+ self.spk_emb_dim = spk_emb_dim
47
+ self.n_feats = n_feats
48
+ self.out_size = out_size
49
+ self.prior_loss = prior_loss
50
+ self.use_precomputed_durations = use_precomputed_durations
51
+
52
+ if n_spks > 1:
53
+ self.spk_emb = torch.nn.Embedding(n_spks, spk_emb_dim)
54
+
55
+ self.encoder = TextEncoder(
56
+ encoder.encoder_type,
57
+ encoder.encoder_params,
58
+ encoder.duration_predictor_params,
59
+ n_vocab,
60
+ n_spks,
61
+ spk_emb_dim,
62
+ )
63
+
64
+ self.decoder = CFM(
65
+ in_channels=2 * encoder.encoder_params.n_feats,
66
+ out_channel=encoder.encoder_params.n_feats,
67
+ cfm_params=cfm,
68
+ decoder_params=decoder,
69
+ n_spks=n_spks,
70
+ spk_emb_dim=spk_emb_dim,
71
+ )
72
+
73
+ self.update_data_statistics(data_statistics)
74
+
75
+ @torch.inference_mode()
76
+ def synthesise(self, x, x_lengths, n_timesteps, temperature=1.0, spks=None, length_scale=1.0):
77
+ """
78
+ Generates mel-spectrogram from text. Returns:
79
+ 1. encoder outputs
80
+ 2. decoder outputs
81
+ 3. generated alignment
82
+
83
+ Args:
84
+ x (torch.Tensor): batch of texts, converted to a tensor with phoneme embedding ids.
85
+ shape: (batch_size, max_text_length)
86
+ x_lengths (torch.Tensor): lengths of texts in batch.
87
+ shape: (batch_size,)
88
+ n_timesteps (int): number of steps to use for reverse diffusion in decoder.
89
+ temperature (float, optional): controls variance of terminal distribution.
90
+ spks (bool, optional): speaker ids.
91
+ shape: (batch_size,)
92
+ length_scale (float, optional): controls speech pace.
93
+ Increase value to slow down generated speech and vice versa.
94
+
95
+ Returns:
96
+ dict: {
97
+ "encoder_outputs": torch.Tensor, shape: (batch_size, n_feats, max_mel_length),
98
+ # Average mel spectrogram generated by the encoder
99
+ "decoder_outputs": torch.Tensor, shape: (batch_size, n_feats, max_mel_length),
100
+ # Refined mel spectrogram improved by the CFM
101
+ "attn": torch.Tensor, shape: (batch_size, max_text_length, max_mel_length),
102
+ # Alignment map between text and mel spectrogram
103
+ "mel": torch.Tensor, shape: (batch_size, n_feats, max_mel_length),
104
+ # Denormalized mel spectrogram
105
+ "mel_lengths": torch.Tensor, shape: (batch_size,),
106
+ # Lengths of mel spectrograms
107
+ "rtf": float,
108
+ # Real-time factor
109
+ }
110
+ """
111
+ # For RTF computation
112
+ t = dt.datetime.now()
113
+
114
+ if self.n_spks > 1:
115
+ # Get speaker embedding
116
+ spks = self.spk_emb(spks.long())
117
+
118
+ # Get encoder_outputs `mu_x` and log-scaled token durations `logw`
119
+ mu_x, logw, x_mask = self.encoder(x, x_lengths, spks)
120
+
121
+ w = torch.exp(logw) * x_mask
122
+ w_ceil = torch.ceil(w) * length_scale
123
+ y_lengths = torch.clamp_min(torch.sum(w_ceil, [1, 2]), 1).long()
124
+ y_max_length = y_lengths.max()
125
+ y_max_length_ = fix_len_compatibility(y_max_length)
126
+
127
+ # Using obtained durations `w` construct alignment map `attn`
128
+ y_mask = sequence_mask(y_lengths, y_max_length_).unsqueeze(1).to(x_mask.dtype)
129
+ attn_mask = x_mask.unsqueeze(-1) * y_mask.unsqueeze(2)
130
+ attn = generate_path(w_ceil.squeeze(1), attn_mask.squeeze(1)).unsqueeze(1)
131
+
132
+ # Align encoded text and get mu_y
133
+ mu_y = torch.matmul(attn.squeeze(1).transpose(1, 2), mu_x.transpose(1, 2))
134
+ mu_y = mu_y.transpose(1, 2)
135
+ encoder_outputs = mu_y[:, :, :y_max_length]
136
+
137
+ # Generate sample tracing the probability flow
138
+ decoder_outputs = self.decoder(mu_y, y_mask, n_timesteps, temperature, spks)
139
+ decoder_outputs = decoder_outputs[:, :, :y_max_length]
140
+
141
+ t = (dt.datetime.now() - t).total_seconds()
142
+ rtf = t * 22050 / (decoder_outputs.shape[-1] * 256)
143
+
144
+ return {
145
+ "encoder_outputs": encoder_outputs,
146
+ "decoder_outputs": decoder_outputs,
147
+ "attn": attn[:, :, :y_max_length],
148
+ "mel": denormalize(decoder_outputs, self.mel_mean, self.mel_std),
149
+ "mel_lengths": y_lengths,
150
+ "rtf": rtf,
151
+ }
152
+
153
+ def forward(self, x, x_lengths, y, y_lengths, spks=None, out_size=None, cond=None, durations=None):
154
+ """
155
+ Computes 3 losses:
156
+ 1. duration loss: loss between predicted token durations and those extracted by Monotonic Alignment Search (MAS).
157
+ 2. prior loss: loss between mel-spectrogram and encoder outputs.
158
+ 3. flow matching loss: loss between mel-spectrogram and decoder outputs.
159
+
160
+ Args:
161
+ x (torch.Tensor): batch of texts, converted to a tensor with phoneme embedding ids.
162
+ shape: (batch_size, max_text_length)
163
+ x_lengths (torch.Tensor): lengths of texts in batch.
164
+ shape: (batch_size,)
165
+ y (torch.Tensor): batch of corresponding mel-spectrograms.
166
+ shape: (batch_size, n_feats, max_mel_length)
167
+ y_lengths (torch.Tensor): lengths of mel-spectrograms in batch.
168
+ shape: (batch_size,)
169
+ out_size (int, optional): length (in mel's sampling rate) of segment to cut, on which decoder will be trained.
170
+ Should be divisible by 2^{num of UNet downsamplings}. Needed to increase batch size.
171
+ spks (torch.Tensor, optional): speaker ids.
172
+ shape: (batch_size,)
173
+ """
174
+ if self.n_spks > 1:
175
+ # Get speaker embedding
176
+ spks = self.spk_emb(spks)
177
+
178
+ # Get encoder_outputs `mu_x` and log-scaled token durations `logw`
179
+ mu_x, logw, x_mask = self.encoder(x, x_lengths, spks)
180
+ y_max_length = y.shape[-1]
181
+
182
+ y_mask = sequence_mask(y_lengths, y_max_length).unsqueeze(1).to(x_mask)
183
+ attn_mask = x_mask.unsqueeze(-1) * y_mask.unsqueeze(2)
184
+
185
+ if self.use_precomputed_durations:
186
+ attn = generate_path(durations.squeeze(1), attn_mask.squeeze(1))
187
+ else:
188
+ # Use MAS to find most likely alignment `attn` between text and mel-spectrogram
189
+ with torch.no_grad():
190
+ const = -0.5 * math.log(2 * math.pi) * self.n_feats
191
+ factor = -0.5 * torch.ones(mu_x.shape, dtype=mu_x.dtype, device=mu_x.device)
192
+ y_square = torch.matmul(factor.transpose(1, 2), y**2)
193
+ y_mu_double = torch.matmul(2.0 * (factor * mu_x).transpose(1, 2), y)
194
+ mu_square = torch.sum(factor * (mu_x**2), 1).unsqueeze(-1)
195
+ log_prior = y_square - y_mu_double + mu_square + const
196
+
197
+ attn = monotonic_align.maximum_path(log_prior, attn_mask.squeeze(1))
198
+ attn = attn.detach() # b, t_text, T_mel
199
+
200
+ # Compute loss between predicted log-scaled durations and those obtained from MAS
201
+ # refered to as prior loss in the paper
202
+ logw_ = torch.log(1e-8 + torch.sum(attn.unsqueeze(1), -1)) * x_mask
203
+ dur_loss = duration_loss(logw, logw_, x_lengths)
204
+
205
+ # Cut a small segment of mel-spectrogram in order to increase batch size
206
+ # - "Hack" taken from Grad-TTS, in case of Grad-TTS, we cannot train batch size 32 on a 24GB GPU without it
207
+ # - Do not need this hack for Matcha-TTS, but it works with it as well
208
+ if not isinstance(out_size, type(None)):
209
+ max_offset = (y_lengths - out_size).clamp(0)
210
+ offset_ranges = list(zip([0] * max_offset.shape[0], max_offset.cpu().numpy()))
211
+ out_offset = torch.LongTensor(
212
+ [torch.tensor(random.choice(range(start, end)) if end > start else 0) for start, end in offset_ranges]
213
+ ).to(y_lengths)
214
+ attn_cut = torch.zeros(attn.shape[0], attn.shape[1], out_size, dtype=attn.dtype, device=attn.device)
215
+ y_cut = torch.zeros(y.shape[0], self.n_feats, out_size, dtype=y.dtype, device=y.device)
216
+
217
+ y_cut_lengths = []
218
+ for i, (y_, out_offset_) in enumerate(zip(y, out_offset)):
219
+ y_cut_length = out_size + (y_lengths[i] - out_size).clamp(None, 0)
220
+ y_cut_lengths.append(y_cut_length)
221
+ cut_lower, cut_upper = out_offset_, out_offset_ + y_cut_length
222
+ y_cut[i, :, :y_cut_length] = y_[:, cut_lower:cut_upper]
223
+ attn_cut[i, :, :y_cut_length] = attn[i, :, cut_lower:cut_upper]
224
+
225
+ y_cut_lengths = torch.LongTensor(y_cut_lengths)
226
+ y_cut_mask = sequence_mask(y_cut_lengths).unsqueeze(1).to(y_mask)
227
+
228
+ attn = attn_cut
229
+ y = y_cut
230
+ y_mask = y_cut_mask
231
+
232
+ # Align encoded text with mel-spectrogram and get mu_y segment
233
+ mu_y = torch.matmul(attn.squeeze(1).transpose(1, 2), mu_x.transpose(1, 2))
234
+ mu_y = mu_y.transpose(1, 2)
235
+
236
+ # Compute loss of the decoder
237
+ diff_loss, _ = self.decoder.compute_loss(x1=y, mask=y_mask, mu=mu_y, spks=spks, cond=cond)
238
+
239
+ if self.prior_loss:
240
+ prior_loss = torch.sum(0.5 * ((y - mu_y) ** 2 + math.log(2 * math.pi)) * y_mask)
241
+ prior_loss = prior_loss / (torch.sum(y_mask) * self.n_feats)
242
+ else:
243
+ prior_loss = 0
244
+
245
+ return dur_loss, prior_loss, diff_loss, attn
whisper/assets/gpt2.tiktoken ADDED
The diff for this file is too large to render. See raw diff
 
whisper/assets/multilingual.tiktoken ADDED
The diff for this file is too large to render. See raw diff
 
whisper/decoding.py ADDED
@@ -0,0 +1,826 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass, field, replace
2
+ from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Sequence, Tuple, Union
3
+
4
+ import numpy as np
5
+ import torch
6
+ import torch.nn.functional as F
7
+ from torch import Tensor
8
+ from torch.distributions import Categorical
9
+
10
+ from .audio import CHUNK_LENGTH
11
+ from .tokenizer import Tokenizer, get_tokenizer
12
+ from .utils import compression_ratio
13
+
14
+ if TYPE_CHECKING:
15
+ from .model import Whisper
16
+
17
+
18
+ @torch.no_grad()
19
+ def detect_language(
20
+ model: "Whisper", mel: Tensor, tokenizer: Tokenizer = None
21
+ ) -> Tuple[Tensor, List[dict]]:
22
+ """
23
+ Detect the spoken language in the audio, and return them as list of strings, along with the ids
24
+ of the most probable language tokens and the probability distribution over all language tokens.
25
+ This is performed outside the main decode loop in order to not interfere with kv-caching.
26
+
27
+ Returns
28
+ -------
29
+ language_tokens : Tensor, shape = (n_audio,)
30
+ ids of the most probable language tokens, which appears after the startoftranscript token.
31
+ language_probs : List[Dict[str, float]], length = n_audio
32
+ list of dictionaries containing the probability distribution over all languages.
33
+ """
34
+ if tokenizer is None:
35
+ tokenizer = get_tokenizer(
36
+ model.is_multilingual, num_languages=model.num_languages
37
+ )
38
+ if (
39
+ tokenizer.language is None
40
+ or tokenizer.language_token not in tokenizer.sot_sequence
41
+ ):
42
+ raise ValueError(
43
+ "This model doesn't have language tokens so it can't perform lang id"
44
+ )
45
+
46
+ single = mel.ndim == 2
47
+ if single:
48
+ mel = mel.unsqueeze(0)
49
+
50
+ # skip encoder forward pass if already-encoded audio features were given
51
+ if mel.shape[-2:] != (model.dims.n_audio_ctx, model.dims.n_audio_state):
52
+ mel = model.encoder(mel)
53
+
54
+ # forward pass using a single token, startoftranscript
55
+ n_audio = mel.shape[0]
56
+ x = torch.tensor([[tokenizer.sot]] * n_audio).to(mel.device) # [n_audio, 1]
57
+ logits = model.logits(x, mel)[:, 0]
58
+
59
+ # collect detected languages; suppress all non-language tokens
60
+ mask = torch.ones(logits.shape[-1], dtype=torch.bool)
61
+ mask[list(tokenizer.all_language_tokens)] = False
62
+ logits[:, mask] = -np.inf
63
+ language_tokens = logits.argmax(dim=-1)
64
+ language_token_probs = logits.softmax(dim=-1).cpu()
65
+ language_probs = [
66
+ {
67
+ c: language_token_probs[i, j].item()
68
+ for j, c in zip(tokenizer.all_language_tokens, tokenizer.all_language_codes)
69
+ }
70
+ for i in range(n_audio)
71
+ ]
72
+
73
+ if single:
74
+ language_tokens = language_tokens[0]
75
+ language_probs = language_probs[0]
76
+
77
+ return language_tokens, language_probs
78
+
79
+
80
+ @dataclass(frozen=True)
81
+ class DecodingOptions:
82
+ # whether to perform X->X "transcribe" or X->English "translate"
83
+ task: str = "transcribe"
84
+
85
+ # language that the audio is in; uses detected language if None
86
+ language: Optional[str] = None
87
+
88
+ # sampling-related options
89
+ temperature: float = 0.0
90
+ sample_len: Optional[int] = None # maximum number of tokens to sample
91
+ best_of: Optional[int] = None # number of independent sample trajectories, if t > 0
92
+ beam_size: Optional[int] = None # number of beams in beam search, if t == 0
93
+ patience: Optional[float] = None # patience in beam search (arxiv:2204.05424)
94
+
95
+ # "alpha" in Google NMT, or None for length norm, when ranking generations
96
+ # to select which to return among the beams or best-of-N samples
97
+ length_penalty: Optional[float] = None
98
+
99
+ # text or tokens to feed as the prompt or the prefix; for more info:
100
+ # https://github.com/openai/whisper/discussions/117#discussioncomment-3727051
101
+ prompt: Optional[Union[str, List[int]]] = None # for the previous context
102
+ prefix: Optional[Union[str, List[int]]] = None # to prefix the current context
103
+
104
+ # list of tokens ids (or comma-separated token ids) to suppress
105
+ # "-1" will suppress a set of symbols as defined in `tokenizer.non_speech_tokens()`
106
+ suppress_tokens: Optional[Union[str, Iterable[int]]] = "-1"
107
+ suppress_blank: bool = True # this will suppress blank outputs
108
+
109
+ # timestamp sampling options
110
+ without_timestamps: bool = False # use <|notimestamps|> to sample text tokens only
111
+ max_initial_timestamp: Optional[float] = 1.0
112
+
113
+ # implementation details
114
+ fp16: bool = True # use fp16 for most of the calculation
115
+
116
+
117
+ @dataclass(frozen=True)
118
+ class DecodingResult:
119
+ audio_features: Tensor
120
+ language: str
121
+ language_probs: Optional[Dict[str, float]] = None
122
+ tokens: List[int] = field(default_factory=list)
123
+ text: str = ""
124
+ avg_logprob: float = np.nan
125
+ no_speech_prob: float = np.nan
126
+ temperature: float = np.nan
127
+ compression_ratio: float = np.nan
128
+
129
+
130
+ class Inference:
131
+ def logits(self, tokens: Tensor, audio_features: Tensor) -> Tensor:
132
+ """Perform a forward pass on the decoder and return per-token logits"""
133
+ raise NotImplementedError
134
+
135
+ def rearrange_kv_cache(self, source_indices) -> None:
136
+ """Update the key-value cache according to the updated beams"""
137
+ raise NotImplementedError
138
+
139
+ def cleanup_caching(self) -> None:
140
+ """Clean up any resources or hooks after decoding is finished"""
141
+ pass
142
+
143
+
144
+ class PyTorchInference(Inference):
145
+ def __init__(self, model: "Whisper", initial_token_length: int):
146
+ self.model: "Whisper" = model
147
+ self.initial_token_length = initial_token_length
148
+ self.kv_cache = {}
149
+ self.hooks = []
150
+
151
+ key_modules = [block.attn.key for block in self.model.decoder.blocks]
152
+ value_modules = [block.attn.value for block in self.model.decoder.blocks]
153
+ self.kv_modules = key_modules + value_modules
154
+
155
+ def logits(self, tokens: Tensor, audio_features: Tensor) -> Tensor:
156
+ if not self.kv_cache:
157
+ self.kv_cache, self.hooks = self.model.install_kv_cache_hooks()
158
+
159
+ if tokens.shape[-1] > self.initial_token_length:
160
+ # only need to use the last token except in the first forward pass
161
+ tokens = tokens[:, -1:]
162
+
163
+ return self.model.decoder(tokens, audio_features, kv_cache=self.kv_cache)
164
+
165
+ def cleanup_caching(self):
166
+ for hook in self.hooks:
167
+ hook.remove()
168
+
169
+ self.kv_cache = {}
170
+ self.hooks = []
171
+
172
+ def rearrange_kv_cache(self, source_indices):
173
+ if source_indices != list(range(len(source_indices))):
174
+ for module in self.kv_modules:
175
+ # update the key/value cache to contain the selected sequences
176
+ self.kv_cache[module] = self.kv_cache[module][source_indices].detach()
177
+
178
+
179
+ class SequenceRanker:
180
+ def rank(
181
+ self, tokens: List[List[Tensor]], sum_logprobs: List[List[float]]
182
+ ) -> List[int]:
183
+ """
184
+ Given a list of groups of samples and their cumulative log probabilities,
185
+ return the indices of the samples in each group to select as the final result
186
+ """
187
+ raise NotImplementedError
188
+
189
+
190
+ class MaximumLikelihoodRanker(SequenceRanker):
191
+ """
192
+ Select the sample with the highest log probabilities, penalized using either
193
+ a simple length normalization or Google NMT paper's length penalty
194
+ """
195
+
196
+ def __init__(self, length_penalty: Optional[float]):
197
+ self.length_penalty = length_penalty
198
+
199
+ def rank(self, tokens: List[List[Tensor]], sum_logprobs: List[List[float]]):
200
+ def scores(logprobs, lengths):
201
+ result = []
202
+ for logprob, length in zip(logprobs, lengths):
203
+ if self.length_penalty is None:
204
+ penalty = length
205
+ else:
206
+ # from the Google NMT paper
207
+ penalty = ((5 + length) / 6) ** self.length_penalty
208
+ result.append(logprob / penalty)
209
+ return result
210
+
211
+ # get the sequence with the highest score
212
+ lengths = [[len(t) for t in s] for s in tokens]
213
+ return [np.argmax(scores(p, l)) for p, l in zip(sum_logprobs, lengths)]
214
+
215
+
216
+ class TokenDecoder:
217
+ def reset(self):
218
+ """Initialize any stateful variables for decoding a new sequence"""
219
+
220
+ def update(
221
+ self, tokens: Tensor, logits: Tensor, sum_logprobs: Tensor
222
+ ) -> Tuple[Tensor, bool]:
223
+ """Specify how to select the next token, based on the current trace and logits
224
+
225
+ Parameters
226
+ ----------
227
+ tokens : Tensor, shape = (n_batch, current_sequence_length)
228
+ all tokens in the context so far, including the prefix and sot_sequence tokens
229
+
230
+ logits : Tensor, shape = (n_batch, vocab_size)
231
+ per-token logits of the probability distribution at the current step
232
+
233
+ sum_logprobs : Tensor, shape = (n_batch)
234
+ cumulative log probabilities for each sequence
235
+
236
+ Returns
237
+ -------
238
+ tokens : Tensor, shape = (n_batch, current_sequence_length + 1)
239
+ the tokens, appended with the selected next token
240
+
241
+ completed : bool
242
+ True if all sequences has reached the end of text
243
+
244
+ """
245
+ raise NotImplementedError
246
+
247
+ def finalize(
248
+ self, tokens: Tensor, sum_logprobs: Tensor
249
+ ) -> Tuple[Sequence[Sequence[Tensor]], List[List[float]]]:
250
+ """Finalize search and return the final candidate sequences
251
+
252
+ Parameters
253
+ ----------
254
+ tokens : Tensor, shape = (n_audio, n_group, current_sequence_length)
255
+ all tokens in the context so far, including the prefix and sot_sequence
256
+
257
+ sum_logprobs : Tensor, shape = (n_audio, n_group)
258
+ cumulative log probabilities for each sequence
259
+
260
+ Returns
261
+ -------
262
+ tokens : Sequence[Sequence[Tensor]], length = n_audio
263
+ sequence of Tensors containing candidate token sequences, for each audio input
264
+
265
+ sum_logprobs : List[List[float]], length = n_audio
266
+ sequence of cumulative log probabilities corresponding to the above
267
+
268
+ """
269
+ raise NotImplementedError
270
+
271
+
272
+ class GreedyDecoder(TokenDecoder):
273
+ def __init__(self, temperature: float, eot: int):
274
+ self.temperature = temperature
275
+ self.eot = eot
276
+
277
+ def update(
278
+ self, tokens: Tensor, logits: Tensor, sum_logprobs: Tensor
279
+ ) -> Tuple[Tensor, bool]:
280
+ if self.temperature == 0:
281
+ next_tokens = logits.argmax(dim=-1)
282
+ else:
283
+ next_tokens = Categorical(logits=logits / self.temperature).sample()
284
+
285
+ logprobs = F.log_softmax(logits.float(), dim=-1)
286
+ current_logprobs = logprobs[torch.arange(logprobs.shape[0]), next_tokens]
287
+ sum_logprobs += current_logprobs * (tokens[:, -1] != self.eot)
288
+
289
+ next_tokens[tokens[:, -1] == self.eot] = self.eot
290
+ tokens = torch.cat([tokens, next_tokens[:, None]], dim=-1)
291
+
292
+ completed = (tokens[:, -1] == self.eot).all()
293
+ return tokens, completed
294
+
295
+ def finalize(self, tokens: Tensor, sum_logprobs: Tensor):
296
+ # make sure each sequence has at least one EOT token at the end
297
+ tokens = F.pad(tokens, (0, 1), value=self.eot)
298
+ return tokens, sum_logprobs.tolist()
299
+
300
+
301
+ class BeamSearchDecoder(TokenDecoder):
302
+ def __init__(
303
+ self,
304
+ beam_size: int,
305
+ eot: int,
306
+ inference: Inference,
307
+ patience: Optional[float] = None,
308
+ ):
309
+ self.beam_size = beam_size
310
+ self.eot = eot
311
+ self.inference = inference
312
+ self.patience = patience or 1.0
313
+ self.max_candidates: int = round(beam_size * self.patience)
314
+ self.finished_sequences = None
315
+
316
+ assert (
317
+ self.max_candidates > 0
318
+ ), f"Invalid beam size ({beam_size}) or patience ({patience})"
319
+
320
+ def reset(self):
321
+ self.finished_sequences = None
322
+
323
+ def update(
324
+ self, tokens: Tensor, logits: Tensor, sum_logprobs: Tensor
325
+ ) -> Tuple[Tensor, bool]:
326
+ if tokens.shape[0] % self.beam_size != 0:
327
+ raise ValueError(f"{tokens.shape}[0] % {self.beam_size} != 0")
328
+
329
+ n_audio = tokens.shape[0] // self.beam_size
330
+ if self.finished_sequences is None: # for the first update
331
+ self.finished_sequences = [{} for _ in range(n_audio)]
332
+
333
+ logprobs = F.log_softmax(logits.float(), dim=-1)
334
+ next_tokens, source_indices, finished_sequences = [], [], []
335
+ for i in range(n_audio):
336
+ scores, sources, finished = {}, {}, {}
337
+
338
+ # STEP 1: calculate the cumulative log probabilities for possible candidates
339
+ for j in range(self.beam_size):
340
+ idx = i * self.beam_size + j
341
+ prefix = tokens[idx].tolist()
342
+ for logprob, token in zip(*logprobs[idx].topk(self.beam_size + 1)):
343
+ new_logprob = (sum_logprobs[idx] + logprob).item()
344
+ sequence = tuple(prefix + [token.item()])
345
+ scores[sequence] = new_logprob
346
+ sources[sequence] = idx
347
+
348
+ # STEP 2: rank the candidates and keep the top beam_size sequences for each audio
349
+ saved = 0
350
+ for sequence in sorted(scores, key=scores.get, reverse=True):
351
+ if sequence[-1] == self.eot:
352
+ finished[sequence] = scores[sequence]
353
+ else:
354
+ sum_logprobs[len(next_tokens)] = scores[sequence]
355
+ next_tokens.append(sequence)
356
+ source_indices.append(sources[sequence])
357
+
358
+ saved += 1
359
+ if saved == self.beam_size:
360
+ break
361
+
362
+ finished_sequences.append(finished)
363
+
364
+ tokens = torch.tensor(next_tokens, device=tokens.device)
365
+ self.inference.rearrange_kv_cache(source_indices)
366
+
367
+ # add newly finished sequences to self.finished_sequences
368
+ assert len(self.finished_sequences) == len(finished_sequences)
369
+ for previously_finished, newly_finished in zip(
370
+ self.finished_sequences, finished_sequences
371
+ ):
372
+ for seq in sorted(newly_finished, key=newly_finished.get, reverse=True):
373
+ if len(previously_finished) >= self.max_candidates:
374
+ break # the candidate list is full
375
+ previously_finished[seq] = newly_finished[seq]
376
+
377
+ # mark as completed if all audio has enough number of samples
378
+ completed = all(
379
+ len(sequences) >= self.max_candidates
380
+ for sequences in self.finished_sequences
381
+ )
382
+ return tokens, completed
383
+
384
+ def finalize(self, preceding_tokens: Tensor, sum_logprobs: Tensor):
385
+ # collect all finished sequences, including patience, and add unfinished ones if not enough
386
+ sum_logprobs = sum_logprobs.cpu()
387
+ for i, sequences in enumerate(self.finished_sequences):
388
+ if (
389
+ len(sequences) < self.beam_size
390
+ ): # when not enough sequences are finished
391
+ for j in list(np.argsort(sum_logprobs[i]))[::-1]:
392
+ sequence = preceding_tokens[i, j].tolist() + [self.eot]
393
+ sequences[tuple(sequence)] = sum_logprobs[i][j].item()
394
+ if len(sequences) >= self.beam_size:
395
+ break
396
+
397
+ tokens: List[List[Tensor]] = [
398
+ [torch.tensor(seq) for seq in sequences.keys()]
399
+ for sequences in self.finished_sequences
400
+ ]
401
+ sum_logprobs: List[List[float]] = [
402
+ list(sequences.values()) for sequences in self.finished_sequences
403
+ ]
404
+ return tokens, sum_logprobs
405
+
406
+
407
+ class LogitFilter:
408
+ def apply(self, logits: Tensor, tokens: Tensor) -> None:
409
+ """Apply any filtering or masking to logits in-place
410
+
411
+ Parameters
412
+ ----------
413
+ logits : Tensor, shape = (n_batch, vocab_size)
414
+ per-token logits of the probability distribution at the current step
415
+
416
+ tokens : Tensor, shape = (n_batch, current_sequence_length)
417
+ all tokens in the context so far, including the prefix and sot_sequence tokens
418
+
419
+ """
420
+ raise NotImplementedError
421
+
422
+
423
+ class SuppressBlank(LogitFilter):
424
+ def __init__(self, tokenizer: Tokenizer, sample_begin: int):
425
+ self.tokenizer = tokenizer
426
+ self.sample_begin = sample_begin
427
+
428
+ def apply(self, logits: Tensor, tokens: Tensor):
429
+ if tokens.shape[1] == self.sample_begin:
430
+ logits[:, self.tokenizer.encode(" ") + [self.tokenizer.eot]] = -np.inf
431
+
432
+
433
+ class SuppressTokens(LogitFilter):
434
+ def __init__(self, suppress_tokens: Sequence[int]):
435
+ self.suppress_tokens = list(suppress_tokens)
436
+
437
+ def apply(self, logits: Tensor, tokens: Tensor):
438
+ logits[:, self.suppress_tokens] = -np.inf
439
+
440
+
441
+ class ApplyTimestampRules(LogitFilter):
442
+ def __init__(
443
+ self,
444
+ tokenizer: Tokenizer,
445
+ sample_begin: int,
446
+ max_initial_timestamp_index: Optional[int],
447
+ ):
448
+ self.tokenizer = tokenizer
449
+ self.sample_begin = sample_begin
450
+ self.max_initial_timestamp_index = max_initial_timestamp_index
451
+
452
+ def apply(self, logits: Tensor, tokens: Tensor):
453
+ # suppress <|notimestamps|> which is handled by without_timestamps
454
+ if self.tokenizer.no_timestamps is not None:
455
+ logits[:, self.tokenizer.no_timestamps] = -np.inf
456
+
457
+ # timestamps have to appear in pairs, except directly before EOT; mask logits accordingly
458
+ for k in range(tokens.shape[0]):
459
+ sampled_tokens = tokens[k, self.sample_begin :]
460
+ seq = [t for t in sampled_tokens.tolist()]
461
+ last_was_timestamp = (
462
+ len(seq) >= 1 and seq[-1] >= self.tokenizer.timestamp_begin
463
+ )
464
+ penultimate_was_timestamp = (
465
+ len(seq) < 2 or seq[-2] >= self.tokenizer.timestamp_begin
466
+ )
467
+
468
+ if last_was_timestamp:
469
+ if penultimate_was_timestamp: # has to be non-timestamp
470
+ logits[k, self.tokenizer.timestamp_begin :] = -np.inf
471
+ else: # cannot be normal text tokens
472
+ logits[k, : self.tokenizer.eot] = -np.inf
473
+
474
+ timestamps = sampled_tokens[
475
+ sampled_tokens.ge(self.tokenizer.timestamp_begin)
476
+ ]
477
+ if timestamps.numel() > 0:
478
+ # timestamps shouldn't decrease; forbid timestamp tokens smaller than the last
479
+ # also force each segment to have a nonzero length, to prevent infinite looping
480
+ if last_was_timestamp and not penultimate_was_timestamp:
481
+ timestamp_last = timestamps[-1]
482
+ else:
483
+ timestamp_last = timestamps[-1] + 1
484
+ logits[k, self.tokenizer.timestamp_begin : timestamp_last] = -np.inf
485
+
486
+ if tokens.shape[1] == self.sample_begin:
487
+ # suppress generating non-timestamp tokens at the beginning
488
+ logits[:, : self.tokenizer.timestamp_begin] = -np.inf
489
+
490
+ # apply the `max_initial_timestamp` option
491
+ if self.max_initial_timestamp_index is not None:
492
+ last_allowed = (
493
+ self.tokenizer.timestamp_begin + self.max_initial_timestamp_index
494
+ )
495
+ logits[:, last_allowed + 1 :] = -np.inf
496
+
497
+ # if sum of probability over timestamps is above any other token, sample timestamp
498
+ logprobs = F.log_softmax(logits.float(), dim=-1)
499
+ for k in range(tokens.shape[0]):
500
+ timestamp_logprob = logprobs[k, self.tokenizer.timestamp_begin :].logsumexp(
501
+ dim=-1
502
+ )
503
+ max_text_token_logprob = logprobs[k, : self.tokenizer.timestamp_begin].max()
504
+ if timestamp_logprob > max_text_token_logprob:
505
+ logits[k, : self.tokenizer.timestamp_begin] = -np.inf
506
+
507
+
508
+ class DecodingTask:
509
+ inference: Inference
510
+ sequence_ranker: SequenceRanker
511
+ decoder: TokenDecoder
512
+ logit_filters: List[LogitFilter]
513
+
514
+ def __init__(self, model: "Whisper", options: DecodingOptions):
515
+ self.model = model
516
+
517
+ language = options.language or "en"
518
+ tokenizer = get_tokenizer(
519
+ model.is_multilingual,
520
+ num_languages=model.num_languages,
521
+ language=language,
522
+ task=options.task,
523
+ )
524
+ self.tokenizer: Tokenizer = tokenizer
525
+ self.options: DecodingOptions = self._verify_options(options)
526
+
527
+ self.n_group: int = options.beam_size or options.best_of or 1
528
+ self.n_ctx: int = model.dims.n_text_ctx
529
+ self.sample_len: int = options.sample_len or model.dims.n_text_ctx // 2
530
+
531
+ self.sot_sequence: Tuple[int] = tokenizer.sot_sequence
532
+ if self.options.without_timestamps:
533
+ self.sot_sequence = tokenizer.sot_sequence_including_notimestamps
534
+
535
+ self.initial_tokens: Tuple[int] = self._get_initial_tokens()
536
+ self.sample_begin: int = len(self.initial_tokens)
537
+ self.sot_index: int = self.initial_tokens.index(tokenizer.sot)
538
+
539
+ # inference: implements the forward pass through the decoder, including kv caching
540
+ self.inference = PyTorchInference(model, len(self.initial_tokens))
541
+
542
+ # sequence ranker: implements how to rank a group of sampled sequences
543
+ self.sequence_ranker = MaximumLikelihoodRanker(options.length_penalty)
544
+
545
+ # decoder: implements how to select the next tokens, given the autoregressive distribution
546
+ if options.beam_size is not None:
547
+ self.decoder = BeamSearchDecoder(
548
+ options.beam_size, tokenizer.eot, self.inference, options.patience
549
+ )
550
+ else:
551
+ self.decoder = GreedyDecoder(options.temperature, tokenizer.eot)
552
+
553
+ # logit filters: applies various rules to suppress or penalize certain tokens
554
+ self.logit_filters = []
555
+ if self.options.suppress_blank:
556
+ self.logit_filters.append(SuppressBlank(self.tokenizer, self.sample_begin))
557
+ if self.options.suppress_tokens:
558
+ self.logit_filters.append(SuppressTokens(self._get_suppress_tokens()))
559
+ if not options.without_timestamps:
560
+ precision = CHUNK_LENGTH / model.dims.n_audio_ctx # usually 0.02 seconds
561
+ max_initial_timestamp_index = None
562
+ if options.max_initial_timestamp:
563
+ max_initial_timestamp_index = round(
564
+ self.options.max_initial_timestamp / precision
565
+ )
566
+ self.logit_filters.append(
567
+ ApplyTimestampRules(
568
+ tokenizer, self.sample_begin, max_initial_timestamp_index
569
+ )
570
+ )
571
+
572
+ def _verify_options(self, options: DecodingOptions) -> DecodingOptions:
573
+ if options.beam_size is not None and options.best_of is not None:
574
+ raise ValueError("beam_size and best_of can't be given together")
575
+ if options.temperature == 0:
576
+ if options.best_of is not None:
577
+ raise ValueError("best_of with greedy sampling (T=0) is not compatible")
578
+ if options.patience is not None and options.beam_size is None:
579
+ raise ValueError("patience requires beam_size to be given")
580
+ if options.length_penalty is not None and not (
581
+ 0 <= options.length_penalty <= 1
582
+ ):
583
+ raise ValueError("length_penalty (alpha) should be a value between 0 and 1")
584
+
585
+ return options
586
+
587
+ def _get_initial_tokens(self) -> Tuple[int]:
588
+ tokens = list(self.sot_sequence)
589
+
590
+ if prefix := self.options.prefix:
591
+ prefix_tokens = (
592
+ self.tokenizer.encode(" " + prefix.strip())
593
+ if isinstance(prefix, str)
594
+ else prefix
595
+ )
596
+ if self.sample_len is not None:
597
+ max_prefix_len = self.n_ctx // 2 - self.sample_len
598
+ prefix_tokens = prefix_tokens[-max_prefix_len:]
599
+ tokens = tokens + prefix_tokens
600
+
601
+ if prompt := self.options.prompt:
602
+ prompt_tokens = (
603
+ self.tokenizer.encode(" " + prompt.strip())
604
+ if isinstance(prompt, str)
605
+ else prompt
606
+ )
607
+ tokens = (
608
+ [self.tokenizer.sot_prev]
609
+ + prompt_tokens[-(self.n_ctx // 2 - 1) :]
610
+ + tokens
611
+ )
612
+
613
+ return tuple(tokens)
614
+
615
+ def _get_suppress_tokens(self) -> Tuple[int]:
616
+ suppress_tokens = self.options.suppress_tokens
617
+
618
+ if isinstance(suppress_tokens, str):
619
+ suppress_tokens = [int(t) for t in suppress_tokens.split(",")]
620
+
621
+ if -1 in suppress_tokens:
622
+ suppress_tokens = [t for t in suppress_tokens if t >= 0]
623
+ suppress_tokens.extend(self.tokenizer.non_speech_tokens)
624
+ elif suppress_tokens is None or len(suppress_tokens) == 0:
625
+ suppress_tokens = [] # interpret empty string as an empty list
626
+ else:
627
+ assert isinstance(suppress_tokens, list), "suppress_tokens must be a list"
628
+
629
+ suppress_tokens.extend(
630
+ [
631
+ self.tokenizer.transcribe,
632
+ self.tokenizer.translate,
633
+ self.tokenizer.sot,
634
+ self.tokenizer.sot_prev,
635
+ self.tokenizer.sot_lm,
636
+ ]
637
+ )
638
+ if self.tokenizer.no_speech is not None:
639
+ # no-speech probability is collected separately
640
+ suppress_tokens.append(self.tokenizer.no_speech)
641
+
642
+ return tuple(sorted(set(suppress_tokens)))
643
+
644
+ def _get_audio_features(self, mel: Tensor):
645
+ if self.options.fp16:
646
+ mel = mel.half()
647
+
648
+ if mel.shape[-2:] == (
649
+ self.model.dims.n_audio_ctx,
650
+ self.model.dims.n_audio_state,
651
+ ):
652
+ # encoded audio features are given; skip audio encoding
653
+ audio_features = mel
654
+ else:
655
+ audio_features = self.model.encoder(mel)
656
+
657
+ if audio_features.dtype != (
658
+ torch.float16 if self.options.fp16 else torch.float32
659
+ ):
660
+ return TypeError(
661
+ f"audio_features has an incorrect dtype: {audio_features.dtype}"
662
+ )
663
+
664
+ return audio_features
665
+
666
+ def _detect_language(self, audio_features: Tensor, tokens: Tensor):
667
+ languages = [self.options.language] * audio_features.shape[0]
668
+ lang_probs = None
669
+
670
+ if self.options.language is None or self.options.task == "lang_id":
671
+ lang_tokens, lang_probs = self.model.detect_language(
672
+ audio_features, self.tokenizer
673
+ )
674
+ languages = [max(probs, key=probs.get) for probs in lang_probs]
675
+ if self.options.language is None:
676
+ tokens[:, self.sot_index + 1] = lang_tokens # write language tokens
677
+
678
+ return languages, lang_probs
679
+
680
+ def _main_loop(self, audio_features: Tensor, tokens: Tensor):
681
+ n_batch = tokens.shape[0]
682
+ sum_logprobs: Tensor = torch.zeros(n_batch, device=audio_features.device)
683
+ no_speech_probs = [np.nan] * n_batch
684
+
685
+ try:
686
+ for i in range(self.sample_len):
687
+ logits = self.inference.logits(tokens, audio_features)
688
+
689
+ if (
690
+ i == 0 and self.tokenizer.no_speech is not None
691
+ ): # save no_speech_probs
692
+ probs_at_sot = logits[:, self.sot_index].float().softmax(dim=-1)
693
+ no_speech_probs = probs_at_sot[:, self.tokenizer.no_speech].tolist()
694
+
695
+ # now we need to consider the logits at the last token only
696
+ logits = logits[:, -1]
697
+
698
+ # apply the logit filters, e.g. for suppressing or applying penalty to
699
+ for logit_filter in self.logit_filters:
700
+ logit_filter.apply(logits, tokens)
701
+
702
+ # expand the tokens tensor with the selected next tokens
703
+ tokens, completed = self.decoder.update(tokens, logits, sum_logprobs)
704
+
705
+ if completed or tokens.shape[-1] > self.n_ctx:
706
+ break
707
+ finally:
708
+ self.inference.cleanup_caching()
709
+
710
+ return tokens, sum_logprobs, no_speech_probs
711
+
712
+ @torch.no_grad()
713
+ def run(self, mel: Tensor) -> List[DecodingResult]:
714
+ self.decoder.reset()
715
+ tokenizer: Tokenizer = self.tokenizer
716
+ n_audio: int = mel.shape[0]
717
+
718
+ audio_features: Tensor = self._get_audio_features(mel) # encoder forward pass
719
+ tokens: Tensor = torch.tensor([self.initial_tokens]).repeat(n_audio, 1)
720
+
721
+ # detect language if requested, overwriting the language token
722
+ languages, language_probs = self._detect_language(audio_features, tokens)
723
+ if self.options.task == "lang_id":
724
+ return [
725
+ DecodingResult(
726
+ audio_features=features, language=language, language_probs=probs
727
+ )
728
+ for features, language, probs in zip(
729
+ audio_features, languages, language_probs
730
+ )
731
+ ]
732
+
733
+ # repeat text tensors by the group size, for beam search or best-of-n sampling
734
+ tokens = tokens.repeat_interleave(self.n_group, dim=0).to(audio_features.device)
735
+
736
+ # call the main sampling loop
737
+ tokens, sum_logprobs, no_speech_probs = self._main_loop(audio_features, tokens)
738
+
739
+ # reshape the tensors to have (n_audio, n_group) as the first two dimensions
740
+ audio_features = audio_features[:: self.n_group]
741
+ no_speech_probs = no_speech_probs[:: self.n_group]
742
+ assert audio_features.shape[0] == len(no_speech_probs) == n_audio
743
+
744
+ tokens = tokens.reshape(n_audio, self.n_group, -1)
745
+ sum_logprobs = sum_logprobs.reshape(n_audio, self.n_group)
746
+
747
+ # get the final candidates for each group, and slice between the first sampled token and EOT
748
+ tokens, sum_logprobs = self.decoder.finalize(tokens, sum_logprobs)
749
+ tokens: List[List[Tensor]] = [
750
+ [t[self.sample_begin : (t == tokenizer.eot).nonzero()[0, 0]] for t in s]
751
+ for s in tokens
752
+ ]
753
+
754
+ # select the top-ranked sample in each group
755
+ selected = self.sequence_ranker.rank(tokens, sum_logprobs)
756
+ tokens: List[List[int]] = [t[i].tolist() for i, t in zip(selected, tokens)]
757
+ texts: List[str] = [tokenizer.decode(t).strip() for t in tokens]
758
+
759
+ sum_logprobs: List[float] = [lp[i] for i, lp in zip(selected, sum_logprobs)]
760
+ avg_logprobs: List[float] = [
761
+ lp / (len(t) + 1) for t, lp in zip(tokens, sum_logprobs)
762
+ ]
763
+
764
+ fields = (
765
+ texts,
766
+ languages,
767
+ tokens,
768
+ audio_features,
769
+ avg_logprobs,
770
+ no_speech_probs,
771
+ )
772
+ if len(set(map(len, fields))) != 1:
773
+ raise RuntimeError(f"inconsistent result lengths: {list(map(len, fields))}")
774
+
775
+ return [
776
+ DecodingResult(
777
+ audio_features=features,
778
+ language=language,
779
+ tokens=tokens,
780
+ text=text,
781
+ avg_logprob=avg_logprob,
782
+ no_speech_prob=no_speech_prob,
783
+ temperature=self.options.temperature,
784
+ compression_ratio=compression_ratio(text),
785
+ )
786
+ for text, language, tokens, features, avg_logprob, no_speech_prob in zip(
787
+ *fields
788
+ )
789
+ ]
790
+
791
+
792
+ @torch.no_grad()
793
+ def decode(
794
+ model: "Whisper",
795
+ mel: Tensor,
796
+ options: DecodingOptions = DecodingOptions(),
797
+ **kwargs,
798
+ ) -> Union[DecodingResult, List[DecodingResult]]:
799
+ """
800
+ Performs decoding of 30-second audio segment(s), provided as Mel spectrogram(s).
801
+
802
+ Parameters
803
+ ----------
804
+ model: Whisper
805
+ the Whisper model instance
806
+
807
+ mel: torch.Tensor, shape = (80, 3000) or (*, 80, 3000)
808
+ A tensor containing the Mel spectrogram(s)
809
+
810
+ options: DecodingOptions
811
+ A dataclass that contains all necessary options for decoding 30-second segments
812
+
813
+ Returns
814
+ -------
815
+ result: Union[DecodingResult, List[DecodingResult]]
816
+ The result(s) of decoding contained in `DecodingResult` dataclass instance(s)
817
+ """
818
+ if single := mel.ndim == 2:
819
+ mel = mel.unsqueeze(0)
820
+
821
+ if kwargs:
822
+ options = replace(options, **kwargs)
823
+
824
+ result = DecodingTask(model, options).run(mel)
825
+
826
+ return result[0] if single else result
whisper/model.py ADDED
@@ -0,0 +1,314 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import base64
2
+ import gzip
3
+ from dataclasses import dataclass
4
+ from typing import Dict, Iterable, Optional
5
+
6
+ import numpy as np
7
+ import torch
8
+ import torch.nn.functional as F
9
+ from torch import Tensor, nn
10
+
11
+ from .decoding import decode as decode_function
12
+ from .decoding import detect_language as detect_language_function
13
+ from .transcribe import transcribe as transcribe_function
14
+
15
+
16
+ @dataclass
17
+ class ModelDimensions:
18
+ n_mels: int
19
+ n_audio_ctx: int
20
+ n_audio_state: int
21
+ n_audio_head: int
22
+ n_audio_layer: int
23
+ n_vocab: int
24
+ n_text_ctx: int
25
+ n_text_state: int
26
+ n_text_head: int
27
+ n_text_layer: int
28
+
29
+
30
+ class LayerNorm(nn.LayerNorm):
31
+ def forward(self, x: Tensor) -> Tensor:
32
+ return super().forward(x.float()).type(x.dtype)
33
+
34
+
35
+ class Linear(nn.Linear):
36
+ def forward(self, x: Tensor) -> Tensor:
37
+ return F.linear(
38
+ x,
39
+ self.weight.to(x.dtype),
40
+ None if self.bias is None else self.bias.to(x.dtype),
41
+ )
42
+
43
+
44
+ class Conv1d(nn.Conv1d):
45
+ def _conv_forward(
46
+ self, x: Tensor, weight: Tensor, bias: Optional[Tensor]
47
+ ) -> Tensor:
48
+ return super()._conv_forward(
49
+ x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype)
50
+ )
51
+
52
+
53
+ def sinusoids(length, channels, max_timescale=10000):
54
+ """Returns sinusoids for positional embedding"""
55
+ assert channels % 2 == 0
56
+ log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1)
57
+ inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2))
58
+ scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :]
59
+ return torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1)
60
+
61
+
62
+ class MultiHeadAttention(nn.Module):
63
+ def __init__(self, n_state: int, n_head: int):
64
+ super().__init__()
65
+ self.n_head = n_head
66
+ self.query = Linear(n_state, n_state)
67
+ self.key = Linear(n_state, n_state, bias=False)
68
+ self.value = Linear(n_state, n_state)
69
+ self.out = Linear(n_state, n_state)
70
+
71
+ def forward(
72
+ self,
73
+ x: Tensor,
74
+ xa: Optional[Tensor] = None,
75
+ mask: Optional[Tensor] = None,
76
+ kv_cache: Optional[dict] = None,
77
+ ):
78
+ q = self.query(x)
79
+
80
+ if kv_cache is None or xa is None or self.key not in kv_cache:
81
+ # hooks, if installed (i.e. kv_cache is not None), will prepend the cached kv tensors;
82
+ # otherwise, perform key/value projections for self- or cross-attention as usual.
83
+ k = self.key(x if xa is None else xa)
84
+ v = self.value(x if xa is None else xa)
85
+ else:
86
+ # for cross-attention, calculate keys and values once and reuse in subsequent calls.
87
+ k = kv_cache[self.key]
88
+ v = kv_cache[self.value]
89
+
90
+ wv, qk = self.qkv_attention(q, k, v, mask)
91
+ return self.out(wv), qk
92
+
93
+ def qkv_attention(
94
+ self, q: Tensor, k: Tensor, v: Tensor, mask: Optional[Tensor] = None
95
+ ):
96
+ n_batch, n_ctx, n_state = q.shape
97
+ scale = (n_state // self.n_head) ** -0.25
98
+ q = q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) * scale
99
+ k = k.view(*k.shape[:2], self.n_head, -1).permute(0, 2, 3, 1) * scale
100
+ v = v.view(*v.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)
101
+
102
+ qk = q @ k
103
+ if mask is not None:
104
+ qk = qk + mask[:n_ctx, :n_ctx]
105
+ qk = qk.float()
106
+
107
+ w = F.softmax(qk, dim=-1).to(q.dtype)
108
+ return (w @ v).permute(0, 2, 1, 3).flatten(start_dim=2), qk.detach()
109
+
110
+
111
+ class ResidualAttentionBlock(nn.Module):
112
+ def __init__(self, n_state: int, n_head: int, cross_attention: bool = False):
113
+ super().__init__()
114
+
115
+ self.attn = MultiHeadAttention(n_state, n_head)
116
+ self.attn_ln = LayerNorm(n_state)
117
+
118
+ self.cross_attn = (
119
+ MultiHeadAttention(n_state, n_head) if cross_attention else None
120
+ )
121
+ self.cross_attn_ln = LayerNorm(n_state) if cross_attention else None
122
+
123
+ n_mlp = n_state * 4
124
+ self.mlp = nn.Sequential(
125
+ Linear(n_state, n_mlp), nn.GELU(), Linear(n_mlp, n_state)
126
+ )
127
+ self.mlp_ln = LayerNorm(n_state)
128
+
129
+ def forward(
130
+ self,
131
+ x: Tensor,
132
+ xa: Optional[Tensor] = None,
133
+ mask: Optional[Tensor] = None,
134
+ kv_cache: Optional[dict] = None,
135
+ ):
136
+ x = x + self.attn(self.attn_ln(x), mask=mask, kv_cache=kv_cache)[0]
137
+ if self.cross_attn:
138
+ x = x + self.cross_attn(self.cross_attn_ln(x), xa, kv_cache=kv_cache)[0]
139
+ x = x + self.mlp(self.mlp_ln(x))
140
+ return x
141
+
142
+
143
+ class AudioEncoder(nn.Module):
144
+ def __init__(
145
+ self, n_mels: int, n_ctx: int, n_state: int, n_head: int, n_layer: int
146
+ ):
147
+ super().__init__()
148
+ self.conv1 = Conv1d(n_mels, n_state, kernel_size=3, padding=1)
149
+ self.conv2 = Conv1d(n_state, n_state, kernel_size=3, stride=2, padding=1)
150
+ self.register_buffer("positional_embedding", sinusoids(n_ctx, n_state))
151
+
152
+ self.blocks: Iterable[ResidualAttentionBlock] = nn.ModuleList(
153
+ [ResidualAttentionBlock(n_state, n_head) for _ in range(n_layer)]
154
+ )
155
+ self.ln_post = LayerNorm(n_state)
156
+
157
+ def forward(self, x: Tensor):
158
+ """
159
+ x : torch.Tensor, shape = (batch_size, n_mels, n_ctx)
160
+ the mel spectrogram of the audio
161
+ """
162
+ x = F.gelu(self.conv1(x))
163
+ x = F.gelu(self.conv2(x))
164
+ x = x.permute(0, 2, 1)
165
+
166
+ assert x.shape[1:] == self.positional_embedding.shape, "incorrect audio shape"
167
+ x = (x + self.positional_embedding).to(x.dtype)
168
+
169
+ for block in self.blocks:
170
+ x = block(x)
171
+
172
+ x = self.ln_post(x)
173
+ return x
174
+
175
+
176
+ class TextDecoder(nn.Module):
177
+ def __init__(
178
+ self, n_vocab: int, n_ctx: int, n_state: int, n_head: int, n_layer: int
179
+ ):
180
+ super().__init__()
181
+
182
+ self.token_embedding = nn.Embedding(n_vocab, n_state)
183
+ self.positional_embedding = nn.Parameter(torch.empty(n_ctx, n_state))
184
+
185
+ self.blocks: Iterable[ResidualAttentionBlock] = nn.ModuleList(
186
+ [
187
+ ResidualAttentionBlock(n_state, n_head, cross_attention=True)
188
+ for _ in range(n_layer)
189
+ ]
190
+ )
191
+ self.ln = LayerNorm(n_state)
192
+
193
+ mask = torch.empty(n_ctx, n_ctx).fill_(-np.inf).triu_(1)
194
+ self.register_buffer("mask", mask, persistent=False)
195
+
196
+ def forward(self, x: Tensor, xa: Tensor, kv_cache: Optional[dict] = None):
197
+ """
198
+ x : torch.LongTensor, shape = (batch_size, <= n_ctx)
199
+ the text tokens
200
+ xa : torch.Tensor, shape = (batch_size, n_audio_ctx, n_audio_state)
201
+ the encoded audio features to be attended on
202
+ """
203
+ offset = next(iter(kv_cache.values())).shape[1] if kv_cache else 0
204
+ x = (
205
+ self.token_embedding(x)
206
+ + self.positional_embedding[offset : offset + x.shape[-1]]
207
+ )
208
+ x = x.to(xa.dtype)
209
+
210
+ for block in self.blocks:
211
+ x = block(x, xa, mask=self.mask, kv_cache=kv_cache)
212
+
213
+ x = self.ln(x)
214
+ logits = (
215
+ x @ torch.transpose(self.token_embedding.weight.to(x.dtype), 0, 1)
216
+ ).float()
217
+
218
+ return logits
219
+
220
+
221
+ class Whisper(nn.Module):
222
+ def __init__(self, dims: ModelDimensions):
223
+ super().__init__()
224
+ self.dims = dims
225
+ self.encoder = AudioEncoder(
226
+ self.dims.n_mels,
227
+ self.dims.n_audio_ctx,
228
+ self.dims.n_audio_state,
229
+ self.dims.n_audio_head,
230
+ self.dims.n_audio_layer,
231
+ )
232
+ self.decoder = TextDecoder(
233
+ self.dims.n_vocab,
234
+ self.dims.n_text_ctx,
235
+ self.dims.n_text_state,
236
+ self.dims.n_text_head,
237
+ self.dims.n_text_layer,
238
+ )
239
+ # use the last half among the decoder layers for time alignment by default;
240
+ # to use a specific set of heads, see `set_alignment_heads()` below.
241
+ all_heads = torch.zeros(
242
+ self.dims.n_text_layer, self.dims.n_text_head, dtype=torch.bool
243
+ )
244
+ all_heads[self.dims.n_text_layer // 2 :] = True
245
+ self.register_buffer("alignment_heads", all_heads.to_sparse(), persistent=False)
246
+
247
+ def set_alignment_heads(self, dump: bytes):
248
+ array = np.frombuffer(
249
+ gzip.decompress(base64.b85decode(dump)), dtype=bool
250
+ ).copy()
251
+ mask = torch.from_numpy(array).reshape(
252
+ self.dims.n_text_layer, self.dims.n_text_head
253
+ )
254
+ self.register_buffer("alignment_heads", mask.to_sparse(), persistent=False)
255
+
256
+ def embed_audio(self, mel: torch.Tensor):
257
+ return self.encoder(mel)
258
+
259
+ def logits(self, tokens: torch.Tensor, audio_features: torch.Tensor):
260
+ return self.decoder(tokens, audio_features)
261
+
262
+ def forward(
263
+ self, mel: torch.Tensor, tokens: torch.Tensor
264
+ ) -> Dict[str, torch.Tensor]:
265
+ return self.decoder(tokens, self.encoder(mel))
266
+
267
+ @property
268
+ def device(self):
269
+ return next(self.parameters()).device
270
+
271
+ @property
272
+ def is_multilingual(self):
273
+ return self.dims.n_vocab >= 51865
274
+
275
+ @property
276
+ def num_languages(self):
277
+ return self.dims.n_vocab - 51765 - int(self.is_multilingual)
278
+
279
+ def install_kv_cache_hooks(self, cache: Optional[dict] = None):
280
+ """
281
+ The `MultiHeadAttention` module optionally accepts `kv_cache` which stores the key and value
282
+ tensors calculated for the previous positions. This method returns a dictionary that stores
283
+ all caches, and the necessary hooks for the key and value projection modules that save the
284
+ intermediate tensors to be reused during later calculations.
285
+
286
+ Returns
287
+ -------
288
+ cache : Dict[nn.Module, torch.Tensor]
289
+ A dictionary object mapping the key/value projection modules to its cache
290
+ hooks : List[RemovableHandle]
291
+ List of PyTorch RemovableHandle objects to stop the hooks to be called
292
+ """
293
+ cache = {**cache} if cache is not None else {}
294
+ hooks = []
295
+
296
+ def save_to_cache(module, _, output):
297
+ if module not in cache or output.shape[1] > self.dims.n_text_ctx:
298
+ # save as-is, for the first token or cross attention
299
+ cache[module] = output
300
+ else:
301
+ cache[module] = torch.cat([cache[module], output], dim=1).detach()
302
+ return cache[module]
303
+
304
+ def install_hooks(layer: nn.Module):
305
+ if isinstance(layer, MultiHeadAttention):
306
+ hooks.append(layer.key.register_forward_hook(save_to_cache))
307
+ hooks.append(layer.value.register_forward_hook(save_to_cache))
308
+
309
+ self.decoder.apply(install_hooks)
310
+ return cache, hooks
311
+
312
+ detect_language = detect_language_function
313
+ transcribe = transcribe_function
314
+ decode = decode_function
whisper/normalizers/english.json ADDED
@@ -0,0 +1,1741 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accessorise": "accessorize",
3
+ "accessorised": "accessorized",
4
+ "accessorises": "accessorizes",
5
+ "accessorising": "accessorizing",
6
+ "acclimatisation": "acclimatization",
7
+ "acclimatise": "acclimatize",
8
+ "acclimatised": "acclimatized",
9
+ "acclimatises": "acclimatizes",
10
+ "acclimatising": "acclimatizing",
11
+ "accoutrements": "accouterments",
12
+ "aeon": "eon",
13
+ "aeons": "eons",
14
+ "aerogramme": "aerogram",
15
+ "aerogrammes": "aerograms",
16
+ "aeroplane": "airplane",
17
+ "aeroplanes": "airplanes",
18
+ "aesthete": "esthete",
19
+ "aesthetes": "esthetes",
20
+ "aesthetic": "esthetic",
21
+ "aesthetically": "esthetically",
22
+ "aesthetics": "esthetics",
23
+ "aetiology": "etiology",
24
+ "ageing": "aging",
25
+ "aggrandisement": "aggrandizement",
26
+ "agonise": "agonize",
27
+ "agonised": "agonized",
28
+ "agonises": "agonizes",
29
+ "agonising": "agonizing",
30
+ "agonisingly": "agonizingly",
31
+ "almanack": "almanac",
32
+ "almanacks": "almanacs",
33
+ "aluminium": "aluminum",
34
+ "amortisable": "amortizable",
35
+ "amortisation": "amortization",
36
+ "amortisations": "amortizations",
37
+ "amortise": "amortize",
38
+ "amortised": "amortized",
39
+ "amortises": "amortizes",
40
+ "amortising": "amortizing",
41
+ "amphitheatre": "amphitheater",
42
+ "amphitheatres": "amphitheaters",
43
+ "anaemia": "anemia",
44
+ "anaemic": "anemic",
45
+ "anaesthesia": "anesthesia",
46
+ "anaesthetic": "anesthetic",
47
+ "anaesthetics": "anesthetics",
48
+ "anaesthetise": "anesthetize",
49
+ "anaesthetised": "anesthetized",
50
+ "anaesthetises": "anesthetizes",
51
+ "anaesthetising": "anesthetizing",
52
+ "anaesthetist": "anesthetist",
53
+ "anaesthetists": "anesthetists",
54
+ "anaesthetize": "anesthetize",
55
+ "anaesthetized": "anesthetized",
56
+ "anaesthetizes": "anesthetizes",
57
+ "anaesthetizing": "anesthetizing",
58
+ "analogue": "analog",
59
+ "analogues": "analogs",
60
+ "analyse": "analyze",
61
+ "analysed": "analyzed",
62
+ "analyses": "analyzes",
63
+ "analysing": "analyzing",
64
+ "anglicise": "anglicize",
65
+ "anglicised": "anglicized",
66
+ "anglicises": "anglicizes",
67
+ "anglicising": "anglicizing",
68
+ "annualised": "annualized",
69
+ "antagonise": "antagonize",
70
+ "antagonised": "antagonized",
71
+ "antagonises": "antagonizes",
72
+ "antagonising": "antagonizing",
73
+ "apologise": "apologize",
74
+ "apologised": "apologized",
75
+ "apologises": "apologizes",
76
+ "apologising": "apologizing",
77
+ "appal": "appall",
78
+ "appals": "appalls",
79
+ "appetiser": "appetizer",
80
+ "appetisers": "appetizers",
81
+ "appetising": "appetizing",
82
+ "appetisingly": "appetizingly",
83
+ "arbour": "arbor",
84
+ "arbours": "arbors",
85
+ "archeological": "archaeological",
86
+ "archaeologically": "archeologically",
87
+ "archaeologist": "archeologist",
88
+ "archaeologists": "archeologists",
89
+ "archaeology": "archeology</span>",
90
+ "ardour": "ardor",
91
+ "armour": "armor",
92
+ "armoured": "armored",
93
+ "armourer": "armorer",
94
+ "armourers": "armorers",
95
+ "armouries": "armories",
96
+ "armoury": "armory",
97
+ "artefact": "artifact",
98
+ "artefacts": "artifacts",
99
+ "authorise": "authorize",
100
+ "authorised": "authorized",
101
+ "authorises": "authorizes",
102
+ "authorising": "authorizing",
103
+ "axe": "ax",
104
+ "backpedalled": "backpedaled",
105
+ "backpedalling": "backpedaling",
106
+ "bannister": "banister",
107
+ "bannisters": "banisters",
108
+ "baptise": "baptize",
109
+ "baptised": "baptized",
110
+ "baptises": "baptizes",
111
+ "baptising": "baptizing",
112
+ "bastardise": "bastardize",
113
+ "bastardised": "bastardized",
114
+ "bastardises": "bastardizes",
115
+ "bastardising": "bastardizing",
116
+ "battleax": "battleaxe",
117
+ "baulk": "balk",
118
+ "baulked": "balked",
119
+ "baulking": "balking",
120
+ "baulks": "balks",
121
+ "bedevilled": "bedeviled",
122
+ "bedevilling": "bedeviling",
123
+ "behaviour": "behavior",
124
+ "behavioural": "behavioral",
125
+ "behaviourism": "behaviorism",
126
+ "behaviourist": "behaviorist",
127
+ "behaviourists": "behaviorists",
128
+ "behaviours": "behaviors",
129
+ "behove": "behoove",
130
+ "behoved": "behooved",
131
+ "behoves": "behooves",
132
+ "bejewelled": "bejeweled",
133
+ "belabour": "belabor",
134
+ "belaboured": "belabored",
135
+ "belabouring": "belaboring",
136
+ "belabours": "belabors",
137
+ "bevelled": "beveled",
138
+ "bevvies": "bevies",
139
+ "bevvy": "bevy",
140
+ "biassed": "biased",
141
+ "biassing": "biasing",
142
+ "bingeing": "binging",
143
+ "bougainvillaea": "bougainvillea",
144
+ "bougainvillaeas": "bougainvilleas",
145
+ "bowdlerise": "bowdlerize",
146
+ "bowdlerised": "bowdlerized",
147
+ "bowdlerises": "bowdlerizes",
148
+ "bowdlerising": "bowdlerizing",
149
+ "breathalyse": "breathalyze",
150
+ "breathalysed": "breathalyzed",
151
+ "breathalyser": "breathalyzer",
152
+ "breathalysers": "breathalyzers",
153
+ "breathalyses": "breathalyzes",
154
+ "breathalysing": "breathalyzing",
155
+ "brutalise": "brutalize",
156
+ "brutalised": "brutalized",
157
+ "brutalises": "brutalizes",
158
+ "brutalising": "brutalizing",
159
+ "busses": "buses",
160
+ "bussing": "busing",
161
+ "caesarean": "cesarean",
162
+ "caesareans": "cesareans",
163
+ "calibre": "caliber",
164
+ "calibres": "calibers",
165
+ "calliper": "caliper",
166
+ "callipers": "calipers",
167
+ "callisthenics": "calisthenics",
168
+ "canalise": "canalize",
169
+ "canalised": "canalized",
170
+ "canalises": "canalizes",
171
+ "canalising": "canalizing",
172
+ "cancelation": "cancellation",
173
+ "cancelations": "cancellations",
174
+ "cancelled": "canceled",
175
+ "cancelling": "canceling",
176
+ "candour": "candor",
177
+ "cannibalise": "cannibalize",
178
+ "cannibalised": "cannibalized",
179
+ "cannibalises": "cannibalizes",
180
+ "cannibalising": "cannibalizing",
181
+ "canonise": "canonize",
182
+ "canonised": "canonized",
183
+ "canonises": "canonizes",
184
+ "canonising": "canonizing",
185
+ "capitalise": "capitalize",
186
+ "capitalised": "capitalized",
187
+ "capitalises": "capitalizes",
188
+ "capitalising": "capitalizing",
189
+ "caramelise": "caramelize",
190
+ "caramelised": "caramelized",
191
+ "caramelises": "caramelizes",
192
+ "caramelising": "caramelizing",
193
+ "carbonise": "carbonize",
194
+ "carbonised": "carbonized",
195
+ "carbonises": "carbonizes",
196
+ "carbonising": "carbonizing",
197
+ "carolled": "caroled",
198
+ "carolling": "caroling",
199
+ "catalogue": "catalog",
200
+ "catalogued": "cataloged",
201
+ "catalogues": "catalogs",
202
+ "cataloguing": "cataloging",
203
+ "catalyse": "catalyze",
204
+ "catalysed": "catalyzed",
205
+ "catalyses": "catalyzes",
206
+ "catalysing": "catalyzing",
207
+ "categorise": "categorize",
208
+ "categorised": "categorized",
209
+ "categorises": "categorizes",
210
+ "categorising": "categorizing",
211
+ "cauterise": "cauterize",
212
+ "cauterised": "cauterized",
213
+ "cauterises": "cauterizes",
214
+ "cauterising": "cauterizing",
215
+ "cavilled": "caviled",
216
+ "cavilling": "caviling",
217
+ "centigramme": "centigram",
218
+ "centigrammes": "centigrams",
219
+ "centilitre": "centiliter",
220
+ "centilitres": "centiliters",
221
+ "centimetre": "centimeter",
222
+ "centimetres": "centimeters",
223
+ "centralise": "centralize",
224
+ "centralised": "centralized",
225
+ "centralises": "centralizes",
226
+ "centralising": "centralizing",
227
+ "centre": "center",
228
+ "centred": "centered",
229
+ "centrefold": "centerfold",
230
+ "centrefolds": "centerfolds",
231
+ "centrepiece": "centerpiece",
232
+ "centrepieces": "centerpieces",
233
+ "centres": "centers",
234
+ "channelled": "channeled",
235
+ "channelling": "channeling",
236
+ "characterise": "characterize",
237
+ "characterised": "characterized",
238
+ "characterises": "characterizes",
239
+ "characterising": "characterizing",
240
+ "cheque": "check",
241
+ "chequebook": "checkbook",
242
+ "chequebooks": "checkbooks",
243
+ "chequered": "checkered",
244
+ "cheques": "checks",
245
+ "chilli": "chili",
246
+ "chimaera": "chimera",
247
+ "chimaeras": "chimeras",
248
+ "chiselled": "chiseled",
249
+ "chiselling": "chiseling",
250
+ "circularise": "circularize",
251
+ "circularised": "circularized",
252
+ "circularises": "circularizes",
253
+ "circularising": "circularizing",
254
+ "civilise": "civilize",
255
+ "civilised": "civilized",
256
+ "civilises": "civilizes",
257
+ "civilising": "civilizing",
258
+ "clamour": "clamor",
259
+ "clamoured": "clamored",
260
+ "clamouring": "clamoring",
261
+ "clamours": "clamors",
262
+ "clangour": "clangor",
263
+ "clarinettist": "clarinetist",
264
+ "clarinettists": "clarinetists",
265
+ "collectivise": "collectivize",
266
+ "collectivised": "collectivized",
267
+ "collectivises": "collectivizes",
268
+ "collectivising": "collectivizing",
269
+ "colonisation": "colonization",
270
+ "colonise": "colonize",
271
+ "colonised": "colonized",
272
+ "coloniser": "colonizer",
273
+ "colonisers": "colonizers",
274
+ "colonises": "colonizes",
275
+ "colonising": "colonizing",
276
+ "colour": "color",
277
+ "colourant": "colorant",
278
+ "colourants": "colorants",
279
+ "coloured": "colored",
280
+ "coloureds": "coloreds",
281
+ "colourful": "colorful",
282
+ "colourfully": "colorfully",
283
+ "colouring": "coloring",
284
+ "colourize": "colorize",
285
+ "colourized": "colorized",
286
+ "colourizes": "colorizes",
287
+ "colourizing": "colorizing",
288
+ "colourless": "colorless",
289
+ "colours": "colors",
290
+ "commercialise": "commercialize",
291
+ "commercialised": "commercialized",
292
+ "commercialises": "commercializes",
293
+ "commercialising": "commercializing",
294
+ "compartmentalise": "compartmentalize",
295
+ "compartmentalised": "compartmentalized",
296
+ "compartmentalises": "compartmentalizes",
297
+ "compartmentalising": "compartmentalizing",
298
+ "computerise": "computerize",
299
+ "computerised": "computerized",
300
+ "computerises": "computerizes",
301
+ "computerising": "computerizing",
302
+ "conceptualise": "conceptualize",
303
+ "conceptualised": "conceptualized",
304
+ "conceptualises": "conceptualizes",
305
+ "conceptualising": "conceptualizing",
306
+ "connexion": "connection",
307
+ "connexions": "connections",
308
+ "contextualise": "contextualize",
309
+ "contextualised": "contextualized",
310
+ "contextualises": "contextualizes",
311
+ "contextualising": "contextualizing",
312
+ "cosier": "cozier",
313
+ "cosies": "cozies",
314
+ "cosiest": "coziest",
315
+ "cosily": "cozily",
316
+ "cosiness": "coziness",
317
+ "cosy": "cozy",
318
+ "councillor": "councilor",
319
+ "councillors": "councilors",
320
+ "counselled": "counseled",
321
+ "counselling": "counseling",
322
+ "counsellor": "counselor",
323
+ "counsellors": "counselors",
324
+ "crenelated": "crenellated",
325
+ "criminalise": "criminalize",
326
+ "criminalised": "criminalized",
327
+ "criminalises": "criminalizes",
328
+ "criminalising": "criminalizing",
329
+ "criticise": "criticize",
330
+ "criticised": "criticized",
331
+ "criticises": "criticizes",
332
+ "criticising": "criticizing",
333
+ "crueller": "crueler",
334
+ "cruellest": "cruelest",
335
+ "crystallisation": "crystallization",
336
+ "crystallise": "crystallize",
337
+ "crystallised": "crystallized",
338
+ "crystallises": "crystallizes",
339
+ "crystallising": "crystallizing",
340
+ "cudgelled": "cudgeled",
341
+ "cudgelling": "cudgeling",
342
+ "customise": "customize",
343
+ "customised": "customized",
344
+ "customises": "customizes",
345
+ "customising": "customizing",
346
+ "cypher": "cipher",
347
+ "cyphers": "ciphers",
348
+ "decentralisation": "decentralization",
349
+ "decentralise": "decentralize",
350
+ "decentralised": "decentralized",
351
+ "decentralises": "decentralizes",
352
+ "decentralising": "decentralizing",
353
+ "decriminalisation": "decriminalization",
354
+ "decriminalise": "decriminalize",
355
+ "decriminalised": "decriminalized",
356
+ "decriminalises": "decriminalizes",
357
+ "decriminalising": "decriminalizing",
358
+ "defence": "defense",
359
+ "defenceless": "defenseless",
360
+ "defences": "defenses",
361
+ "dehumanisation": "dehumanization",
362
+ "dehumanise": "dehumanize",
363
+ "dehumanised": "dehumanized",
364
+ "dehumanises": "dehumanizes",
365
+ "dehumanising": "dehumanizing",
366
+ "demeanour": "demeanor",
367
+ "demilitarisation": "demilitarization",
368
+ "demilitarise": "demilitarize",
369
+ "demilitarised": "demilitarized",
370
+ "demilitarises": "demilitarizes",
371
+ "demilitarising": "demilitarizing",
372
+ "demobilisation": "demobilization",
373
+ "demobilise": "demobilize",
374
+ "demobilised": "demobilized",
375
+ "demobilises": "demobilizes",
376
+ "demobilising": "demobilizing",
377
+ "democratisation": "democratization",
378
+ "democratise": "democratize",
379
+ "democratised": "democratized",
380
+ "democratises": "democratizes",
381
+ "democratising": "democratizing",
382
+ "demonise": "demonize",
383
+ "demonised": "demonized",
384
+ "demonises": "demonizes",
385
+ "demonising": "demonizing",
386
+ "demoralisation": "demoralization",
387
+ "demoralise": "demoralize",
388
+ "demoralised": "demoralized",
389
+ "demoralises": "demoralizes",
390
+ "demoralising": "demoralizing",
391
+ "denationalisation": "denationalization",
392
+ "denationalise": "denationalize",
393
+ "denationalised": "denationalized",
394
+ "denationalises": "denationalizes",
395
+ "denationalising": "denationalizing",
396
+ "deodorise": "deodorize",
397
+ "deodorised": "deodorized",
398
+ "deodorises": "deodorizes",
399
+ "deodorising": "deodorizing",
400
+ "depersonalise": "depersonalize",
401
+ "depersonalised": "depersonalized",
402
+ "depersonalises": "depersonalizes",
403
+ "depersonalising": "depersonalizing",
404
+ "deputise": "deputize",
405
+ "deputised": "deputized",
406
+ "deputises": "deputizes",
407
+ "deputising": "deputizing",
408
+ "desensitisation": "desensitization",
409
+ "desensitise": "desensitize",
410
+ "desensitised": "desensitized",
411
+ "desensitises": "desensitizes",
412
+ "desensitising": "desensitizing",
413
+ "destabilisation": "destabilization",
414
+ "destabilise": "destabilize",
415
+ "destabilised": "destabilized",
416
+ "destabilises": "destabilizes",
417
+ "destabilising": "destabilizing",
418
+ "dialled": "dialed",
419
+ "dialling": "dialing",
420
+ "dialogue": "dialog",
421
+ "dialogues": "dialogs",
422
+ "diarrhoea": "diarrhea",
423
+ "digitise": "digitize",
424
+ "digitised": "digitized",
425
+ "digitises": "digitizes",
426
+ "digitising": "digitizing",
427
+ "disc": "disk",
428
+ "discolour": "discolor",
429
+ "discoloured": "discolored",
430
+ "discolouring": "discoloring",
431
+ "discolours": "discolors",
432
+ "discs": "disks",
433
+ "disembowelled": "disemboweled",
434
+ "disembowelling": "disemboweling",
435
+ "disfavour": "disfavor",
436
+ "dishevelled": "disheveled",
437
+ "dishonour": "dishonor",
438
+ "dishonourable": "dishonorable",
439
+ "dishonourably": "dishonorably",
440
+ "dishonoured": "dishonored",
441
+ "dishonouring": "dishonoring",
442
+ "dishonours": "dishonors",
443
+ "disorganisation": "disorganization",
444
+ "disorganised": "disorganized",
445
+ "distil": "distill",
446
+ "distils": "distills",
447
+ "dramatisation": "dramatization",
448
+ "dramatisations": "dramatizations",
449
+ "dramatise": "dramatize",
450
+ "dramatised": "dramatized",
451
+ "dramatises": "dramatizes",
452
+ "dramatising": "dramatizing",
453
+ "draught": "draft",
454
+ "draughtboard": "draftboard",
455
+ "draughtboards": "draftboards",
456
+ "draughtier": "draftier",
457
+ "draughtiest": "draftiest",
458
+ "draughts": "drafts",
459
+ "draughtsman": "draftsman",
460
+ "draughtsmanship": "draftsmanship",
461
+ "draughtsmen": "draftsmen",
462
+ "draughtswoman": "draftswoman",
463
+ "draughtswomen": "draftswomen",
464
+ "draughty": "drafty",
465
+ "drivelled": "driveled",
466
+ "drivelling": "driveling",
467
+ "duelled": "dueled",
468
+ "duelling": "dueling",
469
+ "economise": "economize",
470
+ "economised": "economized",
471
+ "economises": "economizes",
472
+ "economising": "economizing",
473
+ "edoema": "edema",
474
+ "editorialise": "editorialize",
475
+ "editorialised": "editorialized",
476
+ "editorialises": "editorializes",
477
+ "editorialising": "editorializing",
478
+ "empathise": "empathize",
479
+ "empathised": "empathized",
480
+ "empathises": "empathizes",
481
+ "empathising": "empathizing",
482
+ "emphasise": "emphasize",
483
+ "emphasised": "emphasized",
484
+ "emphasises": "emphasizes",
485
+ "emphasising": "emphasizing",
486
+ "enamelled": "enameled",
487
+ "enamelling": "enameling",
488
+ "enamoured": "enamored",
489
+ "encyclopaedia": "encyclopedia",
490
+ "encyclopaedias": "encyclopedias",
491
+ "encyclopaedic": "encyclopedic",
492
+ "endeavour": "endeavor",
493
+ "endeavoured": "endeavored",
494
+ "endeavouring": "endeavoring",
495
+ "endeavours": "endeavors",
496
+ "energise": "energize",
497
+ "energised": "energized",
498
+ "energises": "energizes",
499
+ "energising": "energizing",
500
+ "enrol": "enroll",
501
+ "enrols": "enrolls",
502
+ "enthral": "enthrall",
503
+ "enthrals": "enthralls",
504
+ "epaulette": "epaulet",
505
+ "epaulettes": "epaulets",
506
+ "epicentre": "epicenter",
507
+ "epicentres": "epicenters",
508
+ "epilogue": "epilog",
509
+ "epilogues": "epilogs",
510
+ "epitomise": "epitomize",
511
+ "epitomised": "epitomized",
512
+ "epitomises": "epitomizes",
513
+ "epitomising": "epitomizing",
514
+ "equalisation": "equalization",
515
+ "equalise": "equalize",
516
+ "equalised": "equalized",
517
+ "equaliser": "equalizer",
518
+ "equalisers": "equalizers",
519
+ "equalises": "equalizes",
520
+ "equalising": "equalizing",
521
+ "eulogise": "eulogize",
522
+ "eulogised": "eulogized",
523
+ "eulogises": "eulogizes",
524
+ "eulogising": "eulogizing",
525
+ "evangelise": "evangelize",
526
+ "evangelised": "evangelized",
527
+ "evangelises": "evangelizes",
528
+ "evangelising": "evangelizing",
529
+ "exorcise": "exorcize",
530
+ "exorcised": "exorcized",
531
+ "exorcises": "exorcizes",
532
+ "exorcising": "exorcizing",
533
+ "extemporisation": "extemporization",
534
+ "extemporise": "extemporize",
535
+ "extemporised": "extemporized",
536
+ "extemporises": "extemporizes",
537
+ "extemporising": "extemporizing",
538
+ "externalisation": "externalization",
539
+ "externalisations": "externalizations",
540
+ "externalise": "externalize",
541
+ "externalised": "externalized",
542
+ "externalises": "externalizes",
543
+ "externalising": "externalizing",
544
+ "factorise": "factorize",
545
+ "factorised": "factorized",
546
+ "factorises": "factorizes",
547
+ "factorising": "factorizing",
548
+ "faecal": "fecal",
549
+ "faeces": "feces",
550
+ "familiarisation": "familiarization",
551
+ "familiarise": "familiarize",
552
+ "familiarised": "familiarized",
553
+ "familiarises": "familiarizes",
554
+ "familiarising": "familiarizing",
555
+ "fantasise": "fantasize",
556
+ "fantasised": "fantasized",
557
+ "fantasises": "fantasizes",
558
+ "fantasising": "fantasizing",
559
+ "favour": "favor",
560
+ "favourable": "favorable",
561
+ "favourably": "favorably",
562
+ "favoured": "favored",
563
+ "favouring": "favoring",
564
+ "favourite": "favorite",
565
+ "favourites": "favorites",
566
+ "favouritism": "favoritism",
567
+ "favours": "favors",
568
+ "feminise": "feminize",
569
+ "feminised": "feminized",
570
+ "feminises": "feminizes",
571
+ "feminising": "feminizing",
572
+ "fertilisation": "fertilization",
573
+ "fertilise": "fertilize",
574
+ "fertilised": "fertilized",
575
+ "fertiliser": "fertilizer",
576
+ "fertilisers": "fertilizers",
577
+ "fertilises": "fertilizes",
578
+ "fertilising": "fertilizing",
579
+ "fervour": "fervor",
580
+ "fibre": "fiber",
581
+ "fibreglass": "fiberglass",
582
+ "fibres": "fibers",
583
+ "fictionalisation": "fictionalization",
584
+ "fictionalisations": "fictionalizations",
585
+ "fictionalise": "fictionalize",
586
+ "fictionalised": "fictionalized",
587
+ "fictionalises": "fictionalizes",
588
+ "fictionalising": "fictionalizing",
589
+ "fillet": "filet",
590
+ "filleted": "fileted",
591
+ "filleting": "fileting",
592
+ "fillets": "filets",
593
+ "finalisation": "finalization",
594
+ "finalise": "finalize",
595
+ "finalised": "finalized",
596
+ "finalises": "finalizes",
597
+ "finalising": "finalizing",
598
+ "flautist": "flutist",
599
+ "flautists": "flutists",
600
+ "flavour": "flavor",
601
+ "flavoured": "flavored",
602
+ "flavouring": "flavoring",
603
+ "flavourings": "flavorings",
604
+ "flavourless": "flavorless",
605
+ "flavours": "flavors",
606
+ "flavoursome": "flavorsome",
607
+ "flyer / flier": "flier / flyer",
608
+ "foetal": "fetal",
609
+ "foetid": "fetid",
610
+ "foetus": "fetus",
611
+ "foetuses": "fetuses",
612
+ "formalisation": "formalization",
613
+ "formalise": "formalize",
614
+ "formalised": "formalized",
615
+ "formalises": "formalizes",
616
+ "formalising": "formalizing",
617
+ "fossilisation": "fossilization",
618
+ "fossilise": "fossilize",
619
+ "fossilised": "fossilized",
620
+ "fossilises": "fossilizes",
621
+ "fossilising": "fossilizing",
622
+ "fraternisation": "fraternization",
623
+ "fraternise": "fraternize",
624
+ "fraternised": "fraternized",
625
+ "fraternises": "fraternizes",
626
+ "fraternising": "fraternizing",
627
+ "fulfil": "fulfill",
628
+ "fulfilment": "fulfillment",
629
+ "fulfils": "fulfills",
630
+ "funnelled": "funneled",
631
+ "funnelling": "funneling",
632
+ "galvanise": "galvanize",
633
+ "galvanised": "galvanized",
634
+ "galvanises": "galvanizes",
635
+ "galvanising": "galvanizing",
636
+ "gambolled": "gamboled",
637
+ "gambolling": "gamboling",
638
+ "gaol": "jail",
639
+ "gaolbird": "jailbird",
640
+ "gaolbirds": "jailbirds",
641
+ "gaolbreak": "jailbreak",
642
+ "gaolbreaks": "jailbreaks",
643
+ "gaoled": "jailed",
644
+ "gaoler": "jailer",
645
+ "gaolers": "jailers",
646
+ "gaoling": "jailing",
647
+ "gaols": "jails",
648
+ "gasses": "gases",
649
+ "gage": "gauge",
650
+ "gaged": "gauged",
651
+ "gages": "gauges",
652
+ "gaging": "gauging",
653
+ "generalisation": "generalization",
654
+ "generalisations": "generalizations",
655
+ "generalise": "generalize",
656
+ "generalised": "generalized",
657
+ "generalises": "generalizes",
658
+ "generalising": "generalizing",
659
+ "ghettoise": "ghettoize",
660
+ "ghettoised": "ghettoized",
661
+ "ghettoises": "ghettoizes",
662
+ "ghettoising": "ghettoizing",
663
+ "gipsies": "gypsies",
664
+ "glamorise": "glamorize",
665
+ "glamorised": "glamorized",
666
+ "glamorises": "glamorizes",
667
+ "glamorising": "glamorizing",
668
+ "glamor": "glamour",
669
+ "globalisation": "globalization",
670
+ "globalise": "globalize",
671
+ "globalised": "globalized",
672
+ "globalises": "globalizes",
673
+ "globalising": "globalizing",
674
+ "glueing": "gluing",
675
+ "goitre": "goiter",
676
+ "goitres": "goiters",
677
+ "gonorrhoea": "gonorrhea",
678
+ "gramme": "gram",
679
+ "grammes": "grams",
680
+ "gravelled": "graveled",
681
+ "grey": "gray",
682
+ "greyed": "grayed",
683
+ "greying": "graying",
684
+ "greyish": "grayish",
685
+ "greyness": "grayness",
686
+ "greys": "grays",
687
+ "grovelled": "groveled",
688
+ "grovelling": "groveling",
689
+ "groyne": "groin",
690
+ "groynes": "groins",
691
+ "gruelling": "grueling",
692
+ "gruellingly": "gruelingly",
693
+ "gryphon": "griffin",
694
+ "gryphons": "griffins",
695
+ "gynaecological": "gynecological",
696
+ "gynaecologist": "gynecologist",
697
+ "gynaecologists": "gynecologists",
698
+ "gynaecology": "gynecology",
699
+ "haematological": "hematological",
700
+ "haematologist": "hematologist",
701
+ "haematologists": "hematologists",
702
+ "haematology": "hematology",
703
+ "haemoglobin": "hemoglobin",
704
+ "haemophilia": "hemophilia",
705
+ "haemophiliac": "hemophiliac",
706
+ "haemophiliacs": "hemophiliacs",
707
+ "haemorrhage": "hemorrhage",
708
+ "haemorrhaged": "hemorrhaged",
709
+ "haemorrhages": "hemorrhages",
710
+ "haemorrhaging": "hemorrhaging",
711
+ "haemorrhoids": "hemorrhoids",
712
+ "harbour": "harbor",
713
+ "harboured": "harbored",
714
+ "harbouring": "harboring",
715
+ "harbours": "harbors",
716
+ "harmonisation": "harmonization",
717
+ "harmonise": "harmonize",
718
+ "harmonised": "harmonized",
719
+ "harmonises": "harmonizes",
720
+ "harmonising": "harmonizing",
721
+ "homoeopath": "homeopath",
722
+ "homoeopathic": "homeopathic",
723
+ "homoeopaths": "homeopaths",
724
+ "homoeopathy": "homeopathy",
725
+ "homogenise": "homogenize",
726
+ "homogenised": "homogenized",
727
+ "homogenises": "homogenizes",
728
+ "homogenising": "homogenizing",
729
+ "honour": "honor",
730
+ "honourable": "honorable",
731
+ "honourably": "honorably",
732
+ "honoured": "honored",
733
+ "honouring": "honoring",
734
+ "honours": "honors",
735
+ "hospitalisation": "hospitalization",
736
+ "hospitalise": "hospitalize",
737
+ "hospitalised": "hospitalized",
738
+ "hospitalises": "hospitalizes",
739
+ "hospitalising": "hospitalizing",
740
+ "humanise": "humanize",
741
+ "humanised": "humanized",
742
+ "humanises": "humanizes",
743
+ "humanising": "humanizing",
744
+ "humour": "humor",
745
+ "humoured": "humored",
746
+ "humouring": "humoring",
747
+ "humourless": "humorless",
748
+ "humours": "humors",
749
+ "hybridise": "hybridize",
750
+ "hybridised": "hybridized",
751
+ "hybridises": "hybridizes",
752
+ "hybridising": "hybridizing",
753
+ "hypnotise": "hypnotize",
754
+ "hypnotised": "hypnotized",
755
+ "hypnotises": "hypnotizes",
756
+ "hypnotising": "hypnotizing",
757
+ "hypothesise": "hypothesize",
758
+ "hypothesised": "hypothesized",
759
+ "hypothesises": "hypothesizes",
760
+ "hypothesising": "hypothesizing",
761
+ "idealisation": "idealization",
762
+ "idealise": "idealize",
763
+ "idealised": "idealized",
764
+ "idealises": "idealizes",
765
+ "idealising": "idealizing",
766
+ "idolise": "idolize",
767
+ "idolised": "idolized",
768
+ "idolises": "idolizes",
769
+ "idolising": "idolizing",
770
+ "immobilisation": "immobilization",
771
+ "immobilise": "immobilize",
772
+ "immobilised": "immobilized",
773
+ "immobiliser": "immobilizer",
774
+ "immobilisers": "immobilizers",
775
+ "immobilises": "immobilizes",
776
+ "immobilising": "immobilizing",
777
+ "immortalise": "immortalize",
778
+ "immortalised": "immortalized",
779
+ "immortalises": "immortalizes",
780
+ "immortalising": "immortalizing",
781
+ "immunisation": "immunization",
782
+ "immunise": "immunize",
783
+ "immunised": "immunized",
784
+ "immunises": "immunizes",
785
+ "immunising": "immunizing",
786
+ "impanelled": "impaneled",
787
+ "impanelling": "impaneling",
788
+ "imperilled": "imperiled",
789
+ "imperilling": "imperiling",
790
+ "individualise": "individualize",
791
+ "individualised": "individualized",
792
+ "individualises": "individualizes",
793
+ "individualising": "individualizing",
794
+ "industrialise": "industrialize",
795
+ "industrialised": "industrialized",
796
+ "industrialises": "industrializes",
797
+ "industrialising": "industrializing",
798
+ "inflexion": "inflection",
799
+ "inflexions": "inflections",
800
+ "initialise": "initialize",
801
+ "initialised": "initialized",
802
+ "initialises": "initializes",
803
+ "initialising": "initializing",
804
+ "initialled": "initialed",
805
+ "initialling": "initialing",
806
+ "instal": "install",
807
+ "instalment": "installment",
808
+ "instalments": "installments",
809
+ "instals": "installs",
810
+ "instil": "instill",
811
+ "instils": "instills",
812
+ "institutionalisation": "institutionalization",
813
+ "institutionalise": "institutionalize",
814
+ "institutionalised": "institutionalized",
815
+ "institutionalises": "institutionalizes",
816
+ "institutionalising": "institutionalizing",
817
+ "intellectualise": "intellectualize",
818
+ "intellectualised": "intellectualized",
819
+ "intellectualises": "intellectualizes",
820
+ "intellectualising": "intellectualizing",
821
+ "internalisation": "internalization",
822
+ "internalise": "internalize",
823
+ "internalised": "internalized",
824
+ "internalises": "internalizes",
825
+ "internalising": "internalizing",
826
+ "internationalisation": "internationalization",
827
+ "internationalise": "internationalize",
828
+ "internationalised": "internationalized",
829
+ "internationalises": "internationalizes",
830
+ "internationalising": "internationalizing",
831
+ "ionisation": "ionization",
832
+ "ionise": "ionize",
833
+ "ionised": "ionized",
834
+ "ioniser": "ionizer",
835
+ "ionisers": "ionizers",
836
+ "ionises": "ionizes",
837
+ "ionising": "ionizing",
838
+ "italicise": "italicize",
839
+ "italicised": "italicized",
840
+ "italicises": "italicizes",
841
+ "italicising": "italicizing",
842
+ "itemise": "itemize",
843
+ "itemised": "itemized",
844
+ "itemises": "itemizes",
845
+ "itemising": "itemizing",
846
+ "jeopardise": "jeopardize",
847
+ "jeopardised": "jeopardized",
848
+ "jeopardises": "jeopardizes",
849
+ "jeopardising": "jeopardizing",
850
+ "jewelled": "jeweled",
851
+ "jeweller": "jeweler",
852
+ "jewellers": "jewelers",
853
+ "jewellery": "jewelry",
854
+ "judgement": "judgment",
855
+ "kilogramme": "kilogram",
856
+ "kilogrammes": "kilograms",
857
+ "kilometre": "kilometer",
858
+ "kilometres": "kilometers",
859
+ "labelled": "labeled",
860
+ "labelling": "labeling",
861
+ "labour": "labor",
862
+ "laboured": "labored",
863
+ "labourer": "laborer",
864
+ "labourers": "laborers",
865
+ "labouring": "laboring",
866
+ "labours": "labors",
867
+ "lacklustre": "lackluster",
868
+ "legalisation": "legalization",
869
+ "legalise": "legalize",
870
+ "legalised": "legalized",
871
+ "legalises": "legalizes",
872
+ "legalising": "legalizing",
873
+ "legitimise": "legitimize",
874
+ "legitimised": "legitimized",
875
+ "legitimises": "legitimizes",
876
+ "legitimising": "legitimizing",
877
+ "leukaemia": "leukemia",
878
+ "levelled": "leveled",
879
+ "leveller": "leveler",
880
+ "levellers": "levelers",
881
+ "levelling": "leveling",
882
+ "libelled": "libeled",
883
+ "libelling": "libeling",
884
+ "libellous": "libelous",
885
+ "liberalisation": "liberalization",
886
+ "liberalise": "liberalize",
887
+ "liberalised": "liberalized",
888
+ "liberalises": "liberalizes",
889
+ "liberalising": "liberalizing",
890
+ "licence": "license",
891
+ "licenced": "licensed",
892
+ "licences": "licenses",
893
+ "licencing": "licensing",
894
+ "likeable": "likable",
895
+ "lionisation": "lionization",
896
+ "lionise": "lionize",
897
+ "lionised": "lionized",
898
+ "lionises": "lionizes",
899
+ "lionising": "lionizing",
900
+ "liquidise": "liquidize",
901
+ "liquidised": "liquidized",
902
+ "liquidiser": "liquidizer",
903
+ "liquidisers": "liquidizers",
904
+ "liquidises": "liquidizes",
905
+ "liquidising": "liquidizing",
906
+ "litre": "liter",
907
+ "litres": "liters",
908
+ "localise": "localize",
909
+ "localised": "localized",
910
+ "localises": "localizes",
911
+ "localising": "localizing",
912
+ "louvre": "louver",
913
+ "louvred": "louvered",
914
+ "louvres": "louvers",
915
+ "lustre": "luster",
916
+ "magnetise": "magnetize",
917
+ "magnetised": "magnetized",
918
+ "magnetises": "magnetizes",
919
+ "magnetising": "magnetizing",
920
+ "manoeuvrability": "maneuverability",
921
+ "manoeuvrable": "maneuverable",
922
+ "manoeuvre": "maneuver",
923
+ "manoeuvred": "maneuvered",
924
+ "manoeuvres": "maneuvers",
925
+ "manoeuvring": "maneuvering",
926
+ "manoeuvrings": "maneuverings",
927
+ "marginalisation": "marginalization",
928
+ "marginalise": "marginalize",
929
+ "marginalised": "marginalized",
930
+ "marginalises": "marginalizes",
931
+ "marginalising": "marginalizing",
932
+ "marshalled": "marshaled",
933
+ "marshalling": "marshaling",
934
+ "marvelled": "marveled",
935
+ "marvelling": "marveling",
936
+ "marvellous": "marvelous",
937
+ "marvellously": "marvelously",
938
+ "materialisation": "materialization",
939
+ "materialise": "materialize",
940
+ "materialised": "materialized",
941
+ "materialises": "materializes",
942
+ "materialising": "materializing",
943
+ "maximisation": "maximization",
944
+ "maximise": "maximize",
945
+ "maximised": "maximized",
946
+ "maximises": "maximizes",
947
+ "maximising": "maximizing",
948
+ "meagre": "meager",
949
+ "mechanisation": "mechanization",
950
+ "mechanise": "mechanize",
951
+ "mechanised": "mechanized",
952
+ "mechanises": "mechanizes",
953
+ "mechanising": "mechanizing",
954
+ "mediaeval": "medieval",
955
+ "memorialise": "memorialize",
956
+ "memorialised": "memorialized",
957
+ "memorialises": "memorializes",
958
+ "memorialising": "memorializing",
959
+ "memorise": "memorize",
960
+ "memorised": "memorized",
961
+ "memorises": "memorizes",
962
+ "memorising": "memorizing",
963
+ "mesmerise": "mesmerize",
964
+ "mesmerised": "mesmerized",
965
+ "mesmerises": "mesmerizes",
966
+ "mesmerising": "mesmerizing",
967
+ "metabolise": "metabolize",
968
+ "metabolised": "metabolized",
969
+ "metabolises": "metabolizes",
970
+ "metabolising": "metabolizing",
971
+ "metre": "meter",
972
+ "metres": "meters",
973
+ "micrometre": "micrometer",
974
+ "micrometres": "micrometers",
975
+ "militarise": "militarize",
976
+ "militarised": "militarized",
977
+ "militarises": "militarizes",
978
+ "militarising": "militarizing",
979
+ "milligramme": "milligram",
980
+ "milligrammes": "milligrams",
981
+ "millilitre": "milliliter",
982
+ "millilitres": "milliliters",
983
+ "millimetre": "millimeter",
984
+ "millimetres": "millimeters",
985
+ "miniaturisation": "miniaturization",
986
+ "miniaturise": "miniaturize",
987
+ "miniaturised": "miniaturized",
988
+ "miniaturises": "miniaturizes",
989
+ "miniaturising": "miniaturizing",
990
+ "minibusses": "minibuses",
991
+ "minimise": "minimize",
992
+ "minimised": "minimized",
993
+ "minimises": "minimizes",
994
+ "minimising": "minimizing",
995
+ "misbehaviour": "misbehavior",
996
+ "misdemeanour": "misdemeanor",
997
+ "misdemeanours": "misdemeanors",
998
+ "misspelt": "misspelled",
999
+ "mitre": "miter",
1000
+ "mitres": "miters",
1001
+ "mobilisation": "mobilization",
1002
+ "mobilise": "mobilize",
1003
+ "mobilised": "mobilized",
1004
+ "mobilises": "mobilizes",
1005
+ "mobilising": "mobilizing",
1006
+ "modelled": "modeled",
1007
+ "modeller": "modeler",
1008
+ "modellers": "modelers",
1009
+ "modelling": "modeling",
1010
+ "modernise": "modernize",
1011
+ "modernised": "modernized",
1012
+ "modernises": "modernizes",
1013
+ "modernising": "modernizing",
1014
+ "moisturise": "moisturize",
1015
+ "moisturised": "moisturized",
1016
+ "moisturiser": "moisturizer",
1017
+ "moisturisers": "moisturizers",
1018
+ "moisturises": "moisturizes",
1019
+ "moisturising": "moisturizing",
1020
+ "monologue": "monolog",
1021
+ "monologues": "monologs",
1022
+ "monopolisation": "monopolization",
1023
+ "monopolise": "monopolize",
1024
+ "monopolised": "monopolized",
1025
+ "monopolises": "monopolizes",
1026
+ "monopolising": "monopolizing",
1027
+ "moralise": "moralize",
1028
+ "moralised": "moralized",
1029
+ "moralises": "moralizes",
1030
+ "moralising": "moralizing",
1031
+ "motorised": "motorized",
1032
+ "mould": "mold",
1033
+ "moulded": "molded",
1034
+ "moulder": "molder",
1035
+ "mouldered": "moldered",
1036
+ "mouldering": "moldering",
1037
+ "moulders": "molders",
1038
+ "mouldier": "moldier",
1039
+ "mouldiest": "moldiest",
1040
+ "moulding": "molding",
1041
+ "mouldings": "moldings",
1042
+ "moulds": "molds",
1043
+ "mouldy": "moldy",
1044
+ "moult": "molt",
1045
+ "moulted": "molted",
1046
+ "moulting": "molting",
1047
+ "moults": "molts",
1048
+ "moustache": "mustache",
1049
+ "moustached": "mustached",
1050
+ "moustaches": "mustaches",
1051
+ "moustachioed": "mustachioed",
1052
+ "multicoloured": "multicolored",
1053
+ "nationalisation": "nationalization",
1054
+ "nationalisations": "nationalizations",
1055
+ "nationalise": "nationalize",
1056
+ "nationalised": "nationalized",
1057
+ "nationalises": "nationalizes",
1058
+ "nationalising": "nationalizing",
1059
+ "naturalisation": "naturalization",
1060
+ "naturalise": "naturalize",
1061
+ "naturalised": "naturalized",
1062
+ "naturalises": "naturalizes",
1063
+ "naturalising": "naturalizing",
1064
+ "neighbour": "neighbor",
1065
+ "neighbourhood": "neighborhood",
1066
+ "neighbourhoods": "neighborhoods",
1067
+ "neighbouring": "neighboring",
1068
+ "neighbourliness": "neighborliness",
1069
+ "neighbourly": "neighborly",
1070
+ "neighbours": "neighbors",
1071
+ "neutralisation": "neutralization",
1072
+ "neutralise": "neutralize",
1073
+ "neutralised": "neutralized",
1074
+ "neutralises": "neutralizes",
1075
+ "neutralising": "neutralizing",
1076
+ "normalisation": "normalization",
1077
+ "normalise": "normalize",
1078
+ "normalised": "normalized",
1079
+ "normalises": "normalizes",
1080
+ "normalising": "normalizing",
1081
+ "odour": "odor",
1082
+ "odourless": "odorless",
1083
+ "odours": "odors",
1084
+ "oesophagus": "esophagus",
1085
+ "oesophaguses": "esophaguses",
1086
+ "oestrogen": "estrogen",
1087
+ "offence": "offense",
1088
+ "offences": "offenses",
1089
+ "omelette": "omelet",
1090
+ "omelettes": "omelets",
1091
+ "optimise": "optimize",
1092
+ "optimised": "optimized",
1093
+ "optimises": "optimizes",
1094
+ "optimising": "optimizing",
1095
+ "organisation": "organization",
1096
+ "organisational": "organizational",
1097
+ "organisations": "organizations",
1098
+ "organise": "organize",
1099
+ "organised": "organized",
1100
+ "organiser": "organizer",
1101
+ "organisers": "organizers",
1102
+ "organises": "organizes",
1103
+ "organising": "organizing",
1104
+ "orthopaedic": "orthopedic",
1105
+ "orthopaedics": "orthopedics",
1106
+ "ostracise": "ostracize",
1107
+ "ostracised": "ostracized",
1108
+ "ostracises": "ostracizes",
1109
+ "ostracising": "ostracizing",
1110
+ "outmanoeuvre": "outmaneuver",
1111
+ "outmanoeuvred": "outmaneuvered",
1112
+ "outmanoeuvres": "outmaneuvers",
1113
+ "outmanoeuvring": "outmaneuvering",
1114
+ "overemphasise": "overemphasize",
1115
+ "overemphasised": "overemphasized",
1116
+ "overemphasises": "overemphasizes",
1117
+ "overemphasising": "overemphasizing",
1118
+ "oxidisation": "oxidization",
1119
+ "oxidise": "oxidize",
1120
+ "oxidised": "oxidized",
1121
+ "oxidises": "oxidizes",
1122
+ "oxidising": "oxidizing",
1123
+ "paederast": "pederast",
1124
+ "paederasts": "pederasts",
1125
+ "paediatric": "pediatric",
1126
+ "paediatrician": "pediatrician",
1127
+ "paediatricians": "pediatricians",
1128
+ "paediatrics": "pediatrics",
1129
+ "paedophile": "pedophile",
1130
+ "paedophiles": "pedophiles",
1131
+ "paedophilia": "pedophilia",
1132
+ "palaeolithic": "paleolithic",
1133
+ "palaeontologist": "paleontologist",
1134
+ "palaeontologists": "paleontologists",
1135
+ "palaeontology": "paleontology",
1136
+ "panelled": "paneled",
1137
+ "panelling": "paneling",
1138
+ "panellist": "panelist",
1139
+ "panellists": "panelists",
1140
+ "paralyse": "paralyze",
1141
+ "paralysed": "paralyzed",
1142
+ "paralyses": "paralyzes",
1143
+ "paralysing": "paralyzing",
1144
+ "parcelled": "parceled",
1145
+ "parcelling": "parceling",
1146
+ "parlour": "parlor",
1147
+ "parlours": "parlors",
1148
+ "particularise": "particularize",
1149
+ "particularised": "particularized",
1150
+ "particularises": "particularizes",
1151
+ "particularising": "particularizing",
1152
+ "passivisation": "passivization",
1153
+ "passivise": "passivize",
1154
+ "passivised": "passivized",
1155
+ "passivises": "passivizes",
1156
+ "passivising": "passivizing",
1157
+ "pasteurisation": "pasteurization",
1158
+ "pasteurise": "pasteurize",
1159
+ "pasteurised": "pasteurized",
1160
+ "pasteurises": "pasteurizes",
1161
+ "pasteurising": "pasteurizing",
1162
+ "patronise": "patronize",
1163
+ "patronised": "patronized",
1164
+ "patronises": "patronizes",
1165
+ "patronising": "patronizing",
1166
+ "patronisingly": "patronizingly",
1167
+ "pedalled": "pedaled",
1168
+ "pedalling": "pedaling",
1169
+ "pedestrianisation": "pedestrianization",
1170
+ "pedestrianise": "pedestrianize",
1171
+ "pedestrianised": "pedestrianized",
1172
+ "pedestrianises": "pedestrianizes",
1173
+ "pedestrianising": "pedestrianizing",
1174
+ "penalise": "penalize",
1175
+ "penalised": "penalized",
1176
+ "penalises": "penalizes",
1177
+ "penalising": "penalizing",
1178
+ "pencilled": "penciled",
1179
+ "pencilling": "penciling",
1180
+ "personalise": "personalize",
1181
+ "personalised": "personalized",
1182
+ "personalises": "personalizes",
1183
+ "personalising": "personalizing",
1184
+ "pharmacopoeia": "pharmacopeia",
1185
+ "pharmacopoeias": "pharmacopeias",
1186
+ "philosophise": "philosophize",
1187
+ "philosophised": "philosophized",
1188
+ "philosophises": "philosophizes",
1189
+ "philosophising": "philosophizing",
1190
+ "philtre": "filter",
1191
+ "philtres": "filters",
1192
+ "phoney": "phony",
1193
+ "plagiarise": "plagiarize",
1194
+ "plagiarised": "plagiarized",
1195
+ "plagiarises": "plagiarizes",
1196
+ "plagiarising": "plagiarizing",
1197
+ "plough": "plow",
1198
+ "ploughed": "plowed",
1199
+ "ploughing": "plowing",
1200
+ "ploughman": "plowman",
1201
+ "ploughmen": "plowmen",
1202
+ "ploughs": "plows",
1203
+ "ploughshare": "plowshare",
1204
+ "ploughshares": "plowshares",
1205
+ "polarisation": "polarization",
1206
+ "polarise": "polarize",
1207
+ "polarised": "polarized",
1208
+ "polarises": "polarizes",
1209
+ "polarising": "polarizing",
1210
+ "politicisation": "politicization",
1211
+ "politicise": "politicize",
1212
+ "politicised": "politicized",
1213
+ "politicises": "politicizes",
1214
+ "politicising": "politicizing",
1215
+ "popularisation": "popularization",
1216
+ "popularise": "popularize",
1217
+ "popularised": "popularized",
1218
+ "popularises": "popularizes",
1219
+ "popularising": "popularizing",
1220
+ "pouffe": "pouf",
1221
+ "pouffes": "poufs",
1222
+ "practise": "practice",
1223
+ "practised": "practiced",
1224
+ "practises": "practices",
1225
+ "practising": "practicing",
1226
+ "praesidium": "presidium",
1227
+ "praesidiums": "presidiums",
1228
+ "pressurisation": "pressurization",
1229
+ "pressurise": "pressurize",
1230
+ "pressurised": "pressurized",
1231
+ "pressurises": "pressurizes",
1232
+ "pressurising": "pressurizing",
1233
+ "pretence": "pretense",
1234
+ "pretences": "pretenses",
1235
+ "primaeval": "primeval",
1236
+ "prioritisation": "prioritization",
1237
+ "prioritise": "prioritize",
1238
+ "prioritised": "prioritized",
1239
+ "prioritises": "prioritizes",
1240
+ "prioritising": "prioritizing",
1241
+ "privatisation": "privatization",
1242
+ "privatisations": "privatizations",
1243
+ "privatise": "privatize",
1244
+ "privatised": "privatized",
1245
+ "privatises": "privatizes",
1246
+ "privatising": "privatizing",
1247
+ "professionalisation": "professionalization",
1248
+ "professionalise": "professionalize",
1249
+ "professionalised": "professionalized",
1250
+ "professionalises": "professionalizes",
1251
+ "professionalising": "professionalizing",
1252
+ "programme": "program",
1253
+ "programmes": "programs",
1254
+ "prologue": "prolog",
1255
+ "prologues": "prologs",
1256
+ "propagandise": "propagandize",
1257
+ "propagandised": "propagandized",
1258
+ "propagandises": "propagandizes",
1259
+ "propagandising": "propagandizing",
1260
+ "proselytise": "proselytize",
1261
+ "proselytised": "proselytized",
1262
+ "proselytiser": "proselytizer",
1263
+ "proselytisers": "proselytizers",
1264
+ "proselytises": "proselytizes",
1265
+ "proselytising": "proselytizing",
1266
+ "psychoanalyse": "psychoanalyze",
1267
+ "psychoanalysed": "psychoanalyzed",
1268
+ "psychoanalyses": "psychoanalyzes",
1269
+ "psychoanalysing": "psychoanalyzing",
1270
+ "publicise": "publicize",
1271
+ "publicised": "publicized",
1272
+ "publicises": "publicizes",
1273
+ "publicising": "publicizing",
1274
+ "pulverisation": "pulverization",
1275
+ "pulverise": "pulverize",
1276
+ "pulverised": "pulverized",
1277
+ "pulverises": "pulverizes",
1278
+ "pulverising": "pulverizing",
1279
+ "pummelled": "pummel",
1280
+ "pummelling": "pummeled",
1281
+ "pyjama": "pajama",
1282
+ "pyjamas": "pajamas",
1283
+ "pzazz": "pizzazz",
1284
+ "quarrelled": "quarreled",
1285
+ "quarrelling": "quarreling",
1286
+ "radicalise": "radicalize",
1287
+ "radicalised": "radicalized",
1288
+ "radicalises": "radicalizes",
1289
+ "radicalising": "radicalizing",
1290
+ "rancour": "rancor",
1291
+ "randomise": "randomize",
1292
+ "randomised": "randomized",
1293
+ "randomises": "randomizes",
1294
+ "randomising": "randomizing",
1295
+ "rationalisation": "rationalization",
1296
+ "rationalisations": "rationalizations",
1297
+ "rationalise": "rationalize",
1298
+ "rationalised": "rationalized",
1299
+ "rationalises": "rationalizes",
1300
+ "rationalising": "rationalizing",
1301
+ "ravelled": "raveled",
1302
+ "ravelling": "raveling",
1303
+ "realisable": "realizable",
1304
+ "realisation": "realization",
1305
+ "realisations": "realizations",
1306
+ "realise": "realize",
1307
+ "realised": "realized",
1308
+ "realises": "realizes",
1309
+ "realising": "realizing",
1310
+ "recognisable": "recognizable",
1311
+ "recognisably": "recognizably",
1312
+ "recognisance": "recognizance",
1313
+ "recognise": "recognize",
1314
+ "recognised": "recognized",
1315
+ "recognises": "recognizes",
1316
+ "recognising": "recognizing",
1317
+ "reconnoitre": "reconnoiter",
1318
+ "reconnoitred": "reconnoitered",
1319
+ "reconnoitres": "reconnoiters",
1320
+ "reconnoitring": "reconnoitering",
1321
+ "refuelled": "refueled",
1322
+ "refuelling": "refueling",
1323
+ "regularisation": "regularization",
1324
+ "regularise": "regularize",
1325
+ "regularised": "regularized",
1326
+ "regularises": "regularizes",
1327
+ "regularising": "regularizing",
1328
+ "remodelled": "remodeled",
1329
+ "remodelling": "remodeling",
1330
+ "remould": "remold",
1331
+ "remoulded": "remolded",
1332
+ "remoulding": "remolding",
1333
+ "remoulds": "remolds",
1334
+ "reorganisation": "reorganization",
1335
+ "reorganisations": "reorganizations",
1336
+ "reorganise": "reorganize",
1337
+ "reorganised": "reorganized",
1338
+ "reorganises": "reorganizes",
1339
+ "reorganising": "reorganizing",
1340
+ "revelled": "reveled",
1341
+ "reveller": "reveler",
1342
+ "revellers": "revelers",
1343
+ "revelling": "reveling",
1344
+ "revitalise": "revitalize",
1345
+ "revitalised": "revitalized",
1346
+ "revitalises": "revitalizes",
1347
+ "revitalising": "revitalizing",
1348
+ "revolutionise": "revolutionize",
1349
+ "revolutionised": "revolutionized",
1350
+ "revolutionises": "revolutionizes",
1351
+ "revolutionising": "revolutionizing",
1352
+ "rhapsodise": "rhapsodize",
1353
+ "rhapsodised": "rhapsodized",
1354
+ "rhapsodises": "rhapsodizes",
1355
+ "rhapsodising": "rhapsodizing",
1356
+ "rigour": "rigor",
1357
+ "rigours": "rigors",
1358
+ "ritualised": "ritualized",
1359
+ "rivalled": "rivaled",
1360
+ "rivalling": "rivaling",
1361
+ "romanticise": "romanticize",
1362
+ "romanticised": "romanticized",
1363
+ "romanticises": "romanticizes",
1364
+ "romanticising": "romanticizing",
1365
+ "rumour": "rumor",
1366
+ "rumoured": "rumored",
1367
+ "rumours": "rumors",
1368
+ "sabre": "saber",
1369
+ "sabres": "sabers",
1370
+ "saltpetre": "saltpeter",
1371
+ "sanitise": "sanitize",
1372
+ "sanitised": "sanitized",
1373
+ "sanitises": "sanitizes",
1374
+ "sanitising": "sanitizing",
1375
+ "satirise": "satirize",
1376
+ "satirised": "satirized",
1377
+ "satirises": "satirizes",
1378
+ "satirising": "satirizing",
1379
+ "saviour": "savior",
1380
+ "saviours": "saviors",
1381
+ "savour": "savor",
1382
+ "savoured": "savored",
1383
+ "savouries": "savories",
1384
+ "savouring": "savoring",
1385
+ "savours": "savors",
1386
+ "savoury": "savory",
1387
+ "scandalise": "scandalize",
1388
+ "scandalised": "scandalized",
1389
+ "scandalises": "scandalizes",
1390
+ "scandalising": "scandalizing",
1391
+ "sceptic": "skeptic",
1392
+ "sceptical": "skeptical",
1393
+ "sceptically": "skeptically",
1394
+ "scepticism": "skepticism",
1395
+ "sceptics": "skeptics",
1396
+ "sceptre": "scepter",
1397
+ "sceptres": "scepters",
1398
+ "scrutinise": "scrutinize",
1399
+ "scrutinised": "scrutinized",
1400
+ "scrutinises": "scrutinizes",
1401
+ "scrutinising": "scrutinizing",
1402
+ "secularisation": "secularization",
1403
+ "secularise": "secularize",
1404
+ "secularised": "secularized",
1405
+ "secularises": "secularizes",
1406
+ "secularising": "secularizing",
1407
+ "sensationalise": "sensationalize",
1408
+ "sensationalised": "sensationalized",
1409
+ "sensationalises": "sensationalizes",
1410
+ "sensationalising": "sensationalizing",
1411
+ "sensitise": "sensitize",
1412
+ "sensitised": "sensitized",
1413
+ "sensitises": "sensitizes",
1414
+ "sensitising": "sensitizing",
1415
+ "sentimentalise": "sentimentalize",
1416
+ "sentimentalised": "sentimentalized",
1417
+ "sentimentalises": "sentimentalizes",
1418
+ "sentimentalising": "sentimentalizing",
1419
+ "sepulchre": "sepulcher",
1420
+ "sepulchres": "sepulchers",
1421
+ "serialisation": "serialization",
1422
+ "serialisations": "serializations",
1423
+ "serialise": "serialize",
1424
+ "serialised": "serialized",
1425
+ "serialises": "serializes",
1426
+ "serialising": "serializing",
1427
+ "sermonise": "sermonize",
1428
+ "sermonised": "sermonized",
1429
+ "sermonises": "sermonizes",
1430
+ "sermonising": "sermonizing",
1431
+ "sheikh": "sheik",
1432
+ "shovelled": "shoveled",
1433
+ "shovelling": "shoveling",
1434
+ "shrivelled": "shriveled",
1435
+ "shrivelling": "shriveling",
1436
+ "signalise": "signalize",
1437
+ "signalised": "signalized",
1438
+ "signalises": "signalizes",
1439
+ "signalising": "signalizing",
1440
+ "signalled": "signaled",
1441
+ "signalling": "signaling",
1442
+ "smoulder": "smolder",
1443
+ "smouldered": "smoldered",
1444
+ "smouldering": "smoldering",
1445
+ "smoulders": "smolders",
1446
+ "snivelled": "sniveled",
1447
+ "snivelling": "sniveling",
1448
+ "snorkelled": "snorkeled",
1449
+ "snorkelling": "snorkeling",
1450
+ "snowplough": "snowplow",
1451
+ "snowploughs": "snowplow",
1452
+ "socialisation": "socialization",
1453
+ "socialise": "socialize",
1454
+ "socialised": "socialized",
1455
+ "socialises": "socializes",
1456
+ "socialising": "socializing",
1457
+ "sodomise": "sodomize",
1458
+ "sodomised": "sodomized",
1459
+ "sodomises": "sodomizes",
1460
+ "sodomising": "sodomizing",
1461
+ "solemnise": "solemnize",
1462
+ "solemnised": "solemnized",
1463
+ "solemnises": "solemnizes",
1464
+ "solemnising": "solemnizing",
1465
+ "sombre": "somber",
1466
+ "specialisation": "specialization",
1467
+ "specialisations": "specializations",
1468
+ "specialise": "specialize",
1469
+ "specialised": "specialized",
1470
+ "specialises": "specializes",
1471
+ "specialising": "specializing",
1472
+ "spectre": "specter",
1473
+ "spectres": "specters",
1474
+ "spiralled": "spiraled",
1475
+ "spiralling": "spiraling",
1476
+ "splendour": "splendor",
1477
+ "splendours": "splendors",
1478
+ "squirrelled": "squirreled",
1479
+ "squirrelling": "squirreling",
1480
+ "stabilisation": "stabilization",
1481
+ "stabilise": "stabilize",
1482
+ "stabilised": "stabilized",
1483
+ "stabiliser": "stabilizer",
1484
+ "stabilisers": "stabilizers",
1485
+ "stabilises": "stabilizes",
1486
+ "stabilising": "stabilizing",
1487
+ "standardisation": "standardization",
1488
+ "standardise": "standardize",
1489
+ "standardised": "standardized",
1490
+ "standardises": "standardizes",
1491
+ "standardising": "standardizing",
1492
+ "stencilled": "stenciled",
1493
+ "stencilling": "stenciling",
1494
+ "sterilisation": "sterilization",
1495
+ "sterilisations": "sterilizations",
1496
+ "sterilise": "sterilize",
1497
+ "sterilised": "sterilized",
1498
+ "steriliser": "sterilizer",
1499
+ "sterilisers": "sterilizers",
1500
+ "sterilises": "sterilizes",
1501
+ "sterilising": "sterilizing",
1502
+ "stigmatisation": "stigmatization",
1503
+ "stigmatise": "stigmatize",
1504
+ "stigmatised": "stigmatized",
1505
+ "stigmatises": "stigmatizes",
1506
+ "stigmatising": "stigmatizing",
1507
+ "storey": "story",
1508
+ "storeys": "stories",
1509
+ "subsidisation": "subsidization",
1510
+ "subsidise": "subsidize",
1511
+ "subsidised": "subsidized",
1512
+ "subsidiser": "subsidizer",
1513
+ "subsidisers": "subsidizers",
1514
+ "subsidises": "subsidizes",
1515
+ "subsidising": "subsidizing",
1516
+ "succour": "succor",
1517
+ "succoured": "succored",
1518
+ "succouring": "succoring",
1519
+ "succours": "succors",
1520
+ "sulphate": "sulfate",
1521
+ "sulphates": "sulfates",
1522
+ "sulphide": "sulfide",
1523
+ "sulphides": "sulfides",
1524
+ "sulphur": "sulfur",
1525
+ "sulphurous": "sulfurous",
1526
+ "summarise": "summarize",
1527
+ "summarised": "summarized",
1528
+ "summarises": "summarizes",
1529
+ "summarising": "summarizing",
1530
+ "swivelled": "swiveled",
1531
+ "swivelling": "swiveling",
1532
+ "symbolise": "symbolize",
1533
+ "symbolised": "symbolized",
1534
+ "symbolises": "symbolizes",
1535
+ "symbolising": "symbolizing",
1536
+ "sympathise": "sympathize",
1537
+ "sympathised": "sympathized",
1538
+ "sympathiser": "sympathizer",
1539
+ "sympathisers": "sympathizers",
1540
+ "sympathises": "sympathizes",
1541
+ "sympathising": "sympathizing",
1542
+ "synchronisation": "synchronization",
1543
+ "synchronise": "synchronize",
1544
+ "synchronised": "synchronized",
1545
+ "synchronises": "synchronizes",
1546
+ "synchronising": "synchronizing",
1547
+ "synthesise": "synthesize",
1548
+ "synthesised": "synthesized",
1549
+ "synthesiser": "synthesizer",
1550
+ "synthesisers": "synthesizers",
1551
+ "synthesises": "synthesizes",
1552
+ "synthesising": "synthesizing",
1553
+ "syphon": "siphon",
1554
+ "syphoned": "siphoned",
1555
+ "syphoning": "siphoning",
1556
+ "syphons": "siphons",
1557
+ "systematisation": "systematization",
1558
+ "systematise": "systematize",
1559
+ "systematised": "systematized",
1560
+ "systematises": "systematizes",
1561
+ "systematising": "systematizing",
1562
+ "tantalise": "tantalize",
1563
+ "tantalised": "tantalized",
1564
+ "tantalises": "tantalizes",
1565
+ "tantalising": "tantalizing",
1566
+ "tantalisingly": "tantalizingly",
1567
+ "tasselled": "tasseled",
1568
+ "technicolour": "technicolor",
1569
+ "temporise": "temporize",
1570
+ "temporised": "temporized",
1571
+ "temporises": "temporizes",
1572
+ "temporising": "temporizing",
1573
+ "tenderise": "tenderize",
1574
+ "tenderised": "tenderized",
1575
+ "tenderises": "tenderizes",
1576
+ "tenderising": "tenderizing",
1577
+ "terrorise": "terrorize",
1578
+ "terrorised": "terrorized",
1579
+ "terrorises": "terrorizes",
1580
+ "terrorising": "terrorizing",
1581
+ "theatre": "theater",
1582
+ "theatregoer": "theatergoer",
1583
+ "theatregoers": "theatergoers",
1584
+ "theatres": "theaters",
1585
+ "theorise": "theorize",
1586
+ "theorised": "theorized",
1587
+ "theorises": "theorizes",
1588
+ "theorising": "theorizing",
1589
+ "tonne": "ton",
1590
+ "tonnes": "tons",
1591
+ "towelled": "toweled",
1592
+ "towelling": "toweling",
1593
+ "toxaemia": "toxemia",
1594
+ "tranquillise": "tranquilize",
1595
+ "tranquillised": "tranquilized",
1596
+ "tranquilliser": "tranquilizer",
1597
+ "tranquillisers": "tranquilizers",
1598
+ "tranquillises": "tranquilizes",
1599
+ "tranquillising": "tranquilizing",
1600
+ "tranquillity": "tranquility",
1601
+ "tranquillize": "tranquilize",
1602
+ "tranquillized": "tranquilized",
1603
+ "tranquillizer": "tranquilizer",
1604
+ "tranquillizers": "tranquilizers",
1605
+ "tranquillizes": "tranquilizes",
1606
+ "tranquillizing": "tranquilizing",
1607
+ "tranquilly": "tranquility",
1608
+ "transistorised": "transistorized",
1609
+ "traumatise": "traumatize",
1610
+ "traumatised": "traumatized",
1611
+ "traumatises": "traumatizes",
1612
+ "traumatising": "traumatizing",
1613
+ "travelled": "traveled",
1614
+ "traveller": "traveler",
1615
+ "travellers": "travelers",
1616
+ "travelling": "traveling",
1617
+ "travelog": "travelogue",
1618
+ "travelogs": "travelogues",
1619
+ "trialled": "trialed",
1620
+ "trialling": "trialing",
1621
+ "tricolour": "tricolor",
1622
+ "tricolours": "tricolors",
1623
+ "trivialise": "trivialize",
1624
+ "trivialised": "trivialized",
1625
+ "trivialises": "trivializes",
1626
+ "trivialising": "trivializing",
1627
+ "tumour": "tumor",
1628
+ "tumours": "tumors",
1629
+ "tunnelled": "tunneled",
1630
+ "tunnelling": "tunneling",
1631
+ "tyrannise": "tyrannize",
1632
+ "tyrannised": "tyrannized",
1633
+ "tyrannises": "tyrannizes",
1634
+ "tyrannising": "tyrannizing",
1635
+ "tyre": "tire",
1636
+ "tyres": "tires",
1637
+ "unauthorised": "unauthorized",
1638
+ "uncivilised": "uncivilized",
1639
+ "underutilised": "underutilized",
1640
+ "unequalled": "unequaled",
1641
+ "unfavourable": "unfavorable",
1642
+ "unfavourably": "unfavorably",
1643
+ "unionisation": "unionization",
1644
+ "unionise": "unionize",
1645
+ "unionised": "unionized",
1646
+ "unionises": "unionizes",
1647
+ "unionising": "unionizing",
1648
+ "unorganised": "unorganized",
1649
+ "unravelled": "unraveled",
1650
+ "unravelling": "unraveling",
1651
+ "unrecognisable": "unrecognizable",
1652
+ "unrecognised": "unrecognized",
1653
+ "unrivalled": "unrivaled",
1654
+ "unsavoury": "unsavory",
1655
+ "untrammelled": "untrammeled",
1656
+ "urbanisation": "urbanization",
1657
+ "urbanise": "urbanize",
1658
+ "urbanised": "urbanized",
1659
+ "urbanises": "urbanizes",
1660
+ "urbanising": "urbanizing",
1661
+ "utilisable": "utilizable",
1662
+ "utilisation": "utilization",
1663
+ "utilise": "utilize",
1664
+ "utilised": "utilized",
1665
+ "utilises": "utilizes",
1666
+ "utilising": "utilizing",
1667
+ "valour": "valor",
1668
+ "vandalise": "vandalize",
1669
+ "vandalised": "vandalized",
1670
+ "vandalises": "vandalizes",
1671
+ "vandalising": "vandalizing",
1672
+ "vaporisation": "vaporization",
1673
+ "vaporise": "vaporize",
1674
+ "vaporised": "vaporized",
1675
+ "vaporises": "vaporizes",
1676
+ "vaporising": "vaporizing",
1677
+ "vapour": "vapor",
1678
+ "vapours": "vapors",
1679
+ "verbalise": "verbalize",
1680
+ "verbalised": "verbalized",
1681
+ "verbalises": "verbalizes",
1682
+ "verbalising": "verbalizing",
1683
+ "victimisation": "victimization",
1684
+ "victimise": "victimize",
1685
+ "victimised": "victimized",
1686
+ "victimises": "victimizes",
1687
+ "victimising": "victimizing",
1688
+ "videodisc": "videodisk",
1689
+ "videodiscs": "videodisks",
1690
+ "vigour": "vigor",
1691
+ "visualisation": "visualization",
1692
+ "visualisations": "visualizations",
1693
+ "visualise": "visualize",
1694
+ "visualised": "visualized",
1695
+ "visualises": "visualizes",
1696
+ "visualising": "visualizing",
1697
+ "vocalisation": "vocalization",
1698
+ "vocalisations": "vocalizations",
1699
+ "vocalise": "vocalize",
1700
+ "vocalised": "vocalized",
1701
+ "vocalises": "vocalizes",
1702
+ "vocalising": "vocalizing",
1703
+ "vulcanised": "vulcanized",
1704
+ "vulgarisation": "vulgarization",
1705
+ "vulgarise": "vulgarize",
1706
+ "vulgarised": "vulgarized",
1707
+ "vulgarises": "vulgarizes",
1708
+ "vulgarising": "vulgarizing",
1709
+ "waggon": "wagon",
1710
+ "waggons": "wagons",
1711
+ "watercolour": "watercolor",
1712
+ "watercolours": "watercolors",
1713
+ "weaselled": "weaseled",
1714
+ "weaselling": "weaseling",
1715
+ "westernisation": "westernization",
1716
+ "westernise": "westernize",
1717
+ "westernised": "westernized",
1718
+ "westernises": "westernizes",
1719
+ "westernising": "westernizing",
1720
+ "womanise": "womanize",
1721
+ "womanised": "womanized",
1722
+ "womaniser": "womanizer",
1723
+ "womanisers": "womanizers",
1724
+ "womanises": "womanizes",
1725
+ "womanising": "womanizing",
1726
+ "woollen": "woolen",
1727
+ "woollens": "woolens",
1728
+ "woollies": "woolies",
1729
+ "woolly": "wooly",
1730
+ "worshipped": "worshiped",
1731
+ "worshipping": "worshiping",
1732
+ "worshipper": "worshiper",
1733
+ "yodelled": "yodeled",
1734
+ "yodelling": "yodeling",
1735
+ "yoghourt": "yogurt",
1736
+ "yoghourts": "yogurts",
1737
+ "yoghurt": "yogurt",
1738
+ "yoghurts": "yogurts",
1739
+ "mhm": "hmm",
1740
+ "mmm": "hmm"
1741
+ }
whisper/normalizers/english.py ADDED
@@ -0,0 +1,550 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+ import re
4
+ from fractions import Fraction
5
+ from typing import Iterator, List, Match, Optional, Union
6
+
7
+ from more_itertools import windowed
8
+
9
+ from .basic import remove_symbols_and_diacritics
10
+
11
+
12
+ class EnglishNumberNormalizer:
13
+ """
14
+ Convert any spelled-out numbers into arabic numbers, while handling:
15
+
16
+ - remove any commas
17
+ - keep the suffixes such as: `1960s`, `274th`, `32nd`, etc.
18
+ - spell out currency symbols after the number. e.g. `$20 million` -> `20000000 dollars`
19
+ - spell out `one` and `ones`
20
+ - interpret successive single-digit numbers as nominal: `one oh one` -> `101`
21
+ """
22
+
23
+ def __init__(self):
24
+ super().__init__()
25
+
26
+ self.zeros = {"o", "oh", "zero"}
27
+ self.ones = {
28
+ name: i
29
+ for i, name in enumerate(
30
+ [
31
+ "one",
32
+ "two",
33
+ "three",
34
+ "four",
35
+ "five",
36
+ "six",
37
+ "seven",
38
+ "eight",
39
+ "nine",
40
+ "ten",
41
+ "eleven",
42
+ "twelve",
43
+ "thirteen",
44
+ "fourteen",
45
+ "fifteen",
46
+ "sixteen",
47
+ "seventeen",
48
+ "eighteen",
49
+ "nineteen",
50
+ ],
51
+ start=1,
52
+ )
53
+ }
54
+ self.ones_plural = {
55
+ "sixes" if name == "six" else name + "s": (value, "s")
56
+ for name, value in self.ones.items()
57
+ }
58
+ self.ones_ordinal = {
59
+ "zeroth": (0, "th"),
60
+ "first": (1, "st"),
61
+ "second": (2, "nd"),
62
+ "third": (3, "rd"),
63
+ "fifth": (5, "th"),
64
+ "twelfth": (12, "th"),
65
+ **{
66
+ name + ("h" if name.endswith("t") else "th"): (value, "th")
67
+ for name, value in self.ones.items()
68
+ if value > 3 and value != 5 and value != 12
69
+ },
70
+ }
71
+ self.ones_suffixed = {**self.ones_plural, **self.ones_ordinal}
72
+
73
+ self.tens = {
74
+ "twenty": 20,
75
+ "thirty": 30,
76
+ "forty": 40,
77
+ "fifty": 50,
78
+ "sixty": 60,
79
+ "seventy": 70,
80
+ "eighty": 80,
81
+ "ninety": 90,
82
+ }
83
+ self.tens_plural = {
84
+ name.replace("y", "ies"): (value, "s") for name, value in self.tens.items()
85
+ }
86
+ self.tens_ordinal = {
87
+ name.replace("y", "ieth"): (value, "th")
88
+ for name, value in self.tens.items()
89
+ }
90
+ self.tens_suffixed = {**self.tens_plural, **self.tens_ordinal}
91
+
92
+ self.multipliers = {
93
+ "hundred": 100,
94
+ "thousand": 1_000,
95
+ "million": 1_000_000,
96
+ "billion": 1_000_000_000,
97
+ "trillion": 1_000_000_000_000,
98
+ "quadrillion": 1_000_000_000_000_000,
99
+ "quintillion": 1_000_000_000_000_000_000,
100
+ "sextillion": 1_000_000_000_000_000_000_000,
101
+ "septillion": 1_000_000_000_000_000_000_000_000,
102
+ "octillion": 1_000_000_000_000_000_000_000_000_000,
103
+ "nonillion": 1_000_000_000_000_000_000_000_000_000_000,
104
+ "decillion": 1_000_000_000_000_000_000_000_000_000_000_000,
105
+ }
106
+ self.multipliers_plural = {
107
+ name + "s": (value, "s") for name, value in self.multipliers.items()
108
+ }
109
+ self.multipliers_ordinal = {
110
+ name + "th": (value, "th") for name, value in self.multipliers.items()
111
+ }
112
+ self.multipliers_suffixed = {
113
+ **self.multipliers_plural,
114
+ **self.multipliers_ordinal,
115
+ }
116
+ self.decimals = {*self.ones, *self.tens, *self.zeros}
117
+
118
+ self.preceding_prefixers = {
119
+ "minus": "-",
120
+ "negative": "-",
121
+ "plus": "+",
122
+ "positive": "+",
123
+ }
124
+ self.following_prefixers = {
125
+ "pound": "£",
126
+ "pounds": "£",
127
+ "euro": "€",
128
+ "euros": "€",
129
+ "dollar": "$",
130
+ "dollars": "$",
131
+ "cent": "¢",
132
+ "cents": "¢",
133
+ }
134
+ self.prefixes = set(
135
+ list(self.preceding_prefixers.values())
136
+ + list(self.following_prefixers.values())
137
+ )
138
+ self.suffixers = {
139
+ "per": {"cent": "%"},
140
+ "percent": "%",
141
+ }
142
+ self.specials = {"and", "double", "triple", "point"}
143
+
144
+ self.words = set(
145
+ [
146
+ key
147
+ for mapping in [
148
+ self.zeros,
149
+ self.ones,
150
+ self.ones_suffixed,
151
+ self.tens,
152
+ self.tens_suffixed,
153
+ self.multipliers,
154
+ self.multipliers_suffixed,
155
+ self.preceding_prefixers,
156
+ self.following_prefixers,
157
+ self.suffixers,
158
+ self.specials,
159
+ ]
160
+ for key in mapping
161
+ ]
162
+ )
163
+ self.literal_words = {"one", "ones"}
164
+
165
+ def process_words(self, words: List[str]) -> Iterator[str]:
166
+ prefix: Optional[str] = None
167
+ value: Optional[Union[str, int]] = None
168
+ skip = False
169
+
170
+ def to_fraction(s: str):
171
+ try:
172
+ return Fraction(s)
173
+ except ValueError:
174
+ return None
175
+
176
+ def output(result: Union[str, int]):
177
+ nonlocal prefix, value
178
+ result = str(result)
179
+ if prefix is not None:
180
+ result = prefix + result
181
+ value = None
182
+ prefix = None
183
+ return result
184
+
185
+ if len(words) == 0:
186
+ return
187
+
188
+ for prev, current, next in windowed([None] + words + [None], 3):
189
+ if skip:
190
+ skip = False
191
+ continue
192
+
193
+ next_is_numeric = next is not None and re.match(r"^\d+(\.\d+)?$", next)
194
+ has_prefix = current[0] in self.prefixes
195
+ current_without_prefix = current[1:] if has_prefix else current
196
+ if re.match(r"^\d+(\.\d+)?$", current_without_prefix):
197
+ # arabic numbers (potentially with signs and fractions)
198
+ f = to_fraction(current_without_prefix)
199
+ assert f is not None
200
+ if value is not None:
201
+ if isinstance(value, str) and value.endswith("."):
202
+ # concatenate decimals / ip address components
203
+ value = str(value) + str(current)
204
+ continue
205
+ else:
206
+ yield output(value)
207
+
208
+ prefix = current[0] if has_prefix else prefix
209
+ if f.denominator == 1:
210
+ value = f.numerator # store integers as int
211
+ else:
212
+ value = current_without_prefix
213
+ elif current not in self.words:
214
+ # non-numeric words
215
+ if value is not None:
216
+ yield output(value)
217
+ yield output(current)
218
+ elif current in self.zeros:
219
+ value = str(value or "") + "0"
220
+ elif current in self.ones:
221
+ ones = self.ones[current]
222
+
223
+ if value is None:
224
+ value = ones
225
+ elif isinstance(value, str) or prev in self.ones:
226
+ if (
227
+ prev in self.tens and ones < 10
228
+ ): # replace the last zero with the digit
229
+ assert value[-1] == "0"
230
+ value = value[:-1] + str(ones)
231
+ else:
232
+ value = str(value) + str(ones)
233
+ elif ones < 10:
234
+ if value % 10 == 0:
235
+ value += ones
236
+ else:
237
+ value = str(value) + str(ones)
238
+ else: # eleven to nineteen
239
+ if value % 100 == 0:
240
+ value += ones
241
+ else:
242
+ value = str(value) + str(ones)
243
+ elif current in self.ones_suffixed:
244
+ # ordinal or cardinal; yield the number right away
245
+ ones, suffix = self.ones_suffixed[current]
246
+ if value is None:
247
+ yield output(str(ones) + suffix)
248
+ elif isinstance(value, str) or prev in self.ones:
249
+ if prev in self.tens and ones < 10:
250
+ assert value[-1] == "0"
251
+ yield output(value[:-1] + str(ones) + suffix)
252
+ else:
253
+ yield output(str(value) + str(ones) + suffix)
254
+ elif ones < 10:
255
+ if value % 10 == 0:
256
+ yield output(str(value + ones) + suffix)
257
+ else:
258
+ yield output(str(value) + str(ones) + suffix)
259
+ else: # eleven to nineteen
260
+ if value % 100 == 0:
261
+ yield output(str(value + ones) + suffix)
262
+ else:
263
+ yield output(str(value) + str(ones) + suffix)
264
+ value = None
265
+ elif current in self.tens:
266
+ tens = self.tens[current]
267
+ if value is None:
268
+ value = tens
269
+ elif isinstance(value, str):
270
+ value = str(value) + str(tens)
271
+ else:
272
+ if value % 100 == 0:
273
+ value += tens
274
+ else:
275
+ value = str(value) + str(tens)
276
+ elif current in self.tens_suffixed:
277
+ # ordinal or cardinal; yield the number right away
278
+ tens, suffix = self.tens_suffixed[current]
279
+ if value is None:
280
+ yield output(str(tens) + suffix)
281
+ elif isinstance(value, str):
282
+ yield output(str(value) + str(tens) + suffix)
283
+ else:
284
+ if value % 100 == 0:
285
+ yield output(str(value + tens) + suffix)
286
+ else:
287
+ yield output(str(value) + str(tens) + suffix)
288
+ elif current in self.multipliers:
289
+ multiplier = self.multipliers[current]
290
+ if value is None:
291
+ value = multiplier
292
+ elif isinstance(value, str) or value == 0:
293
+ f = to_fraction(value)
294
+ p = f * multiplier if f is not None else None
295
+ if f is not None and p.denominator == 1:
296
+ value = p.numerator
297
+ else:
298
+ yield output(value)
299
+ value = multiplier
300
+ else:
301
+ before = value // 1000 * 1000
302
+ residual = value % 1000
303
+ value = before + residual * multiplier
304
+ elif current in self.multipliers_suffixed:
305
+ multiplier, suffix = self.multipliers_suffixed[current]
306
+ if value is None:
307
+ yield output(str(multiplier) + suffix)
308
+ elif isinstance(value, str):
309
+ f = to_fraction(value)
310
+ p = f * multiplier if f is not None else None
311
+ if f is not None and p.denominator == 1:
312
+ yield output(str(p.numerator) + suffix)
313
+ else:
314
+ yield output(value)
315
+ yield output(str(multiplier) + suffix)
316
+ else: # int
317
+ before = value // 1000 * 1000
318
+ residual = value % 1000
319
+ value = before + residual * multiplier
320
+ yield output(str(value) + suffix)
321
+ value = None
322
+ elif current in self.preceding_prefixers:
323
+ # apply prefix (positive, minus, etc.) if it precedes a number
324
+ if value is not None:
325
+ yield output(value)
326
+
327
+ if next in self.words or next_is_numeric:
328
+ prefix = self.preceding_prefixers[current]
329
+ else:
330
+ yield output(current)
331
+ elif current in self.following_prefixers:
332
+ # apply prefix (dollars, cents, etc.) only after a number
333
+ if value is not None:
334
+ prefix = self.following_prefixers[current]
335
+ yield output(value)
336
+ else:
337
+ yield output(current)
338
+ elif current in self.suffixers:
339
+ # apply suffix symbols (percent -> '%')
340
+ if value is not None:
341
+ suffix = self.suffixers[current]
342
+ if isinstance(suffix, dict):
343
+ if next in suffix:
344
+ yield output(str(value) + suffix[next])
345
+ skip = True
346
+ else:
347
+ yield output(value)
348
+ yield output(current)
349
+ else:
350
+ yield output(str(value) + suffix)
351
+ else:
352
+ yield output(current)
353
+ elif current in self.specials:
354
+ if next not in self.words and not next_is_numeric:
355
+ # apply special handling only if the next word can be numeric
356
+ if value is not None:
357
+ yield output(value)
358
+ yield output(current)
359
+ elif current == "and":
360
+ # ignore "and" after hundreds, thousands, etc.
361
+ if prev not in self.multipliers:
362
+ if value is not None:
363
+ yield output(value)
364
+ yield output(current)
365
+ elif current == "double" or current == "triple":
366
+ if next in self.ones or next in self.zeros:
367
+ repeats = 2 if current == "double" else 3
368
+ ones = self.ones.get(next, 0)
369
+ value = str(value or "") + str(ones) * repeats
370
+ skip = True
371
+ else:
372
+ if value is not None:
373
+ yield output(value)
374
+ yield output(current)
375
+ elif current == "point":
376
+ if next in self.decimals or next_is_numeric:
377
+ value = str(value or "") + "."
378
+ else:
379
+ # should all have been covered at this point
380
+ raise ValueError(f"Unexpected token: {current}")
381
+ else:
382
+ # all should have been covered at this point
383
+ raise ValueError(f"Unexpected token: {current}")
384
+
385
+ if value is not None:
386
+ yield output(value)
387
+
388
+ def preprocess(self, s: str):
389
+ # replace "<number> and a half" with "<number> point five"
390
+ results = []
391
+
392
+ segments = re.split(r"\band\s+a\s+half\b", s)
393
+ for i, segment in enumerate(segments):
394
+ if len(segment.strip()) == 0:
395
+ continue
396
+ if i == len(segments) - 1:
397
+ results.append(segment)
398
+ else:
399
+ results.append(segment)
400
+ last_word = segment.rsplit(maxsplit=2)[-1]
401
+ if last_word in self.decimals or last_word in self.multipliers:
402
+ results.append("point five")
403
+ else:
404
+ results.append("and a half")
405
+
406
+ s = " ".join(results)
407
+
408
+ # put a space at number/letter boundary
409
+ s = re.sub(r"([a-z])([0-9])", r"\1 \2", s)
410
+ s = re.sub(r"([0-9])([a-z])", r"\1 \2", s)
411
+
412
+ # but remove spaces which could be a suffix
413
+ s = re.sub(r"([0-9])\s+(st|nd|rd|th|s)\b", r"\1\2", s)
414
+
415
+ return s
416
+
417
+ def postprocess(self, s: str):
418
+ def combine_cents(m: Match):
419
+ try:
420
+ currency = m.group(1)
421
+ integer = m.group(2)
422
+ cents = int(m.group(3))
423
+ return f"{currency}{integer}.{cents:02d}"
424
+ except ValueError:
425
+ return m.string
426
+
427
+ def extract_cents(m: Match):
428
+ try:
429
+ return f"¢{int(m.group(1))}"
430
+ except ValueError:
431
+ return m.string
432
+
433
+ # apply currency postprocessing; "$2 and ¢7" -> "$2.07"
434
+ s = re.sub(r"([€£$])([0-9]+) (?:and )?¢([0-9]{1,2})\b", combine_cents, s)
435
+ s = re.sub(r"[€£$]0.([0-9]{1,2})\b", extract_cents, s)
436
+
437
+ # write "one(s)" instead of "1(s)", just for the readability
438
+ s = re.sub(r"\b1(s?)\b", r"one\1", s)
439
+
440
+ return s
441
+
442
+ def __call__(self, s: str):
443
+ s = self.preprocess(s)
444
+ s = " ".join(word for word in self.process_words(s.split()) if word is not None)
445
+ s = self.postprocess(s)
446
+
447
+ return s
448
+
449
+
450
+ class EnglishSpellingNormalizer:
451
+ """
452
+ Applies British-American spelling mappings as listed in [1].
453
+
454
+ [1] https://www.tysto.com/uk-us-spelling-list.html
455
+ """
456
+
457
+ def __init__(self):
458
+ mapping_path = os.path.join(os.path.dirname(__file__), "english.json")
459
+ self.mapping = json.load(open(mapping_path))
460
+
461
+ def __call__(self, s: str):
462
+ return " ".join(self.mapping.get(word, word) for word in s.split())
463
+
464
+
465
+ class EnglishTextNormalizer:
466
+ def __init__(self):
467
+ self.ignore_patterns = r"\b(hmm|mm|mhm|mmm|uh|um)\b"
468
+ self.replacers = {
469
+ # common contractions
470
+ r"\bwon't\b": "will not",
471
+ r"\bcan't\b": "can not",
472
+ r"\blet's\b": "let us",
473
+ r"\bain't\b": "aint",
474
+ r"\by'all\b": "you all",
475
+ r"\bwanna\b": "want to",
476
+ r"\bgotta\b": "got to",
477
+ r"\bgonna\b": "going to",
478
+ r"\bi'ma\b": "i am going to",
479
+ r"\bimma\b": "i am going to",
480
+ r"\bwoulda\b": "would have",
481
+ r"\bcoulda\b": "could have",
482
+ r"\bshoulda\b": "should have",
483
+ r"\bma'am\b": "madam",
484
+ # contractions in titles/prefixes
485
+ r"\bmr\b": "mister ",
486
+ r"\bmrs\b": "missus ",
487
+ r"\bst\b": "saint ",
488
+ r"\bdr\b": "doctor ",
489
+ r"\bprof\b": "professor ",
490
+ r"\bcapt\b": "captain ",
491
+ r"\bgov\b": "governor ",
492
+ r"\bald\b": "alderman ",
493
+ r"\bgen\b": "general ",
494
+ r"\bsen\b": "senator ",
495
+ r"\brep\b": "representative ",
496
+ r"\bpres\b": "president ",
497
+ r"\brev\b": "reverend ",
498
+ r"\bhon\b": "honorable ",
499
+ r"\basst\b": "assistant ",
500
+ r"\bassoc\b": "associate ",
501
+ r"\blt\b": "lieutenant ",
502
+ r"\bcol\b": "colonel ",
503
+ r"\bjr\b": "junior ",
504
+ r"\bsr\b": "senior ",
505
+ r"\besq\b": "esquire ",
506
+ # prefect tenses, ideally it should be any past participles, but it's harder..
507
+ r"'d been\b": " had been",
508
+ r"'s been\b": " has been",
509
+ r"'d gone\b": " had gone",
510
+ r"'s gone\b": " has gone",
511
+ r"'d done\b": " had done", # "'s done" is ambiguous
512
+ r"'s got\b": " has got",
513
+ # general contractions
514
+ r"n't\b": " not",
515
+ r"'re\b": " are",
516
+ r"'s\b": " is",
517
+ r"'d\b": " would",
518
+ r"'ll\b": " will",
519
+ r"'t\b": " not",
520
+ r"'ve\b": " have",
521
+ r"'m\b": " am",
522
+ }
523
+ self.standardize_numbers = EnglishNumberNormalizer()
524
+ self.standardize_spellings = EnglishSpellingNormalizer()
525
+
526
+ def __call__(self, s: str):
527
+ s = s.lower()
528
+
529
+ s = re.sub(r"[<\[][^>\]]*[>\]]", "", s) # remove words between brackets
530
+ s = re.sub(r"\(([^)]+?)\)", "", s) # remove words between parenthesis
531
+ s = re.sub(self.ignore_patterns, "", s)
532
+ s = re.sub(r"\s+'", "'", s) # when there's a space before an apostrophe
533
+
534
+ for pattern, replacement in self.replacers.items():
535
+ s = re.sub(pattern, replacement, s)
536
+
537
+ s = re.sub(r"(\d),(\d)", r"\1\2", s) # remove commas between digits
538
+ s = re.sub(r"\.([^0-9]|$)", r" \1", s) # remove periods not followed by numbers
539
+ s = remove_symbols_and_diacritics(s, keep=".%$¢€£") # keep numeric symbols
540
+
541
+ s = self.standardize_numbers(s)
542
+ s = self.standardize_spellings(s)
543
+
544
+ # now remove prefix/suffix symbols that are not preceded/followed by numbers
545
+ s = re.sub(r"[.$¢€£]([^0-9])", r" \1", s)
546
+ s = re.sub(r"([^0-9])%", r"\1 ", s)
547
+
548
+ s = re.sub(r"\s+", " ", s) # replace any successive whitespaces with a space
549
+
550
+ return s
whisper/timing.py ADDED
@@ -0,0 +1,385 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import itertools
2
+ import subprocess
3
+ import warnings
4
+ from dataclasses import dataclass
5
+ from typing import TYPE_CHECKING, List
6
+
7
+ import numba
8
+ import numpy as np
9
+ import torch
10
+ import torch.nn.functional as F
11
+
12
+ from .audio import HOP_LENGTH, SAMPLE_RATE, TOKENS_PER_SECOND
13
+ from .tokenizer import Tokenizer
14
+
15
+ if TYPE_CHECKING:
16
+ from .model import Whisper
17
+
18
+
19
+ def median_filter(x: torch.Tensor, filter_width: int):
20
+ """Apply a median filter of width `filter_width` along the last dimension of `x`"""
21
+ pad_width = filter_width // 2
22
+ if x.shape[-1] <= pad_width:
23
+ # F.pad requires the padding width to be smaller than the input dimension
24
+ return x
25
+
26
+ if (ndim := x.ndim) <= 2:
27
+ # `F.pad` does not support 1D or 2D inputs for reflect padding but supports 3D and 4D
28
+ x = x[None, None, :]
29
+
30
+ assert (
31
+ filter_width > 0 and filter_width % 2 == 1
32
+ ), "`filter_width` should be an odd number"
33
+
34
+ result = None
35
+ x = F.pad(x, (filter_width // 2, filter_width // 2, 0, 0), mode="reflect")
36
+ if x.is_cuda:
37
+ try:
38
+ from .triton_ops import median_filter_cuda
39
+
40
+ result = median_filter_cuda(x, filter_width)
41
+ except (RuntimeError, subprocess.CalledProcessError):
42
+ warnings.warn(
43
+ "Failed to launch Triton kernels, likely due to missing CUDA toolkit; "
44
+ "falling back to a slower median kernel implementation..."
45
+ )
46
+
47
+ if result is None:
48
+ # sort() is faster than torch.median (https://github.com/pytorch/pytorch/issues/51450)
49
+ result = x.unfold(-1, filter_width, 1).sort()[0][..., filter_width // 2]
50
+
51
+ if ndim <= 2:
52
+ result = result[0, 0]
53
+
54
+ return result
55
+
56
+
57
+ @numba.jit(nopython=True)
58
+ def backtrace(trace: np.ndarray):
59
+ i = trace.shape[0] - 1
60
+ j = trace.shape[1] - 1
61
+ trace[0, :] = 2
62
+ trace[:, 0] = 1
63
+
64
+ result = []
65
+ while i > 0 or j > 0:
66
+ result.append((i - 1, j - 1))
67
+
68
+ if trace[i, j] == 0:
69
+ i -= 1
70
+ j -= 1
71
+ elif trace[i, j] == 1:
72
+ i -= 1
73
+ elif trace[i, j] == 2:
74
+ j -= 1
75
+ else:
76
+ raise ValueError("Unexpected trace[i, j]")
77
+
78
+ result = np.array(result)
79
+ return result[::-1, :].T
80
+
81
+
82
+ @numba.jit(nopython=True, parallel=True)
83
+ def dtw_cpu(x: np.ndarray):
84
+ N, M = x.shape
85
+ cost = np.ones((N + 1, M + 1), dtype=np.float32) * np.inf
86
+ trace = -np.ones((N + 1, M + 1), dtype=np.float32)
87
+
88
+ cost[0, 0] = 0
89
+ for j in range(1, M + 1):
90
+ for i in range(1, N + 1):
91
+ c0 = cost[i - 1, j - 1]
92
+ c1 = cost[i - 1, j]
93
+ c2 = cost[i, j - 1]
94
+
95
+ if c0 < c1 and c0 < c2:
96
+ c, t = c0, 0
97
+ elif c1 < c0 and c1 < c2:
98
+ c, t = c1, 1
99
+ else:
100
+ c, t = c2, 2
101
+
102
+ cost[i, j] = x[i - 1, j - 1] + c
103
+ trace[i, j] = t
104
+
105
+ return backtrace(trace)
106
+
107
+
108
+ def dtw_cuda(x, BLOCK_SIZE=1024):
109
+ from .triton_ops import dtw_kernel
110
+
111
+ M, N = x.shape
112
+ assert M < BLOCK_SIZE, f"M should be smaller than {BLOCK_SIZE=}"
113
+
114
+ x_skew = (
115
+ F.pad(x, (0, M + 1), value=np.inf).flatten()[: M * (N + M)].reshape(M, N + M)
116
+ )
117
+ x_skew = x_skew.T.contiguous()
118
+ cost = torch.ones(N + M + 2, M + 2) * np.inf
119
+ cost[0, 0] = 0
120
+ cost = cost.cuda()
121
+ trace = torch.zeros_like(cost, dtype=torch.int32)
122
+
123
+ dtw_kernel[(1,)](
124
+ cost,
125
+ trace,
126
+ x_skew,
127
+ x_skew.stride(0),
128
+ cost.stride(0),
129
+ trace.stride(0),
130
+ N,
131
+ M,
132
+ BLOCK_SIZE=BLOCK_SIZE,
133
+ )
134
+
135
+ trace = trace.T.flatten()[: (M + 1) * (M + N + 3)].reshape(M + 1, M + N + 3)[
136
+ :, : N + 1
137
+ ]
138
+ return backtrace(trace.cpu().numpy())
139
+
140
+
141
+ def dtw(x: torch.Tensor) -> np.ndarray:
142
+ if x.is_cuda:
143
+ try:
144
+ return dtw_cuda(x)
145
+ except (RuntimeError, subprocess.CalledProcessError):
146
+ warnings.warn(
147
+ "Failed to launch Triton kernels, likely due to missing CUDA toolkit; "
148
+ "falling back to a slower DTW implementation..."
149
+ )
150
+
151
+ return dtw_cpu(x.double().cpu().numpy())
152
+
153
+
154
+ @dataclass
155
+ class WordTiming:
156
+ word: str
157
+ tokens: List[int]
158
+ start: float
159
+ end: float
160
+ probability: float
161
+
162
+
163
+ def find_alignment(
164
+ model: "Whisper",
165
+ tokenizer: Tokenizer,
166
+ text_tokens: List[int],
167
+ mel: torch.Tensor,
168
+ num_frames: int,
169
+ *,
170
+ medfilt_width: int = 7,
171
+ qk_scale: float = 1.0,
172
+ ) -> List[WordTiming]:
173
+ if len(text_tokens) == 0:
174
+ return []
175
+
176
+ tokens = torch.tensor(
177
+ [
178
+ *tokenizer.sot_sequence,
179
+ tokenizer.no_timestamps,
180
+ *text_tokens,
181
+ tokenizer.eot,
182
+ ]
183
+ ).to(model.device)
184
+
185
+ # install hooks on the cross attention layers to retrieve the attention weights
186
+ QKs = [None] * model.dims.n_text_layer
187
+ hooks = [
188
+ block.cross_attn.register_forward_hook(
189
+ lambda _, ins, outs, index=i: QKs.__setitem__(index, outs[-1][0])
190
+ )
191
+ for i, block in enumerate(model.decoder.blocks)
192
+ ]
193
+
194
+ with torch.no_grad():
195
+ logits = model(mel.unsqueeze(0), tokens.unsqueeze(0))[0]
196
+ sampled_logits = logits[len(tokenizer.sot_sequence) :, : tokenizer.eot]
197
+ token_probs = sampled_logits.softmax(dim=-1)
198
+ text_token_probs = token_probs[np.arange(len(text_tokens)), text_tokens]
199
+ text_token_probs = text_token_probs.tolist()
200
+
201
+ for hook in hooks:
202
+ hook.remove()
203
+
204
+ # heads * tokens * frames
205
+ weights = torch.stack([QKs[_l][_h] for _l, _h in model.alignment_heads.indices().T])
206
+ weights = weights[:, :, : num_frames // 2]
207
+ weights = (weights * qk_scale).softmax(dim=-1)
208
+ std, mean = torch.std_mean(weights, dim=-2, keepdim=True, unbiased=False)
209
+ weights = (weights - mean) / std
210
+ weights = median_filter(weights, medfilt_width)
211
+
212
+ matrix = weights.mean(axis=0)
213
+ matrix = matrix[len(tokenizer.sot_sequence) : -1]
214
+ text_indices, time_indices = dtw(-matrix)
215
+
216
+ words, word_tokens = tokenizer.split_to_word_tokens(text_tokens + [tokenizer.eot])
217
+ if len(word_tokens) <= 1:
218
+ # return on eot only
219
+ # >>> np.pad([], (1, 0))
220
+ # array([0.])
221
+ # This results in crashes when we lookup jump_times with float, like
222
+ # IndexError: arrays used as indices must be of integer (or boolean) type
223
+ return []
224
+ word_boundaries = np.pad(np.cumsum([len(t) for t in word_tokens[:-1]]), (1, 0))
225
+
226
+ jumps = np.pad(np.diff(text_indices), (1, 0), constant_values=1).astype(bool)
227
+ jump_times = time_indices[jumps] / TOKENS_PER_SECOND
228
+ start_times = jump_times[word_boundaries[:-1]]
229
+ end_times = jump_times[word_boundaries[1:]]
230
+ word_probabilities = [
231
+ np.mean(text_token_probs[i:j])
232
+ for i, j in zip(word_boundaries[:-1], word_boundaries[1:])
233
+ ]
234
+
235
+ return [
236
+ WordTiming(word, tokens, start, end, probability)
237
+ for word, tokens, start, end, probability in zip(
238
+ words, word_tokens, start_times, end_times, word_probabilities
239
+ )
240
+ ]
241
+
242
+
243
+ def merge_punctuations(alignment: List[WordTiming], prepended: str, appended: str):
244
+ # merge prepended punctuations
245
+ i = len(alignment) - 2
246
+ j = len(alignment) - 1
247
+ while i >= 0:
248
+ previous = alignment[i]
249
+ following = alignment[j]
250
+ if previous.word.startswith(" ") and previous.word.strip() in prepended:
251
+ # prepend it to the following word
252
+ following.word = previous.word + following.word
253
+ following.tokens = previous.tokens + following.tokens
254
+ previous.word = ""
255
+ previous.tokens = []
256
+ else:
257
+ j = i
258
+ i -= 1
259
+
260
+ # merge appended punctuations
261
+ i = 0
262
+ j = 1
263
+ while j < len(alignment):
264
+ previous = alignment[i]
265
+ following = alignment[j]
266
+ if not previous.word.endswith(" ") and following.word in appended:
267
+ # append it to the previous word
268
+ previous.word = previous.word + following.word
269
+ previous.tokens = previous.tokens + following.tokens
270
+ following.word = ""
271
+ following.tokens = []
272
+ else:
273
+ i = j
274
+ j += 1
275
+
276
+
277
+ def add_word_timestamps(
278
+ *,
279
+ segments: List[dict],
280
+ model: "Whisper",
281
+ tokenizer: Tokenizer,
282
+ mel: torch.Tensor,
283
+ num_frames: int,
284
+ prepend_punctuations: str = "\"'“¿([{-",
285
+ append_punctuations: str = "\"'.。,,!!??::”)]}、",
286
+ last_speech_timestamp: float,
287
+ **kwargs,
288
+ ):
289
+ if len(segments) == 0:
290
+ return
291
+
292
+ text_tokens_per_segment = [
293
+ [token for token in segment["tokens"] if token < tokenizer.eot]
294
+ for segment in segments
295
+ ]
296
+
297
+ text_tokens = list(itertools.chain.from_iterable(text_tokens_per_segment))
298
+ alignment = find_alignment(model, tokenizer, text_tokens, mel, num_frames, **kwargs)
299
+ word_durations = np.array([t.end - t.start for t in alignment])
300
+ word_durations = word_durations[word_durations.nonzero()]
301
+ median_duration = np.median(word_durations) if len(word_durations) > 0 else 0.0
302
+ max_duration = median_duration * 2
303
+
304
+ # hack: truncate long words at sentence boundaries.
305
+ # a better segmentation algorithm based on VAD should be able to replace this.
306
+ if len(word_durations) > 0:
307
+ sentence_end_marks = ".。!!??"
308
+ # ensure words at sentence boundaries are not longer than twice the median word duration.
309
+ for i in range(1, len(alignment)):
310
+ if alignment[i].end - alignment[i].start > max_duration:
311
+ if alignment[i].word in sentence_end_marks:
312
+ alignment[i].end = alignment[i].start + max_duration
313
+ elif alignment[i - 1].word in sentence_end_marks:
314
+ alignment[i].start = alignment[i].end - max_duration
315
+
316
+ merge_punctuations(alignment, prepend_punctuations, append_punctuations)
317
+
318
+ time_offset = segments[0]["seek"] * HOP_LENGTH / SAMPLE_RATE
319
+ word_index = 0
320
+
321
+ for segment, text_tokens in zip(segments, text_tokens_per_segment):
322
+ saved_tokens = 0
323
+ words = []
324
+
325
+ while word_index < len(alignment) and saved_tokens < len(text_tokens):
326
+ timing = alignment[word_index]
327
+
328
+ if timing.word:
329
+ words.append(
330
+ dict(
331
+ word=timing.word,
332
+ start=round(time_offset + timing.start, 2),
333
+ end=round(time_offset + timing.end, 2),
334
+ probability=timing.probability,
335
+ )
336
+ )
337
+
338
+ saved_tokens += len(timing.tokens)
339
+ word_index += 1
340
+
341
+ # hack: truncate long words at segment boundaries.
342
+ # a better segmentation algorithm based on VAD should be able to replace this.
343
+ if len(words) > 0:
344
+ # ensure the first and second word after a pause is not longer than
345
+ # twice the median word duration.
346
+ if words[0]["end"] - last_speech_timestamp > median_duration * 4 and (
347
+ words[0]["end"] - words[0]["start"] > max_duration
348
+ or (
349
+ len(words) > 1
350
+ and words[1]["end"] - words[0]["start"] > max_duration * 2
351
+ )
352
+ ):
353
+ if (
354
+ len(words) > 1
355
+ and words[1]["end"] - words[1]["start"] > max_duration
356
+ ):
357
+ boundary = max(words[1]["end"] / 2, words[1]["end"] - max_duration)
358
+ words[0]["end"] = words[1]["start"] = boundary
359
+ words[0]["start"] = max(0, words[0]["end"] - max_duration)
360
+
361
+ # prefer the segment-level start timestamp if the first word is too long.
362
+ if (
363
+ segment["start"] < words[0]["end"]
364
+ and segment["start"] - 0.5 > words[0]["start"]
365
+ ):
366
+ words[0]["start"] = max(
367
+ 0, min(words[0]["end"] - median_duration, segment["start"])
368
+ )
369
+ else:
370
+ segment["start"] = words[0]["start"]
371
+
372
+ # prefer the segment-level end timestamp if the last word is too long.
373
+ if (
374
+ segment["end"] > words[-1]["start"]
375
+ and segment["end"] + 0.5 < words[-1]["end"]
376
+ ):
377
+ words[-1]["end"] = max(
378
+ words[-1]["start"] + median_duration, segment["end"]
379
+ )
380
+ else:
381
+ segment["end"] = words[-1]["end"]
382
+
383
+ last_speech_timestamp = segment["end"]
384
+
385
+ segment["words"] = words
whisper/tokenizer.py ADDED
@@ -0,0 +1,395 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import base64
2
+ import os
3
+ import string
4
+ from dataclasses import dataclass, field
5
+ from functools import cached_property, lru_cache
6
+ from typing import Dict, List, Optional, Tuple
7
+
8
+ import tiktoken
9
+
10
+ LANGUAGES = {
11
+ "en": "english",
12
+ "zh": "chinese",
13
+ "de": "german",
14
+ "es": "spanish",
15
+ "ru": "russian",
16
+ "ko": "korean",
17
+ "fr": "french",
18
+ "ja": "japanese",
19
+ "pt": "portuguese",
20
+ "tr": "turkish",
21
+ "pl": "polish",
22
+ "ca": "catalan",
23
+ "nl": "dutch",
24
+ "ar": "arabic",
25
+ "sv": "swedish",
26
+ "it": "italian",
27
+ "id": "indonesian",
28
+ "hi": "hindi",
29
+ "fi": "finnish",
30
+ "vi": "vietnamese",
31
+ "he": "hebrew",
32
+ "uk": "ukrainian",
33
+ "el": "greek",
34
+ "ms": "malay",
35
+ "cs": "czech",
36
+ "ro": "romanian",
37
+ "da": "danish",
38
+ "hu": "hungarian",
39
+ "ta": "tamil",
40
+ "no": "norwegian",
41
+ "th": "thai",
42
+ "ur": "urdu",
43
+ "hr": "croatian",
44
+ "bg": "bulgarian",
45
+ "lt": "lithuanian",
46
+ "la": "latin",
47
+ "mi": "maori",
48
+ "ml": "malayalam",
49
+ "cy": "welsh",
50
+ "sk": "slovak",
51
+ "te": "telugu",
52
+ "fa": "persian",
53
+ "lv": "latvian",
54
+ "bn": "bengali",
55
+ "sr": "serbian",
56
+ "az": "azerbaijani",
57
+ "sl": "slovenian",
58
+ "kn": "kannada",
59
+ "et": "estonian",
60
+ "mk": "macedonian",
61
+ "br": "breton",
62
+ "eu": "basque",
63
+ "is": "icelandic",
64
+ "hy": "armenian",
65
+ "ne": "nepali",
66
+ "mn": "mongolian",
67
+ "bs": "bosnian",
68
+ "kk": "kazakh",
69
+ "sq": "albanian",
70
+ "sw": "swahili",
71
+ "gl": "galician",
72
+ "mr": "marathi",
73
+ "pa": "punjabi",
74
+ "si": "sinhala",
75
+ "km": "khmer",
76
+ "sn": "shona",
77
+ "yo": "yoruba",
78
+ "so": "somali",
79
+ "af": "afrikaans",
80
+ "oc": "occitan",
81
+ "ka": "georgian",
82
+ "be": "belarusian",
83
+ "tg": "tajik",
84
+ "sd": "sindhi",
85
+ "gu": "gujarati",
86
+ "am": "amharic",
87
+ "yi": "yiddish",
88
+ "lo": "lao",
89
+ "uz": "uzbek",
90
+ "fo": "faroese",
91
+ "ht": "haitian creole",
92
+ "ps": "pashto",
93
+ "tk": "turkmen",
94
+ "nn": "nynorsk",
95
+ "mt": "maltese",
96
+ "sa": "sanskrit",
97
+ "lb": "luxembourgish",
98
+ "my": "myanmar",
99
+ "bo": "tibetan",
100
+ "tl": "tagalog",
101
+ "mg": "malagasy",
102
+ "as": "assamese",
103
+ "tt": "tatar",
104
+ "haw": "hawaiian",
105
+ "ln": "lingala",
106
+ "ha": "hausa",
107
+ "ba": "bashkir",
108
+ "jw": "javanese",
109
+ "su": "sundanese",
110
+ "yue": "cantonese",
111
+ }
112
+
113
+ # language code lookup by name, with a few language aliases
114
+ TO_LANGUAGE_CODE = {
115
+ **{language: code for code, language in LANGUAGES.items()},
116
+ "burmese": "my",
117
+ "valencian": "ca",
118
+ "flemish": "nl",
119
+ "haitian": "ht",
120
+ "letzeburgesch": "lb",
121
+ "pushto": "ps",
122
+ "panjabi": "pa",
123
+ "moldavian": "ro",
124
+ "moldovan": "ro",
125
+ "sinhalese": "si",
126
+ "castilian": "es",
127
+ "mandarin": "zh",
128
+ }
129
+
130
+
131
+ @dataclass
132
+ class Tokenizer:
133
+ """A thin wrapper around `tiktoken` providing quick access to special tokens"""
134
+
135
+ encoding: tiktoken.Encoding
136
+ num_languages: int
137
+ language: Optional[str] = None
138
+ task: Optional[str] = None
139
+ sot_sequence: Tuple[int] = ()
140
+ special_tokens: Dict[str, int] = field(default_factory=dict)
141
+
142
+ def __post_init__(self):
143
+ for special in self.encoding.special_tokens_set:
144
+ special_token = self.encoding.encode_single_token(special)
145
+ self.special_tokens[special] = special_token
146
+
147
+ sot: int = self.special_tokens["<|startoftranscript|>"]
148
+ translate: int = self.special_tokens["<|translate|>"]
149
+ transcribe: int = self.special_tokens["<|transcribe|>"]
150
+
151
+ langs = tuple(LANGUAGES.keys())[: self.num_languages]
152
+ sot_sequence = [sot]
153
+ if self.language is not None:
154
+ sot_sequence.append(sot + 1 + langs.index(self.language))
155
+ if self.task is not None:
156
+ task_token: int = transcribe if self.task == "transcribe" else translate
157
+ sot_sequence.append(task_token)
158
+
159
+ self.sot_sequence = tuple(sot_sequence)
160
+
161
+ def encode(self, text, **kwargs):
162
+ return self.encoding.encode(text, **kwargs)
163
+
164
+ def decode(self, token_ids: List[int], **kwargs) -> str:
165
+ token_ids = [t for t in token_ids if t < self.timestamp_begin]
166
+ return self.encoding.decode(token_ids, **kwargs)
167
+
168
+ def decode_with_timestamps(self, token_ids: List[int], **kwargs) -> str:
169
+ """
170
+ Timestamp tokens are above other special tokens' id range and are ignored by `decode()`.
171
+ This method decodes given tokens with timestamps tokens annotated, e.g. "<|1.08|>".
172
+ """
173
+ return self.encoding.decode(token_ids, **kwargs)
174
+
175
+ @cached_property
176
+ def eot(self) -> int:
177
+ return self.encoding.eot_token
178
+
179
+ @cached_property
180
+ def transcribe(self) -> int:
181
+ return self.special_tokens["<|transcribe|>"]
182
+
183
+ @cached_property
184
+ def translate(self) -> int:
185
+ return self.special_tokens["<|translate|>"]
186
+
187
+ @cached_property
188
+ def sot(self) -> int:
189
+ return self.special_tokens["<|startoftranscript|>"]
190
+
191
+ @cached_property
192
+ def sot_lm(self) -> int:
193
+ return self.special_tokens["<|startoflm|>"]
194
+
195
+ @cached_property
196
+ def sot_prev(self) -> int:
197
+ return self.special_tokens["<|startofprev|>"]
198
+
199
+ @cached_property
200
+ def no_speech(self) -> int:
201
+ return self.special_tokens["<|nospeech|>"]
202
+
203
+ @cached_property
204
+ def no_timestamps(self) -> int:
205
+ return self.special_tokens["<|notimestamps|>"]
206
+
207
+ @cached_property
208
+ def timestamp_begin(self) -> int:
209
+ return self.special_tokens["<|0.00|>"]
210
+
211
+ @cached_property
212
+ def language_token(self) -> int:
213
+ """Returns the token id corresponding to the value of the `language` field"""
214
+ if self.language is None:
215
+ raise ValueError("This tokenizer does not have language token configured")
216
+
217
+ return self.to_language_token(self.language)
218
+
219
+ def to_language_token(self, language):
220
+ if token := self.special_tokens.get(f"<|{language}|>", None):
221
+ return token
222
+
223
+ raise KeyError(f"Language {language} not found in tokenizer.")
224
+
225
+ @cached_property
226
+ def all_language_tokens(self) -> Tuple[int]:
227
+ result = []
228
+ for token, token_id in self.special_tokens.items():
229
+ if token.strip("<|>") in LANGUAGES:
230
+ result.append(token_id)
231
+ return tuple(result)[: self.num_languages]
232
+
233
+ @cached_property
234
+ def all_language_codes(self) -> Tuple[str]:
235
+ return tuple(self.decode([_l]).strip("<|>") for _l in self.all_language_tokens)
236
+
237
+ @cached_property
238
+ def sot_sequence_including_notimestamps(self) -> Tuple[int]:
239
+ return tuple(list(self.sot_sequence) + [self.no_timestamps])
240
+
241
+ @cached_property
242
+ def non_speech_tokens(self) -> Tuple[int]:
243
+ """
244
+ Returns the list of tokens to suppress in order to avoid any speaker tags or non-speech
245
+ annotations, to prevent sampling texts that are not actually spoken in the audio, e.g.
246
+
247
+ - ♪♪♪
248
+ - ( SPEAKING FOREIGN LANGUAGE )
249
+ - [DAVID] Hey there,
250
+
251
+ keeping basic punctuations like commas, periods, question marks, exclamation points, etc.
252
+ """
253
+ symbols = list('"#()*+/:;<=>@[\\]^_`{|}~「」『』')
254
+ symbols += (
255
+ "<< >> <<< >>> -- --- -( -[ (' (\" (( )) ((( ))) [[ ]] {{ }} ♪♪ ♪♪♪".split()
256
+ )
257
+
258
+ # symbols that may be a single token or multiple tokens depending on the tokenizer.
259
+ # In case they're multiple tokens, suppress the first token, which is safe because:
260
+ # These are between U+2640 and U+267F miscellaneous symbols that are okay to suppress
261
+ # in generations, and in the 3-byte UTF-8 representation they share the first two bytes.
262
+ miscellaneous = set("♩♪♫♬♭♮♯")
263
+ assert all(0x2640 <= ord(c) <= 0x267F for c in miscellaneous)
264
+
265
+ # allow hyphens "-" and single quotes "'" between words, but not at the beginning of a word
266
+ result = {self.encoding.encode(" -")[0], self.encoding.encode(" '")[0]}
267
+ for symbol in symbols + list(miscellaneous):
268
+ for tokens in [
269
+ self.encoding.encode(symbol),
270
+ self.encoding.encode(" " + symbol),
271
+ ]:
272
+ if len(tokens) == 1 or symbol in miscellaneous:
273
+ result.add(tokens[0])
274
+
275
+ return tuple(sorted(result))
276
+
277
+ def split_to_word_tokens(self, tokens: List[int]):
278
+ if self.language in {"zh", "ja", "th", "lo", "my", "yue"}:
279
+ # These languages don't typically use spaces, so it is difficult to split words
280
+ # without morpheme analysis. Here, we instead split words at any
281
+ # position where the tokens are decoded as valid unicode points
282
+ return self.split_tokens_on_unicode(tokens)
283
+
284
+ return self.split_tokens_on_spaces(tokens)
285
+
286
+ def split_tokens_on_unicode(self, tokens: List[int]):
287
+ decoded_full = self.decode_with_timestamps(tokens)
288
+ replacement_char = "\ufffd"
289
+
290
+ words = []
291
+ word_tokens = []
292
+ current_tokens = []
293
+ unicode_offset = 0
294
+
295
+ for token in tokens:
296
+ current_tokens.append(token)
297
+ decoded = self.decode_with_timestamps(current_tokens)
298
+
299
+ if (
300
+ replacement_char not in decoded
301
+ or decoded_full[unicode_offset + decoded.index(replacement_char)]
302
+ == replacement_char
303
+ ):
304
+ words.append(decoded)
305
+ word_tokens.append(current_tokens)
306
+ current_tokens = []
307
+ unicode_offset += len(decoded)
308
+
309
+ return words, word_tokens
310
+
311
+ def split_tokens_on_spaces(self, tokens: List[int]):
312
+ subwords, subword_tokens_list = self.split_tokens_on_unicode(tokens)
313
+ words = []
314
+ word_tokens = []
315
+
316
+ for subword, subword_tokens in zip(subwords, subword_tokens_list):
317
+ special = subword_tokens[0] >= self.eot
318
+ with_space = subword.startswith(" ")
319
+ punctuation = subword.strip() in string.punctuation
320
+ if special or with_space or punctuation or len(words) == 0:
321
+ words.append(subword)
322
+ word_tokens.append(subword_tokens)
323
+ else:
324
+ words[-1] = words[-1] + subword
325
+ word_tokens[-1].extend(subword_tokens)
326
+
327
+ return words, word_tokens
328
+
329
+
330
+ @lru_cache(maxsize=None)
331
+ def get_encoding(name: str = "gpt2", num_languages: int = 99):
332
+ vocab_path = os.path.join(os.path.dirname(__file__), "assets", f"{name}.tiktoken")
333
+ ranks = {
334
+ base64.b64decode(token): int(rank)
335
+ for token, rank in (line.split() for line in open(vocab_path) if line)
336
+ }
337
+ n_vocab = len(ranks)
338
+ special_tokens = {}
339
+
340
+ specials = [
341
+ "<|endoftext|>",
342
+ "<|startoftranscript|>",
343
+ *[f"<|{lang}|>" for lang in list(LANGUAGES.keys())[:num_languages]],
344
+ "<|translate|>",
345
+ "<|transcribe|>",
346
+ "<|startoflm|>",
347
+ "<|startofprev|>",
348
+ "<|nospeech|>",
349
+ "<|notimestamps|>",
350
+ *[f"<|{i * 0.02:.2f}|>" for i in range(1501)],
351
+ ]
352
+
353
+ for token in specials:
354
+ special_tokens[token] = n_vocab
355
+ n_vocab += 1
356
+
357
+ return tiktoken.Encoding(
358
+ name=os.path.basename(vocab_path),
359
+ explicit_n_vocab=n_vocab,
360
+ pat_str=r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""",
361
+ mergeable_ranks=ranks,
362
+ special_tokens=special_tokens,
363
+ )
364
+
365
+
366
+ @lru_cache(maxsize=None)
367
+ def get_tokenizer(
368
+ multilingual: bool,
369
+ *,
370
+ num_languages: int = 99,
371
+ language: Optional[str] = None,
372
+ task: Optional[str] = None, # Literal["transcribe", "translate", None]
373
+ ) -> Tokenizer:
374
+ if language is not None:
375
+ language = language.lower()
376
+ if language not in LANGUAGES:
377
+ if language in TO_LANGUAGE_CODE:
378
+ language = TO_LANGUAGE_CODE[language]
379
+ else:
380
+ raise ValueError(f"Unsupported language: {language}")
381
+
382
+ if multilingual:
383
+ encoding_name = "multilingual"
384
+ language = language or "en"
385
+ task = task or "transcribe"
386
+ else:
387
+ encoding_name = "gpt2"
388
+ language = None
389
+ task = None
390
+
391
+ encoding = get_encoding(name=encoding_name, num_languages=num_languages)
392
+
393
+ return Tokenizer(
394
+ encoding=encoding, num_languages=num_languages, language=language, task=task
395
+ )
whisper/transcribe.py ADDED
@@ -0,0 +1,486 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import os
3
+ import traceback
4
+ import warnings
5
+ from typing import TYPE_CHECKING, Optional, Tuple, Union
6
+
7
+ import numpy as np
8
+ import torch
9
+ import tqdm
10
+
11
+ from .audio import (
12
+ FRAMES_PER_SECOND,
13
+ HOP_LENGTH,
14
+ N_FRAMES,
15
+ N_SAMPLES,
16
+ SAMPLE_RATE,
17
+ log_mel_spectrogram,
18
+ pad_or_trim,
19
+ )
20
+ from .decoding import DecodingOptions, DecodingResult
21
+ from .timing import add_word_timestamps
22
+ from .tokenizer import LANGUAGES, TO_LANGUAGE_CODE, get_tokenizer
23
+ from .utils import (
24
+ exact_div,
25
+ format_timestamp,
26
+ get_writer,
27
+ make_safe,
28
+ optional_float,
29
+ optional_int,
30
+ str2bool,
31
+ )
32
+
33
+ if TYPE_CHECKING:
34
+ from .model import Whisper
35
+
36
+
37
+ def transcribe(
38
+ model: "Whisper",
39
+ audio: Union[str, np.ndarray, torch.Tensor],
40
+ *,
41
+ verbose: Optional[bool] = None,
42
+ temperature: Union[float, Tuple[float, ...]] = (0.0, 0.2, 0.4, 0.6, 0.8, 1.0),
43
+ compression_ratio_threshold: Optional[float] = 2.4,
44
+ logprob_threshold: Optional[float] = -1.0,
45
+ no_speech_threshold: Optional[float] = 0.6,
46
+ condition_on_previous_text: bool = True,
47
+ initial_prompt: Optional[str] = None,
48
+ word_timestamps: bool = False,
49
+ prepend_punctuations: str = "\"'“¿([{-",
50
+ append_punctuations: str = "\"'.。,,!!??::”)]}、",
51
+ **decode_options,
52
+ ):
53
+ """
54
+ Transcribe an audio file using Whisper
55
+
56
+ Parameters
57
+ ----------
58
+ model: Whisper
59
+ The Whisper model instance
60
+
61
+ audio: Union[str, np.ndarray, torch.Tensor]
62
+ The path to the audio file to open, or the audio waveform
63
+
64
+ verbose: bool
65
+ Whether to display the text being decoded to the console. If True, displays all the details,
66
+ If False, displays minimal details. If None, does not display anything
67
+
68
+ temperature: Union[float, Tuple[float, ...]]
69
+ Temperature for sampling. It can be a tuple of temperatures, which will be successively used
70
+ upon failures according to either `compression_ratio_threshold` or `logprob_threshold`.
71
+
72
+ compression_ratio_threshold: float
73
+ If the gzip compression ratio is above this value, treat as failed
74
+
75
+ logprob_threshold: float
76
+ If the average log probability over sampled tokens is below this value, treat as failed
77
+
78
+ no_speech_threshold: float
79
+ If the no_speech probability is higher than this value AND the average log probability
80
+ over sampled tokens is below `logprob_threshold`, consider the segment as silent
81
+
82
+ condition_on_previous_text: bool
83
+ if True, the previous output of the model is provided as a prompt for the next window;
84
+ disabling may make the text inconsistent across windows, but the model becomes less prone to
85
+ getting stuck in a failure loop, such as repetition looping or timestamps going out of sync.
86
+
87
+ word_timestamps: bool
88
+ Extract word-level timestamps using the cross-attention pattern and dynamic time warping,
89
+ and include the timestamps for each word in each segment.
90
+
91
+ prepend_punctuations: str
92
+ If word_timestamps is True, merge these punctuation symbols with the next word
93
+
94
+ append_punctuations: str
95
+ If word_timestamps is True, merge these punctuation symbols with the previous word
96
+
97
+ initial_prompt: Optional[str]
98
+ Optional text to provide as a prompt for the first window. This can be used to provide, or
99
+ "prompt-engineer" a context for transcription, e.g. custom vocabularies or proper nouns
100
+ to make it more likely to predict those word correctly.
101
+
102
+ decode_options: dict
103
+ Keyword arguments to construct `DecodingOptions` instances
104
+
105
+ Returns
106
+ -------
107
+ A dictionary containing the resulting text ("text") and segment-level details ("segments"), and
108
+ the spoken language ("language"), which is detected when `decode_options["language"]` is None.
109
+ """
110
+ dtype = torch.float16 if decode_options.get("fp16", True) else torch.float32
111
+ if model.device == torch.device("cpu"):
112
+ if torch.cuda.is_available():
113
+ warnings.warn("Performing inference on CPU when CUDA is available")
114
+ if dtype == torch.float16:
115
+ warnings.warn("FP16 is not supported on CPU; using FP32 instead")
116
+ dtype = torch.float32
117
+
118
+ if dtype == torch.float32:
119
+ decode_options["fp16"] = False
120
+
121
+ # Pad 30-seconds of silence to the input audio, for slicing
122
+ mel = log_mel_spectrogram(audio, model.dims.n_mels, padding=N_SAMPLES)
123
+ content_frames = mel.shape[-1] - N_FRAMES
124
+
125
+ if decode_options.get("language", None) is None:
126
+ if not model.is_multilingual:
127
+ decode_options["language"] = "en"
128
+ else:
129
+ if verbose:
130
+ print(
131
+ "Detecting language using up to the first 30 seconds. Use `--language` to specify the language"
132
+ )
133
+ mel_segment = pad_or_trim(mel, N_FRAMES).to(model.device).to(dtype)
134
+ _, probs = model.detect_language(mel_segment)
135
+ decode_options["language"] = max(probs, key=probs.get)
136
+ if verbose is not None:
137
+ print(
138
+ f"Detected language: {LANGUAGES[decode_options['language']].title()}"
139
+ )
140
+
141
+ language: str = decode_options["language"]
142
+ task: str = decode_options.get("task", "transcribe")
143
+ tokenizer = get_tokenizer(
144
+ model.is_multilingual,
145
+ num_languages=model.num_languages,
146
+ language=language,
147
+ task=task,
148
+ )
149
+
150
+ if word_timestamps and task == "translate":
151
+ warnings.warn("Word-level timestamps on translations may not be reliable.")
152
+
153
+ def decode_with_fallback(segment: torch.Tensor) -> DecodingResult:
154
+ temperatures = (
155
+ [temperature] if isinstance(temperature, (int, float)) else temperature
156
+ )
157
+ decode_result = None
158
+
159
+ for t in temperatures:
160
+ kwargs = {**decode_options}
161
+ if t > 0:
162
+ # disable beam_size and patience when t > 0
163
+ kwargs.pop("beam_size", None)
164
+ kwargs.pop("patience", None)
165
+ else:
166
+ # disable best_of when t == 0
167
+ kwargs.pop("best_of", None)
168
+
169
+ options = DecodingOptions(**kwargs, temperature=t)
170
+ decode_result = model.decode(segment, options)
171
+
172
+ needs_fallback = False
173
+ if (
174
+ compression_ratio_threshold is not None
175
+ and decode_result.compression_ratio > compression_ratio_threshold
176
+ ):
177
+ needs_fallback = True # too repetitive
178
+ if (
179
+ logprob_threshold is not None
180
+ and decode_result.avg_logprob < logprob_threshold
181
+ ):
182
+ needs_fallback = True # average log probability is too low
183
+ if (
184
+ no_speech_threshold is not None
185
+ and decode_result.no_speech_prob > no_speech_threshold
186
+ ):
187
+ needs_fallback = False # silence
188
+ if not needs_fallback:
189
+ break
190
+
191
+ return decode_result
192
+
193
+ seek = 0
194
+ input_stride = exact_div(
195
+ N_FRAMES, model.dims.n_audio_ctx
196
+ ) # mel frames per output token: 2
197
+ time_precision = (
198
+ input_stride * HOP_LENGTH / SAMPLE_RATE
199
+ ) # time per output token: 0.02 (seconds)
200
+ all_tokens = []
201
+ all_segments = []
202
+ prompt_reset_since = 0
203
+
204
+ if initial_prompt is not None:
205
+ initial_prompt_tokens = tokenizer.encode(" " + initial_prompt.strip())
206
+ all_tokens.extend(initial_prompt_tokens)
207
+ else:
208
+ initial_prompt_tokens = []
209
+
210
+ def new_segment(
211
+ *, start: float, end: float, tokens: torch.Tensor, result: DecodingResult
212
+ ):
213
+ tokens = tokens.tolist()
214
+ text_tokens = [token for token in tokens if token < tokenizer.eot]
215
+ return {
216
+ "seek": seek,
217
+ "start": start,
218
+ "end": end,
219
+ "text": tokenizer.decode(text_tokens),
220
+ "tokens": tokens,
221
+ "temperature": result.temperature,
222
+ "avg_logprob": result.avg_logprob,
223
+ "compression_ratio": result.compression_ratio,
224
+ "no_speech_prob": result.no_speech_prob,
225
+ }
226
+
227
+ # show the progress bar when verbose is False (if True, transcribed text will be printed)
228
+ with tqdm.tqdm(
229
+ total=content_frames, unit="frames", disable=verbose is not False
230
+ ) as pbar:
231
+ last_speech_timestamp = 0.0
232
+ while seek < content_frames:
233
+ time_offset = float(seek * HOP_LENGTH / SAMPLE_RATE)
234
+ mel_segment = mel[:, seek : seek + N_FRAMES]
235
+ segment_size = min(N_FRAMES, content_frames - seek)
236
+ segment_duration = segment_size * HOP_LENGTH / SAMPLE_RATE
237
+ mel_segment = pad_or_trim(mel_segment, N_FRAMES).to(model.device).to(dtype)
238
+
239
+ decode_options["prompt"] = all_tokens[prompt_reset_since:]
240
+ result: DecodingResult = decode_with_fallback(mel_segment)
241
+ tokens = torch.tensor(result.tokens)
242
+
243
+ if no_speech_threshold is not None:
244
+ # no voice activity check
245
+ should_skip = result.no_speech_prob > no_speech_threshold
246
+ if (
247
+ logprob_threshold is not None
248
+ and result.avg_logprob > logprob_threshold
249
+ ):
250
+ # don't skip if the logprob is high enough, despite the no_speech_prob
251
+ should_skip = False
252
+
253
+ if should_skip:
254
+ seek += segment_size # fast-forward to the next segment boundary
255
+ continue
256
+
257
+ previous_seek = seek
258
+ current_segments = []
259
+
260
+ timestamp_tokens: torch.Tensor = tokens.ge(tokenizer.timestamp_begin)
261
+ single_timestamp_ending = timestamp_tokens[-2:].tolist() == [False, True]
262
+
263
+ consecutive = torch.where(timestamp_tokens[:-1] & timestamp_tokens[1:])[0]
264
+ consecutive.add_(1)
265
+ if len(consecutive) > 0:
266
+ # if the output contains two consecutive timestamp tokens
267
+ slices = consecutive.tolist()
268
+ if single_timestamp_ending:
269
+ slices.append(len(tokens))
270
+
271
+ last_slice = 0
272
+ for current_slice in slices:
273
+ sliced_tokens = tokens[last_slice:current_slice]
274
+ start_timestamp_pos = (
275
+ sliced_tokens[0].item() - tokenizer.timestamp_begin
276
+ )
277
+ end_timestamp_pos = (
278
+ sliced_tokens[-1].item() - tokenizer.timestamp_begin
279
+ )
280
+ current_segments.append(
281
+ new_segment(
282
+ start=time_offset + start_timestamp_pos * time_precision,
283
+ end=time_offset + end_timestamp_pos * time_precision,
284
+ tokens=sliced_tokens,
285
+ result=result,
286
+ )
287
+ )
288
+ last_slice = current_slice
289
+
290
+ if single_timestamp_ending:
291
+ # single timestamp at the end means no speech after the last timestamp.
292
+ seek += segment_size
293
+ else:
294
+ # otherwise, ignore the unfinished segment and seek to the last timestamp
295
+ last_timestamp_pos = (
296
+ tokens[last_slice - 1].item() - tokenizer.timestamp_begin
297
+ )
298
+ seek += last_timestamp_pos * input_stride
299
+ else:
300
+ duration = segment_duration
301
+ timestamps = tokens[timestamp_tokens.nonzero().flatten()]
302
+ if (
303
+ len(timestamps) > 0
304
+ and timestamps[-1].item() != tokenizer.timestamp_begin
305
+ ):
306
+ # no consecutive timestamps but it has a timestamp; use the last one.
307
+ last_timestamp_pos = (
308
+ timestamps[-1].item() - tokenizer.timestamp_begin
309
+ )
310
+ duration = last_timestamp_pos * time_precision
311
+
312
+ current_segments.append(
313
+ new_segment(
314
+ start=time_offset,
315
+ end=time_offset + duration,
316
+ tokens=tokens,
317
+ result=result,
318
+ )
319
+ )
320
+ seek += segment_size
321
+
322
+ if word_timestamps:
323
+ add_word_timestamps(
324
+ segments=current_segments,
325
+ model=model,
326
+ tokenizer=tokenizer,
327
+ mel=mel_segment,
328
+ num_frames=segment_size,
329
+ prepend_punctuations=prepend_punctuations,
330
+ append_punctuations=append_punctuations,
331
+ last_speech_timestamp=last_speech_timestamp,
332
+ )
333
+ word_end_timestamps = [
334
+ w["end"] for s in current_segments for w in s["words"]
335
+ ]
336
+ if len(word_end_timestamps) > 0:
337
+ last_speech_timestamp = word_end_timestamps[-1]
338
+ if not single_timestamp_ending and len(word_end_timestamps) > 0:
339
+ seek_shift = round(
340
+ (word_end_timestamps[-1] - time_offset) * FRAMES_PER_SECOND
341
+ )
342
+ if seek_shift > 0:
343
+ seek = previous_seek + seek_shift
344
+
345
+ if verbose:
346
+ for segment in current_segments:
347
+ start, end, text = segment["start"], segment["end"], segment["text"]
348
+ line = f"[{format_timestamp(start)} --> {format_timestamp(end)}] {text}"
349
+ print(make_safe(line))
350
+
351
+ # if a segment is instantaneous or does not contain text, clear it
352
+ for i, segment in enumerate(current_segments):
353
+ if segment["start"] == segment["end"] or segment["text"].strip() == "":
354
+ segment["text"] = ""
355
+ segment["tokens"] = []
356
+ segment["words"] = []
357
+
358
+ all_segments.extend(
359
+ [
360
+ {"id": i, **segment}
361
+ for i, segment in enumerate(
362
+ current_segments, start=len(all_segments)
363
+ )
364
+ ]
365
+ )
366
+ all_tokens.extend(
367
+ [token for segment in current_segments for token in segment["tokens"]]
368
+ )
369
+
370
+ if not condition_on_previous_text or result.temperature > 0.5:
371
+ # do not feed the prompt tokens if a high temperature was used
372
+ prompt_reset_since = len(all_tokens)
373
+
374
+ # update progress bar
375
+ pbar.update(min(content_frames, seek) - previous_seek)
376
+
377
+ return dict(
378
+ text=tokenizer.decode(all_tokens[len(initial_prompt_tokens) :]),
379
+ segments=all_segments,
380
+ language=language,
381
+ )
382
+
383
+
384
+ def cli():
385
+ from . import available_models
386
+
387
+ def valid_model_name(name):
388
+ if name in available_models() or os.path.exists(name):
389
+ return name
390
+ raise ValueError(
391
+ f"model should be one of {available_models()} or path to a model checkpoint"
392
+ )
393
+
394
+ # fmt: off
395
+ parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
396
+ parser.add_argument("audio", nargs="+", type=str, help="audio file(s) to transcribe")
397
+ parser.add_argument("--model", default="small", type=valid_model_name, help="name of the Whisper model to use")
398
+ parser.add_argument("--model_dir", type=str, default=None, help="the path to save model files; uses ~/.cache/whisper by default")
399
+ parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu", help="device to use for PyTorch inference")
400
+ parser.add_argument("--output_dir", "-o", type=str, default=".", help="directory to save the outputs")
401
+ parser.add_argument("--output_format", "-f", type=str, default="all", choices=["txt", "vtt", "srt", "tsv", "json", "all"], help="format of the output file; if not specified, all available formats will be produced")
402
+ parser.add_argument("--verbose", type=str2bool, default=True, help="whether to print out the progress and debug messages")
403
+
404
+ parser.add_argument("--task", type=str, default="transcribe", choices=["transcribe", "translate"], help="whether to perform X->X speech recognition ('transcribe') or X->English translation ('translate')")
405
+ parser.add_argument("--language", type=str, default=None, choices=sorted(LANGUAGES.keys()) + sorted([k.title() for k in TO_LANGUAGE_CODE.keys()]), help="language spoken in the audio, specify None to perform language detection")
406
+
407
+ parser.add_argument("--temperature", type=float, default=0, help="temperature to use for sampling")
408
+ parser.add_argument("--best_of", type=optional_int, default=5, help="number of candidates when sampling with non-zero temperature")
409
+ parser.add_argument("--beam_size", type=optional_int, default=5, help="number of beams in beam search, only applicable when temperature is zero")
410
+ parser.add_argument("--patience", type=float, default=None, help="optional patience value to use in beam decoding, as in https://arxiv.org/abs/2204.05424, the default (1.0) is equivalent to conventional beam search")
411
+ parser.add_argument("--length_penalty", type=float, default=None, help="optional token length penalty coefficient (alpha) as in https://arxiv.org/abs/1609.08144, uses simple length normalization by default")
412
+
413
+ parser.add_argument("--suppress_tokens", type=str, default="-1", help="comma-separated list of token ids to suppress during sampling; '-1' will suppress most special characters except common punctuations")
414
+ parser.add_argument("--initial_prompt", type=str, default=None, help="optional text to provide as a prompt for the first window.")
415
+ parser.add_argument("--condition_on_previous_text", type=str2bool, default=True, help="if True, provide the previous output of the model as a prompt for the next window; disabling may make the text inconsistent across windows, but the model becomes less prone to getting stuck in a failure loop")
416
+ parser.add_argument("--fp16", type=str2bool, default=True, help="whether to perform inference in fp16; True by default")
417
+
418
+ parser.add_argument("--temperature_increment_on_fallback", type=optional_float, default=0.2, help="temperature to increase when falling back when the decoding fails to meet either of the thresholds below")
419
+ parser.add_argument("--compression_ratio_threshold", type=optional_float, default=2.4, help="if the gzip compression ratio is higher than this value, treat the decoding as failed")
420
+ parser.add_argument("--logprob_threshold", type=optional_float, default=-1.0, help="if the average log probability is lower than this value, treat the decoding as failed")
421
+ parser.add_argument("--no_speech_threshold", type=optional_float, default=0.6, help="if the probability of the <|nospeech|> token is higher than this value AND the decoding has failed due to `logprob_threshold`, consider the segment as silence")
422
+ parser.add_argument("--word_timestamps", type=str2bool, default=False, help="(experimental) extract word-level timestamps and refine the results based on them")
423
+ parser.add_argument("--prepend_punctuations", type=str, default="\"\'“¿([{-", help="if word_timestamps is True, merge these punctuation symbols with the next word")
424
+ parser.add_argument("--append_punctuations", type=str, default="\"\'.。,,!!??::”)]}、", help="if word_timestamps is True, merge these punctuation symbols with the previous word")
425
+ parser.add_argument("--highlight_words", type=str2bool, default=False, help="(requires --word_timestamps True) underline each word as it is spoken in srt and vtt")
426
+ parser.add_argument("--max_line_width", type=optional_int, default=None, help="(requires --word_timestamps True) the maximum number of characters in a line before breaking the line")
427
+ parser.add_argument("--max_line_count", type=optional_int, default=None, help="(requires --word_timestamps True) the maximum number of lines in a segment")
428
+ parser.add_argument("--max_words_per_line", type=optional_int, default=None, help="(requires --word_timestamps True, no effect with --max_line_width) the maximum number of words in a segment")
429
+ parser.add_argument("--threads", type=optional_int, default=0, help="number of threads used by torch for CPU inference; supercedes MKL_NUM_THREADS/OMP_NUM_THREADS")
430
+ # fmt: on
431
+
432
+ args = parser.parse_args().__dict__
433
+ model_name: str = args.pop("model")
434
+ model_dir: str = args.pop("model_dir")
435
+ output_dir: str = args.pop("output_dir")
436
+ output_format: str = args.pop("output_format")
437
+ device: str = args.pop("device")
438
+ os.makedirs(output_dir, exist_ok=True)
439
+
440
+ if model_name.endswith(".en") and args["language"] not in {"en", "English"}:
441
+ if args["language"] is not None:
442
+ warnings.warn(
443
+ f"{model_name} is an English-only model but receipted '{args['language']}'; using English instead."
444
+ )
445
+ args["language"] = "en"
446
+
447
+ temperature = args.pop("temperature")
448
+ if (increment := args.pop("temperature_increment_on_fallback")) is not None:
449
+ temperature = tuple(np.arange(temperature, 1.0 + 1e-6, increment))
450
+ else:
451
+ temperature = [temperature]
452
+
453
+ if (threads := args.pop("threads")) > 0:
454
+ torch.set_num_threads(threads)
455
+
456
+ from . import load_model
457
+
458
+ model = load_model(model_name, device=device, download_root=model_dir)
459
+
460
+ writer = get_writer(output_format, output_dir)
461
+ word_options = [
462
+ "highlight_words",
463
+ "max_line_count",
464
+ "max_line_width",
465
+ "max_words_per_line",
466
+ ]
467
+ if not args["word_timestamps"]:
468
+ for option in word_options:
469
+ if args[option]:
470
+ parser.error(f"--{option} requires --word_timestamps True")
471
+ if args["max_line_count"] and not args["max_line_width"]:
472
+ warnings.warn("--max_line_count has no effect without --max_line_width")
473
+ if args["max_words_per_line"] and args["max_line_width"]:
474
+ warnings.warn("--max_words_per_line has no effect with --max_line_width")
475
+ writer_args = {arg: args.pop(arg) for arg in word_options}
476
+ for audio_path in args.pop("audio"):
477
+ try:
478
+ result = transcribe(model, audio_path, temperature=temperature, **args)
479
+ writer(result, audio_path, **writer_args)
480
+ except Exception as e:
481
+ traceback.print_exc()
482
+ print(f"Skipping {audio_path} due to {type(e).__name__}: {str(e)}")
483
+
484
+
485
+ if __name__ == "__main__":
486
+ cli()
whisper/utils.py ADDED
@@ -0,0 +1,302 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+ import re
4
+ import sys
5
+ import zlib
6
+ from typing import Callable, Optional, TextIO
7
+
8
+ system_encoding = sys.getdefaultencoding()
9
+
10
+ if system_encoding != "utf-8":
11
+
12
+ def make_safe(string):
13
+ # replaces any character not representable using the system default encoding with an '?',
14
+ # avoiding UnicodeEncodeError (https://github.com/openai/whisper/discussions/729).
15
+ return string.encode(system_encoding, errors="replace").decode(system_encoding)
16
+
17
+ else:
18
+
19
+ def make_safe(string):
20
+ # utf-8 can encode any Unicode code point, so no need to do the round-trip encoding
21
+ return string
22
+
23
+
24
+ def exact_div(x, y):
25
+ assert x % y == 0
26
+ return x // y
27
+
28
+
29
+ def str2bool(string):
30
+ str2val = {"True": True, "False": False}
31
+ if string in str2val:
32
+ return str2val[string]
33
+ else:
34
+ raise ValueError(f"Expected one of {set(str2val.keys())}, got {string}")
35
+
36
+
37
+ def optional_int(string):
38
+ return None if string == "None" else int(string)
39
+
40
+
41
+ def optional_float(string):
42
+ return None if string == "None" else float(string)
43
+
44
+
45
+ def compression_ratio(text) -> float:
46
+ text_bytes = text.encode("utf-8")
47
+ return len(text_bytes) / len(zlib.compress(text_bytes))
48
+
49
+
50
+ def format_timestamp(
51
+ seconds: float, always_include_hours: bool = False, decimal_marker: str = "."
52
+ ):
53
+ assert seconds >= 0, "non-negative timestamp expected"
54
+ milliseconds = round(seconds * 1000.0)
55
+
56
+ hours = milliseconds // 3_600_000
57
+ milliseconds -= hours * 3_600_000
58
+
59
+ minutes = milliseconds // 60_000
60
+ milliseconds -= minutes * 60_000
61
+
62
+ seconds = milliseconds // 1_000
63
+ milliseconds -= seconds * 1_000
64
+
65
+ hours_marker = f"{hours:02d}:" if always_include_hours or hours > 0 else ""
66
+ return (
67
+ f"{hours_marker}{minutes:02d}:{seconds:02d}{decimal_marker}{milliseconds:03d}"
68
+ )
69
+
70
+
71
+ class ResultWriter:
72
+ extension: str
73
+
74
+ def __init__(self, output_dir: str):
75
+ self.output_dir = output_dir
76
+
77
+ def __call__(
78
+ self, result: dict, audio_path: str, options: Optional[dict] = None, **kwargs
79
+ ):
80
+ audio_basename = os.path.basename(audio_path)
81
+ audio_basename = os.path.splitext(audio_basename)[0]
82
+ output_path = os.path.join(
83
+ self.output_dir, audio_basename + "." + self.extension
84
+ )
85
+
86
+ with open(output_path, "w", encoding="utf-8") as f:
87
+ self.write_result(result, file=f, options=options, **kwargs)
88
+
89
+ def write_result(
90
+ self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs
91
+ ):
92
+ raise NotImplementedError
93
+
94
+
95
+ class WriteTXT(ResultWriter):
96
+ extension: str = "txt"
97
+
98
+ def write_result(
99
+ self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs
100
+ ):
101
+ for segment in result["segments"]:
102
+ print(segment["text"].strip(), file=file, flush=True)
103
+
104
+
105
+ class SubtitlesWriter(ResultWriter):
106
+ always_include_hours: bool
107
+ decimal_marker: str
108
+
109
+ def iterate_result(
110
+ self,
111
+ result: dict,
112
+ options: Optional[dict] = None,
113
+ *,
114
+ max_line_width: Optional[int] = None,
115
+ max_line_count: Optional[int] = None,
116
+ highlight_words: bool = False,
117
+ max_words_per_line: Optional[int] = None,
118
+ ):
119
+ options = options or {}
120
+ max_line_width = max_line_width or options.get("max_line_width")
121
+ max_line_count = max_line_count or options.get("max_line_count")
122
+ highlight_words = highlight_words or options.get("highlight_words", False)
123
+ max_words_per_line = max_words_per_line or options.get("max_words_per_line")
124
+ preserve_segments = max_line_count is None or max_line_width is None
125
+ max_line_width = max_line_width or 1000
126
+ max_words_per_line = max_words_per_line or 1000
127
+
128
+ def iterate_subtitles():
129
+ line_len = 0
130
+ line_count = 1
131
+ # the next subtitle to yield (a list of word timings with whitespace)
132
+ subtitle: list[dict] = []
133
+ last = result["segments"][0]["words"][0]["start"]
134
+ for segment in result["segments"]:
135
+ chunk_index = 0
136
+ words_count = max_words_per_line
137
+ while chunk_index < len(segment["words"]):
138
+ remaining_words = len(segment["words"]) - chunk_index
139
+ if max_words_per_line > len(segment["words"]) - chunk_index:
140
+ words_count = remaining_words
141
+ for i, original_timing in enumerate(
142
+ segment["words"][chunk_index : chunk_index + words_count]
143
+ ):
144
+ timing = original_timing.copy()
145
+ long_pause = (
146
+ not preserve_segments and timing["start"] - last > 3.0
147
+ )
148
+ has_room = line_len + len(timing["word"]) <= max_line_width
149
+ seg_break = i == 0 and len(subtitle) > 0 and preserve_segments
150
+ if (
151
+ line_len > 0
152
+ and has_room
153
+ and not long_pause
154
+ and not seg_break
155
+ ):
156
+ # line continuation
157
+ line_len += len(timing["word"])
158
+ else:
159
+ # new line
160
+ timing["word"] = timing["word"].strip()
161
+ if (
162
+ len(subtitle) > 0
163
+ and max_line_count is not None
164
+ and (long_pause or line_count >= max_line_count)
165
+ or seg_break
166
+ ):
167
+ # subtitle break
168
+ yield subtitle
169
+ subtitle = []
170
+ line_count = 1
171
+ elif line_len > 0:
172
+ # line break
173
+ line_count += 1
174
+ timing["word"] = "\n" + timing["word"]
175
+ line_len = len(timing["word"].strip())
176
+ subtitle.append(timing)
177
+ last = timing["start"]
178
+ chunk_index += max_words_per_line
179
+ if len(subtitle) > 0:
180
+ yield subtitle
181
+
182
+ if len(result["segments"]) > 0 and "words" in result["segments"][0]:
183
+ for subtitle in iterate_subtitles():
184
+ subtitle_start = self.format_timestamp(subtitle[0]["start"])
185
+ subtitle_end = self.format_timestamp(subtitle[-1]["end"])
186
+ subtitle_text = "".join([word["word"] for word in subtitle])
187
+ if highlight_words:
188
+ last = subtitle_start
189
+ all_words = [timing["word"] for timing in subtitle]
190
+ for i, this_word in enumerate(subtitle):
191
+ start = self.format_timestamp(this_word["start"])
192
+ end = self.format_timestamp(this_word["end"])
193
+ if last != start:
194
+ yield last, start, subtitle_text
195
+
196
+ yield start, end, "".join(
197
+ [
198
+ re.sub(r"^(\s*)(.*)$", r"\1<u>\2</u>", word)
199
+ if j == i
200
+ else word
201
+ for j, word in enumerate(all_words)
202
+ ]
203
+ )
204
+ last = end
205
+ else:
206
+ yield subtitle_start, subtitle_end, subtitle_text
207
+ else:
208
+ for segment in result["segments"]:
209
+ segment_start = self.format_timestamp(segment["start"])
210
+ segment_end = self.format_timestamp(segment["end"])
211
+ segment_text = segment["text"].strip().replace("-->", "->")
212
+ yield segment_start, segment_end, segment_text
213
+
214
+ def format_timestamp(self, seconds: float):
215
+ return format_timestamp(
216
+ seconds=seconds,
217
+ always_include_hours=self.always_include_hours,
218
+ decimal_marker=self.decimal_marker,
219
+ )
220
+
221
+
222
+ class WriteVTT(SubtitlesWriter):
223
+ extension: str = "vtt"
224
+ always_include_hours: bool = False
225
+ decimal_marker: str = "."
226
+
227
+ def write_result(
228
+ self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs
229
+ ):
230
+ print("WEBVTT\n", file=file)
231
+ for start, end, text in self.iterate_result(result, options, **kwargs):
232
+ print(f"{start} --> {end}\n{text}\n", file=file, flush=True)
233
+
234
+
235
+ class WriteSRT(SubtitlesWriter):
236
+ extension: str = "srt"
237
+ always_include_hours: bool = True
238
+ decimal_marker: str = ","
239
+
240
+ def write_result(
241
+ self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs
242
+ ):
243
+ for i, (start, end, text) in enumerate(
244
+ self.iterate_result(result, options, **kwargs), start=1
245
+ ):
246
+ print(f"{i}\n{start} --> {end}\n{text}\n", file=file, flush=True)
247
+
248
+
249
+ class WriteTSV(ResultWriter):
250
+ """
251
+ Write a transcript to a file in TSV (tab-separated values) format containing lines like:
252
+ <start time in integer milliseconds>\t<end time in integer milliseconds>\t<transcript text>
253
+
254
+ Using integer milliseconds as start and end times means there's no chance of interference from
255
+ an environment setting a language encoding that causes the decimal in a floating point number
256
+ to appear as a comma; also is faster and more efficient to parse & store, e.g., in C++.
257
+ """
258
+
259
+ extension: str = "tsv"
260
+
261
+ def write_result(
262
+ self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs
263
+ ):
264
+ print("start", "end", "text", sep="\t", file=file)
265
+ for segment in result["segments"]:
266
+ print(round(1000 * segment["start"]), file=file, end="\t")
267
+ print(round(1000 * segment["end"]), file=file, end="\t")
268
+ print(segment["text"].strip().replace("\t", " "), file=file, flush=True)
269
+
270
+
271
+ class WriteJSON(ResultWriter):
272
+ extension: str = "json"
273
+
274
+ def write_result(
275
+ self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs
276
+ ):
277
+ json.dump(result, file)
278
+
279
+
280
+ def get_writer(
281
+ output_format: str, output_dir: str
282
+ ) -> Callable[[dict, TextIO, dict], None]:
283
+ writers = {
284
+ "txt": WriteTXT,
285
+ "vtt": WriteVTT,
286
+ "srt": WriteSRT,
287
+ "tsv": WriteTSV,
288
+ "json": WriteJSON,
289
+ }
290
+
291
+ if output_format == "all":
292
+ all_writers = [writer(output_dir) for writer in writers.values()]
293
+
294
+ def write_all(
295
+ result: dict, file: TextIO, options: Optional[dict] = None, **kwargs
296
+ ):
297
+ for writer in all_writers:
298
+ writer(result, file, options, **kwargs)
299
+
300
+ return write_all
301
+
302
+ return writers[output_format](output_dir)