pinecoresystems commited on
Commit
ea69d96
·
verified ·
1 Parent(s): 4aba593

Mirror configuration_moss_audio_tokenizer.py

Browse files
Files changed (1) hide show
  1. configuration_moss_audio_tokenizer.py +475 -0
configuration_moss_audio_tokenizer.py ADDED
@@ -0,0 +1,475 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2026 OpenMOSS and the HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """MossAudioTokenizer model configuration."""
16
+
17
+ from typing import Any
18
+
19
+ try:
20
+ from transformers.configuration_utils import PreTrainedConfig
21
+ except ImportError:
22
+ from transformers.configuration_utils import PretrainedConfig as PreTrainedConfig
23
+ from transformers.utils import logging
24
+
25
+
26
+ logger = logging.get_logger(__name__)
27
+
28
+
29
+ class MossAudioTokenizerConfig(PreTrainedConfig):
30
+ r"""
31
+ This is the configuration class to store the configuration of a [`MossAudioTokenizerModel`]. It is used to instantiate a
32
+ MossAudioTokenizer model according to the specified arguments, defining the model architecture.
33
+
34
+ Instantiating a configuration with the defaults will yield a similar configuration to that of the
35
+ [OpenMOSS-Team/MOSS-Audio-Tokenizer-v2](https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer-v2) architecture.
36
+
37
+ Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
38
+ documentation from [`PreTrainedConfig`] for more information.
39
+
40
+ Args:
41
+ sampling_rate (`int`, *optional*, defaults to 48000):
42
+ The sampling rate at which the audio waveform should be digitalized expressed in hertz (Hz).
43
+ downsample_rate (`int`, *optional*, defaults to 3840):
44
+ Total downsampling rate from waveform to tokens.
45
+ causal_transformer_context_duration (`float`, *optional*, defaults to 10.0):
46
+ Legacy global fallback context duration in seconds for causal transformer. If an individual transformer
47
+ entry in `encoder_kwargs` or `decoder_kwargs` provides `context_duration`, that per-module value takes
48
+ precedence.
49
+ encoder_kwargs (`list[dict]`, *optional*):
50
+ List of encoder module configurations. Each dict specifies a module type and its parameters.
51
+ decoder_kwargs (`list[dict]`, *optional*):
52
+ List of decoder module configurations in execution order.
53
+ number_channels (`int`, *optional*, defaults to 2):
54
+ Number of audio channels exposed by the public waveform interface.
55
+ enable_channel_interleave (`bool`, *optional*, defaults to `True`):
56
+ Whether to flatten multi-channel waveforms into a single internal stream before codec inference.
57
+ attention_implementation (`str`, *optional*, defaults to `"sdpa"`):
58
+ Attention implementation to prefer for transformer layers. Supported values are `"sdpa"` and
59
+ `"flash_attention_2"`.
60
+ compute_dtype (`str`, *optional*, defaults to `"fp32"`):
61
+ Inference compute dtype for non-quantizer modules. Supported values are `"fp32"`, `"bf16"`.
62
+ codec_weight_dtype (`str`, *optional*, defaults to `"fp32"`):
63
+ Parameter dtype for encoder and decoder modules. The quantizer remains fp32 because it explicitly disables
64
+ autocast and performs numerically sensitive codebook operations in fp32.
65
+ quantizer_type (`str`, *optional*, defaults to `"rlfq"`):
66
+ Quantizer type. Options include `"rvq"`, `"spec_rvq"`, `"rlfq"`, `"random_prefix_rlfq"`.
67
+ quantizer_kwargs (`dict`, *optional*):
68
+ Configuration for the quantizer including `input_dim`, `rvq_dim`, `output_dim`, `num_quantizers`,
69
+ `codebook_size`, and `codebook_dim`.
70
+
71
+ Example:
72
+
73
+ ```python
74
+ >>> from transformers import MossAudioTokenizerModel, MossAudioTokenizerConfig
75
+
76
+ >>> # Initializing a MossAudioTokenizer style configuration
77
+ >>> configuration = MossAudioTokenizerConfig()
78
+
79
+ >>> # Initializing a model (with random weights) from the configuration
80
+ >>> model = MossAudioTokenizerModel(configuration)
81
+
82
+ >>> # Accessing the model configuration
83
+ >>> configuration = model.config
84
+ ```
85
+ """
86
+
87
+ model_type = "moss-audio-tokenizer"
88
+
89
+ # Backward-compatible alias used by some checkpoints.
90
+ attribute_map = {"sample_rate": "sampling_rate"}
91
+
92
+ sampling_rate: int
93
+ downsample_rate: int
94
+ causal_transformer_context_duration: float
95
+ encoder_kwargs: list[dict[str, Any]]
96
+ decoder_kwargs: list[dict[str, Any]]
97
+ number_channels: int
98
+ enable_channel_interleave: bool
99
+ attention_implementation: str
100
+ compute_dtype: str
101
+ codec_weight_dtype: str
102
+ quantizer_type: str
103
+ quantizer_kwargs: dict[str, Any]
104
+
105
+ def __init__(
106
+ self,
107
+ version: str | None = None,
108
+ sampling_rate: int = 48000,
109
+ downsample_rate: int = 3840,
110
+ causal_transformer_context_duration: float = 10.0,
111
+ encoder_kwargs: list[dict[str, Any]] | None = None,
112
+ decoder_kwargs: list[dict[str, Any]] | None = None,
113
+ number_channels: int = 2,
114
+ enable_channel_interleave: bool = True,
115
+ attention_implementation: str = "sdpa",
116
+ compute_dtype: str = "fp32",
117
+ codec_weight_dtype: str = "fp32",
118
+ quantizer_type: str = "rlfq",
119
+ quantizer_kwargs: dict[str, Any] | None = None,
120
+ **kwargs,
121
+ ):
122
+ # Some checkpoints might include an incorrect/legacy `model_type` (e.g. "speech_tokenizer").
123
+ # We drop it to avoid overriding the class-level `model_type`.
124
+ kwargs.pop("model_type", None)
125
+ if "channels_numbers" in kwargs:
126
+ number_channels = kwargs.pop("channels_numbers")
127
+ if "enable_channel_interleave" in kwargs:
128
+ enable_channel_interleave = kwargs.pop("enable_channel_interleave")
129
+ if "attention_backend" in kwargs and attention_implementation == "sdpa":
130
+ attention_implementation = kwargs.pop("attention_backend")
131
+ if "codec_compute_dtype" in kwargs and compute_dtype == "fp32":
132
+ compute_dtype = kwargs.pop("codec_compute_dtype")
133
+ if "codec_load_dtype" in kwargs and codec_weight_dtype == "fp32":
134
+ codec_weight_dtype = kwargs.pop("codec_load_dtype")
135
+ reversed_decoder_kwargs = kwargs.pop("reversed_decoder_kwargs", None)
136
+
137
+ # `version` is accepted for compatibility but not used in modeling.
138
+ self.version = version
139
+ self.sampling_rate = sampling_rate
140
+ self.downsample_rate = downsample_rate
141
+ self.causal_transformer_context_duration = causal_transformer_context_duration
142
+ self.number_channels = number_channels
143
+ self.enable_channel_interleave = enable_channel_interleave
144
+ self.attention_implementation = attention_implementation
145
+ self.compute_dtype = compute_dtype
146
+ self.codec_weight_dtype = codec_weight_dtype
147
+ # Default encoder configuration
148
+ if encoder_kwargs is None:
149
+ encoder_kwargs = [
150
+ {
151
+ "module_type": "PatchedPretransform",
152
+ "patch_size": 240,
153
+ },
154
+ {
155
+ "module_type": "Transformer",
156
+ "input_dimension": 240,
157
+ "output_dimension": 384,
158
+ "d_model": 768,
159
+ "num_heads": 12,
160
+ "num_layers": 12,
161
+ "dim_feedforward": 3072,
162
+ "causal": True,
163
+ "norm": "layer_norm",
164
+ "positional_embedding": "rope",
165
+ "max_period": 10000,
166
+ "gating": "none",
167
+ "layer_scale": 0.01,
168
+ "conv_layout": True,
169
+ "context_duration": 1.0,
170
+ },
171
+ {
172
+ "module_type": "PatchedPretransform",
173
+ "patch_size": 2,
174
+ },
175
+ {
176
+ "module_type": "Transformer",
177
+ "input_dimension": 768,
178
+ "output_dimension": 384,
179
+ "d_model": 768,
180
+ "num_heads": 12,
181
+ "num_layers": 12,
182
+ "dim_feedforward": 3072,
183
+ "causal": True,
184
+ "norm": "layer_norm",
185
+ "positional_embedding": "rope",
186
+ "max_period": 10000,
187
+ "gating": "none",
188
+ "layer_scale": 0.01,
189
+ "conv_layout": True,
190
+ "context_duration": 2.0,
191
+ },
192
+ {
193
+ "module_type": "PatchedPretransform",
194
+ "patch_size": 2,
195
+ },
196
+ {
197
+ "module_type": "Transformer",
198
+ "input_dimension": 768,
199
+ "output_dimension": 384,
200
+ "d_model": 768,
201
+ "num_heads": 12,
202
+ "num_layers": 12,
203
+ "dim_feedforward": 3072,
204
+ "causal": True,
205
+ "norm": "layer_norm",
206
+ "positional_embedding": "rope",
207
+ "max_period": 10000,
208
+ "gating": "none",
209
+ "layer_scale": 0.01,
210
+ "conv_layout": True,
211
+ "context_duration": 4.0,
212
+ },
213
+ {
214
+ "module_type": "PatchedPretransform",
215
+ "patch_size": 2,
216
+ },
217
+ {
218
+ "module_type": "Transformer",
219
+ "input_dimension": 768,
220
+ "output_dimension": 384,
221
+ "d_model": 768,
222
+ "num_heads": 12,
223
+ "num_layers": 12,
224
+ "dim_feedforward": 3072,
225
+ "causal": True,
226
+ "norm": "layer_norm",
227
+ "positional_embedding": "rope",
228
+ "max_period": 10000,
229
+ "gating": "none",
230
+ "layer_scale": 0.01,
231
+ "conv_layout": True,
232
+ "context_duration": 8.0,
233
+ },
234
+ {
235
+ "module_type": "PatchedPretransform",
236
+ "patch_size": 2,
237
+ },
238
+ {
239
+ "module_type": "Transformer",
240
+ "input_dimension": 768,
241
+ "output_dimension": 640,
242
+ "d_model": 768,
243
+ "num_heads": 12,
244
+ "num_layers": 12,
245
+ "dim_feedforward": 3072,
246
+ "causal": True,
247
+ "norm": "layer_norm",
248
+ "positional_embedding": "rope",
249
+ "max_period": 10000,
250
+ "gating": "none",
251
+ "layer_scale": 0.01,
252
+ "conv_layout": True,
253
+ "context_duration": 10.0,
254
+ },
255
+ {
256
+ "module_type": "PatchedPretransform",
257
+ "patch_size": 2,
258
+ },
259
+ {
260
+ "module_type": "Transformer",
261
+ "input_dimension": 1280,
262
+ "output_dimension": 768,
263
+ "d_model": 1280,
264
+ "num_heads": 20,
265
+ "num_layers": 32,
266
+ "dim_feedforward": 5120,
267
+ "causal": True,
268
+ "norm": "layer_norm",
269
+ "positional_embedding": "rope",
270
+ "max_period": 10000,
271
+ "gating": "none",
272
+ "layer_scale": 0.01,
273
+ "conv_layout": True,
274
+ "context_duration": 10.0,
275
+ },
276
+ ]
277
+ else:
278
+ encoder_kwargs = [dict(module_kwargs) for module_kwargs in encoder_kwargs]
279
+ for module_kwargs in encoder_kwargs:
280
+ if module_kwargs.get("module_type") == "Transformer":
281
+ module_kwargs.setdefault("context_duration", causal_transformer_context_duration)
282
+ self.encoder_kwargs = encoder_kwargs
283
+
284
+ # Default decoder configuration (execution order)
285
+ if decoder_kwargs is None and reversed_decoder_kwargs is not None:
286
+ reversed_decoder_kwargs = [dict(module_kwargs) for module_kwargs in reversed_decoder_kwargs]
287
+ decoder_kwargs = []
288
+ for module_kwargs in reversed_decoder_kwargs[::-1]:
289
+ if module_kwargs.get("module_type") != "Transformer":
290
+ decoder_kwargs.append(module_kwargs)
291
+ continue
292
+ module_kwargs = dict(module_kwargs)
293
+ module_kwargs["input_dimension"], module_kwargs["output_dimension"] = (
294
+ module_kwargs["output_dimension"],
295
+ module_kwargs["input_dimension"],
296
+ )
297
+ decoder_kwargs.append(module_kwargs)
298
+
299
+ if decoder_kwargs is None:
300
+ decoder_kwargs = [
301
+ {
302
+ "module_type": "Transformer",
303
+ "input_dimension": 768,
304
+ "output_dimension": 1280,
305
+ "d_model": 1280,
306
+ "num_heads": 20,
307
+ "num_layers": 32,
308
+ "dim_feedforward": 5120,
309
+ "causal": True,
310
+ "norm": "layer_norm",
311
+ "positional_embedding": "rope",
312
+ "max_period": 10000,
313
+ "gating": "none",
314
+ "layer_scale": 0.01,
315
+ "conv_layout": True,
316
+ "context_duration": 10.0,
317
+ },
318
+ {
319
+ "module_type": "PatchedPretransform",
320
+ "patch_size": 2,
321
+ },
322
+ {
323
+ "module_type": "Transformer",
324
+ "input_dimension": 640,
325
+ "output_dimension": 768,
326
+ "d_model": 768,
327
+ "num_heads": 12,
328
+ "num_layers": 12,
329
+ "dim_feedforward": 3072,
330
+ "causal": True,
331
+ "norm": "layer_norm",
332
+ "positional_embedding": "rope",
333
+ "max_period": 10000,
334
+ "gating": "none",
335
+ "layer_scale": 0.01,
336
+ "conv_layout": True,
337
+ "context_duration": 10.0,
338
+ },
339
+ {
340
+ "module_type": "PatchedPretransform",
341
+ "patch_size": 2,
342
+ },
343
+ {
344
+ "module_type": "Transformer",
345
+ "input_dimension": 384,
346
+ "output_dimension": 768,
347
+ "d_model": 768,
348
+ "num_heads": 12,
349
+ "num_layers": 12,
350
+ "dim_feedforward": 3072,
351
+ "causal": True,
352
+ "norm": "layer_norm",
353
+ "positional_embedding": "rope",
354
+ "max_period": 10000,
355
+ "gating": "none",
356
+ "layer_scale": 0.01,
357
+ "conv_layout": True,
358
+ "context_duration": 8.0,
359
+ },
360
+ {
361
+ "module_type": "PatchedPretransform",
362
+ "patch_size": 2,
363
+ },
364
+ {
365
+ "module_type": "Transformer",
366
+ "input_dimension": 384,
367
+ "output_dimension": 768,
368
+ "d_model": 768,
369
+ "num_heads": 12,
370
+ "num_layers": 12,
371
+ "dim_feedforward": 3072,
372
+ "causal": True,
373
+ "norm": "layer_norm",
374
+ "positional_embedding": "rope",
375
+ "max_period": 10000,
376
+ "gating": "none",
377
+ "layer_scale": 0.01,
378
+ "conv_layout": True,
379
+ "context_duration": 4.0,
380
+ },
381
+ {
382
+ "module_type": "PatchedPretransform",
383
+ "patch_size": 2,
384
+ },
385
+ {
386
+ "module_type": "Transformer",
387
+ "input_dimension": 384,
388
+ "output_dimension": 768,
389
+ "d_model": 768,
390
+ "num_heads": 12,
391
+ "num_layers": 12,
392
+ "dim_feedforward": 3072,
393
+ "causal": True,
394
+ "norm": "layer_norm",
395
+ "positional_embedding": "rope",
396
+ "max_period": 10000,
397
+ "gating": "none",
398
+ "layer_scale": 0.01,
399
+ "conv_layout": True,
400
+ "context_duration": 2.0,
401
+ },
402
+ {
403
+ "module_type": "PatchedPretransform",
404
+ "patch_size": 2,
405
+ },
406
+ {
407
+ "module_type": "Transformer",
408
+ "input_dimension": 384,
409
+ "output_dimension": 240,
410
+ "d_model": 768,
411
+ "num_heads": 12,
412
+ "num_layers": 12,
413
+ "dim_feedforward": 3072,
414
+ "causal": True,
415
+ "norm": "layer_norm",
416
+ "positional_embedding": "rope",
417
+ "max_period": 10000,
418
+ "gating": "none",
419
+ "layer_scale": 0.01,
420
+ "conv_layout": True,
421
+ "context_duration": 1.0,
422
+ },
423
+ {
424
+ "module_type": "PatchedPretransform",
425
+ "patch_size": 240,
426
+ },
427
+ ]
428
+ else:
429
+ decoder_kwargs = [dict(module_kwargs) for module_kwargs in decoder_kwargs]
430
+ for module_kwargs in decoder_kwargs:
431
+ if module_kwargs.get("module_type") == "Transformer":
432
+ module_kwargs.setdefault("context_duration", causal_transformer_context_duration)
433
+ self.decoder_kwargs = decoder_kwargs
434
+
435
+ # Default quantizer configuration
436
+ if quantizer_kwargs is None:
437
+ quantizer_kwargs = {
438
+ "input_dim": 768,
439
+ "rvq_dim": 512,
440
+ "output_dim": 768,
441
+ "num_quantizers": 32,
442
+ "codebook_size": 1024,
443
+ "codebook_dim": 8,
444
+ "quantizer_type": "rlfq",
445
+ }
446
+
447
+ # Handle quantizer_type from kwargs or config
448
+ kw_qtype = quantizer_kwargs.get("quantizer_type", None)
449
+ if kw_qtype is not None:
450
+ self.quantizer_type = kw_qtype
451
+ else:
452
+ self.quantizer_type = quantizer_type
453
+ quantizer_kwargs["quantizer_type"] = quantizer_type
454
+
455
+ self.quantizer_kwargs = quantizer_kwargs
456
+
457
+ super().__init__(**kwargs)
458
+
459
+ @property
460
+ def num_quantizers(self) -> int:
461
+ """Return the number of quantizers from quantizer_kwargs."""
462
+ return self.quantizer_kwargs.get("num_quantizers", 32)
463
+
464
+ @property
465
+ def codebook_size(self) -> int:
466
+ """Return the codebook size from quantizer_kwargs."""
467
+ return self.quantizer_kwargs.get("codebook_size", 4096)
468
+
469
+ @property
470
+ def frame_rate(self) -> float:
471
+ """Return the frame rate (tokens per second)."""
472
+ return self.sampling_rate / self.downsample_rate
473
+
474
+
475
+ __all__ = ["MossAudioTokenizerConfig"]