TedZhangHao commited on
Commit
5f986ca
·
0 Parent(s):

DirectS2ST checkpoints

Browse files
.gitattributes ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ tags:
3
+ - speech-to-speech-translation
4
+ - expressive-speech-translation
5
+ - mimi
6
+ language:
7
+ - en
8
+ - de
9
+ - es
10
+ ---
11
+
12
+ # DirectS2ST
13
+
14
+ Checkpoints of DirectS2ST, the end-to-end expressive speech-to-speech translation
15
+ model from *Holistic Parallel Supervision for Expressive Speech-to-Speech
16
+ Translation*, trained on VC-Dub (professional dubbing + multilingual voice
17
+ conversion).
18
+
19
+ | Folder | Direction |
20
+ | --- | --- |
21
+ | `en-de/` | English → German |
22
+ | `en-es/` | English → Spanish |
23
+
24
+ Each folder holds `pytorch_model.bin`, `model_config.json` and the
25
+ SentencePiece-8k text tokenizer (`text_tokenizer/`).
26
+
27
+ Given English source speech, the model predicts all 16 Mimi codec streams of
28
+ the target speech directly: a frozen w2v-BERT 2.0 encoder, an AR temporal
29
+ decoder (source text, target text and codec stream c0 with one shared head), and
30
+ an NAR depth decoder (c1-c15) conditioned on a CAMPPlus speaker prefix and a
31
+ source codec prompt.
32
+
33
+ ## Usage
34
+
35
+ Code, setup and training: <https://github.com/hzted/truly-e2e-expressive-s2st-demo/tree/main/DirectS2ST>
36
+
37
+ ```
38
+ hf download TedZhangHao/DirectS2ST --include "en-es/*" --local-dir checkpoints
39
+ python inference/infer.py \
40
+ --checkpoint-dir checkpoints/en-es \
41
+ --campplus-model-root /path/to/seed-vc \
42
+ --campplus-checkpoint-path /path/to/campplus_cn_en_common.pt \
43
+ --source-wav input_en.wav --output-wav output_es.wav
44
+ ```
45
+
46
+ Demo: <https://hzted.github.io/truly-e2e-expressive-s2st-demo/>
47
+
48
+ ## Third-party components
49
+
50
+ The checkpoints contain the frozen weights of
51
+ [w2v-BERT 2.0](https://huggingface.co/facebook/w2v-bert-2.0) and
52
+ [Mimi](https://huggingface.co/kyutai/mimi); their licenses apply to those
53
+ parts. The CAMPPlus speaker encoder
54
+ ([iic/speech_campplus_sv_zh_en_16k-common_advanced](https://modelscope.cn/models/iic/speech_campplus_sv_zh_en_16k-common_advanced))
55
+ is not included and is loaded at run time.
en-de/model_config.json ADDED
@@ -0,0 +1,221 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "c0_auxiliary_loss_weight": 0.0,
3
+ "c0_gold_anchor_mass": 0.75,
4
+ "c0_gold_anchored_distill_temperature": 1.0,
5
+ "c0_gold_anchored_distill_weight": 0.0,
6
+ "c0_teacher_distill_temperature": 1.0,
7
+ "c0_teacher_distill_weight": 0.0,
8
+ "c0_teacher_hidden_dim": 1024,
9
+ "c0_teacher_hidden_distill_weight": 0.0,
10
+ "c0_teacher_hidden_loss_type": "cosine",
11
+ "c0_teacher_hidden_use_projection": true,
12
+ "campplus_depth_conditioning_mode": "prefix_token",
13
+ "campplus_depth_identity_loss_weight": 0.1,
14
+ "campplus_depth_start_level": 1,
15
+ "campplus_embedding_size": 192,
16
+ "campplus_film_scale": 1.0,
17
+ "campplus_identity_level_aggregation": "conditioned_mean",
18
+ "campplus_identity_supervision": "target",
19
+ "codebook_loss_weights": null,
20
+ "codebook_model_order": null,
21
+ "codebook_size": 2048,
22
+ "codec_mass_loss_weight": 0.0,
23
+ "depth_auto_adjust_heads": true,
24
+ "depth_chain_scheduled_sampling_mode": "argmax",
25
+ "depth_chain_scheduled_sampling_prob": 0.0,
26
+ "depth_chain_scheduled_sampling_start_step": 0,
27
+ "depth_chain_scheduled_sampling_temperature": 1.0,
28
+ "depth_chain_scheduled_sampling_topk": 0,
29
+ "depth_chain_scheduled_sampling_warmup_steps": 0,
30
+ "depth_dropout": 0.1,
31
+ "depth_ffn_dim": 4096,
32
+ "depth_hidden_size": 1024,
33
+ "depth_loss_weight": 1.0,
34
+ "depth_num_heads": 16,
35
+ "depth_num_layers": 8,
36
+ "depth_objective_start_level": 1,
37
+ "depth_scheduled_sampling_mode": "argmax",
38
+ "depth_scheduled_sampling_prob": 0.0,
39
+ "depth_scheduled_sampling_start_step": 0,
40
+ "depth_scheduled_sampling_temperature": 1.0,
41
+ "depth_scheduled_sampling_topk": 0,
42
+ "depth_scheduled_sampling_warmup_steps": 0,
43
+ "depth_source_conditioning_detach": true,
44
+ "depth_source_conditioning_dropout": 0.1,
45
+ "depth_source_conditioning_num_heads": 16,
46
+ "depth_source_conditioning_residual_scale": 0.2,
47
+ "detach_temporal_for_depth_loss": true,
48
+ "direct_text_to_c0_input": "target",
49
+ "direct_text_to_c0_kd_temperature": 1.0,
50
+ "direct_text_to_c0_kd_weight": 0.0,
51
+ "direct_text_to_c0_loss_weight": 0.0,
52
+ "direct_text_to_c0_zero_speech_memory": false,
53
+ "enable_c0_auxiliary_loss": false,
54
+ "enable_c0_gold_anchored_distill": false,
55
+ "enable_c0_teacher_distill": false,
56
+ "enable_c0_teacher_hidden_distill": false,
57
+ "enable_campplus_depth_conditioning": true,
58
+ "enable_codec_mass_loss": false,
59
+ "enable_depth_source_conditioning": false,
60
+ "enable_direct_text_to_c0_auxiliary": false,
61
+ "enable_mimi_nar_depth_prompt": true,
62
+ "enable_mimi_source_acoustic_conditioning": false,
63
+ "enable_mimi_source_acoustic_temporal_conditioning": false,
64
+ "enable_mimi_source_speaker_prompt": true,
65
+ "enable_mimi_source_style_token_bank": false,
66
+ "enable_mixed_source_target_auxiliary": false,
67
+ "enable_predicted_source_target_auxiliary": false,
68
+ "enable_quality_cot": true,
69
+ "enable_semantic_prefix_ar": false,
70
+ "enable_source_unit_auxiliary": false,
71
+ "enable_source_unit_transformer_auxiliary": false,
72
+ "enable_target_only_speech_auxiliary": false,
73
+ "enable_text_codec_ar": true,
74
+ "enable_text_codec_kd_loss": false,
75
+ "enable_text_codec_text_path_loss": false,
76
+ "enable_text_prefix_ar": true,
77
+ "enable_transvip_style_loss_aggregation": false,
78
+ "enable_uniss_content_controls": true,
79
+ "enable_unity_source_memory_fusion": false,
80
+ "input_num_quantizers": 16,
81
+ "level0_depth_objective": true,
82
+ "level0_eos_loss_weight": 1.0,
83
+ "level0_label_smoothing": 0.0,
84
+ "level0_label_smoothing_train_only": false,
85
+ "level0_loss_weight": 1.0,
86
+ "level0_tail_loss_last_n": 0,
87
+ "level0_tail_loss_weight": 1.0,
88
+ "load_text_path_encoder": false,
89
+ "mimi_nar_depth_attention_mode": "full",
90
+ "mimi_nar_depth_max_positions": 4096,
91
+ "mimi_nar_depth_teacher_force_level_batch_size": 1,
92
+ "mimi_source_acoustic_depth_start_level": 99,
93
+ "mimi_source_acoustic_detach": true,
94
+ "mimi_source_acoustic_input_sample_rate": 16000,
95
+ "mimi_source_acoustic_model_name": "kyutai/mimi",
96
+ "mimi_source_acoustic_pooling": "mean_std",
97
+ "mimi_source_acoustic_residual_scale": 0.0,
98
+ "mimi_source_acoustic_temporal_detach": true,
99
+ "mimi_source_acoustic_temporal_residual_scale": 0.0,
100
+ "mimi_source_speaker_prompt_detach": true,
101
+ "mimi_source_style_token_c1_gate_init": 0.03,
102
+ "mimi_source_style_token_count": 8,
103
+ "mimi_source_style_token_depth_start_level": 1,
104
+ "mimi_source_style_token_dropout": 0.1,
105
+ "mimi_source_style_token_gate_init": 0.1,
106
+ "mimi_source_style_token_num_heads": 8,
107
+ "mimi_source_style_token_residual_scale": 0.7,
108
+ "mixed_source_auxiliary_seed": 59,
109
+ "mixed_source_prediction_prob": 0.0,
110
+ "mixed_source_start_step": 0,
111
+ "mixed_source_target_loss_weight": 0.0,
112
+ "mixed_source_warmup_steps": 0,
113
+ "num_codebook_levels": 16,
114
+ "num_temporal_codebook_levels": 1,
115
+ "predicted_source_target_batch_size": 2,
116
+ "predicted_source_target_loss_weight": 0.0,
117
+ "predicted_source_target_max_source_tokens": 64,
118
+ "predicted_source_target_start_step": 0,
119
+ "predicted_source_target_warmup_steps": 0,
120
+ "quality_cot_code0_loss_weight": 0.0,
121
+ "quality_cot_code1_loss_weight": 0.0,
122
+ "quality_cot_depth_loss_weight": 3.0,
123
+ "quality_cot_prompt_all_seps": false,
124
+ "quality_cot_source_text_loss_weight": 0.0,
125
+ "quality_cot_source_text_max_tokens": 128,
126
+ "quality_cot_source_text_prefix_extra_token_ids_after_bos": [
127
+ 4
128
+ ],
129
+ "quality_cot_target_text_loss_weight": 0.0,
130
+ "quality_cot_target_text_max_tokens": 128,
131
+ "skip_mimi_source_acoustic_encoder_when_precomputed": false,
132
+ "source_adapter_dropout": 0.1,
133
+ "source_adapter_ffn_dim": 4096,
134
+ "source_adapter_num_heads": 16,
135
+ "source_adapter_num_layers": 0,
136
+ "source_adapter_residual": true,
137
+ "source_adapter_residual_scale": 0.1,
138
+ "source_speaker_prompt_backend": "mimi_mean_std",
139
+ "source_speaker_prompt_dropout": 0.1,
140
+ "source_speaker_prompt_ffn_dim": 4096,
141
+ "source_speaker_prompt_num_heads": 16,
142
+ "source_speaker_prompt_num_layers": 2,
143
+ "source_speaker_prompt_speech_layer": 12,
144
+ "source_unit_blank_id": 1000,
145
+ "source_unit_loss_weight": 0.2,
146
+ "source_unit_transformer_dropout": 0.1,
147
+ "source_unit_transformer_ffn_dim": 4096,
148
+ "source_unit_transformer_loss_weight": 0.5,
149
+ "source_unit_transformer_max_positions": 513,
150
+ "source_unit_transformer_num_heads": 8,
151
+ "source_unit_transformer_num_layers": 2,
152
+ "source_unit_vocab_size": 1000,
153
+ "speech_encoder_name": "facebook/w2v-bert-2.0",
154
+ "speech_encoder_output_layer": 12,
155
+ "speech_encoder_type": "w2vbert2",
156
+ "target_only_speech_code0_loss_weight": 0.0,
157
+ "target_only_speech_text_loss_weight": 0.0,
158
+ "temporal_activation": "gelu",
159
+ "temporal_auto_adjust_heads": true,
160
+ "temporal_context_levels": 1,
161
+ "temporal_decoder_backend": "direct",
162
+ "temporal_dropout": 0.1,
163
+ "temporal_ffn_dim": 4096,
164
+ "temporal_hidden_size": 1024,
165
+ "temporal_max_positions": 2048,
166
+ "temporal_num_heads": 16,
167
+ "temporal_num_layers": 12,
168
+ "temporal_scheduled_sampling_decay_start_step": -1,
169
+ "temporal_scheduled_sampling_decay_steps": 0,
170
+ "temporal_scheduled_sampling_final_prob": 0.0,
171
+ "temporal_scheduled_sampling_mode": "argmax",
172
+ "temporal_scheduled_sampling_preserve_last_n": 0,
173
+ "temporal_scheduled_sampling_prob": 0.0,
174
+ "temporal_scheduled_sampling_start_step": 0,
175
+ "temporal_scheduled_sampling_temperature": 1.0,
176
+ "temporal_scheduled_sampling_topk": 0,
177
+ "temporal_scheduled_sampling_warmup_steps": 0,
178
+ "text_bos_token_id": 8000,
179
+ "text_codec_kd_loss_weight": 0.0,
180
+ "text_codec_t2c_loss_weight": 0.0,
181
+ "text_codec_t2t_loss_weight": 0.0,
182
+ "text_codec_text_path_input": "source",
183
+ "text_label_smoothing": 0.0,
184
+ "text_loss_weight": 1.0,
185
+ "text_max_positions": 255,
186
+ "text_pad_token_id": 0,
187
+ "text_path_source_token_dropout": 0.0,
188
+ "text_prefix_beam_size": 5,
189
+ "text_prefix_extra_token_ids_after_bos": [
190
+ 5
191
+ ],
192
+ "text_prefix_length_penalty": 1.0,
193
+ "text_prefix_min_tokens": 4,
194
+ "text_prefix_no_repeat_ngram_size": 3,
195
+ "text_prefix_sep_penalty": 0.0,
196
+ "text_sep_token_id": 8001,
197
+ "text_tokenizer_name": "./text_tokenizer",
198
+ "text_vocab_size": 8004,
199
+ "tie_text_embeddings": true,
200
+ "train_only_depth_decoder": true,
201
+ "transvip_load_source_encoder_from_checkpoint": false,
202
+ "transvip_load_text_decoder_from_checkpoint": false,
203
+ "transvip_model_name": "",
204
+ "transvip_num_new_tokens": null,
205
+ "transvip_style_depth_loss_weight": 0.0,
206
+ "transvip_style_kd_loss_weight": 0.0,
207
+ "transvip_style_speech_loss_weight": 0.0,
208
+ "transvip_style_text_loss_weight": 0.0,
209
+ "transvip_text_decoder_text_vocab_size": null,
210
+ "transvip_use_length_control": false,
211
+ "uniss_end_content_token_id": 8003,
212
+ "uniss_start_content_token_id": 8002,
213
+ "unity_source_memory_detach": true,
214
+ "unity_source_memory_fusion_mode": "concat",
215
+ "unity_source_memory_layer": 8,
216
+ "unity_source_memory_scale": 0.5,
217
+ "unity_source_memory_source": "inner",
218
+ "use_precomputed_source_acoustic_embeddings": true,
219
+ "use_unified_code0_loss_in_objective": true,
220
+ "vocab_size": 2050
221
+ }
en-de/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5f996500058680f7797a3a2b16b3fbc6f7ca0d27ed25aead5dc29e03a966cb7
3
+ size 4995031222
en-de/text_tokenizer/added_tokens.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "<|end_content|>": 8003,
3
+ "<|start_content|>": 8002,
4
+ "<|text_bos|>": 8000,
5
+ "<|text_sep|>": 8001
6
+ }
en-de/text_tokenizer/special_tokens_map.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<en>",
4
+ "<de>",
5
+ "<|text_bos|>",
6
+ "<|text_sep|>",
7
+ "<|start_content|>",
8
+ "<|end_content|>"
9
+ ],
10
+ "bos_token": {
11
+ "content": "<s>",
12
+ "lstrip": false,
13
+ "normalized": false,
14
+ "rstrip": false,
15
+ "single_word": false
16
+ },
17
+ "eos_token": {
18
+ "content": "</s>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ },
24
+ "pad_token": {
25
+ "content": "<pad>",
26
+ "lstrip": false,
27
+ "normalized": false,
28
+ "rstrip": false,
29
+ "single_word": false
30
+ },
31
+ "unk_token": {
32
+ "content": "<unk>",
33
+ "lstrip": false,
34
+ "normalized": false,
35
+ "rstrip": false,
36
+ "single_word": false
37
+ }
38
+ }
en-de/text_tokenizer/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
en-de/text_tokenizer/tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:18627b96648618340a1ab873ac814639fe397fb87eac7ce0cdfa91a597769a44
3
+ size 364791
en-de/text_tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": true,
5
+ "added_tokens_decoder": {
6
+ "0": {
7
+ "content": "<pad>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "1": {
15
+ "content": "<unk>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "2": {
23
+ "content": "<s>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": false,
27
+ "single_word": false,
28
+ "special": true
29
+ },
30
+ "3": {
31
+ "content": "</s>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false,
36
+ "special": true
37
+ },
38
+ "4": {
39
+ "content": "<en>",
40
+ "lstrip": false,
41
+ "normalized": false,
42
+ "rstrip": false,
43
+ "single_word": false,
44
+ "special": true
45
+ },
46
+ "5": {
47
+ "content": "<de>",
48
+ "lstrip": false,
49
+ "normalized": false,
50
+ "rstrip": false,
51
+ "single_word": false,
52
+ "special": true
53
+ },
54
+ "8000": {
55
+ "content": "<|text_bos|>",
56
+ "lstrip": false,
57
+ "normalized": false,
58
+ "rstrip": false,
59
+ "single_word": false,
60
+ "special": true
61
+ },
62
+ "8001": {
63
+ "content": "<|text_sep|>",
64
+ "lstrip": false,
65
+ "normalized": false,
66
+ "rstrip": false,
67
+ "single_word": false,
68
+ "special": true
69
+ },
70
+ "8002": {
71
+ "content": "<|start_content|>",
72
+ "lstrip": false,
73
+ "normalized": false,
74
+ "rstrip": false,
75
+ "single_word": false,
76
+ "special": true
77
+ },
78
+ "8003": {
79
+ "content": "<|end_content|>",
80
+ "lstrip": false,
81
+ "normalized": false,
82
+ "rstrip": false,
83
+ "single_word": false,
84
+ "special": true
85
+ }
86
+ },
87
+ "additional_special_tokens": [
88
+ "<en>",
89
+ "<de>",
90
+ "<|text_bos|>",
91
+ "<|text_sep|>",
92
+ "<|start_content|>",
93
+ "<|end_content|>"
94
+ ],
95
+ "bos_token": "<s>",
96
+ "clean_up_tokenization_spaces": false,
97
+ "eos_token": "</s>",
98
+ "extra_special_tokens": {},
99
+ "legacy": true,
100
+ "model_max_length": 2048,
101
+ "pad_token": "<pad>",
102
+ "sp_model_kwargs": {},
103
+ "spaces_between_special_tokens": false,
104
+ "tokenizer_class": "LlamaTokenizer",
105
+ "unk_token": "<unk>",
106
+ "use_default_system_prompt": false
107
+ }
en-es/model_config.json ADDED
@@ -0,0 +1,221 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "c0_auxiliary_loss_weight": 0.0,
3
+ "c0_gold_anchor_mass": 0.75,
4
+ "c0_gold_anchored_distill_temperature": 1.0,
5
+ "c0_gold_anchored_distill_weight": 0.0,
6
+ "c0_teacher_distill_temperature": 1.0,
7
+ "c0_teacher_distill_weight": 0.0,
8
+ "c0_teacher_hidden_dim": 1024,
9
+ "c0_teacher_hidden_distill_weight": 0.0,
10
+ "c0_teacher_hidden_loss_type": "cosine",
11
+ "c0_teacher_hidden_use_projection": true,
12
+ "campplus_depth_conditioning_mode": "prefix_token",
13
+ "campplus_depth_identity_loss_weight": 0.1,
14
+ "campplus_depth_start_level": 1,
15
+ "campplus_embedding_size": 192,
16
+ "campplus_film_scale": 1.0,
17
+ "campplus_identity_level_aggregation": "conditioned_mean",
18
+ "campplus_identity_supervision": "target",
19
+ "codebook_loss_weights": null,
20
+ "codebook_model_order": null,
21
+ "codebook_size": 2048,
22
+ "codec_mass_loss_weight": 0.0,
23
+ "depth_auto_adjust_heads": true,
24
+ "depth_chain_scheduled_sampling_mode": "argmax",
25
+ "depth_chain_scheduled_sampling_prob": 0.0,
26
+ "depth_chain_scheduled_sampling_start_step": 0,
27
+ "depth_chain_scheduled_sampling_temperature": 1.0,
28
+ "depth_chain_scheduled_sampling_topk": 0,
29
+ "depth_chain_scheduled_sampling_warmup_steps": 0,
30
+ "depth_dropout": 0.1,
31
+ "depth_ffn_dim": 4096,
32
+ "depth_hidden_size": 1024,
33
+ "depth_loss_weight": 1.0,
34
+ "depth_num_heads": 16,
35
+ "depth_num_layers": 8,
36
+ "depth_objective_start_level": 1,
37
+ "depth_scheduled_sampling_mode": "argmax",
38
+ "depth_scheduled_sampling_prob": 0.0,
39
+ "depth_scheduled_sampling_start_step": 0,
40
+ "depth_scheduled_sampling_temperature": 1.0,
41
+ "depth_scheduled_sampling_topk": 0,
42
+ "depth_scheduled_sampling_warmup_steps": 0,
43
+ "depth_source_conditioning_detach": true,
44
+ "depth_source_conditioning_dropout": 0.1,
45
+ "depth_source_conditioning_num_heads": 16,
46
+ "depth_source_conditioning_residual_scale": 0.2,
47
+ "detach_temporal_for_depth_loss": true,
48
+ "direct_text_to_c0_input": "target",
49
+ "direct_text_to_c0_kd_temperature": 1.0,
50
+ "direct_text_to_c0_kd_weight": 0.0,
51
+ "direct_text_to_c0_loss_weight": 0.0,
52
+ "direct_text_to_c0_zero_speech_memory": false,
53
+ "enable_c0_auxiliary_loss": false,
54
+ "enable_c0_gold_anchored_distill": false,
55
+ "enable_c0_teacher_distill": false,
56
+ "enable_c0_teacher_hidden_distill": false,
57
+ "enable_campplus_depth_conditioning": true,
58
+ "enable_codec_mass_loss": false,
59
+ "enable_depth_source_conditioning": false,
60
+ "enable_direct_text_to_c0_auxiliary": false,
61
+ "enable_mimi_nar_depth_prompt": true,
62
+ "enable_mimi_source_acoustic_conditioning": false,
63
+ "enable_mimi_source_acoustic_temporal_conditioning": false,
64
+ "enable_mimi_source_speaker_prompt": true,
65
+ "enable_mimi_source_style_token_bank": false,
66
+ "enable_mixed_source_target_auxiliary": false,
67
+ "enable_predicted_source_target_auxiliary": false,
68
+ "enable_quality_cot": true,
69
+ "enable_semantic_prefix_ar": false,
70
+ "enable_source_unit_auxiliary": false,
71
+ "enable_source_unit_transformer_auxiliary": false,
72
+ "enable_target_only_speech_auxiliary": false,
73
+ "enable_text_codec_ar": true,
74
+ "enable_text_codec_kd_loss": false,
75
+ "enable_text_codec_text_path_loss": true,
76
+ "enable_text_prefix_ar": true,
77
+ "enable_transvip_style_loss_aggregation": false,
78
+ "enable_uniss_content_controls": true,
79
+ "enable_unity_source_memory_fusion": false,
80
+ "input_num_quantizers": 16,
81
+ "level0_depth_objective": true,
82
+ "level0_eos_loss_weight": 1.0,
83
+ "level0_label_smoothing": 0.0,
84
+ "level0_label_smoothing_train_only": false,
85
+ "level0_loss_weight": 1.0,
86
+ "level0_tail_loss_last_n": 0,
87
+ "level0_tail_loss_weight": 1.0,
88
+ "load_text_path_encoder": false,
89
+ "mimi_nar_depth_attention_mode": "full",
90
+ "mimi_nar_depth_max_positions": 4096,
91
+ "mimi_nar_depth_teacher_force_level_batch_size": 1,
92
+ "mimi_source_acoustic_depth_start_level": 99,
93
+ "mimi_source_acoustic_detach": true,
94
+ "mimi_source_acoustic_input_sample_rate": 16000,
95
+ "mimi_source_acoustic_model_name": "kyutai/mimi",
96
+ "mimi_source_acoustic_pooling": "mean_std",
97
+ "mimi_source_acoustic_residual_scale": 0.0,
98
+ "mimi_source_acoustic_temporal_detach": true,
99
+ "mimi_source_acoustic_temporal_residual_scale": 0.0,
100
+ "mimi_source_speaker_prompt_detach": true,
101
+ "mimi_source_style_token_c1_gate_init": 0.03,
102
+ "mimi_source_style_token_count": 8,
103
+ "mimi_source_style_token_depth_start_level": 1,
104
+ "mimi_source_style_token_dropout": 0.1,
105
+ "mimi_source_style_token_gate_init": 0.1,
106
+ "mimi_source_style_token_num_heads": 8,
107
+ "mimi_source_style_token_residual_scale": 0.7,
108
+ "mixed_source_auxiliary_seed": 59,
109
+ "mixed_source_prediction_prob": 0.0,
110
+ "mixed_source_start_step": 0,
111
+ "mixed_source_target_loss_weight": 0.0,
112
+ "mixed_source_warmup_steps": 0,
113
+ "num_codebook_levels": 16,
114
+ "num_temporal_codebook_levels": 1,
115
+ "predicted_source_target_batch_size": 2,
116
+ "predicted_source_target_loss_weight": 0.0,
117
+ "predicted_source_target_max_source_tokens": 64,
118
+ "predicted_source_target_start_step": 0,
119
+ "predicted_source_target_warmup_steps": 0,
120
+ "quality_cot_code0_loss_weight": 1.0,
121
+ "quality_cot_code1_loss_weight": 0.0,
122
+ "quality_cot_depth_loss_weight": 3.0,
123
+ "quality_cot_prompt_all_seps": false,
124
+ "quality_cot_source_text_loss_weight": 0.4,
125
+ "quality_cot_source_text_max_tokens": 128,
126
+ "quality_cot_source_text_prefix_extra_token_ids_after_bos": [
127
+ 4
128
+ ],
129
+ "quality_cot_target_text_loss_weight": 0.6,
130
+ "quality_cot_target_text_max_tokens": 128,
131
+ "skip_mimi_source_acoustic_encoder_when_precomputed": false,
132
+ "source_adapter_dropout": 0.1,
133
+ "source_adapter_ffn_dim": 4096,
134
+ "source_adapter_num_heads": 16,
135
+ "source_adapter_num_layers": 0,
136
+ "source_adapter_residual": true,
137
+ "source_adapter_residual_scale": 0.1,
138
+ "source_speaker_prompt_backend": "mimi_mean_std",
139
+ "source_speaker_prompt_dropout": 0.1,
140
+ "source_speaker_prompt_ffn_dim": 4096,
141
+ "source_speaker_prompt_num_heads": 16,
142
+ "source_speaker_prompt_num_layers": 2,
143
+ "source_speaker_prompt_speech_layer": 12,
144
+ "source_unit_blank_id": 1000,
145
+ "source_unit_loss_weight": 0.2,
146
+ "source_unit_transformer_dropout": 0.1,
147
+ "source_unit_transformer_ffn_dim": 4096,
148
+ "source_unit_transformer_loss_weight": 0.5,
149
+ "source_unit_transformer_max_positions": 513,
150
+ "source_unit_transformer_num_heads": 8,
151
+ "source_unit_transformer_num_layers": 2,
152
+ "source_unit_vocab_size": 1000,
153
+ "speech_encoder_name": "facebook/w2v-bert-2.0",
154
+ "speech_encoder_output_layer": 12,
155
+ "speech_encoder_type": "w2vbert2",
156
+ "target_only_speech_code0_loss_weight": 0.0,
157
+ "target_only_speech_text_loss_weight": 0.0,
158
+ "temporal_activation": "gelu",
159
+ "temporal_auto_adjust_heads": true,
160
+ "temporal_context_levels": 1,
161
+ "temporal_decoder_backend": "direct",
162
+ "temporal_dropout": 0.1,
163
+ "temporal_ffn_dim": 4096,
164
+ "temporal_hidden_size": 1024,
165
+ "temporal_max_positions": 2048,
166
+ "temporal_num_heads": 16,
167
+ "temporal_num_layers": 12,
168
+ "temporal_scheduled_sampling_decay_start_step": -1,
169
+ "temporal_scheduled_sampling_decay_steps": 0,
170
+ "temporal_scheduled_sampling_final_prob": 0.0,
171
+ "temporal_scheduled_sampling_mode": "argmax",
172
+ "temporal_scheduled_sampling_preserve_last_n": 0,
173
+ "temporal_scheduled_sampling_prob": 0.0,
174
+ "temporal_scheduled_sampling_start_step": 0,
175
+ "temporal_scheduled_sampling_temperature": 1.0,
176
+ "temporal_scheduled_sampling_topk": 0,
177
+ "temporal_scheduled_sampling_warmup_steps": 0,
178
+ "text_bos_token_id": 8000,
179
+ "text_codec_kd_loss_weight": 0.0,
180
+ "text_codec_t2c_loss_weight": 0.5,
181
+ "text_codec_t2t_loss_weight": 0.3,
182
+ "text_codec_text_path_input": "source",
183
+ "text_label_smoothing": 0.0,
184
+ "text_loss_weight": 1.0,
185
+ "text_max_positions": 255,
186
+ "text_pad_token_id": 0,
187
+ "text_path_source_token_dropout": 0.0,
188
+ "text_prefix_beam_size": 5,
189
+ "text_prefix_extra_token_ids_after_bos": [
190
+ 5
191
+ ],
192
+ "text_prefix_length_penalty": 1.0,
193
+ "text_prefix_min_tokens": 4,
194
+ "text_prefix_no_repeat_ngram_size": 3,
195
+ "text_prefix_sep_penalty": 0.0,
196
+ "text_sep_token_id": 8001,
197
+ "text_tokenizer_name": "./text_tokenizer",
198
+ "text_vocab_size": 8004,
199
+ "tie_text_embeddings": true,
200
+ "train_only_depth_decoder": true,
201
+ "transvip_load_source_encoder_from_checkpoint": false,
202
+ "transvip_load_text_decoder_from_checkpoint": false,
203
+ "transvip_model_name": "",
204
+ "transvip_num_new_tokens": null,
205
+ "transvip_style_depth_loss_weight": 0.0,
206
+ "transvip_style_kd_loss_weight": 0.0,
207
+ "transvip_style_speech_loss_weight": 0.0,
208
+ "transvip_style_text_loss_weight": 0.0,
209
+ "transvip_text_decoder_text_vocab_size": null,
210
+ "transvip_use_length_control": false,
211
+ "uniss_end_content_token_id": 8003,
212
+ "uniss_start_content_token_id": 8002,
213
+ "unity_source_memory_detach": true,
214
+ "unity_source_memory_fusion_mode": "concat",
215
+ "unity_source_memory_layer": 8,
216
+ "unity_source_memory_scale": 0.5,
217
+ "unity_source_memory_source": "inner",
218
+ "use_precomputed_source_acoustic_embeddings": true,
219
+ "use_unified_code0_loss_in_objective": true,
220
+ "vocab_size": 2050
221
+ }
en-es/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5bdb865b68a6c9cd8f442d9369bf14d5559fb7ec9e36f88f706f8f805373446c
3
+ size 4995031222
en-es/text_tokenizer/added_tokens.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "<|end_content|>": 8003,
3
+ "<|start_content|>": 8002,
4
+ "<|text_bos|>": 8000,
5
+ "<|text_sep|>": 8001
6
+ }
en-es/text_tokenizer/special_tokens_map.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<en>",
4
+ "<es>",
5
+ "<|text_bos|>",
6
+ "<|text_sep|>",
7
+ "<|start_content|>",
8
+ "<|end_content|>"
9
+ ],
10
+ "bos_token": {
11
+ "content": "<s>",
12
+ "lstrip": false,
13
+ "normalized": false,
14
+ "rstrip": false,
15
+ "single_word": false
16
+ },
17
+ "eos_token": {
18
+ "content": "</s>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ },
24
+ "pad_token": {
25
+ "content": "<pad>",
26
+ "lstrip": false,
27
+ "normalized": false,
28
+ "rstrip": false,
29
+ "single_word": false
30
+ },
31
+ "unk_token": {
32
+ "content": "<unk>",
33
+ "lstrip": false,
34
+ "normalized": false,
35
+ "rstrip": false,
36
+ "single_word": false
37
+ }
38
+ }
en-es/text_tokenizer/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
en-es/text_tokenizer/tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0111d3c52fd33496b60a079ff0ced24af6ebf66e849c67c7d63d6c93de19f320
3
+ size 367186
en-es/text_tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": true,
5
+ "added_tokens_decoder": {
6
+ "0": {
7
+ "content": "<pad>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "1": {
15
+ "content": "<unk>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "2": {
23
+ "content": "<s>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": false,
27
+ "single_word": false,
28
+ "special": true
29
+ },
30
+ "3": {
31
+ "content": "</s>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false,
36
+ "special": true
37
+ },
38
+ "4": {
39
+ "content": "<en>",
40
+ "lstrip": false,
41
+ "normalized": false,
42
+ "rstrip": false,
43
+ "single_word": false,
44
+ "special": true
45
+ },
46
+ "5": {
47
+ "content": "<es>",
48
+ "lstrip": false,
49
+ "normalized": false,
50
+ "rstrip": false,
51
+ "single_word": false,
52
+ "special": true
53
+ },
54
+ "8000": {
55
+ "content": "<|text_bos|>",
56
+ "lstrip": false,
57
+ "normalized": false,
58
+ "rstrip": false,
59
+ "single_word": false,
60
+ "special": true
61
+ },
62
+ "8001": {
63
+ "content": "<|text_sep|>",
64
+ "lstrip": false,
65
+ "normalized": false,
66
+ "rstrip": false,
67
+ "single_word": false,
68
+ "special": true
69
+ },
70
+ "8002": {
71
+ "content": "<|start_content|>",
72
+ "lstrip": false,
73
+ "normalized": false,
74
+ "rstrip": false,
75
+ "single_word": false,
76
+ "special": true
77
+ },
78
+ "8003": {
79
+ "content": "<|end_content|>",
80
+ "lstrip": false,
81
+ "normalized": false,
82
+ "rstrip": false,
83
+ "single_word": false,
84
+ "special": true
85
+ }
86
+ },
87
+ "additional_special_tokens": [
88
+ "<en>",
89
+ "<es>",
90
+ "<|text_bos|>",
91
+ "<|text_sep|>",
92
+ "<|start_content|>",
93
+ "<|end_content|>"
94
+ ],
95
+ "bos_token": "<s>",
96
+ "clean_up_tokenization_spaces": false,
97
+ "eos_token": "</s>",
98
+ "extra_special_tokens": {},
99
+ "legacy": true,
100
+ "model_max_length": 2048,
101
+ "pad_token": "<pad>",
102
+ "sp_model_kwargs": {},
103
+ "spaces_between_special_tokens": false,
104
+ "tokenizer_class": "LlamaTokenizer",
105
+ "unk_token": "<unk>",
106
+ "use_default_system_prompt": false
107
+ }