multimodalart HF Staff commited on
Commit
154ba0b
·
verified ·
1 Parent(s): 96a219a

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +6 -0
  2. README.md +14 -8
  3. app.py +532 -0
  4. asr_0.wav +3 -0
  5. chat_template.txt +2 -0
  6. omni_diffusion/__init__.py +2 -0
  7. omni_diffusion/constants.py +105 -0
  8. omni_diffusion/data/__init__.py +3 -0
  9. omni_diffusion/data/build.py +110 -0
  10. omni_diffusion/data/data_collator.py +122 -0
  11. omni_diffusion/data/dataset_base.py +424 -0
  12. omni_diffusion/data/dataset_qwen2.py +981 -0
  13. omni_diffusion/data/processor/__init__.py +0 -0
  14. omni_diffusion/data/processor/audio_processor.py +142 -0
  15. omni_diffusion/data/processor/image_processor.py +358 -0
  16. omni_diffusion/data/utils.py +212 -0
  17. omni_diffusion/models/__init__.py +3 -0
  18. omni_diffusion/models/dream/__init__.py +12 -0
  19. omni_diffusion/models/dream/config.yaml +98 -0
  20. omni_diffusion/models/dream/config_dream_resume.json +37 -0
  21. omni_diffusion/models/dream/configuration.json +14 -0
  22. omni_diffusion/models/dream/configuration_dream.py +86 -0
  23. omni_diffusion/models/dream/generation_utils.py +623 -0
  24. omni_diffusion/models/dream/modeling_dream.py +1136 -0
  25. omni_diffusion/models/dream/modeling_sensevoice.py +1258 -0
  26. omni_diffusion/models/dream/resampler_projector.py +40 -0
  27. omni_diffusion/models/dream/tokenization_dream.py +351 -0
  28. omni_diffusion/models/dream/tokenizer_config.json +219 -0
  29. omni_diffusion/models/magvit/common_modules.py +393 -0
  30. omni_diffusion/models/magvit/misc.py +55 -0
  31. omni_diffusion/models/magvit/modeling_magvitv2.py +445 -0
  32. omni_diffusion/models/magvit/modeling_utils.py +1184 -0
  33. omni_diffusion/tokenizer.py +74 -0
  34. omni_diffusion/tokenizer_magvitv2.py +65 -0
  35. omni_diffusion/tokenizer_sensevoice_glm4voice.py +280 -0
  36. requirements.txt +17 -0
  37. s2i_0.wav +3 -0
  38. svqa_0.jpg +3 -0
  39. svqa_0.wav +0 -0
  40. third_party/GLM-4-Voice/.gitignore +4 -0
  41. third_party/GLM-4-Voice/LICENSE +201 -0
  42. third_party/GLM-4-Voice/README.md +159 -0
  43. third_party/GLM-4-Voice/README_en.md +148 -0
  44. third_party/GLM-4-Voice/audio_process.py +93 -0
  45. third_party/GLM-4-Voice/cosyvoice/__init__.py +0 -0
  46. third_party/GLM-4-Voice/cosyvoice/bin/inference.py +114 -0
  47. third_party/GLM-4-Voice/cosyvoice/bin/train.py +140 -0
  48. third_party/GLM-4-Voice/cosyvoice/cli/__init__.py +0 -0
  49. third_party/GLM-4-Voice/cosyvoice/cli/cosyvoice.py +83 -0
  50. third_party/GLM-4-Voice/cosyvoice/cli/frontend.py +168 -0
.gitattributes CHANGED
@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ asr_0.wav filter=lfs diff=lfs merge=lfs -text
37
+ s2i_0.wav filter=lfs diff=lfs merge=lfs -text
38
+ svqa_0.jpg filter=lfs diff=lfs merge=lfs -text
39
+ third_party/GLM-4-Voice/resources/architecture.jpeg filter=lfs diff=lfs merge=lfs -text
40
+ third_party/GLM-4-Voice/resources/web_demo.png filter=lfs diff=lfs merge=lfs -text
41
+ vqa_0.png filter=lfs diff=lfs merge=lfs -text
README.md CHANGED
@@ -1,13 +1,19 @@
1
  ---
2
- title: Omni Diffusion
3
- emoji: 🏆
4
- colorFrom: pink
5
- colorTo: yellow
6
  sdk: gradio
7
- sdk_version: 6.20.0
8
- python_version: '3.12'
9
  app_file: app.py
10
- pinned: false
 
 
11
  ---
12
 
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
1
  ---
2
+ title: Omni-Diffusion
3
+ emoji: 🌐
4
+ colorFrom: yellow
5
+ colorTo: purple
6
  sdk: gradio
7
+ sdk_version: 6.15.1
 
8
  app_file: app.py
9
+ short_description: Any-to-any multimodal model with discrete diffusion
10
+ python_version: "3.10"
11
+ startup_duration_timeout: 1h
12
  ---
13
 
14
+ # Omni-Diffusion
15
+
16
+ Interactive demo for **Omni-Diffusion**, a unified any-to-any multimodal model based on masked discrete diffusion.
17
+ Supports text-to-image, speech-to-image, text-to-speech, ASR, visual question answering, and spoken VQA.
18
+
19
+ Paper: [Omni-Diffusion: Unified Multimodal Understanding and Generation with Masked Discrete Diffusion](https://arxiv.org/abs/2603.06577)
app.py ADDED
@@ -0,0 +1,532 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
3
+ import spaces
4
+ import torch
5
+ import gradio as gr
6
+ import numpy as np
7
+ import re
8
+ import sys
9
+ import random
10
+ import tempfile
11
+ import torchaudio
12
+ import cv2
13
+ from transformers import AutoConfig, AutoModel, AutoTokenizer
14
+ from transformers.generation import GenerationConfig
15
+
16
+ # Add third_party paths for GLM-4-Voice
17
+ _HERE = os.path.dirname(os.path.abspath(__file__))
18
+ sys.path.insert(0, os.path.join(_HERE, "third_party", "GLM-4-Voice"))
19
+ sys.path.insert(0, os.path.join(_HERE, "third_party", "GLM-4-Voice", "cosyvoice"))
20
+ sys.path.insert(0, os.path.join(_HERE, "third_party", "GLM-4-Voice", "third_party", "Matcha-TTS"))
21
+
22
+ from omni_diffusion.data.processor.audio_processor import add_audio_input_contiguous
23
+ from omni_diffusion.tokenizer import get_audio_tokenizer
24
+ from omni_diffusion.data.processor.image_processor import ImageProcessor
25
+
26
+ MODEL_ID = "lijiang/Omni-Diffusion"
27
+ AUDIO_TOKENIZER_PATH = "THUDM/glm-4-voice-tokenizer"
28
+ AUDIO_DECODER_PATH = "THUDM/glm-4-voice-decoder"
29
+ IMAGE_TOKENIZER_PATH = "showlab/magvitv2"
30
+ AUDIO_TOKENIZER_TYPE = "sensevoice_glm4voice"
31
+
32
+ # Load chat template from file to avoid escaping issues
33
+ _TEMPLATE_PATH = os.path.join(_HERE, "chat_template.txt")
34
+ with open(_TEMPLATE_PATH, "r") as _f:
35
+ qwen2_chat_template = _f.read()
36
+
37
+ def set_seed(seed=42):
38
+ random.seed(seed)
39
+ np.random.seed(seed)
40
+ torch.manual_seed(seed)
41
+ if torch.cuda.is_available():
42
+ torch.cuda.manual_seed(seed)
43
+ torch.cuda.manual_seed_all(seed)
44
+
45
+ set_seed()
46
+
47
+ print("Loading model...", flush=True)
48
+
49
+ config = AutoConfig.from_pretrained(MODEL_ID, trust_remote_code=True)
50
+
51
+ tokenizer = AutoTokenizer.from_pretrained(
52
+ MODEL_ID, trust_remote_code=True, chat_template=qwen2_chat_template
53
+ )
54
+
55
+ model = AutoModel.from_pretrained(
56
+ MODEL_ID,
57
+ trust_remote_code=True,
58
+ torch_dtype=torch.bfloat16,
59
+ attn_implementation="sdpa",
60
+ ).to("cuda").eval()
61
+
62
+ model.generation_config = GenerationConfig.from_pretrained(MODEL_ID, trust_remote_code=True)
63
+ model.generation_config.max_new_tokens = 8192
64
+ model.generation_config.chat_format = "chatml"
65
+ model.generation_config.max_window_size = 8192
66
+ model.generation_config.use_cache = True
67
+ model.generation_config.do_sample = False
68
+ model.generation_config.temperature = 1.0
69
+ model.generation_config.top_k = 50
70
+ model.generation_config.top_p = 1.0
71
+ model.generation_config.num_beams = 1
72
+ model.generation_config.pad_token_id = tokenizer.pad_token_id
73
+
74
+ print("Loading audio tokenizer...", flush=True)
75
+ audio_tokenizer = get_audio_tokenizer(
76
+ AUDIO_TOKENIZER_PATH,
77
+ AUDIO_TOKENIZER_TYPE,
78
+ flow_path=AUDIO_DECODER_PATH,
79
+ rank=0,
80
+ )
81
+
82
+ print("Loading image processor...", flush=True)
83
+ image_processor = ImageProcessor(
84
+ IMAGE_TOKENIZER_PATH,
85
+ "dynamic",
86
+ image_size=512,
87
+ normalize_type="imagenet",
88
+ min_patch_grid=1,
89
+ max_patch_grid=12,
90
+ )
91
+ image_processor.image_tokenizer.rank = 0
92
+ image_processor.load_model()
93
+
94
+ print("All models loaded successfully!", flush=True)
95
+
96
+
97
+ def find_audio_segments_regex(text):
98
+ pattern = re.compile(r"<\|begin_of_audio\|>(.*?)<\|end_of_audio\|>", re.DOTALL)
99
+ segments = pattern.findall(text)
100
+ return [segment.strip() for segment in segments]
101
+
102
+
103
+ def run_inference(
104
+ task,
105
+ text_prompt="",
106
+ audio_path=None,
107
+ image_path=None,
108
+ max_tokens=128,
109
+ steps=128,
110
+ alg="entropy",
111
+ cfg=0.0,
112
+ max_position_penalty=1.0,
113
+ repeat_penalty=1.0,
114
+ add_boa_token=0,
115
+ output_text_only=False,
116
+ ):
117
+ """Run Omni-Diffusion inference for a given task.
118
+
119
+ Args:
120
+ task: One of T2I, S2I, TTS, ASR, VQA, SVQA.
121
+ text_prompt: Text input for the task.
122
+ audio_path: Path to input audio file (for ASR, S2I, SVQA).
123
+ image_path: Path to input image file (for VQA, SVQA).
124
+ max_tokens: Maximum number of new tokens to generate.
125
+ steps: Number of diffusion steps.
126
+ alg: Algorithm for generation ("entropy", "entropy-penalty", "origin").
127
+ cfg: Classifier-free guidance scale.
128
+ max_position_penalty: Position penalty for generation.
129
+ repeat_penalty: Repeat penalty for generation.
130
+ add_boa_token: Whether to add beginning-of-audio token.
131
+ output_text_only: Whether to output text only.
132
+
133
+ Returns:
134
+ Tuple of (text_output, audio_output, image_output).
135
+ """
136
+ system_message = []
137
+
138
+ if task == "S2I":
139
+ system_message = [{"role": "system", "content": "Please generate an image based on the input audio."}]
140
+ elif task == "SQA":
141
+ system_message = [{"role": "system", "content": "Please response the input audio."}]
142
+ elif task == "SVQA":
143
+ system_message = [{"role": "system", "content": "Please response the input audio based on the given image."}]
144
+
145
+ if audio_path is not None and image_path is not None:
146
+ messages = system_message + [{"role": "user", "content": "<|audio|>\n<|image|>"}]
147
+ elif audio_path is not None:
148
+ messages = system_message + [{"role": "user", "content": text_prompt + "\n<|audio|>"}]
149
+ elif image_path is not None:
150
+ messages = system_message + [{"role": "user", "content": text_prompt + "\n<|image|>"}]
151
+ else:
152
+ messages = system_message + [{"role": "user", "content": text_prompt}]
153
+
154
+ if audio_path is not None and audio_tokenizer.apply_to_role("user", is_discrete=True):
155
+ audio_tokens = audio_tokenizer.encode(audio_path)
156
+ audio_tokens_str = "".join(f"<|audio_{i}|>" for i in audio_tokens)
157
+ messages[-1]["content"] = messages[-1]["content"].replace("<|audio|>", audio_tokens_str)
158
+
159
+ if image_path is not None:
160
+ image_tokens = image_processor.process_images_with_subpatch(image_path, 512)
161
+ image_tokens = image_processor.get_image_token(image_tokens)
162
+ image_tokens = image_tokens[0].tolist()
163
+ image_tokens_str = "".join(f"<|image_{i}|>" for i in image_tokens)
164
+ messages[-1]["content"] = messages[-1]["content"].replace("<|image|>", image_tokens_str)
165
+
166
+ input_ids = tokenizer.apply_chat_template(
167
+ messages, tokenize=True, add_generation_prompt=True
168
+ )
169
+
170
+ audios = None
171
+ audio_indices = None
172
+ if audio_path is not None and audio_tokenizer.apply_to_role("user", is_contiguous=True):
173
+ audio_paths = [audio_path]
174
+ input_ids, audios, audio_indices = add_audio_input_contiguous(
175
+ input_ids, audio_paths, tokenizer, audio_tokenizer
176
+ )
177
+
178
+ input_ids_tensor = torch.tensor([input_ids], dtype=torch.long).to("cuda")
179
+
180
+ with torch.no_grad():
181
+ outputs, histories = model.generate(
182
+ input_ids_tensor,
183
+ audios=audios,
184
+ audio_indices=audio_indices,
185
+ temperature=0.0,
186
+ top_p=0.9,
187
+ steps=steps,
188
+ max_new_tokens=max_tokens,
189
+ alg=alg,
190
+ cfg=cfg,
191
+ tokenizer=tokenizer,
192
+ add_boa_token=add_boa_token,
193
+ max_position_penalty=max_position_penalty,
194
+ repeat_penalty=repeat_penalty,
195
+ output_text_only=output_text_only,
196
+ task=task,
197
+ )
198
+
199
+ audio_offset = tokenizer.convert_tokens_to_ids("<|audio_0|>")
200
+ image_offset = tokenizer.convert_tokens_to_ids("<|image_0|>")
201
+ audio_tokens_out = []
202
+ image_tokens_out = []
203
+ text_tokens_out = []
204
+
205
+ for token_id in outputs[0][input_ids_tensor.shape[1]:]:
206
+ tid = token_id.item()
207
+ if tid >= audio_offset and tid < audio_offset + 16384:
208
+ audio_tokens_out.append(tid - audio_offset)
209
+ elif tid >= image_offset:
210
+ image_tokens_out.append(tid - image_offset)
211
+ else:
212
+ text_tokens_out.append(tid)
213
+
214
+ # Decode audio
215
+ tts_speech = None
216
+ if len(audio_tokens_out) > 0:
217
+ tts_speech = audio_tokenizer.decode(audio_tokens_out, source_speech_16k=None)
218
+
219
+ # Decode image
220
+ image_out = None
221
+ if len(image_tokens_out) < 256 and len(image_tokens_out) > 0:
222
+ image_tokens_out += [image_tokens_out[-1]] * (256 - len(image_tokens_out))
223
+
224
+ if len(image_tokens_out) > 0:
225
+ gen_token_ids = torch.stack([torch.tensor(t) for t in image_tokens_out], dim=0).unsqueeze(0)
226
+ gen_token_ids = torch.clamp(gen_token_ids, max=8192 - 1, min=0)
227
+ with torch.no_grad():
228
+ image_out = image_processor.image_tokenizer.image_tokenizer.decode_code(
229
+ gen_token_ids[:, :256]
230
+ )
231
+ image_out = torch.clamp((image_out + 1.0) / 2.0, min=0.0, max=1.0)
232
+ image_out *= 255.0
233
+ image_out = image_out.permute(0, 2, 3, 1).cpu().numpy().astype(np.uint8)
234
+ image_out = image_out[:, :, :, [2, 1, 0]][0]
235
+
236
+ # Decode text
237
+ text_out = tokenizer.decode(text_tokens_out, skip_special_tokens=True)
238
+
239
+ return text_out, tts_speech, image_out
240
+
241
+
242
+ @spaces.GPU(duration=300)
243
+ def generate_t2i(prompt, steps=260, alg="entropy-penalty", repeat_penalty=1.2):
244
+ """Generate an image from a text prompt.
245
+
246
+ Args:
247
+ prompt: Text description of the image to generate.
248
+ steps: Number of diffusion denoising steps.
249
+ alg: Generation algorithm.
250
+ repeat_penalty: Penalty for repeated tokens.
251
+ """
252
+ message = "Generate an image based on the provided text description.\n" + prompt
253
+ text_out, audio_out, image_out = run_inference(
254
+ task="T2I",
255
+ text_prompt=message,
256
+ max_tokens=260,
257
+ steps=steps,
258
+ alg=alg,
259
+ repeat_penalty=repeat_penalty,
260
+ max_position_penalty=2.0,
261
+ )
262
+ return image_out, text_out
263
+
264
+
265
+ @spaces.GPU(duration=300)
266
+ def generate_tts(text, steps=25, alg="entropy", repeat_penalty=1.0):
267
+ """Convert text to speech.
268
+
269
+ Args:
270
+ text: Text to synthesize.
271
+ steps: Number of diffusion steps.
272
+ alg: Generation algorithm.
273
+ repeat_penalty: Penalty for repeated tokens.
274
+ """
275
+ message = "Convert the text to speech.\n" + text
276
+ text_out, audio_out, image_out = run_inference(
277
+ task="TTS",
278
+ text_prompt=message,
279
+ max_tokens=50,
280
+ steps=steps,
281
+ alg=alg,
282
+ repeat_penalty=repeat_penalty,
283
+ )
284
+ if audio_out is not None:
285
+ audio_path = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name
286
+ torchaudio.save(audio_path, audio_out.unsqueeze(0), 22050, format="wav")
287
+ return audio_path, text_out
288
+ return None, text_out
289
+
290
+
291
+ @spaces.GPU(duration=300)
292
+ def generate_asr(audio_file, steps=25, alg="entropy", repeat_penalty=1.0):
293
+ """Convert speech to text (ASR).
294
+
295
+ Args:
296
+ audio_file: Input audio file path.
297
+ steps: Number of diffusion steps.
298
+ alg: Generation algorithm.
299
+ repeat_penalty: Penalty for repeated tokens.
300
+ """
301
+ text_out, audio_out, image_out = run_inference(
302
+ task="ASR",
303
+ text_prompt="Convert the speech to text.",
304
+ audio_path=audio_file,
305
+ max_tokens=50,
306
+ steps=steps,
307
+ alg=alg,
308
+ repeat_penalty=repeat_penalty,
309
+ )
310
+ return text_out
311
+
312
+
313
+ @spaces.GPU(duration=300)
314
+ def generate_vqa(image_file, question, steps=64, alg="entropy", repeat_penalty=1.0):
315
+ """Answer a question about an image (Visual QA).
316
+
317
+ Args:
318
+ image_file: Input image file path.
319
+ question: Question about the image.
320
+ steps: Number of diffusion steps.
321
+ alg: Generation algorithm.
322
+ repeat_penalty: Penalty for repeated tokens.
323
+ """
324
+ text_out, audio_out, image_out = run_inference(
325
+ task="VQA",
326
+ text_prompt=question,
327
+ image_path=image_file,
328
+ max_tokens=64,
329
+ steps=steps,
330
+ alg=alg,
331
+ repeat_penalty=repeat_penalty,
332
+ )
333
+ return text_out
334
+
335
+
336
+ @spaces.GPU(duration=300)
337
+ def generate_s2i(audio_file, steps=260, alg="entropy-penalty", repeat_penalty=1.5, cfg=2.0):
338
+ """Generate an image from an audio input (Speech-to-Image).
339
+
340
+ Args:
341
+ audio_file: Input audio file path.
342
+ steps: Number of diffusion steps.
343
+ alg: Generation algorithm.
344
+ repeat_penalty: Penalty for repeated tokens.
345
+ cfg: Classifier-free guidance scale.
346
+ """
347
+ text_out, audio_out, image_out = run_inference(
348
+ task="S2I",
349
+ audio_path=audio_file,
350
+ max_tokens=260,
351
+ steps=steps,
352
+ alg=alg,
353
+ repeat_penalty=repeat_penalty,
354
+ max_position_penalty=2.0,
355
+ cfg=cfg,
356
+ )
357
+ return image_out, text_out
358
+
359
+
360
+ @spaces.GPU(duration=300)
361
+ def generate_svqa(image_file, audio_file, steps=128, alg="entropy-penalty", repeat_penalty=1.2):
362
+ """Answer a spoken question about an image (Spoken Visual QA).
363
+
364
+ Args:
365
+ image_file: Input image file path.
366
+ audio_file: Input audio (spoken question) file path.
367
+ steps: Number of diffusion steps.
368
+ alg: Generation algorithm.
369
+ repeat_penalty: Penalty for repeated tokens.
370
+ """
371
+ text_out, audio_out, image_out = run_inference(
372
+ task="SVQA",
373
+ audio_path=audio_file,
374
+ image_path=image_file,
375
+ add_boa_token=1,
376
+ max_tokens=128,
377
+ steps=steps,
378
+ alg=alg,
379
+ repeat_penalty=repeat_penalty,
380
+ )
381
+ result_audio = None
382
+ if audio_out is not None:
383
+ result_audio = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name
384
+ torchaudio.save(result_audio, audio_out.unsqueeze(0), 22050, format="wav")
385
+ return text_out, result_audio
386
+
387
+
388
+ CSS = """
389
+ #col-container { max-width: 1100px; margin: 0 auto; }
390
+ .dark .gradio-container { color: var(--body-text-color); }
391
+ """
392
+
393
+ with gr.Blocks(theme=gr.themes.Citrus(), css=CSS) as demo:
394
+ gr.Markdown("# Omni-Diffusion: Unified Multimodal Understanding and Generation")
395
+ gr.Markdown("Interactive demo for any-to-any multimodal generation with masked discrete diffusion. Supports text-to-image, speech-to-image, TTS, ASR, visual QA, and spoken visual QA.")
396
+ gr.Markdown("[Paper](https://arxiv.org/abs/2603.06577) | [Model](https://huggingface.co/lijiang/Omni-Diffusion) | [GitHub](https://github.com/VITA-MLLM/Omni-Diffusion)")
397
+
398
+ with gr.Tabs():
399
+ with gr.Tab("Text-to-Image"):
400
+ with gr.Row():
401
+ with gr.Column(scale=1):
402
+ t2i_prompt = gr.Textbox(label="Prompt", placeholder="Describe the image to generate...", lines=3)
403
+ t2i_btn = gr.Button("Generate Image", variant="primary")
404
+ with gr.Accordion("Advanced", open=False):
405
+ t2i_steps = gr.Slider(50, 512, value=260, step=10, label="Steps")
406
+ t2i_alg = gr.Dropdown(["entropy-penalty", "entropy", "origin"], value="entropy-penalty", label="Algorithm")
407
+ t2i_rp = gr.Slider(0.5, 3.0, value=1.2, step=0.1, label="Repeat Penalty")
408
+ with gr.Column(scale=1):
409
+ t2i_image = gr.Image(label="Generated Image", type="numpy")
410
+ t2i_text = gr.Textbox(label="Output Text", lines=2)
411
+ gr.Examples(
412
+ examples=[["The image shows a landscape background with double exposure glasses of wine, displaying a hyperealistic and detailed view of the subject."]],
413
+ inputs=[t2i_prompt],
414
+ outputs=[t2i_image, t2i_text],
415
+ fn=generate_t2i,
416
+ cache_examples=False,
417
+ run_on_click=True,
418
+ )
419
+ t2i_btn.click(generate_t2i, inputs=[t2i_prompt, t2i_steps, t2i_alg, t2i_rp], outputs=[t2i_image, t2i_text])
420
+
421
+ with gr.Tab("Text-to-Speech"):
422
+ with gr.Row():
423
+ with gr.Column(scale=1):
424
+ tts_text = gr.Textbox(label="Text", placeholder="Text to synthesize...", lines=3)
425
+ tts_btn = gr.Button("Generate Speech", variant="primary")
426
+ with gr.Accordion("Advanced", open=False):
427
+ tts_steps = gr.Slider(10, 100, value=25, step=5, label="Steps")
428
+ tts_alg = gr.Dropdown(["entropy", "entropy-penalty", "origin"], value="entropy", label="Algorithm")
429
+ tts_rp = gr.Slider(0.5, 3.0, value=1.0, step=0.1, label="Repeat Penalty")
430
+ with gr.Column(scale=1):
431
+ tts_audio = gr.Audio(label="Generated Audio")
432
+ tts_text_out = gr.Textbox(label="Output Text", lines=2)
433
+ gr.Examples(
434
+ examples=[["Get the trust fund to the bank early."], ["It is a nice walk though, with trees and a river and stuff."]],
435
+ inputs=[tts_text],
436
+ outputs=[tts_audio, tts_text_out],
437
+ fn=generate_tts,
438
+ cache_examples=False,
439
+ run_on_click=True,
440
+ )
441
+ tts_btn.click(generate_tts, inputs=[tts_text, tts_steps, tts_alg, tts_rp], outputs=[tts_audio, tts_text_out])
442
+
443
+ with gr.Tab("Speech-to-Text (ASR)"):
444
+ with gr.Row():
445
+ with gr.Column(scale=1):
446
+ asr_audio = gr.Audio(label="Input Audio", type="filepath")
447
+ asr_btn = gr.Button("Transcribe", variant="primary")
448
+ with gr.Accordion("Advanced", open=False):
449
+ asr_steps = gr.Slider(10, 100, value=25, step=5, label="Steps")
450
+ asr_alg = gr.Dropdown(["entropy", "entropy-penalty", "origin"], value="entropy", label="Algorithm")
451
+ asr_rp = gr.Slider(0.5, 3.0, value=1.0, step=0.1, label="Repeat Penalty")
452
+ with gr.Column(scale=1):
453
+ asr_text = gr.Textbox(label="Transcription", lines=5)
454
+ gr.Examples(
455
+ examples=[["asr_0.wav"]],
456
+ inputs=[asr_audio],
457
+ outputs=[asr_text],
458
+ fn=generate_asr,
459
+ cache_examples=False,
460
+ run_on_click=True,
461
+ )
462
+ asr_btn.click(generate_asr, inputs=[asr_audio, asr_steps, asr_alg, asr_rp], outputs=[asr_text])
463
+
464
+ with gr.Tab("Visual QA"):
465
+ with gr.Row():
466
+ with gr.Column(scale=1):
467
+ vqa_image = gr.Image(label="Input Image", type="filepath")
468
+ vqa_question = gr.Textbox(label="Question", placeholder="Ask about the image...", lines=2)
469
+ vqa_btn = gr.Button("Answer", variant="primary")
470
+ with gr.Accordion("Advanced", open=False):
471
+ vqa_steps = gr.Slider(20, 200, value=64, step=10, label="Steps")
472
+ vqa_alg = gr.Dropdown(["entropy", "entropy-penalty", "origin"], value="entropy", label="Algorithm")
473
+ vqa_rp = gr.Slider(0.5, 3.0, value=1.0, step=0.1, label="Repeat Penalty")
474
+ with gr.Column(scale=1):
475
+ vqa_answer = gr.Textbox(label="Answer", lines=5)
476
+ gr.Examples(
477
+ examples=[["vqa_0.png", "Is the glass of orange juice half empty or half full?"]],
478
+ inputs=[vqa_image, vqa_question],
479
+ outputs=[vqa_answer],
480
+ fn=generate_vqa,
481
+ cache_examples=False,
482
+ run_on_click=True,
483
+ )
484
+ vqa_btn.click(generate_vqa, inputs=[vqa_image, vqa_question, vqa_steps, vqa_alg, vqa_rp], outputs=[vqa_answer])
485
+
486
+ with gr.Tab("Speech-to-Image"):
487
+ with gr.Row():
488
+ with gr.Column(scale=1):
489
+ s2i_audio = gr.Audio(label="Input Audio", type="filepath")
490
+ s2i_btn = gr.Button("Generate Image", variant="primary")
491
+ with gr.Accordion("Advanced", open=False):
492
+ s2i_steps = gr.Slider(50, 512, value=260, step=10, label="Steps")
493
+ s2i_alg = gr.Dropdown(["entropy-penalty", "entropy", "origin"], value="entropy-penalty", label="Algorithm")
494
+ s2i_rp = gr.Slider(0.5, 3.0, value=1.5, step=0.1, label="Repeat Penalty")
495
+ s2i_cfg = gr.Slider(0.0, 5.0, value=2.0, step=0.5, label="CFG Scale")
496
+ with gr.Column(scale=1):
497
+ s2i_image = gr.Image(label="Generated Image", type="numpy")
498
+ s2i_text = gr.Textbox(label="Output Text", lines=2)
499
+ gr.Examples(
500
+ examples=[["s2i_0.wav"]],
501
+ inputs=[s2i_audio],
502
+ outputs=[s2i_image, s2i_text],
503
+ fn=generate_s2i,
504
+ cache_examples=False,
505
+ run_on_click=True,
506
+ )
507
+ s2i_btn.click(generate_s2i, inputs=[s2i_audio, s2i_steps, s2i_alg, s2i_rp, s2i_cfg], outputs=[s2i_image, s2i_text])
508
+
509
+ with gr.Tab("Spoken Visual QA"):
510
+ with gr.Row():
511
+ with gr.Column(scale=1):
512
+ svqa_image = gr.Image(label="Input Image", type="filepath")
513
+ svqa_audio = gr.Audio(label="Input Audio (spoken question)", type="filepath")
514
+ svqa_btn = gr.Button("Answer", variant="primary")
515
+ with gr.Accordion("Advanced", open=False):
516
+ svqa_steps = gr.Slider(20, 300, value=128, step=10, label="Steps")
517
+ svqa_alg = gr.Dropdown(["entropy-penalty", "entropy", "origin"], value="entropy-penalty", label="Algorithm")
518
+ svqa_rp = gr.Slider(0.5, 3.0, value=1.2, step=0.1, label="Repeat Penalty")
519
+ with gr.Column(scale=1):
520
+ svqa_text = gr.Textbox(label="Answer (Text)", lines=3)
521
+ svqa_audio_out = gr.Audio(label="Answer (Audio)")
522
+ gr.Examples(
523
+ examples=[["svqa_0.jpg", "svqa_0.wav"]],
524
+ inputs=[svqa_image, svqa_audio],
525
+ outputs=[svqa_text, svqa_audio_out],
526
+ fn=generate_svqa,
527
+ cache_examples=False,
528
+ run_on_click=True,
529
+ )
530
+ svqa_btn.click(generate_svqa, inputs=[svqa_image, svqa_audio, svqa_steps, svqa_alg, svqa_rp], outputs=[svqa_text, svqa_audio_out])
531
+
532
+ demo.launch(mcp_server=True)
asr_0.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8cc5488d43dadd87e8fa93f8c2a418b6771b7a9201c87ef39fff7b10e5650803
3
+ size 1108218
chat_template.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+
2
+ {%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0]['role'] == 'system' %}\n {{- messages[0]['content'] }}\n {%- endif %}\n {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role }}\n {%- if message.content %}\n {{- '\\n' + message.content }}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- message.content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n
omni_diffusion/__init__.py ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ # from .models import *
2
+ from .data import *
omni_diffusion/constants.py ADDED
@@ -0,0 +1,105 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+
3
+ logger = logging.getLogger(__name__)
4
+
5
+
6
+ if False:
7
+ IMG_TAG_TOKEN = "<image>"
8
+ IMG_CONTEXT_TOKEN = "<IMG_CONTEXT>"
9
+ IMG_START_TOKEN = "<img>"
10
+ IMG_END_TOKEN = "</img>"
11
+
12
+ VID_TAG_TOKEN = "<video>"
13
+ VID_CONTEXT_TOKEN = "<VID_CONTEXT>"
14
+ VID_START_TOKEN = "<vid>"
15
+ VID_END_TOKEN = "</vid>"
16
+
17
+ PATCH_CONTEXT_TOKEN = "<PATCH_CONTEXT>"
18
+ PATCH_START_TOKEN = "<patch>"
19
+ PATCH_END_TOKEN = "</patch>"
20
+
21
+ AUD_TAG_TOKEN = "<audio>"
22
+ AUD_START_TOKEN = "<|begin_of_audio|>"
23
+ AUD_END_TOKEN = "<|end_of_audio|>"
24
+
25
+ QUAD_START_TOKEN = "<quad>"
26
+ QUAD_END_TOKEN = "</quad>"
27
+ REF_START_TOKEN = "<ref>"
28
+ REF_END_TOKEN = "</ref>"
29
+ BOX_START_TOKEN = "<box>"
30
+ BOX_END_TOKEN = "</box>"
31
+
32
+
33
+ if True:
34
+
35
+ IMG_TAG_TOKEN = "<|image|>"
36
+ IMG_CONTEXT_TOKEN = "<|context_of_image|>"
37
+ IMG_START_TOKEN = "<|begin_of_image|>"
38
+ IMG_END_TOKEN = "<|end_of_image|>"
39
+
40
+ VID_TAG_TOKEN = "<|video|>"
41
+ VID_CONTEXT_TOKEN = "<|context_of_video|>"
42
+ VID_START_TOKEN = "<|begin_of_video|>"
43
+ VID_END_TOKEN = "<|end_of_video|>"
44
+
45
+ PATCH_CONTEXT_TOKEN = "<|context_of_patch|>"
46
+ PATCH_START_TOKEN = "<|begin_of_patch|>"
47
+ PATCH_END_TOKEN = "<|end_of_patch|>"
48
+
49
+ AUD_TAG_TOKEN = "<|audio|>"
50
+ AUD_CONTEXT_TOKEN = "<|context_of_audio|>"
51
+ AUD_START_TOKEN = "<|begin_of_audio|>"
52
+ AUD_END_TOKEN = "<|end_of_audio|>"
53
+
54
+ QUAD_START_TOKEN = "<|begin_of_quad|>"
55
+ QUAD_END_TOKEN = "<|end_of_quad|>"
56
+ REF_START_TOKEN = "<|begin_of_ref|>"
57
+ REF_END_TOKEN = "<|end_of_ref|>"
58
+ BOX_START_TOKEN = "<|begin_of_box|>"
59
+ BOX_END_TOKEN = "<|end_of_box|>"
60
+
61
+
62
+ logger.info(f"{IMG_TAG_TOKEN=}")
63
+ logger.info(f"{IMG_CONTEXT_TOKEN=}")
64
+ logger.info(f"{IMG_START_TOKEN=}")
65
+ logger.info(f"{IMG_END_TOKEN=}")
66
+
67
+ logger.info(f"{VID_TAG_TOKEN=}")
68
+ logger.info(f"{VID_CONTEXT_TOKEN=}")
69
+ logger.info(f"{VID_START_TOKEN=}")
70
+ logger.info(f"{VID_END_TOKEN=}")
71
+
72
+ logger.info(f"{PATCH_CONTEXT_TOKEN=}")
73
+ logger.info(f"{PATCH_START_TOKEN=}")
74
+ logger.info(f"{PATCH_END_TOKEN=}")
75
+
76
+ logger.info(f"{AUD_TAG_TOKEN=}")
77
+ logger.info(f"{AUD_CONTEXT_TOKEN=}")
78
+ logger.info(f"{AUD_START_TOKEN=}")
79
+ logger.info(f"{AUD_END_TOKEN=}")
80
+
81
+ # IMAGENET_MEAN = (0.485, 0.456, 0.406)
82
+ # IMAGENET_STD = (0.229, 0.224, 0.225)
83
+
84
+ # CLIP_MEAN = (0.4814546, 0.4578275, 0.40821073)
85
+ # CLIP_STD = (0.2686295, 0.2613025, 0.2757711)
86
+
87
+ # SIGLIP_MEAN = (0.5, 0.5, 0.5)
88
+ # SIGLIP_STD = (0.5, 0.5, 0.5)
89
+
90
+
91
+ IMAGENET_DEFAULT_MEAN = [0.485, 0.456, 0.406]
92
+ IMAGENET_DEFAULT_STD = [0.229, 0.224, 0.225]
93
+ IMAGENET_STANDARD_MEAN = [0.5, 0.5, 0.5]
94
+ IMAGENET_STANDARD_STD = [0.5, 0.5, 0.5]
95
+ OPENAI_CLIP_MEAN = [0.48145466, 0.4578275, 0.40821073]
96
+ OPENAI_CLIP_STD = [0.26862954, 0.26130258, 0.27577711]
97
+
98
+
99
+ # Model Constants
100
+ IGNORE_INDEX = -100
101
+ IMAGE_TOKEN_INDEX = -200
102
+ DEFAULT_IMAGE_TOKEN = IMG_CONTEXT_TOKEN
103
+ DEFAULT_IMAGE_PATCH_TOKEN = PATCH_CONTEXT_TOKEN
104
+ DEFAULT_IM_START_TOKEN = IMG_START_TOKEN
105
+ DEFAULT_IM_END_TOKEN = IMG_END_TOKEN
omni_diffusion/data/__init__.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ # data.__init__
2
+
3
+ from .build import build_supervised_dataset_deepspeed
omni_diffusion/data/build.py ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+ import os
3
+ from dataclasses import dataclass
4
+
5
+ from datasets import concatenate_datasets, load_dataset
6
+
7
+ from .data_collator import DataCollatorForSupervisedDataset, collate_fn_deepspeed
8
+ from .dataset_qwen2 import Qwen2Dataset
9
+
10
+ logger = logging.getLogger(__name__)
11
+ logger.setLevel(logging.INFO)
12
+
13
+
14
+ def build_supervised_dataset_deepspeed(
15
+ model_config,
16
+ model_args,
17
+ data_args,
18
+ training_args,
19
+ tokenizer,
20
+ create_position_ids=True,
21
+ create_loss_mask=False,
22
+ shift_token=False,
23
+ ):
24
+ """
25
+ Builds the supervised dataset and data collator specifically for DeepSpeed training.
26
+
27
+ Args:
28
+ model_config: Configuration object for the model.
29
+ model_args: Arguments related to model architecture and parameters.
30
+ data_args: Arguments related to data processing and loading.
31
+ training_args: Arguments related to the training loop (e.g., output dir, seed).
32
+ tokenizer: The tokenizer instance to process text.
33
+ create_position_ids (bool): Whether to generate position IDs.
34
+ create_loss_mask (bool): Whether to generate a mask for loss calculation.
35
+ shift_token (bool): Whether to shift tokens for causal language modeling.
36
+
37
+ Returns:
38
+ dict: A dictionary containing 'train', 'validation' datasets and the 'data_collator'.
39
+ """
40
+
41
+ logging.info("building dataset...")
42
+
43
+ cfg_path = data_args.dataset_name
44
+ max_padding_length = model_args.model_max_length
45
+ output_dir = training_args.output_dir
46
+
47
+ create_attention_mask = data_args.create_attention_mask
48
+ create_attention_mask_2d = data_args.create_attention_mask_2d
49
+
50
+ image_size = model_args.image_size
51
+ image_token_length = model_args.image_token_length
52
+
53
+ max_num_frame = model_args.max_num_frame
54
+ max_fps = model_args.max_fps
55
+
56
+ reset_position_ids = data_args.reset_position_ids
57
+ reset_attention_mask = data_args.reset_attention_mask
58
+ variable_length = data_args.variable_length
59
+
60
+ min_patch_grid = model_args.min_patch_grid
61
+ max_patch_grid = model_args.max_patch_grid
62
+ process_type = model_args.vision_process_type
63
+ normalize_type = model_args.vision_normalize_type
64
+
65
+ audio_tokenizer_path = model_args.audio_tokenizer_path
66
+ audio_tokenizer_type = model_args.audio_tokenizer_type
67
+ image_tokenizer_path = model_args.image_tokenizer_path
68
+
69
+ seed = training_args.seed
70
+ cross_dataset_joint = data_args.cross_dataset_joint
71
+ dataset_joint = data_args.dataset_joint
72
+
73
+ TrainDataset = Qwen2Dataset
74
+
75
+ train_dataset = TrainDataset(
76
+ cfg_path,
77
+ tokenizer,
78
+ image_size=image_size,
79
+ image_token_length=image_token_length,
80
+ max_padding_length=max_padding_length,
81
+ variable_length=variable_length,
82
+ output_dir=output_dir,
83
+ training_args=None,
84
+ shift_token=shift_token,
85
+ create_position_ids=create_position_ids,
86
+ create_attention_mask=create_attention_mask,
87
+ create_attention_mask_2d=create_attention_mask_2d,
88
+ create_loss_mask=create_loss_mask,
89
+ max_num_frame=max_num_frame,
90
+ max_fps=max_fps,
91
+ reset_position_ids=reset_position_ids,
92
+ reset_attention_mask=reset_attention_mask,
93
+ min_patch_grid=min_patch_grid,
94
+ max_patch_grid=max_patch_grid,
95
+ process_type=process_type,
96
+ normalize_type=normalize_type,
97
+ seed=seed,
98
+ cross_dataset_joint=cross_dataset_joint,
99
+ dataset_joint=dataset_joint,
100
+ audio_tokenizer_type=audio_tokenizer_type,
101
+ audio_tokenizer_path=audio_tokenizer_path,
102
+ image_tokenizer_path=image_tokenizer_path,
103
+ use_megatron=False,
104
+ )
105
+ eval_dataset = None
106
+
107
+ data_collator = collate_fn_deepspeed
108
+
109
+ return dict(train=train_dataset, validation=eval_dataset, data_collator=data_collator)
110
+
omni_diffusion/data/data_collator.py ADDED
@@ -0,0 +1,122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import itertools
2
+ import json
3
+ import logging
4
+ import math
5
+ import os
6
+ import re
7
+ from dataclasses import dataclass, field
8
+ from typing import Dict, List, Optional, Sequence
9
+
10
+ import torch
11
+ import transformers
12
+ from torch.utils.data import default_collate
13
+ from transformers.trainer_pt_utils import LabelSmoother
14
+
15
+ IGNORE_TOKEN_ID = LabelSmoother.ignore_index
16
+
17
+
18
+ @dataclass
19
+ class DataCollatorForSupervisedDataset(object):
20
+ """
21
+ Data collator for supervised fine-tuning.
22
+ Handles padding of input IDs and labels, creation of attention masks,
23
+ and concatenation of multimodal inputs (images).
24
+ """
25
+ tokenizer: transformers.PreTrainedTokenizer
26
+
27
+ def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
28
+ input_ids = [instance["input_ids"] for instance in instances]
29
+ labels = [instance["labels"] for instance in instances]
30
+
31
+ input_ids = torch.nn.utils.rnn.pad_sequence(
32
+ input_ids,
33
+ batch_first=True,
34
+ padding_value=self.tokenizer.pad_token_id,
35
+ )
36
+ labels = torch.nn.utils.rnn.pad_sequence(
37
+ labels,
38
+ batch_first=True,
39
+ padding_value=IGNORE_TOKEN_ID,
40
+ )
41
+
42
+ input_ids = input_ids[:, : self.tokenizer.model_max_length]
43
+ labels = labels[:, : self.tokenizer.model_max_length]
44
+
45
+ batch = dict(
46
+ input_ids=input_ids,
47
+ labels=labels,
48
+ attention_mask=input_ids.ne(self.tokenizer.pad_token_id),
49
+ )
50
+
51
+ if "images" in instances[0]:
52
+ images = [instance["images"] for instance in instances]
53
+ batch["images"] = torch.cat(images, dim=0)
54
+
55
+ if "doclm_images" in instances[0]:
56
+ images = [instance["doclm_images"] for instance in instances]
57
+ batch["doclm_images"] = torch.cat(images, dim=0)
58
+
59
+ if "image_paths" in instances[0]:
60
+ image_paths = [instance["image_paths"] for instance in instances]
61
+ batch["image_paths"] = image_paths
62
+
63
+ if "pixel_values" in instances[0]:
64
+ pixel_values = torch.cat([instance["pixel_values"] for instance in instances])
65
+ batch["pixel_values"] = pixel_values
66
+
67
+ if "image_flags" in instances[0]:
68
+ image_flags = torch.cat([instance["image_flags"] for instance in instances])
69
+ batch["image_flags"] = image_flags
70
+
71
+ return batch
72
+
73
+ def collate_fn_deepspeed(batch):
74
+ """
75
+ Custom collation function designed for DeepSpeed training.
76
+ Handles complex multimodal data (images/audio) and potentially packed sequences (cu_seq_lens).
77
+ """
78
+ tmp_batch = [{} for _ in range(len(batch))]
79
+ if "cu_seq_lens" in batch[0]:
80
+ cu_seq_lens = [x["cu_seq_lens"] for x in batch]
81
+ max_seq_len = [x["max_seq_len"] for x in batch]
82
+ else:
83
+ cu_seq_lens = None
84
+ max_seq_len = None
85
+
86
+ if "images" in batch[0].keys():
87
+ for new_x, x in zip(tmp_batch, batch):
88
+ new_x["images"] = x.pop("images")
89
+ new_x["image_indices"] = x.pop("image_indices")
90
+
91
+ if "audios" in batch[0].keys():
92
+ for new_x, x in zip(tmp_batch, batch):
93
+ new_x["audios"] = x.pop("audios")
94
+ new_x["audio_indices"] = x.pop("audio_indices")
95
+
96
+ new_batch = default_collate(batch)
97
+
98
+ if "images" in tmp_batch[0].keys():
99
+
100
+ new_batch["images"] = torch.cat([x["images"] for x in tmp_batch], dim=0)
101
+
102
+ for sample_idx, sample in enumerate(tmp_batch):
103
+ sample["image_indices"][0, :, :] = sample_idx
104
+
105
+ new_batch["image_indices"] = torch.cat([x["image_indices"] for x in tmp_batch], dim=1)
106
+
107
+ if "audios" in tmp_batch[0].keys():
108
+
109
+ new_batch["audios"] = list(itertools.chain.from_iterable([x["audios"] for x in tmp_batch]))
110
+ # print(f"{[x.size() for x in sample['audios']]}")
111
+
112
+ for sample_idx, sample in enumerate(tmp_batch):
113
+ for j in range(len(sample["audio_indices"])):
114
+ sample["audio_indices"][j][0, :, :] = sample_idx
115
+
116
+ new_batch["audio_indices"] = list(
117
+ itertools.chain.from_iterable([x["audio_indices"] for x in tmp_batch])
118
+ )
119
+ # print(f"{[x.size() for x in sample['audio_indices']]}")
120
+
121
+
122
+ return new_batch
omni_diffusion/data/dataset_base.py ADDED
@@ -0,0 +1,424 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import contextlib
2
+ import json
3
+ import logging
4
+ import os
5
+ import pdb
6
+ import re
7
+ import traceback
8
+ import uuid
9
+
10
+ import numpy as np
11
+ import torch
12
+ import yaml
13
+ from PIL import Image
14
+
15
+ from torchvision import transforms
16
+ from torchvision.transforms import InterpolationMode
17
+
18
+ from .processor.audio_processor import AudioProcessor
19
+ from .processor.image_processor import ImageProcessor
20
+ from .utils import draw_data
21
+
22
+ logger = logging.getLogger(__name__)
23
+ logger.setLevel(logging.INFO)
24
+
25
+
26
+ class BaseDataset(torch.utils.data.Dataset):
27
+ def __init__(
28
+ self,
29
+ cfg_path,
30
+ tokenizer,
31
+ image_size=448,
32
+ image_token_length=1024,
33
+ max_padding_length=32768,
34
+ variable_length=False,
35
+ output_dir="",
36
+ add_task_symbol=True,
37
+ training_args=None,
38
+ shift_token=False,
39
+ create_position_ids=True,
40
+ create_attention_mask=True,
41
+ create_attention_mask_2d=False,
42
+ create_loss_mask=False,
43
+ max_num_frame=8,
44
+ max_fps=1,
45
+ reset_position_ids=False,
46
+ reset_attention_mask=False,
47
+ min_patch_grid=1,
48
+ max_patch_grid=6,
49
+ process_type="anyres",
50
+ normalize_type="imagenet",
51
+ seed=42,
52
+ cross_dataset_joint=False,
53
+ dataset_joint=True,
54
+ audio_tokenizer_type=None,
55
+ audio_tokenizer_path=None,
56
+ image_tokenizer_path=None,
57
+ text_audio_interval_ratio=None,
58
+ use_megatron=True,
59
+ skip_samples=0,
60
+ ):
61
+ super(BaseDataset, self).__init__()
62
+
63
+ self.cfg_path = cfg_path
64
+ with open(self.cfg_path, "r", encoding="utf8") as cfg_file:
65
+ cfg_data = cfg_file.read()
66
+
67
+ self.cfg = yaml.load(cfg_data, Loader=yaml.CLoader)
68
+ logger.info(f"cfg {self.cfg}")
69
+
70
+ self.tokenizer = tokenizer
71
+ self.max_padding_length = max_padding_length
72
+ self.variable_length = variable_length
73
+ self.output_dir = output_dir
74
+ self.training_args = training_args
75
+ self.shift_token = shift_token
76
+ self.create_position_ids = create_position_ids
77
+ self.create_attention_mask = create_attention_mask
78
+ self.create_attention_mask_2d = create_attention_mask_2d
79
+ self.create_loss_mask = create_loss_mask
80
+ self.max_num_frame = max_num_frame
81
+ self.max_fps = max_fps
82
+ self.reset_position_ids = reset_position_ids
83
+ self.reset_attention_mask = reset_attention_mask
84
+
85
+ self.seed = seed
86
+ self.cross_dataset_joint = cross_dataset_joint
87
+ self.dataset_joint = dataset_joint
88
+
89
+ self.image_size = image_size
90
+ self.image_token_length = image_token_length
91
+
92
+ self.do_dataset_format = self.cfg.get("do_dataset_format", False)
93
+ self.do_dataset_cast = self.cfg.get("do_dataset_cast", False)
94
+ self.xlsx_sample_num = self.cfg.get("xlsx_sample_num", 5)
95
+
96
+ self.processor = {}
97
+ self.processor["image"] = ImageProcessor(
98
+ image_tokenizer_path,
99
+ process_type,
100
+ image_size=self.image_size,
101
+ normalize_type=normalize_type,
102
+ min_patch_grid=min_patch_grid,
103
+ max_patch_grid=max_patch_grid,
104
+ )
105
+
106
+ self.processor["audio"] = AudioProcessor(
107
+ audio_tokenizer_path=audio_tokenizer_path,
108
+ audio_tokenizer_type=audio_tokenizer_type,
109
+ text_audio_interval_ratio=text_audio_interval_ratio
110
+ )
111
+
112
+ self.total_samples = {}
113
+ self.processed_samples = {}
114
+ self.unjoint_samples = {}
115
+ self.joint_samples = {}
116
+ self.source2jsonpath = {}
117
+ self.source2prefixpath = {}
118
+ self.skip_samples = skip_samples
119
+
120
+ if use_megatron:
121
+ self.load_data()
122
+ else:
123
+ with main_process_first(local=False, desc="Loading data"):
124
+ self.load_data()
125
+
126
+
127
+ def load_data(self):
128
+ from datasets import Dataset, DatasetDict, concatenate_datasets, load_dataset
129
+
130
+ raw_data = None
131
+
132
+ sampled_data = {}
133
+ source_idx = 0
134
+ for data_name, data_info in self.cfg["dataset"].items():
135
+ data_ratio = data_info.get("ratio", 1)
136
+ data_num = data_info.get("num", 999999999)
137
+
138
+ prefix_path = data_info.get("prefix_path", "")
139
+
140
+ if data_ratio == 0:
141
+ continue
142
+
143
+ if data_num == 0:
144
+ continue
145
+
146
+ for data_idx, data_path in enumerate(data_info["json_paths"]):
147
+ logger.info("-" * 100)
148
+
149
+ if not os.path.isfile(data_path) and not os.path.isdir(data_path):
150
+ logger.warning(f"Data file no found {data_path}")
151
+ continue
152
+
153
+ this_data = load_json(data_path, self.output_dir)
154
+ # this_data = load_data_one(data_path, self.outout_dir)
155
+ if this_data is None:
156
+ logger.warning(f"Failed to load {data_path}")
157
+ continue
158
+ # print(f"this_data {this_data}")
159
+
160
+ column_names = list(this_data.features)
161
+ if "id" in column_names:
162
+ this_data = this_data.remove_columns("id")
163
+
164
+ # sources = [data_path] * len(this_data)
165
+ sources = [source_idx] * len(this_data)
166
+ # sources = [data_name] * len(this_data)
167
+ this_data = this_data.add_column("source", sources)
168
+
169
+ self.source2jsonpath[source_idx] = data_path
170
+ self.source2prefixpath[source_idx] = prefix_path
171
+ self.total_samples[source_idx] = len(this_data)
172
+ self.processed_samples[source_idx] = 0
173
+ self.unjoint_samples[source_idx] = 0
174
+ self.joint_samples[source_idx] = 0
175
+
176
+ source_idx += 1
177
+
178
+ if "images" not in column_names:
179
+ # images = [[]] * len(this_data)
180
+ images = [None] * len(this_data)
181
+ this_data = this_data.add_column("images", images)
182
+
183
+ if "videos" not in column_names:
184
+ # videos = [[]] * len(this_data)
185
+ videos = [None] * len(this_data)
186
+ this_data = this_data.add_column("videos", videos)
187
+
188
+ if "audios" not in column_names:
189
+ # audios = [[]] * len(this_data)
190
+ audios = [None] * len(this_data)
191
+ this_data = this_data.add_column("audios", audios)
192
+
193
+ if False:
194
+ column_names = list(this_data.features)
195
+ this_data = this_data.map(
196
+ format_function_general,
197
+ batched=True,
198
+ batch_size=2560,
199
+ num_proc=1,
200
+ # batch_size=1,
201
+ # num_proc=1,
202
+ remove_columns=column_names,
203
+ keep_in_memory=False,
204
+ desc="Running format on dataset",
205
+ )
206
+
207
+ this_data = this_data.shuffle(seed=self.seed)
208
+ # this_data = this_data.flatten_indices()
209
+ this_data = this_data.shuffle(seed=self.seed)
210
+ # this_data = this_data.flatten_indices()
211
+
212
+ data_ratio = float(data_ratio)
213
+ total_num = len(this_data)
214
+ used_num = min(int(total_num * data_ratio), data_num)
215
+ logger.info(f"total_num {total_num}")
216
+ logger.info(f"data_ratio {data_ratio}")
217
+ logger.info(f"data_num {data_num}")
218
+ logger.info(f"used_num {used_num}")
219
+
220
+ indices = [x % total_num for x in range(used_num)]
221
+
222
+ this_data = this_data.select(indices)
223
+
224
+ if raw_data is None:
225
+ raw_data = this_data
226
+ else:
227
+ if self.do_dataset_cast:
228
+ this_data = this_data.cast(raw_data.features)
229
+ raw_data = concatenate_datasets([raw_data, this_data])
230
+
231
+ sampled_data[data_path] = {}
232
+ sampled_data[data_path]["data"] = this_data.select(
233
+ range(min(self.xlsx_sample_num, used_num))
234
+ )
235
+ sampled_data[data_path]["total_num"] = total_num
236
+ sampled_data[data_path]["used_num"] = used_num
237
+
238
+ logger.info(f"this_data {this_data}")
239
+ logger.info(f"raw_data {raw_data}")
240
+ # logger.info(f"raw_data {raw_data[0]}")
241
+ # logger.info(f"raw_data {raw_data[-1]}")
242
+ logger.info(f"Successful load {data_path}")
243
+
244
+ raw_data = raw_data.shuffle(seed=self.seed)
245
+ # raw_data = raw_data.flatten_indices()
246
+ raw_data = raw_data.shuffle(seed=self.seed)
247
+ # raw_data = raw_data.flatten_indices()
248
+
249
+ self.raw_data = raw_data
250
+
251
+ if torch.distributed.is_initialized() and torch.distributed.get_rank() == 0:
252
+ output_xlsx = os.path.basename(self.cfg_path).replace("yaml", "xlsx")
253
+ output_xlsx = os.path.join(self.output_dir, output_xlsx)
254
+ logger.info(f"output_xlsx {output_xlsx}")
255
+ draw_data(
256
+ sampled_data,
257
+ output_xlsx,
258
+ tokenizer=self.tokenizer,
259
+ image_processor=self.processor["image"],
260
+ )
261
+
262
+ logger.info(f"raw_data {raw_data}")
263
+
264
+ if torch.distributed.is_initialized() and torch.distributed.get_rank() == 0:
265
+ logger.info(f"raw_data {raw_data[:10]}")
266
+ logger.info(f"raw_data {raw_data[-10:]}")
267
+
268
+ def __len__(self):
269
+ return len(self.raw_data)
270
+
271
+ def update_data_path(self, sample):
272
+ source = sample["source"]
273
+ prefix_path = self.source2prefixpath[source]
274
+
275
+ if has_image(sample):
276
+ sample["images"] = [os.path.join(prefix_path, image) for image in sample["images"]]
277
+
278
+ if has_video(sample):
279
+ sample["videos"] = [os.path.join(prefix_path, video) for video in sample["videos"]]
280
+
281
+ if has_audio(sample):
282
+ sample["audios"] = [os.path.join(prefix_path, audio) for audio in sample["audios"]]
283
+
284
+ return sample
285
+
286
+
287
+
288
+ def format_function_general(examples):
289
+ messages = [x for x in examples["messages"]]
290
+
291
+ if "images" in examples:
292
+ images = [x for x in examples["images"]]
293
+ else:
294
+ images = [None for _ in messages]
295
+
296
+ if "videos" in examples:
297
+ videos = [x for x in examples["videos"]]
298
+ else:
299
+ videos = [None for _ in messages]
300
+
301
+ if "audios" in examples:
302
+ audios = [x for x in examples["audios"]]
303
+ else:
304
+ audios = [None for _ in messages]
305
+
306
+ return {
307
+ "messages": messages,
308
+ "images": images,
309
+ "videos": videos,
310
+ "audios": audios,
311
+ }
312
+
313
+
314
+ def load_json_A(data_file):
315
+ from datasets import Dataset, DatasetDict, concatenate_datasets, load_dataset
316
+
317
+ with open(data_file, "r") as f:
318
+ raw_data = json.load(f)
319
+ this_data = Dataset.from_list(raw_data)
320
+ return this_data
321
+
322
+
323
+ def load_json_B(data_file):
324
+ from datasets import Dataset, DatasetDict, concatenate_datasets, load_dataset
325
+
326
+ this_data = load_dataset("json", data_files=data_file, keep_in_memory=False)
327
+ return this_data["train"]
328
+
329
+
330
+ def load_json_C(data_file):
331
+ from datasets import Dataset, DatasetDict, concatenate_datasets, load_dataset
332
+
333
+ raw_data = []
334
+ with open(data_file, "r") as f:
335
+ for line in f.readlines():
336
+ d = json.loads(line)
337
+ # raw_data.append({"conversations": d["conversations"], "id": d["id"]})
338
+ if "conversations" in d:
339
+ raw_data.append({"conversations": d["conversations"]})
340
+ if "messages" in d:
341
+ raw_data.append({"messages": d["messages"]})
342
+ this_data = Dataset.from_list(raw_data)
343
+ return this_data
344
+
345
+
346
+ def load_json(data_file, output_dir):
347
+ for func in [load_json_B, load_json_A, load_json_C]:
348
+ try:
349
+ this_data = func(data_file)
350
+ return this_data
351
+ except Exception as error:
352
+ with open(os.path.join(output_dir, "data_error.log"), "a") as f:
353
+ print("-" * 100, file=f)
354
+ print(f"{data_file=}", file=f)
355
+ # print(error, file=f)
356
+ print(traceback.format_exc(), file=f)
357
+ continue
358
+ return None
359
+
360
+
361
+ def load_data_one(data_file, output_dir):
362
+ if data_file.endswith("json") or data_file.endswith("jsonl"):
363
+ return load_json(data_file, output_dir)
364
+ from datasets import Dataset, DatasetDict, concatenate_datasets, load_dataset
365
+
366
+ this_data = load_dataset(data_file, keep_in_memory=False)
367
+ return this_data["train"]
368
+
369
+
370
+ @contextlib.contextmanager
371
+ def main_process_first(local=True, desc="work"):
372
+
373
+ if torch.distributed.is_initialized() and torch.distributed.get_world_size() > 1:
374
+ if local:
375
+ rank = int(os.environ["LOCAL_RANK"])
376
+ else:
377
+ rank = torch.distributed.get_rank()
378
+ is_main_process = rank == 0
379
+
380
+ try:
381
+ if not is_main_process:
382
+ torch.distributed.barrier()
383
+ yield
384
+ finally:
385
+ if is_main_process:
386
+ torch.distributed.barrier()
387
+ else:
388
+ yield
389
+
390
+ def has_video(sample):
391
+ # video
392
+ if (
393
+ "videos" in sample
394
+ and isinstance(sample["videos"], list)
395
+ and None not in sample["videos"]
396
+ and len(sample["videos"])
397
+ ):
398
+ return True
399
+ return False
400
+
401
+
402
+ def has_image(sample):
403
+ # image
404
+ if (
405
+ "images" in sample
406
+ and isinstance(sample["images"], list)
407
+ and None not in sample["images"]
408
+ and len(sample["images"])
409
+ ):
410
+ return True
411
+ return False
412
+
413
+
414
+ def has_audio(sample):
415
+ # audio
416
+ if (
417
+ "audios" in sample
418
+ and isinstance(sample["audios"], list)
419
+ and None not in sample["audios"]
420
+ and len(sample["audios"])
421
+ ):
422
+ return True
423
+ return False
424
+
omni_diffusion/data/dataset_qwen2.py ADDED
@@ -0,0 +1,981 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import logging
3
+ import math
4
+ import os
5
+ import pdb
6
+ import random
7
+ import re
8
+ import sys
9
+ import time
10
+ import traceback
11
+ from collections import defaultdict
12
+ from typing import Dict, List, Optional, Sequence
13
+ from typing import List, Optional, Tuple, Union
14
+ import copy
15
+ import numpy as np
16
+ import torch
17
+ import transformers
18
+ from transformers.trainer_pt_utils import LabelSmoother
19
+ import soundfile as sf
20
+
21
+ from .dataset_base import BaseDataset
22
+
23
+ logger = logging.getLogger(__name__)
24
+ logger.setLevel(logging.INFO)
25
+
26
+ IGNORE_TOKEN_ID = LabelSmoother.ignore_index
27
+
28
+ def forward_process(
29
+ bsz: int,
30
+ seq_len: int,
31
+ device: torch.device,
32
+ labels: torch.Tensor,
33
+ eps: float = 1e-3,
34
+ special_token_id: int = 151643,
35
+ special_mask_ratio: float = 0.1
36
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
37
+ """
38
+ Generates a mask for the input sequences, applying different masking probabilities
39
+ for normal tokens and special tokens within the valid (non-padded) regions.
40
+ """
41
+
42
+ # Initialize the boolean mask matrix and the overall mask ratio for each sample
43
+ b, l = bsz, seq_len
44
+ masked_indices = torch.zeros((b, l), device=device, dtype=torch.bool)
45
+ p_mask = torch.rand(b, device=device)
46
+ p_mask = (1 - eps) * p_mask + eps
47
+ p_mask = p_mask.unsqueeze(1) # [b,1]
48
+
49
+ # Find the first and last valid positions (where label is not -100) for each sequence
50
+ first_idxs = []
51
+ last_idxs = []
52
+ for i in range(b):
53
+ # -100 is the ignore_index
54
+ nonneg = (labels[i] != -100).nonzero(as_tuple=True)[0]
55
+ if nonneg.numel() == 0:
56
+ first_idxs.append(None)
57
+ last_idxs.append(None)
58
+ else:
59
+ first_idxs.append(int(nonneg[0]))
60
+ last_idxs.append(int(nonneg[-1]))
61
+
62
+ # Generate masks specifically for the valid interval of each sequence
63
+ for i in range(b):
64
+ start = first_idxs[i]
65
+ end = last_idxs[i]
66
+
67
+ # Skip if the sequence has no valid tokens or invalid boundaries
68
+ if start is None or end is None or end < start:
69
+ continue
70
+
71
+ # Generate base thresholds for each position in the valid interval
72
+ valid_len = end - start + 1
73
+ t = torch.rand(valid_len, device=device)
74
+ mask_threshold = (1 - eps) * t + eps # [valid_len]
75
+
76
+ # Generate random values to make masking decisions
77
+ rand_vals = torch.rand(valid_len, device=device)
78
+ # Normal token masking decision
79
+ normal_mask = rand_vals <= mask_threshold
80
+ # Special token masking threshold is lower
81
+ special_thresh = mask_threshold * special_mask_ratio
82
+ special_mask = rand_vals <= special_thresh
83
+
84
+ labels_slice = labels[i, start : end + 1]
85
+ # Final mask: special tokens use special_mask, others use normal_mask
86
+ final_mask = torch.where(
87
+ labels_slice == special_token_id,
88
+ special_mask,
89
+ normal_mask
90
+ )
91
+
92
+ masked_indices[i, start : end + 1] = final_mask
93
+
94
+ total_masked = int(masked_indices.sum().item())
95
+ special_masked = int((masked_indices & (labels == special_token_id)).sum().item())
96
+
97
+ return masked_indices, p_mask
98
+
99
+ def update_labels(input_ids, labels, eos_id, max_n=20):
100
+ """
101
+ Finds the first occurrence of the EOS token in each sequence and updates
102
+ up to `max_n` subsequent labels to the EOS token ID.
103
+ """
104
+ batch_size, seq_len = input_ids.shape
105
+ first_occurrence_indices = []
106
+
107
+ # Record the first occurrence position of eos_id in each batch sample
108
+ for idx in range(batch_size):
109
+ eos_positions = (input_ids[idx] == eos_id).nonzero(as_tuple=True)[0]
110
+ if len(eos_positions) > 0:
111
+ first_occurrence_indices.append(eos_positions[0].item())
112
+ else:
113
+ first_occurrence_indices.append(-1)
114
+
115
+ # Select a random number of sequential positions (up to max_n) starting from first_idx to update
116
+ for i in range(batch_size):
117
+ first_idx = first_occurrence_indices[i]
118
+ if first_idx == -1:
119
+ continue
120
+ max_possible = seq_len - first_idx
121
+
122
+ if max_possible <= 0:
123
+ continue
124
+ num_to_select = random.randint(1, min(max_n, max_possible))
125
+
126
+ selected_indices = torch.arange(first_idx, first_idx + num_to_select)
127
+
128
+ labels[i, selected_indices] = eos_id
129
+
130
+ return labels
131
+
132
+
133
+ import torch
134
+ import random
135
+
136
+ def update_labels_and_inputs(input_ids, labels, eos_id, max_n=20):
137
+ input_ids = torch.tensor(input_ids).unsqueeze(0)
138
+ labels = torch.tensor(labels).unsqueeze(0)
139
+ batch_size, seq_len = input_ids.shape
140
+ input_ids = input_ids.clone()
141
+ labels = labels.clone()
142
+ new_input_ids = []
143
+ new_labels = []
144
+
145
+ for idx in range(batch_size):
146
+ eos_positions = (input_ids[idx] == eos_id).nonzero(as_tuple=True)[0]
147
+ if len(eos_positions) > 0:
148
+ first_idx = eos_positions[0].item()
149
+ cur_input_ids = input_ids[idx]
150
+ cur_labels = labels[idx]
151
+ else:
152
+ random_max_n = random.randint(1, max_n)
153
+ eos_ids = torch.full((random_max_n,), eos_id, device=input_ids.device, dtype=input_ids.dtype)
154
+ cur_input_ids = torch.cat([input_ids[idx], eos_ids])
155
+ pad_labels = torch.full((random_max_n,), eos_id, device=labels.device, dtype=labels.dtype)
156
+ cur_labels = torch.cat([labels[idx], pad_labels])
157
+ # first_idx = len(cur_input_ids) - random_max_n
158
+
159
+
160
+ new_input_ids.append(cur_input_ids)
161
+ new_labels.append(cur_labels)
162
+
163
+ max_len = max(len(x) for x in new_input_ids)
164
+ padded_input_ids = torch.stack([
165
+ torch.cat([x, torch.full((max_len - len(x),), eos_id, device=x.device, dtype=x.dtype)])
166
+ for x in new_input_ids
167
+ ])
168
+ padded_labels = torch.stack([
169
+ torch.cat([x, torch.full((max_len - len(x),), eos_id, device=x.device, dtype=x.dtype)])
170
+ for x in new_labels
171
+ ])
172
+
173
+ return padded_input_ids, padded_labels
174
+
175
+
176
+
177
+ def pad_or_truncate_to_512(
178
+ input_ids,
179
+ labels,
180
+ eos_id,
181
+ target_len: int = 512,
182
+ ):
183
+
184
+ input_ids = torch.as_tensor(input_ids).unsqueeze(0) # (B, L)
185
+ labels = torch.as_tensor(labels ).unsqueeze(0)
186
+
187
+ batch_size = input_ids.size(0)
188
+ new_input_ids, new_labels = [], []
189
+
190
+ for i in range(batch_size):
191
+ cur_input = input_ids[i]
192
+ cur_label = labels[i]
193
+
194
+ cur_input = cur_input[:target_len]
195
+ cur_label = cur_label[:target_len]
196
+
197
+ pad_len = target_len - cur_input.size(0)
198
+ if pad_len > 0:
199
+ eos_pad = torch.full((pad_len,), eos_id, device=cur_input.device, dtype=cur_input.dtype)
200
+ label_pad = torch.full((pad_len,), eos_id, device=cur_label.device, dtype=cur_label.dtype)
201
+ cur_input = torch.cat([cur_input, eos_pad])
202
+ cur_label = torch.cat([cur_label, label_pad])
203
+
204
+ new_input_ids.append(cur_input)
205
+ new_labels.append(cur_label)
206
+
207
+ return torch.stack(new_input_ids), torch.stack(new_labels)
208
+
209
+
210
+
211
+ class Qwen2Dataset(BaseDataset):
212
+ def __init__(
213
+ self,
214
+ *args,
215
+ **kwargs,
216
+ ):
217
+ super().__init__(
218
+ *args,
219
+ **kwargs,
220
+ )
221
+
222
+ self.default_system_message = "You are a helpful AI assistant."
223
+ self.default_system_message = None
224
+
225
+ self.ret = defaultdict(dict)
226
+ self.is_cat = True
227
+
228
+ if self.cross_dataset_joint:
229
+ for i in range(2):
230
+ self.maybe_init_ret(f"default_{i}")
231
+
232
+ def maybe_init_ret(self, source, force=False):
233
+ """Initializes the return dictionary buffer for a specific source if it doesn't exist."""
234
+ if source not in self.ret or force:
235
+ self.ret[source] = {}
236
+
237
+ self.ret[source]["tokens"] = []
238
+ self.ret[source]["labels"] = []
239
+ self.ret[source]["cu_seq_lens"] = [0]
240
+ self.ret[source]["max_seq_len"] = self.max_padding_length
241
+
242
+ if self.create_position_ids:
243
+ self.ret[source]["position_ids"] = []
244
+
245
+ if self.create_attention_mask:
246
+ self.ret[source]["attention_mask"] = []
247
+
248
+ if self.create_attention_mask_2d:
249
+ self.ret[source]["attention_mask_2d"] = torch.tril(
250
+ torch.ones(
251
+ (1, self.max_padding_length, self.max_padding_length), dtype=torch.bool
252
+ )
253
+ )
254
+ return len(self.ret[source]["tokens"]) == 0
255
+
256
+ def get_max_min_ret_length(self):
257
+ """Finds the buffer with the maximum and minimum current lengths."""
258
+ max_ret_lengh = 0
259
+ min_ret_lengh = self.max_padding_length + 1
260
+
261
+ max_ret_key = None
262
+ min_ret_key = None
263
+
264
+ for k, v in self.ret.items():
265
+ cur_length = len(v["tokens"])
266
+
267
+ if cur_length > max_ret_lengh:
268
+ max_ret_lengh = cur_length
269
+ max_ret_key = k
270
+
271
+ if cur_length < min_ret_lengh:
272
+ min_ret_lengh = cur_length
273
+ min_ret_key = k
274
+
275
+ return max_ret_lengh, max_ret_key, min_ret_lengh, min_ret_key
276
+
277
+ def add_ret(self, ret, source):
278
+ """
279
+ Appends a processed sample (tokens, images, audios) to the buffer (`self.ret`).
280
+ Updates indices for images/audios to account for the offset in the concatenated sequence.
281
+ """
282
+ cur_length = len(ret["input_ids"])
283
+ cur_image_length = len(ret["images"])
284
+ cur_audio_length = len(ret["audios"])
285
+
286
+ all_length = len(self.ret[source]["tokens"])
287
+
288
+ if "images" in self.ret[source]:
289
+ all_image_length = len(self.ret[source]["images"])
290
+ else:
291
+ all_image_length = 0
292
+
293
+ if cur_image_length > 0:
294
+ if all_image_length > 0:
295
+ self.ret[source]["images"] = torch.cat(
296
+ [self.ret[source]["images"], ret["images"]], dim=0
297
+ )
298
+ ret["image_indices"][1, :, :] += all_length
299
+ self.ret[source]["image_indices"] = torch.cat(
300
+ [self.ret[source]["image_indices"], ret["image_indices"]], dim=1
301
+ )
302
+ else:
303
+ self.ret[source]["images"] = ret["images"]
304
+ self.ret[source]["image_indices"] = ret["image_indices"]
305
+
306
+ if "audios" in self.ret[source]:
307
+ all_audio_length = len(self.ret[source]["audios"])
308
+ else:
309
+ all_audio_length = 0
310
+
311
+ if cur_audio_length > 0:
312
+ if all_audio_length > 0:
313
+ self.ret[source]["audios"].extend(ret["audios"])
314
+ for audio_indice in ret["audio_indices"]:
315
+ audio_indice[1, :, :] += all_length
316
+ self.ret[source]["audio_indices"].extend(ret["audio_indices"])
317
+ else:
318
+ self.ret[source]["audios"] = ret["audios"]
319
+ self.ret[source]["audio_indices"] = ret["audio_indices"]
320
+
321
+ # print(self.ret[source]["audios"])
322
+
323
+ if self.create_attention_mask:
324
+ self.ret[source]["attention_mask"] += ret["attention_mask"]
325
+
326
+ if self.create_attention_mask_2d:
327
+ self.ret[source]["attention_mask_2d"][:, all_length:, :all_length] = 0
328
+
329
+ if self.create_position_ids:
330
+ self.ret[source]["position_ids"] += list(range(cur_length))
331
+
332
+ self.ret[source]["tokens"] += ret["input_ids"]
333
+ self.ret[source]["labels"] += ret["labels"]
334
+ self.ret[source]["cu_seq_lens"] += [all_length + cur_length]
335
+
336
+ def process_ret(self, to_ret):
337
+ """
338
+ Finalizes the buffer for return. Handles padding, truncation, tensor conversion,
339
+ and attention mask generation.
340
+ """
341
+ if "tokens" in to_ret and len(to_ret["tokens"]) > 0:
342
+ pass
343
+ else:
344
+ return to_ret
345
+
346
+ if self.create_position_ids:
347
+ if self.reset_position_ids:
348
+ pass
349
+ else:
350
+ to_ret["position_ids"] = list(range(len(to_ret["tokens"])))
351
+
352
+ if self.create_attention_mask_2d:
353
+ if self.reset_attention_mask:
354
+ pass
355
+ else:
356
+ to_ret["attention_mask_2d"] = torch.tril(
357
+ torch.ones(
358
+ (1, self.max_padding_length, self.max_padding_length), dtype=torch.bool
359
+ )
360
+ )
361
+
362
+ if self.shift_token:
363
+ to_ret["tokens"] = to_ret["tokens"][:-1]
364
+ to_ret["labels"] = to_ret["labels"][1:]
365
+ to_ret["cu_seq_lens"][-1] -= 1
366
+ if self.create_position_ids:
367
+ to_ret["position_ids"] = to_ret["position_ids"][:-1]
368
+ if self.create_attention_mask:
369
+ to_ret["attention_mask"] = to_ret["attention_mask"][:-1]
370
+
371
+ if self.create_attention_mask_2d:
372
+ to_ret["attention_mask_2d"][:, :, -1] = 0
373
+ to_ret["attention_mask_2d"][:, -1, :] = 0
374
+
375
+ assert len(to_ret["tokens"]) == len(
376
+ to_ret["labels"]
377
+ ), f"{len(to_ret['tokens'])} {len(to_ret['labels'])}"
378
+
379
+ if not self.variable_length and self.max_padding_length > len(to_ret["tokens"]):
380
+ to_ret["tokens"] += [self.tokenizer.pad_token_id] * (
381
+ self.max_padding_length - len(to_ret["tokens"])
382
+ )
383
+ to_ret["labels"] += [IGNORE_TOKEN_ID] * (
384
+ self.max_padding_length - len(to_ret["labels"])
385
+ )
386
+ to_ret["cu_seq_lens"][-1] = self.max_padding_length
387
+ if self.create_position_ids:
388
+ # to_ret["position_ids"] += to_ret["position_ids"][-1:] * (
389
+ # self.max_padding_length - len(to_ret["position_ids"])
390
+ # )
391
+ to_ret["position_ids"] += list(
392
+ range(to_ret["position_ids"][-1] + 1, self.max_padding_length)
393
+ )
394
+ if self.create_attention_mask:
395
+ to_ret["attention_mask"] += [0] * (
396
+ self.max_padding_length - len(to_ret["attention_mask"])
397
+ )
398
+
399
+ to_ret["tokens"] = to_ret["tokens"][: self.max_padding_length]
400
+ to_ret["labels"] = to_ret["labels"][: self.max_padding_length]
401
+ to_ret["cu_seq_lens"][-1] = self.max_padding_length
402
+
403
+ if self.create_position_ids:
404
+ to_ret["position_ids"] = to_ret["position_ids"][: self.max_padding_length]
405
+ if self.create_attention_mask:
406
+ to_ret["attention_mask"] = to_ret["attention_mask"][: self.max_padding_length]
407
+
408
+ to_ret["tokens"] = torch.tensor(to_ret["tokens"], dtype=torch.int64)
409
+ to_ret["labels"] = torch.tensor(to_ret["labels"], dtype=torch.int64)
410
+ to_ret["cu_seq_lens"] = torch.tensor(to_ret["cu_seq_lens"], dtype=torch.int64)
411
+ if self.create_position_ids:
412
+ to_ret["position_ids"] = torch.tensor(to_ret["position_ids"], dtype=torch.int64)
413
+ if self.create_attention_mask:
414
+ to_ret["attention_mask"] = torch.tensor(to_ret["attention_mask"], dtype=torch.int64)
415
+
416
+ if self.create_attention_mask_2d:
417
+ attention_mask_2d = to_ret.pop("attention_mask_2d")
418
+ attention_mask_2d = attention_mask_2d.masked_fill(
419
+ (to_ret["attention_mask"] < 0.5).view(1, 1, self.max_padding_length), value=0
420
+ )
421
+ attention_mask_2d = attention_mask_2d < 0.5
422
+
423
+ to_ret["attention_mask"] = attention_mask_2d
424
+
425
+ if self.create_loss_mask:
426
+ loss_mask = torch.where(to_ret["labels"] == IGNORE_TOKEN_ID, 0, 1)
427
+ to_ret["loss_mask"] = loss_mask.to(torch.float32)
428
+
429
+ if not self.reset_position_ids and not self.reset_attention_mask:
430
+ to_ret.pop("cu_seq_lens")
431
+ else:
432
+ max_seq_len = max(to_ret["cu_seq_lens"][1:] - to_ret["cu_seq_lens"][:-1])
433
+ to_ret["max_seq_len"] = max_seq_len
434
+ to_ret["input_ids"] = to_ret["tokens"]
435
+
436
+ return to_ret
437
+
438
+ def is_skip(self):
439
+
440
+ processed_samples = sum(self.processed_samples.values())
441
+ if processed_samples < self.skip_samples:
442
+ if processed_samples % 1e3 == 0:
443
+ print(
444
+ f"processed_samples {processed_samples} skip_samples {self.skip_samples}"
445
+ )
446
+ return True
447
+
448
+ def show_statistic(self):
449
+ log_interval = 2000
450
+ if self.max_padding_length >= 2**17:
451
+ log_interval = 1000
452
+ if self.max_padding_length >= 2**20:
453
+ log_interval = 200
454
+
455
+ processed_samples = sum(self.processed_samples.values())
456
+ unjoint_samples = sum(self.unjoint_samples.values())
457
+ joint_samples = sum(self.joint_samples.values())
458
+ if unjoint_samples % log_interval == 1:
459
+ pass
460
+ else:
461
+ return
462
+
463
+ with open(os.path.join(self.output_dir, "data_statistics.log"), "a") as f:
464
+ print("-" * 100, file=f)
465
+ print(
466
+ f"processed_samples {processed_samples}" +
467
+ f" unjoint_samples {unjoint_samples}" +
468
+ f" joint_samples {joint_samples}" +
469
+ f" {[len(v['tokens']) for _, v in self.ret.items()]}", file=f,
470
+ )
471
+
472
+ print("source processed_samples unjoint_samples joint_samples data_path", file=f)
473
+ for source, data_path in self.source2jsonpath.items():
474
+ print(f"{source: >6} {self.processed_samples[source]: >16} \
475
+ {self.unjoint_samples[source]: >16} {self.joint_samples[source]: >16} {data_path}", file=f)
476
+
477
+ def __getitem__(self, index):
478
+ """
479
+ Main data loading function.
480
+ 1. Fetches raw data.
481
+ 2. Preprocesses it (tokenization, multimodal handling).
482
+ 3. Packs multiple samples into a single buffer (`self.ret`) until `max_padding_length` is reached.
483
+ 4. Returns the packed batch when full.
484
+ """
485
+ self.processor["audio"].load_model()
486
+ self.processor["image"].load_model()
487
+
488
+ while True:
489
+ try:
490
+ sample = self.raw_data[index]
491
+ sample = copy.deepcopy(sample)
492
+ sample = self.update_data_path(sample)
493
+ source = sample["source"]
494
+
495
+ self.processed_samples[source] += 1
496
+ if self.is_skip():
497
+ return {}
498
+
499
+ if self.cross_dataset_joint:
500
+ is_empty = False
501
+ (
502
+ max_ret_lengh,
503
+ max_ret_key,
504
+ min_ret_lengh,
505
+ min_ret_key,
506
+ ) = self.get_max_min_ret_length()
507
+ else:
508
+ is_empty = self.maybe_init_ret(source)
509
+
510
+ max_ret_lengh = min_ret_lengh = len(self.ret[source]["tokens"])
511
+ max_ret_key = min_ret_key = source
512
+
513
+ is_begin = is_empty or self.reset_position_ids or self.reset_attention_mask
514
+
515
+ #logger.info("preprocess begin " + str(self.processor["audio"].audio_tokenizer.device))
516
+
517
+ ret = preprocess(
518
+ sample,
519
+ self.tokenizer,
520
+ self.image_token_length,
521
+ default_system_message=self.default_system_message,
522
+ processor=self.processor,
523
+ is_begin=is_begin,
524
+ max_num_frame=self.max_num_frame,
525
+ max_fps=self.max_fps,
526
+ )
527
+
528
+ #logger.info("preprocess end " + str(self.processor["audio"].audio_tokenizer.device))
529
+
530
+ if ret is None:
531
+ #logger.info("ret is None " + str(self.processor["audio"].audio_tokenizer.device))
532
+ return {}
533
+
534
+ cur_length = len(ret["input_ids"])
535
+
536
+ if cur_length > self.max_padding_length:
537
+ return {}
538
+
539
+ self.unjoint_samples[source] += 1
540
+
541
+ if not self.dataset_joint:
542
+ import pdb; pdb.set_trace()
543
+ to_ret = self.ret.pop(max_ret_key)
544
+
545
+ self.maybe_init_ret(max_ret_key, force=True)
546
+ self.add_ret(ret, max_ret_key)
547
+
548
+ elif min_ret_lengh + cur_length > self.max_padding_length:
549
+ #logger.info("data too long " + str(self.processor["audio"].audio_tokenizer.device))
550
+ to_ret = self.ret.pop(max_ret_key)
551
+ self.joint_samples[source] += 1
552
+
553
+ self.maybe_init_ret(max_ret_key, force=True)
554
+ self.add_ret(ret, max_ret_key)
555
+
556
+ else:
557
+ to_ret = {}
558
+ self.add_ret(ret, min_ret_key)
559
+
560
+ to_ret = self.process_ret(to_ret)
561
+
562
+ self.show_statistic()
563
+ #logger.info("output ret " + str(len(ret)) + " " + str(self.processor["audio"].audio_tokenizer.device))
564
+ return to_ret
565
+
566
+ except Exception as error:
567
+ try:
568
+ with open(os.path.join(self.output_dir, "data_error.log"), "a") as f:
569
+ #print("-" * 100)
570
+ #print(traceback.format_exc())
571
+ #print(self.raw_data[index])
572
+ print("-" * 100, file=f)
573
+ print(traceback.format_exc(), file=f)
574
+ print(self.raw_data[index], file=f)
575
+ except Exception as error:
576
+ print(error)
577
+ return {}
578
+
579
+
580
+ def preprocess(
581
+ sample,
582
+ tokenizer: transformers.PreTrainedTokenizer,
583
+ image_token_length: int,
584
+ default_system_message: str = "You are a helpful assistant.",
585
+ processor=None,
586
+ is_begin: bool = True,
587
+ max_num_frame: int = 8,
588
+ max_fps: int = 1,
589
+ ) -> Dict:
590
+
591
+ from ..constants import (
592
+ IMG_START_TOKEN,
593
+ IMG_END_TOKEN,
594
+ IMG_CONTEXT_TOKEN,
595
+ VID_START_TOKEN,
596
+ VID_END_TOKEN,
597
+ VID_CONTEXT_TOKEN,
598
+ PATCH_START_TOKEN,
599
+ PATCH_END_TOKEN,
600
+ PATCH_CONTEXT_TOKEN,
601
+ AUD_START_TOKEN,
602
+ AUD_END_TOKEN,
603
+ IMG_TAG_TOKEN,
604
+ VID_TAG_TOKEN,
605
+ AUD_TAG_TOKEN,
606
+ AUD_CONTEXT_TOKEN,
607
+ )
608
+
609
+ human_roles = ["user", "human"]
610
+ gpt_roles = ["assistant", "gpt"]
611
+ system_roles = ["system"]
612
+
613
+ # Ensure special tokens map to exactly one ID
614
+ IMG_CONTEXT_ID = tokenizer(IMG_CONTEXT_TOKEN, add_special_tokens=False).input_ids
615
+ IMG_START_ID = tokenizer(IMG_START_TOKEN, add_special_tokens=False).input_ids
616
+ IMG_END_ID = tokenizer(IMG_END_TOKEN, add_special_tokens=False).input_ids
617
+
618
+ VID_CONTEXT_ID = tokenizer(VID_CONTEXT_TOKEN, add_special_tokens=False).input_ids
619
+ VID_START_ID = tokenizer(VID_START_TOKEN, add_special_tokens=False).input_ids
620
+ VID_END_ID = tokenizer(VID_END_TOKEN, add_special_tokens=False).input_ids
621
+
622
+ PATCH_CONTEXT_ID = tokenizer(PATCH_CONTEXT_TOKEN, add_special_tokens=False).input_ids
623
+ PATCH_START_ID = tokenizer(PATCH_START_TOKEN, add_special_tokens=False).input_ids
624
+ PATCH_END_ID = tokenizer(PATCH_END_TOKEN, add_special_tokens=False).input_ids
625
+
626
+ AUD_CONTEXT_ID = tokenizer(AUD_CONTEXT_TOKEN, add_special_tokens=False).input_ids
627
+ AUD_START_ID = tokenizer(AUD_START_TOKEN, add_special_tokens=False).input_ids
628
+ AUD_END_ID = tokenizer(AUD_END_TOKEN, add_special_tokens=False).input_ids
629
+
630
+ IMG_TAG_ID = tokenizer(IMG_TAG_TOKEN, add_special_tokens=False).input_ids
631
+ VID_TAG_ID = tokenizer(VID_TAG_TOKEN, add_special_tokens=False).input_ids
632
+ AUD_TAG_ID = tokenizer(AUD_TAG_TOKEN, add_special_tokens=False).input_ids
633
+
634
+ assert len(IMG_CONTEXT_ID) == 1
635
+ assert len(IMG_START_ID) == 1
636
+ assert len(IMG_END_ID) == 1
637
+
638
+ assert len(VID_CONTEXT_ID) == 1
639
+ assert len(VID_START_ID) == 1
640
+ assert len(VID_END_ID) == 1
641
+
642
+ assert len(PATCH_CONTEXT_ID) == 1
643
+ assert len(PATCH_START_ID) == 1
644
+ assert len(PATCH_END_ID) == 1
645
+
646
+ IMG_CONTEXT_ID = IMG_CONTEXT_ID[0]
647
+ IMG_START_ID = IMG_START_ID[0]
648
+ IMG_END_ID = IMG_END_ID[0]
649
+
650
+ VID_CONTEXT_ID = VID_CONTEXT_ID[0]
651
+ VID_START_ID = VID_START_ID[0]
652
+ VID_END_ID = VID_END_ID[0]
653
+
654
+ PATCH_CONTEXT_ID = PATCH_CONTEXT_ID[0]
655
+ PATCH_START_ID = PATCH_START_ID[0]
656
+ PATCH_END_ID = PATCH_END_ID[0]
657
+
658
+ AUD_CONTEXT_ID = AUD_CONTEXT_ID[0]
659
+ AUD_START_ID = AUD_START_ID[0]
660
+ AUD_END_ID = AUD_END_ID[0]
661
+
662
+ IMG_TAG_ID = IMG_TAG_ID[0]
663
+ VID_TAG_ID = VID_TAG_ID[0]
664
+ AUD_TAG_ID = AUD_TAG_ID[0]
665
+
666
+ BOS_ID = tokenizer.bos_token_id
667
+ EOS_ID = tokenizer.eos_token_id
668
+
669
+ # ChatML format special tokens
670
+ IM_START = "<|im_start|>"
671
+ IM_END = "<|im_end|>"
672
+ USER = "user"
673
+ ASSISTANT = "assistant"
674
+ SYSTEM = "system"
675
+
676
+ nl_tokens = tokenizer("\n", add_special_tokens=False).input_ids
677
+ IM_START_IDS = tokenizer(IM_START, add_special_tokens=False).input_ids
678
+ IM_END_IDS = tokenizer(IM_END, add_special_tokens=False).input_ids
679
+ USER_IDS = tokenizer(USER, add_special_tokens=False).input_ids
680
+ ASSISTANT_IDS = tokenizer(ASSISTANT, add_special_tokens=False).input_ids
681
+ SYSTEM_IDS = tokenizer(SYSTEM, add_special_tokens=False).input_ids
682
+
683
+ input_ids, targets = [], []
684
+ images = []
685
+ image_indices = []
686
+ audios = []
687
+ audio_indices = []
688
+
689
+ messages = []
690
+ if "conversations" in sample:
691
+ messages = sample["conversations"]
692
+ if len(messages) == 0 and "messages" in sample:
693
+ messages = sample["messages"]
694
+
695
+ # ----------------------------------------------------------------
696
+ # 1. System Prompt Processing
697
+ # ----------------------------------------------------------------
698
+ has_system = False
699
+ if is_begin:
700
+ if messages[0]["role"] == "system":
701
+ has_system = True
702
+ else:
703
+ has_system = False
704
+
705
+ if (
706
+ not has_system
707
+ and default_system_message is not None
708
+ and len(default_system_message) > 0
709
+ ):
710
+ messages = [{"role": "system", "content": default_system_message}] + messages
711
+ has_system = True
712
+
713
+ # ----------------------------------------------------------------
714
+ # 2. Audio Processing
715
+ # ----------------------------------------------------------------
716
+ if has_audio(sample) and processor["audio"].is_discrete:
717
+ unused_audio_idxs = list(range(len(sample["audios"])))
718
+
719
+ audio_tokens_list = [
720
+ processor["audio"].process_audios(x, is_discrete=True) for x in sample["audios"]
721
+ ]
722
+ audio_tokens_list = ["".join(f"<|audio_{i}|>" for i in x) for x in audio_tokens_list]
723
+
724
+ audio_idx = 0
725
+ for j, sentence in enumerate(messages):
726
+ content = sentence["content"]
727
+ role = sentence["role"]
728
+ # whether apply discrete tokenize to this role
729
+ if processor["audio"].apply_to_role(role, is_discrete=True):
730
+ while AUD_TAG_TOKEN in content:
731
+ content = content.replace(
732
+ AUD_TAG_TOKEN,
733
+ f"{AUD_START_TOKEN}{audio_tokens_list[audio_idx]}{AUD_END_TOKEN}",
734
+ 1,
735
+ )
736
+ # <|begin_of_audio|> <|audio_0|> <|audio_1|> ... <|audio_n|> <|end_of_audio|>
737
+ unused_audio_idxs.remove(audio_idx)
738
+ audio_idx += 1
739
+ else:
740
+ audio_idx += content.count(AUD_TAG_TOKEN)
741
+
742
+ sentence["content"] = content
743
+
744
+ # ----------------------------------------------------------------
745
+ # 3. Image Processing
746
+ # ----------------------------------------------------------------
747
+ if has_image(sample):
748
+ # for visual question answering & captioning
749
+ image_tokens_512_list = [
750
+ processor["image"].process_images_with_subpatch(x, 512) for x in sample["images"]
751
+ ]
752
+ image_tokens_512_list = [
753
+ processor["image"].get_image_token(x) for x in image_tokens_512_list
754
+ ]
755
+ image_tokens_512_list = [x[0].tolist() for x in image_tokens_512_list]
756
+ image_tokens_512_list = ["".join(f"<|image_{i}|>" for i in x) for x in image_tokens_512_list]
757
+
758
+ # for image generation
759
+ image_tokens_256_list = [
760
+ processor["image"].process_images_with_subpatch(x, 256) for x in sample["images"]
761
+ ]
762
+ image_tokens_256_list = [
763
+ processor["image"].get_image_token(x) for x in image_tokens_256_list
764
+ ]
765
+ image_tokens_256_list = [x[0].tolist() for x in image_tokens_256_list]
766
+ image_tokens_256_list = ["".join(f"<|image_{i}|>" for i in x) for x in image_tokens_256_list]
767
+
768
+ image_idx = 0
769
+ for j, sentence in enumerate(messages):
770
+ content = sentence["content"]
771
+ role = sentence["role"]
772
+ # for image, always apply discrete tokenize to this role
773
+ # if processor["image"].apply_to_role(role) or True:
774
+ if role == "user":
775
+ image_resolution = 512
776
+ else:
777
+ image_resolution = 256
778
+
779
+ while IMG_TAG_TOKEN in content:
780
+ if image_resolution == 256:
781
+ content = content.replace(
782
+ IMG_TAG_TOKEN,
783
+ f"{IMG_START_TOKEN}{image_tokens_256_list[image_idx]}{IMG_END_TOKEN}",
784
+ 1,
785
+ )
786
+ # <|begin_of_image|> <|image_0|> <|image_1|> ... <|image_n|> <|end_of_image|>
787
+ else:
788
+ content = content.replace(
789
+ IMG_TAG_TOKEN,
790
+ f"{IMG_START_TOKEN}{image_tokens_512_list[image_idx]}{IMG_END_TOKEN}",
791
+ 1,
792
+ )
793
+ # <|begin_of_image|> <|image_0|> <|image_1|> ... <|image_n|> <|end_of_image|>
794
+ image_idx += 1
795
+ else:
796
+ image_idx += content.count(IMG_TAG_TOKEN)
797
+
798
+ sentence["content"] = content
799
+
800
+ # ----------------------------------------------------------------
801
+ # 4. Text Processing
802
+ # ----------------------------------------------------------------
803
+ for j, sentence in enumerate(messages):
804
+ role = sentence["role"]
805
+ content = sentence["content"]
806
+
807
+ if role in human_roles:
808
+ _input_id = (
809
+ IM_START_IDS
810
+ + USER_IDS
811
+ + nl_tokens
812
+ + tokenizer(content, add_special_tokens=False).input_ids
813
+ + IM_END_IDS
814
+ + nl_tokens
815
+ )
816
+ _target = [IGNORE_TOKEN_ID] * len(_input_id)
817
+
818
+ elif role in gpt_roles:
819
+ content_input_id = tokenizer(content, add_special_tokens=False).input_ids
820
+
821
+ _input_id = (
822
+ IM_START_IDS + ASSISTANT_IDS + nl_tokens + content_input_id + IM_END_IDS + nl_tokens
823
+ )
824
+ _target = (
825
+ [IGNORE_TOKEN_ID] * len(IM_START_IDS)
826
+ + [IGNORE_TOKEN_ID] * len(ASSISTANT_IDS)
827
+ + [IGNORE_TOKEN_ID] * len(nl_tokens)
828
+ + content_input_id
829
+ + IM_END_IDS
830
+ + nl_tokens
831
+ )
832
+
833
+ dream_pad_token = tokenizer.encode("<|endoftext|>")
834
+ _input_id_dream = (
835
+ IM_START_IDS + ASSISTANT_IDS + nl_tokens + content_input_id + IM_END_IDS + dream_pad_token
836
+ )
837
+ _target_dream = (
838
+ [IGNORE_TOKEN_ID] * len(IM_START_IDS)
839
+ + [IGNORE_TOKEN_ID] * len(ASSISTANT_IDS)
840
+ + [IGNORE_TOKEN_ID] * len(nl_tokens)
841
+ + content_input_id
842
+ + IM_END_IDS
843
+ + dream_pad_token
844
+ )
845
+
846
+ elif role in system_roles:
847
+ _input_id = (
848
+ IM_START_IDS
849
+ + SYSTEM_IDS
850
+ + nl_tokens
851
+ + tokenizer(content, add_special_tokens=False).input_ids
852
+ + IM_END_IDS
853
+ + nl_tokens
854
+ )
855
+ _target = [IGNORE_TOKEN_ID] * len(_input_id)
856
+
857
+ else:
858
+ raise NotImplementedError
859
+
860
+ input_ids += _input_id
861
+ targets += _target
862
+
863
+
864
+ # ----------------------------------------------------------------
865
+ # 5. Contiguous Audio Processing
866
+ # ----------------------------------------------------------------
867
+ if has_audio(sample) and processor["audio"].is_contiguous:
868
+ aud_positions = [i for i, x in enumerate(input_ids) if x == AUD_TAG_ID]
869
+ assert len(aud_positions) == len(unused_audio_idxs), sample
870
+
871
+ new_input_ids = []
872
+ new_targets = []
873
+ st = 0
874
+
875
+ for aud_idx, aud_pos in enumerate(aud_positions):
876
+ aud_idx = unused_audio_idxs[aud_idx]
877
+ audio = processor["audio"].process_audios(sample["audios"][aud_idx], is_contiguous=True)
878
+ audios.append(audio)
879
+ audio_token_length = audio.size(0) + 4
880
+ # audio_token_length = audio.size(0)
881
+
882
+ new_input_ids += input_ids[st:aud_pos]
883
+ new_targets += targets[st:aud_pos]
884
+
885
+ new_input_ids += [AUD_START_ID]
886
+ new_targets += [IGNORE_TOKEN_ID]
887
+
888
+ audio_indice_b = torch.zeros(
889
+ 1, audio_token_length, dtype=torch.int64
890
+ ) # This will change in collate_fn
891
+ audio_indice_s = (
892
+ torch.arange(len(new_input_ids), len(new_input_ids) + audio_token_length)
893
+ .unsqueeze(0)
894
+ .repeat(1, 1)
895
+ )
896
+ audio_indice_b_s = torch.stack(
897
+ [audio_indice_b, audio_indice_s], dim=0
898
+ ) # 2, num_image, image_length
899
+ audio_indices.append(audio_indice_b_s)
900
+
901
+ new_input_ids += [AUD_CONTEXT_ID] * audio_token_length
902
+ new_targets += [IGNORE_TOKEN_ID] * audio_token_length
903
+
904
+ new_input_ids += [AUD_END_ID]
905
+ new_targets += [IGNORE_TOKEN_ID]
906
+
907
+ st = aud_pos + 1
908
+
909
+ new_input_ids += input_ids[st:]
910
+ new_targets += targets[st:]
911
+
912
+ input_ids = new_input_ids
913
+ targets = new_targets
914
+
915
+ if len(images) > 0:
916
+ images = torch.cat(images, dim=0)
917
+
918
+ if len(image_indices) > 0:
919
+ image_indices = torch.cat(image_indices, dim=1)
920
+
921
+ # ----------------------------------------------------------------
922
+ # 6. Final Masking and Label Updating
923
+ # ----------------------------------------------------------------
924
+ origin_input = input_ids
925
+ labels = targets
926
+ eos_id = 151643
927
+ mask_id = 151666
928
+ input_ids, labels = update_labels_and_inputs(input_ids,labels,eos_id,16)
929
+
930
+ labels_mask = ~(labels == -100)
931
+ bsz, seq_len = labels_mask.shape
932
+ masked_indices, p_mask = forward_process(
933
+ bsz, seq_len, input_ids.device, labels,special_mask_ratio=0.6, special_token_id=eos_id
934
+ )
935
+ final_masked_indices = masked_indices & labels_mask
936
+ final_masked_indices_inv = (~masked_indices) & labels_mask
937
+
938
+ mask_id_tensor = torch.full_like(input_ids, mask_id)
939
+ input_ids = torch.where(final_masked_indices, mask_id_tensor, input_ids)
940
+
941
+ new_labels = labels.clone()
942
+ new_labels[final_masked_indices_inv] = -100
943
+
944
+ input_ids = input_ids.squeeze(0).cpu().tolist()
945
+ new_labels = new_labels.squeeze(0).cpu().tolist()
946
+ attention_mask = [1] * len(input_ids)
947
+ assert len(new_labels) == len(input_ids)
948
+
949
+ return dict(
950
+ input_ids=input_ids,
951
+ labels=new_labels,
952
+ attention_mask=attention_mask,
953
+ images=images,
954
+ image_indices=image_indices,
955
+ audios=audios,
956
+ audio_indices=audio_indices,
957
+ )
958
+
959
+
960
+ def has_image(sample):
961
+ # image
962
+ if (
963
+ "images" in sample
964
+ and isinstance(sample["images"], list)
965
+ and None not in sample["images"]
966
+ and len(sample["images"])
967
+ ):
968
+ return True
969
+ return False
970
+
971
+
972
+ def has_audio(sample):
973
+ # audio
974
+ if (
975
+ "audios" in sample
976
+ and isinstance(sample["audios"], list)
977
+ and None not in sample["audios"]
978
+ and len(sample["audios"])
979
+ ):
980
+ return True
981
+ return False
omni_diffusion/data/processor/__init__.py ADDED
File without changes
omni_diffusion/data/processor/audio_processor.py ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import math
3
+ import os
4
+
5
+ import numpy as np
6
+ import torch
7
+
8
+ import natsort
9
+ from omni_diffusion.tokenizer import get_audio_tokenizer
10
+
11
+
12
+ class AudioProcessor:
13
+ """
14
+ Handles the initialization of audio tokenizers and processing of audio files.
15
+ """
16
+ def __init__(
17
+ self,
18
+ audio_tokenizer_path=None,
19
+ audio_tokenizer_type=None,
20
+ text_audio_interval_ratio=None,
21
+ ):
22
+
23
+ self.audio_tokenizer = get_audio_tokenizer(
24
+ audio_tokenizer_path,
25
+ audio_tokenizer_type,
26
+ )
27
+
28
+ self.audio_tokenizer_type = audio_tokenizer_type
29
+
30
+ self.text_audio_interval_ratio = text_audio_interval_ratio
31
+
32
+ # self.load_model()
33
+
34
+ def load_model(self):
35
+ """Loads the underlying tokenizer model weights."""
36
+ if self.audio_tokenizer is not None:
37
+ self.audio_tokenizer.load_model()
38
+
39
+ def process_audios(self, audio_path, is_discrete=False, is_contiguous=False, **kwargs):
40
+
41
+ assert not (is_discrete and is_contiguous)
42
+ assert is_discrete or is_contiguous
43
+
44
+ if is_discrete:
45
+ audio_tokenizer_type = self.audio_tokenizer_type.split("_")[-1]
46
+ cache_path = os.path.splitext(audio_path)[0] + f"_{audio_tokenizer_type}.json"
47
+ try:
48
+ if os.path.isfile(cache_path):
49
+ with open(cache_path, "r") as f:
50
+ audio_data = json.load(f)
51
+ return audio_data
52
+ except Exception as e:
53
+ pass
54
+
55
+ audio_data = self.audio_tokenizer.encode(
56
+ audio_path, is_discrete=is_discrete, is_contiguous=is_contiguous, **kwargs
57
+ )
58
+ # print(f"{len(audio_data)=}")
59
+
60
+ if is_discrete:
61
+ try:
62
+ if isinstance(audio_data, list):
63
+ with open(cache_path, "w") as f:
64
+ json.dump(audio_data, f)
65
+ except Exception as e:
66
+ pass
67
+
68
+ return audio_data
69
+
70
+ @property
71
+ def is_discrete(self):
72
+ return self.audio_tokenizer.is_discrete
73
+
74
+ @property
75
+ def is_contiguous(self):
76
+ return self.audio_tokenizer.is_contiguous
77
+
78
+ def apply_to_role(self, role, **kwargs):
79
+ return self.audio_tokenizer.apply_to_role(role, **kwargs)
80
+
81
+
82
+ def add_audio_input_contiguous(input_ids, audio_paths, tokenizer, audio_tokenizer):
83
+ """
84
+ Injects contiguous audio embeddings into a text sequence.
85
+ Replaces specific audio tag tokens in the input_ids with a sequence of
86
+ [AUD_START] + [AUD_CONTEXT]... + [AUD_END].
87
+ """
88
+ from ...constants import (
89
+ AUD_START_TOKEN,
90
+ AUD_END_TOKEN,
91
+ AUD_TAG_TOKEN,
92
+ AUD_CONTEXT_TOKEN,
93
+ )
94
+
95
+ AUD_CONTEXT_ID = tokenizer(AUD_CONTEXT_TOKEN, add_special_tokens=False).input_ids
96
+ AUD_TAG_ID = tokenizer(AUD_TAG_TOKEN, add_special_tokens=False).input_ids
97
+ AUD_START_ID = tokenizer(AUD_START_TOKEN, add_special_tokens=False).input_ids
98
+ AUD_END_ID = tokenizer(AUD_END_TOKEN, add_special_tokens=False).input_ids
99
+
100
+ AUD_CONTEXT_ID = AUD_CONTEXT_ID[0]
101
+ AUD_TAG_ID = AUD_TAG_ID[0]
102
+ AUD_START_ID = AUD_START_ID[0]
103
+ AUD_END_ID = AUD_END_ID[0]
104
+
105
+ aud_positions = [i for i, x in enumerate(input_ids) if x == AUD_TAG_ID]
106
+
107
+ audios = []
108
+ audio_indices = []
109
+ new_input_ids = []
110
+ st = 0
111
+ for aud_idx, aud_pos in enumerate(aud_positions):
112
+ audio = audio_tokenizer.encode(audio_paths[aud_idx], is_contiguous=True)
113
+ audios.append(audio)
114
+ audio_token_length = audio.size(0) + 4
115
+
116
+ new_input_ids += input_ids[st:aud_pos]
117
+
118
+ new_input_ids += [AUD_START_ID]
119
+
120
+ audio_indice_b = torch.zeros(
121
+ 1, audio_token_length, dtype=torch.int64
122
+ ) # This will change in collate_fn
123
+ audio_indice_s = (
124
+ torch.arange(len(new_input_ids), len(new_input_ids) + audio_token_length)
125
+ .unsqueeze(0)
126
+ .repeat(1, 1)
127
+ )
128
+ audio_indice_b_s = torch.stack(
129
+ [audio_indice_b, audio_indice_s], dim=0
130
+ ) # 2, num_image, image_length
131
+ audio_indices.append(audio_indice_b_s)
132
+
133
+ new_input_ids += [AUD_CONTEXT_ID] * audio_token_length
134
+
135
+ new_input_ids += [AUD_END_ID]
136
+
137
+ st = aud_pos + 1
138
+
139
+ new_input_ids += input_ids[st:]
140
+ inputs_ids = new_input_ids
141
+
142
+ return inputs_ids, audios, audio_indices
omni_diffusion/data/processor/image_processor.py ADDED
@@ -0,0 +1,358 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ import os
3
+
4
+ import numpy as np
5
+ import torch
6
+ from PIL import Image
7
+
8
+ import decord
9
+ import natsort
10
+ from omni_diffusion.constants import (
11
+ IMAGENET_DEFAULT_MEAN,
12
+ IMAGENET_DEFAULT_STD,
13
+ IMAGENET_STANDARD_MEAN,
14
+ IMAGENET_STANDARD_STD,
15
+ OPENAI_CLIP_MEAN,
16
+ OPENAI_CLIP_STD,
17
+ )
18
+ from torchvision import transforms
19
+ from omni_diffusion.tokenizer_magvitv2 import MagVITV2Tokenizer
20
+
21
+ import logging
22
+ logger = logging.getLogger(__name__)
23
+ logger.setLevel(logging.INFO)
24
+
25
+ def image_transform(image, resolution=256):
26
+ image = transforms.Resize(resolution, interpolation=transforms.InterpolationMode.BICUBIC)(image)
27
+ image = transforms.CenterCrop((resolution, resolution))(image)
28
+ image = transforms.ToTensor()(image)
29
+ image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image)
30
+ return image
31
+
32
+
33
+ class ImageProcessor:
34
+ def __init__(
35
+ self,
36
+ model_path,
37
+ process_type,
38
+ image_size=256,
39
+ normalize_type="imagenet",
40
+ min_patch_grid=1,
41
+ max_patch_grid=6,
42
+ ):
43
+ self.process_type = process_type
44
+ self.image_size = image_size
45
+
46
+ if normalize_type == "imagenet":
47
+ MEAN, STD = IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD
48
+ elif normalize_type == "clip":
49
+ MEAN, STD = OPENAI_CLIP_MEAN, OPENAI_CLIP_STD
50
+ elif normalize_type == "siglip":
51
+ MEAN, STD = IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD
52
+ else:
53
+ raise NotImplementedError(normalize_type)
54
+ self.mean = MEAN
55
+ self.std = STD
56
+
57
+ self.patch_size = image_size
58
+ self.min_patch_grid = min_patch_grid
59
+ self.max_patch_grid = max_patch_grid
60
+
61
+ if self.process_type == "anyres":
62
+ self.grid_pinpoints = [
63
+ (i, j)
64
+ for i in range(min_patch_grid, max_patch_grid + 1)
65
+ for j in range(min_patch_grid, max_patch_grid + 1)
66
+ ]
67
+ self.possible_resolutions = [
68
+ [dim * self.patch_size for dim in pair] for pair in self.grid_pinpoints
69
+ ]
70
+ print(f"grid_pinpoints {self.grid_pinpoints}")
71
+ print(f"possible_resolutions {self.possible_resolutions}")
72
+
73
+ if self.process_type == "dynamic":
74
+ max_num = self.max_patch_grid
75
+ min_num = self.min_patch_grid
76
+ # calculate the existing image aspect ratio
77
+ target_ratios = set(
78
+ (i, j)
79
+ for n in range(min_num, max_num + 1)
80
+ for i in range(1, n + 1)
81
+ for j in range(1, n + 1)
82
+ if i * j <= max_num and i * j >= min_num
83
+ )
84
+ self.target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
85
+ self.possible_resolutions = [
86
+ [dim * self.patch_size for dim in pair] for pair in self.target_ratios
87
+ ]
88
+ print(f"target_ratios {self.target_ratios}")
89
+ print(f"possible_resolutions {self.possible_resolutions}")
90
+
91
+ # self.image_tokenizer = MAGVITv2()
92
+ self.image_tokenizer = MagVITV2Tokenizer(model_path=model_path)
93
+
94
+ def load_model(self):
95
+ if self.image_tokenizer is not None:
96
+ self.image_tokenizer.load_model()
97
+
98
+ def process_images(self, img_or_path_list, image_resolution):
99
+
100
+ if isinstance(img_or_path_list[0], str):
101
+ images = [Image.open(x).convert("RGB") for x in img_or_path_list]
102
+ elif isinstance(img_or_path_list[0], Image.Image):
103
+ images = [x.convert("RGB") for x in img_or_path_list]
104
+ else:
105
+ images = img_or_path_list
106
+
107
+ # image_tensor = torch.ones([len(images), 3, self.image_size, self.image_size])
108
+ image_tensor = torch.ones([len(images), 3, image_resolution, image_resolution])
109
+
110
+ for i, image in enumerate(images):
111
+ image = image_transform(image, resolution=image_resolution)
112
+
113
+ image_tensor[i] = image
114
+
115
+ return image_tensor
116
+
117
+ def process_images_with_subpatch(self, img_or_path, image_resolution):
118
+ return self.process_images([img_or_path], image_resolution)
119
+
120
+ def get_image_token(self, image):
121
+ return self.image_tokenizer.encode(image)
122
+
123
+
124
+ def select_best_resolution(original_size, possible_resolutions):
125
+ """
126
+ Selects the best resolution from a list of possible resolutions based on the original size.
127
+
128
+ Args:
129
+ original_size (tuple): The original size of the image in the format (width, height).
130
+ possible_resolutions (list): A list of possible resolutions in the format [(width1, height1),
131
+ (width2, height2), ...].
132
+
133
+ Returns:
134
+ tuple: The best fit resolution in the format (width, height).
135
+ """
136
+ original_width, original_height = original_size
137
+ best_fit = None
138
+ max_effective_resolution = 0
139
+ min_wasted_resolution = float("inf")
140
+
141
+ for width, height in possible_resolutions:
142
+ # Calculate the downscaled size to keep the aspect ratio
143
+ scale = min(width / original_width, height / original_height)
144
+ downscaled_width, downscaled_height = int(original_width * scale), int(
145
+ original_height * scale
146
+ )
147
+
148
+ # Calculate effective and wasted resolutions
149
+ effective_resolution = min(
150
+ downscaled_width * downscaled_height, original_width * original_height
151
+ )
152
+ wasted_resolution = (width * height) - effective_resolution
153
+
154
+ if effective_resolution > max_effective_resolution or (
155
+ effective_resolution == max_effective_resolution
156
+ and wasted_resolution < min_wasted_resolution
157
+ ):
158
+ max_effective_resolution = effective_resolution
159
+ min_wasted_resolution = wasted_resolution
160
+ best_fit = (width, height)
161
+
162
+ return best_fit
163
+
164
+
165
+ def resize_and_pad_image(image, target_resolution):
166
+ """
167
+ Resize and pad an image to a target resolution while maintaining aspect ratio.
168
+
169
+ Args:
170
+ image (PIL.Image.Image): The input image.
171
+ target_resolution (tuple): The target resolution (width, height) of the image.
172
+
173
+ Returns:
174
+ PIL.Image.Image: The resized and padded image.
175
+ """
176
+ original_width, original_height = image.size
177
+ target_width, target_height = target_resolution
178
+
179
+ # Determine which dimension (width or height) to fill
180
+ scale_w = target_width / original_width
181
+ scale_h = target_height / original_height
182
+
183
+ if scale_w < scale_h:
184
+ # Width will be filled completely
185
+ new_width = target_width
186
+ new_height = min(math.ceil(original_height * scale_w), target_height)
187
+ else:
188
+ # Height will be filled completely
189
+ new_height = target_height
190
+ new_width = min(math.ceil(original_width * scale_h), target_width)
191
+
192
+ # Resize the image
193
+ resized_image = image.resize((new_width, new_height))
194
+
195
+ # Create a new image with the target size and paste the resized image onto it
196
+ new_image = Image.new("RGB", (target_width, target_height), (0, 0, 0))
197
+ paste_x = (target_width - new_width) // 2
198
+ paste_y = (target_height - new_height) // 2
199
+ new_image.paste(resized_image, (paste_x, paste_y))
200
+
201
+ return new_image
202
+
203
+
204
+ def add_image_input_contiguous(input_ids, image_paths, tokenizer):
205
+
206
+ image_processor = ImageProcessor(
207
+ process_type="dynamic",
208
+ image_size=448,
209
+ normalize_type="imagenet",
210
+ min_patch_grid=1,
211
+ max_patch_grid=12,
212
+ )
213
+
214
+ image_token_length = 256
215
+ max_num_frame = 4096
216
+ max_fps = 1
217
+
218
+ from ...constants import (
219
+ IMG_START_TOKEN,
220
+ IMG_END_TOKEN,
221
+ IMG_TAG_TOKEN,
222
+ IMG_CONTEXT_TOKEN,
223
+ VID_START_TOKEN,
224
+ VID_END_TOKEN,
225
+ VID_TAG_TOKEN,
226
+ VID_CONTEXT_TOKEN,
227
+ PATCH_START_TOKEN,
228
+ PATCH_END_TOKEN,
229
+ PATCH_CONTEXT_TOKEN,
230
+ )
231
+
232
+ IMG_CONTEXT_ID = tokenizer(IMG_CONTEXT_TOKEN, add_special_tokens=False).input_ids
233
+ IMG_START_ID = tokenizer(IMG_START_TOKEN, add_special_tokens=False).input_ids
234
+ IMG_END_ID = tokenizer(IMG_END_TOKEN, add_special_tokens=False).input_ids
235
+
236
+ VID_CONTEXT_ID = tokenizer(VID_CONTEXT_TOKEN, add_special_tokens=False).input_ids
237
+ VID_START_ID = tokenizer(VID_START_TOKEN, add_special_tokens=False).input_ids
238
+ VID_END_ID = tokenizer(VID_END_TOKEN, add_special_tokens=False).input_ids
239
+
240
+ PATCH_CONTEXT_ID = tokenizer(PATCH_CONTEXT_TOKEN, add_special_tokens=False).input_ids
241
+ PATCH_START_ID = tokenizer(PATCH_START_TOKEN, add_special_tokens=False).input_ids
242
+ PATCH_END_ID = tokenizer(PATCH_END_TOKEN, add_special_tokens=False).input_ids
243
+
244
+ IMG_TAG_ID = tokenizer(IMG_TAG_TOKEN, add_special_tokens=False).input_ids
245
+ VID_TAG_ID = tokenizer(VID_TAG_TOKEN, add_special_tokens=False).input_ids
246
+
247
+ assert len(IMG_CONTEXT_ID) == 1
248
+ assert len(IMG_START_ID) == 1
249
+ assert len(IMG_END_ID) == 1
250
+
251
+ assert len(VID_CONTEXT_ID) == 1
252
+ assert len(VID_START_ID) == 1
253
+ assert len(VID_END_ID) == 1
254
+
255
+ assert len(PATCH_CONTEXT_ID) == 1
256
+ assert len(PATCH_START_ID) == 1
257
+ assert len(PATCH_END_ID) == 1
258
+
259
+ IMG_CONTEXT_ID = IMG_CONTEXT_ID[0]
260
+ IMG_START_ID = IMG_START_ID[0]
261
+ IMG_END_ID = IMG_END_ID[0]
262
+
263
+ VID_CONTEXT_ID = VID_CONTEXT_ID[0]
264
+ VID_START_ID = VID_START_ID[0]
265
+ VID_END_ID = VID_END_ID[0]
266
+
267
+ PATCH_CONTEXT_ID = PATCH_CONTEXT_ID[0]
268
+ PATCH_START_ID = PATCH_START_ID[0]
269
+ PATCH_END_ID = PATCH_END_ID[0]
270
+
271
+ IMG_TAG_ID = IMG_TAG_ID[0]
272
+ VID_TAG_ID = VID_TAG_ID[0]
273
+
274
+ nl_tokens = tokenizer("\n", add_special_tokens=False).input_ids
275
+
276
+ img_positions = [i for i, x in enumerate(input_ids) if x == IMG_TAG_ID]
277
+
278
+ images = []
279
+ image_indices = []
280
+ new_input_ids = []
281
+ st = 0
282
+ for img_idx, img_pos in enumerate(img_positions):
283
+ image_patches, (
284
+ best_width,
285
+ best_height,
286
+ ) = image_processor.process_images_with_subpatch(image_paths[img_idx])
287
+ images.append(image_patches)
288
+ print(f"add_image_input_contiguous best_width {best_width} best_height {best_height}")
289
+
290
+ new_input_ids += input_ids[st:img_pos]
291
+
292
+ new_input_ids += [IMG_START_ID]
293
+
294
+ image_indice_b = torch.zeros(
295
+ 1, image_token_length, dtype=torch.int64
296
+ ) # This will change in collate_fn
297
+ image_indice_s = (
298
+ torch.arange(len(new_input_ids), len(new_input_ids) + image_token_length)
299
+ .unsqueeze(0)
300
+ .repeat(1, 1)
301
+ )
302
+ image_indice_b_s = torch.stack(
303
+ [image_indice_b, image_indice_s], dim=0
304
+ ) # 2, num_image, image_length
305
+ image_indices.append(image_indice_b_s)
306
+
307
+ new_input_ids += [IMG_CONTEXT_ID] * image_token_length
308
+
309
+ new_input_ids += [IMG_END_ID]
310
+
311
+ if len(image_patches) > 1:
312
+ for i in range(0, best_height, image_processor.patch_size):
313
+ new_input_ids += nl_tokens
314
+
315
+ for j in range(0, best_width, image_processor.patch_size):
316
+ new_input_ids += [PATCH_START_ID]
317
+
318
+ image_indice_b = torch.zeros(
319
+ 1, image_token_length, dtype=torch.int64
320
+ ) # This will change in collate_fn
321
+ image_indice_s = (
322
+ torch.arange(
323
+ len(new_input_ids), len(new_input_ids) + image_token_length
324
+ )
325
+ .unsqueeze(0)
326
+ .repeat(1, 1)
327
+ )
328
+ image_indice_b_s = torch.stack(
329
+ [image_indice_b, image_indice_s], dim=0
330
+ ) # 2, num_image, image_length
331
+ image_indices.append(image_indice_b_s)
332
+
333
+ new_input_ids += [PATCH_CONTEXT_ID] * image_token_length
334
+
335
+ new_input_ids += [PATCH_END_ID]
336
+ # print(f"get_external_dict i {i} j {j} new_input_ids {len(new_input_ids)}")
337
+
338
+ st = img_pos + 1
339
+
340
+ new_input_ids += input_ids[st:]
341
+
342
+ inputs_ids = new_input_ids
343
+
344
+ images = torch.cat(images, dim=0)
345
+ image_indices = torch.cat(image_indices, dim=1)
346
+
347
+ image_indices = image_indices.contiguous().to(torch.cuda.current_device())
348
+ if True:
349
+ images = (
350
+ torch.tensor(images, dtype=torch.bfloat16).contiguous().to(torch.cuda.current_device())
351
+ )
352
+
353
+ else:
354
+ images = (
355
+ torch.tensor(images, dtype=torch.float16).contiguous().to(torch.cuda.current_device())
356
+ )
357
+
358
+ return inputs_ids, images, image_indices
omni_diffusion/data/utils.py ADDED
@@ -0,0 +1,212 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import copy
3
+ import hashlib
4
+ import io
5
+ import itertools
6
+ import json
7
+ import logging
8
+ import os
9
+ import re
10
+ from typing import Dict, Sequence, Tuple
11
+
12
+ import numpy as np
13
+ import xlsxwriter
14
+ from datasets import concatenate_datasets, load_dataset
15
+ from PIL import Image
16
+ from tabulate import tabulate
17
+ from termcolor import colored
18
+
19
+
20
+ def buffer_image(image: Image, format: str = "JPEG"):
21
+ # Store image in buffer, so we don't have to write it to disk.
22
+ buffer = io.BytesIO()
23
+ image.save(buffer, format=format)
24
+ return buffer, image
25
+
26
+
27
+ def resize(img_or_path: str, size: Tuple[int, int], format="JPEG"):
28
+ """
29
+ Opens an image (from a file path or existing object), creates a thumbnail
30
+ preserving aspect ratio using LANCZOS filter, and returns the image buffer.
31
+ """
32
+ if isinstance(img_or_path, str):
33
+ image = Image.open(img_or_path)
34
+ else:
35
+ image = img_or_path
36
+ # image = image.resize(size)
37
+ image.thumbnail(size, Image.LANCZOS)
38
+ image = image.convert("RGB")
39
+
40
+ return buffer_image(image, format)
41
+
42
+
43
+ def calculate_scale(file_path, bound_size):
44
+ # check the image size without loading it into memory
45
+ im = Image.open(file_path)
46
+ original_width, original_height = im.size
47
+
48
+ # calculate the resize factor, keeping original aspect and staying within boundary
49
+ bound_width, bound_height = bound_size
50
+ ratios = (float(bound_width) / original_width, float(bound_height) / original_height)
51
+ return min(ratios)
52
+
53
+
54
+ def draw_data(all_datasets, output_path, tokenizer=None, image_processor=None):
55
+ """
56
+ Visualizes dataset contents by exporting them to an Excel file (.xlsx).
57
+ It creates a summary sheet and individual sheets for each dataset,
58
+ embedding text, images, and video frames into the cells.
59
+ """
60
+ if hasattr(tokenizer, "image_start_tag"):
61
+ image_start_tag = tokenizer.image_start_tag
62
+ image_end_tag = tokenizer.image_end_tag
63
+ else:
64
+ from ..constants import IMG_START_TOKEN, IMG_END_TOKEN
65
+
66
+ image_start_tag = IMG_START_TOKEN
67
+ image_end_tag = IMG_END_TOKEN
68
+
69
+ # Initialize Excel Workbook
70
+ workbook = xlsxwriter.Workbook(output_path)
71
+ cell_format = workbook.add_format({"text_wrap": True, "font_size": 12})
72
+
73
+ worksheet = workbook.add_worksheet("ALL")
74
+ worksheet.set_column(1, 2, 20, cell_format)
75
+ worksheet.set_column(3, 3, 240, cell_format)
76
+ worksheet.write(0, 1, "total_num")
77
+ worksheet.write(0, 2, "used_num")
78
+ worksheet.write(0, 3, "name")
79
+ row = 1
80
+
81
+ # Write statistics for each dataset
82
+ for this_name, this_dataset in all_datasets.items():
83
+ total_num = this_dataset["total_num"]
84
+ used_num = this_dataset["used_num"]
85
+ worksheet.write(row, 1, total_num)
86
+ worksheet.write(row, 2, used_num)
87
+ worksheet.write(row, 3, this_name)
88
+ row += 1
89
+ worksheet.write(
90
+ row, 2, sum([this_dataset["used_num"] for this_dataset in all_datasets.values()])
91
+ )
92
+
93
+ all_base_name = ["all"]
94
+ for this_name, this_dataset in all_datasets.items():
95
+ base_name = os.path.basename(this_name)
96
+ base_name = os.path.splitext(base_name)[0]
97
+
98
+ base_name = base_name[:24]
99
+ all_base_name.append(base_name)
100
+
101
+ if all_base_name.count(base_name) > 1:
102
+ base_name = base_name + "_" + str(all_base_name.count(base_name))
103
+
104
+ worksheet = workbook.add_worksheet(base_name)
105
+ worksheet.set_column(1, 2, 120, cell_format)
106
+ worksheet.write(0, 1, "user")
107
+ worksheet.write(0, 2, "assistant")
108
+ row = 1
109
+
110
+ data = this_dataset["data"]
111
+ for this_data in data:
112
+ # print(this_data)
113
+ if isinstance(this_data, Dict):
114
+ # print(this_data.keys())
115
+ messages = this_data["messages"]
116
+ if "images" in this_data:
117
+ images = this_data["images"]
118
+ if "videos" in this_data:
119
+ videos = this_data["videos"]
120
+ else:
121
+ messages = this_data
122
+
123
+ image_count = 0
124
+ video_count = 0
125
+ for message in messages:
126
+ content = message["content"]
127
+ role = message["role"]
128
+ if role == "user" or role == "human":
129
+ col = 1
130
+ else:
131
+ col = 2
132
+ worksheet.write(row, col, content)
133
+ row += 1
134
+
135
+ bos_pos = [m.start() for m in re.finditer(image_start_tag, content)]
136
+ eos_pos = [m.start() for m in re.finditer(image_end_tag, content)]
137
+ # print(bos_pos, eos_pos)
138
+ for a, b in zip(bos_pos, eos_pos):
139
+ # print(content[a+len(image_start_tag:b])
140
+ img_path = content[a + len(image_start_tag) : b]
141
+ # print(img_path)
142
+ worksheet.set_row(row, 200)
143
+
144
+ try:
145
+ image_buffer, image = resize(img_path, (512, 512), format="JPEG")
146
+ except:
147
+ continue
148
+
149
+ scale = min(256 / image.width, 256 / image.height)
150
+ data = {"x_scale": scale, "y_scale": scale, "object_position": 1}
151
+
152
+ worksheet.insert_image(row, col, img_path, {"image_data": image_buffer, **data})
153
+
154
+ row += 1
155
+
156
+ for _ in range(content.count("<image>") + content.count("<|image|>")):
157
+ if images is None:
158
+ continue
159
+ img_path = images[image_count]
160
+ # print(img_path)
161
+ worksheet.set_row(row, 200)
162
+
163
+ try:
164
+ image_buffer, image = resize(img_path, (512, 512), format="JPEG")
165
+ except:
166
+ continue
167
+
168
+ scale = min(256 / image.width, 256 / image.height)
169
+ data = {"x_scale": scale, "y_scale": scale, "object_position": 1}
170
+
171
+ worksheet.insert_image(row, col, img_path, {"image_data": image_buffer, **data})
172
+
173
+ row += 1
174
+ image_count += 1
175
+
176
+ for _ in range(content.count("<video>") + content.count("<|video|>")):
177
+ if videos is None:
178
+ continue
179
+ vid_path = videos[video_count]
180
+ try:
181
+ _, video_frames = image_processor.process_video(vid_path, max_num_frame=4)
182
+ # print(vid_path)
183
+ except:
184
+ continue
185
+
186
+ for video_frame in video_frames:
187
+ worksheet.set_row(row, 200)
188
+ try:
189
+ image_buffer, image = resize(video_frame, (512, 512), format="JPEG")
190
+ except:
191
+ continue
192
+
193
+ scale = min(256 / image.width, 256 / image.height)
194
+ data = {"x_scale": scale, "y_scale": scale, "object_position": 1}
195
+
196
+ if isinstance(video_frame, str):
197
+ video_path = video_frame
198
+ else:
199
+ video_file = hashlib.md5(video_frame.tobytes()).hexdigest() + ".png"
200
+ video_path = os.path.join("/tmp/", video_file)
201
+ video_frame.save(video_path)
202
+
203
+ worksheet.insert_image(
204
+ row, col, video_path, {"image_data": image_buffer, **data}
205
+ )
206
+
207
+ row += 1
208
+ video_count += 1
209
+
210
+ row += 8
211
+
212
+ workbook.close()
omni_diffusion/models/__init__.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ # models.__init__
2
+
3
+ from .dream import DreamConfig, DreamModel, DreamTokenizer
omni_diffusion/models/dream/__init__.py ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # models.dream.__init__
2
+
3
+ from .modeling_dream import DreamModel
4
+ from .configuration_dream import DreamConfig
5
+ from .tokenization_dream import DreamTokenizer
6
+ from transformers import AutoConfig, AutoTokenizer, AutoModel, AutoModelForCausalLM
7
+
8
+ AutoConfig.register("Dream", DreamConfig)
9
+ AutoModelForCausalLM.register(DreamConfig, DreamModel)
10
+
11
+ DreamConfig.register_for_auto_class()
12
+ DreamModel.register_for_auto_class("AutoModelForCausalLM")
omni_diffusion/models/dream/config.yaml ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ encoder: SenseVoiceEncoderSmall
2
+ encoder_conf:
3
+ output_size: 512
4
+ attention_heads: 4
5
+ linear_units: 2048
6
+ num_blocks: 50
7
+ tp_blocks: 20
8
+ dropout_rate: 0.1
9
+ positional_dropout_rate: 0.1
10
+ attention_dropout_rate: 0.1
11
+ input_layer: pe
12
+ pos_enc_class: SinusoidalPositionEncoder
13
+ normalize_before: true
14
+ kernel_size: 11
15
+ sanm_shfit: 0
16
+ selfattention_layer_type: sanm
17
+
18
+
19
+ model: SenseVoiceSmall
20
+ model_conf:
21
+ length_normalized_loss: true
22
+ sos: 1
23
+ eos: 2
24
+ ignore_id: -1
25
+
26
+ tokenizer: SentencepiecesTokenizer
27
+ tokenizer_conf:
28
+ bpemodel: null
29
+ unk_symbol: <unk>
30
+ split_with_space: true
31
+
32
+ frontend: WavFrontend
33
+ frontend_conf:
34
+ fs: 16000
35
+ window: hamming
36
+ n_mels: 80
37
+ frame_length: 25
38
+ frame_shift: 10
39
+ lfr_m: 7
40
+ lfr_n: 6
41
+ cmvn_file: null
42
+
43
+
44
+ dataset: SenseVoiceCTCDataset
45
+ dataset_conf:
46
+ index_ds: IndexDSJsonl
47
+ batch_sampler: EspnetStyleBatchSampler
48
+ data_split_num: 32
49
+ batch_type: token
50
+ batch_size: 14000
51
+ max_token_length: 2000
52
+ min_token_length: 60
53
+ max_source_length: 2000
54
+ min_source_length: 60
55
+ max_target_length: 200
56
+ min_target_length: 0
57
+ shuffle: true
58
+ num_workers: 4
59
+ sos: ${model_conf.sos}
60
+ eos: ${model_conf.eos}
61
+ IndexDSJsonl: IndexDSJsonl
62
+ retry: 20
63
+
64
+ train_conf:
65
+ accum_grad: 1
66
+ grad_clip: 5
67
+ max_epoch: 20
68
+ keep_nbest_models: 10
69
+ avg_nbest_model: 10
70
+ log_interval: 100
71
+ resume: true
72
+ validate_interval: 10000
73
+ save_checkpoint_interval: 10000
74
+
75
+ optim: adamw
76
+ optim_conf:
77
+ lr: 0.00002
78
+ scheduler: warmuplr
79
+ scheduler_conf:
80
+ warmup_steps: 25000
81
+
82
+ specaug: SpecAugLFR
83
+ specaug_conf:
84
+ apply_time_warp: false
85
+ time_warp_window: 5
86
+ time_warp_mode: bicubic
87
+ apply_freq_mask: true
88
+ freq_mask_width_range:
89
+ - 0
90
+ - 30
91
+ lfr_rate: 6
92
+ num_freq_mask: 1
93
+ apply_time_mask: true
94
+ time_mask_width_range:
95
+ - 0
96
+ - 12
97
+ num_time_mask: 1
98
+
omni_diffusion/models/dream/config_dream_resume.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "Instruct-7B",
3
+ "architectures": [
4
+ "DreamModel"
5
+ ],
6
+ "attention_dropout": 0.0,
7
+ "auto_map": {
8
+ "AutoConfig": "configuration_dream.DreamConfig",
9
+ "AutoModel": "modeling_dream.DreamModel"
10
+ },
11
+ "bos_token_id": 151643,
12
+ "eos_token_id": 151643,
13
+ "hidden_act": "silu",
14
+ "hidden_size": 3584,
15
+ "initializer_range": 0.02,
16
+ "intermediate_size": 18944,
17
+ "mask_token_id": 151666,
18
+ "max_position_embeddings": 131072,
19
+ "max_window_layers": 28,
20
+ "model_type": "Dream",
21
+ "num_attention_heads": 28,
22
+ "num_hidden_layers": 28,
23
+ "num_key_value_heads": 4,
24
+ "pad_token_id": 151643,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_scaling": null,
27
+ "rope_theta": 1000000.0,
28
+ "sliding_window": null,
29
+ "tie_word_embeddings": false,
30
+ "torch_dtype": "bfloat16",
31
+ "transformers_version": "4.46.2",
32
+ "use_cache": true,
33
+ "use_mrope": false,
34
+ "use_sliding_window": false,
35
+ "vocab_size": 176264,
36
+ "chunk_size": -1
37
+ }
omni_diffusion/models/dream/configuration.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "framework": "pytorch",
3
+ "task" : "auto-speech-recognition",
4
+ "model": {"type" : "funasr"},
5
+ "pipeline": {"type":"funasr-pipeline"},
6
+ "model_name_in_hub": {
7
+ "ms":"",
8
+ "hf":""},
9
+ "file_path_metas": {
10
+ "config":"config.yaml",
11
+ "tokenizer_conf": {"bpemodel": "chn_jpn_yue_eng_ko_spectok.bpe.model"},
12
+ "frontend_conf":{"cmvn_file": "am.mvn"}}
13
+ }
14
+
omni_diffusion/models/dream/configuration_dream.py ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2024 The Dream team, HKUNLP Group and the HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """Dream model configuration"""
16
+
17
+ from transformers.configuration_utils import PretrainedConfig
18
+ from transformers.modeling_rope_utils import rope_config_validation
19
+ from transformers.utils import logging
20
+
21
+
22
+ logger = logging.get_logger(__name__)
23
+
24
+
25
+ class DreamConfig(PretrainedConfig):
26
+ model_type = "Dream"
27
+ keys_to_ignore_at_inference = ["past_key_values"]
28
+
29
+ def __init__(
30
+ self,
31
+ vocab_size=151936,
32
+ hidden_size=4096,
33
+ intermediate_size=22016,
34
+ num_hidden_layers=32,
35
+ num_attention_heads=32,
36
+ num_key_value_heads=32,
37
+ hidden_act="silu",
38
+ max_position_embeddings=32768,
39
+ initializer_range=0.02,
40
+ rms_norm_eps=1e-6,
41
+ use_cache=False, # cache not used in diffusion
42
+ tie_word_embeddings=False,
43
+ rope_theta=10000.0,
44
+ rope_scaling=None,
45
+ use_sliding_window=False,
46
+ sliding_window=4096,
47
+ max_window_layers=28,
48
+ attention_dropout=0.0,
49
+ mask_token_id=151666,
50
+ pad_token_id=151643,
51
+ **kwargs,
52
+ ):
53
+ self.vocab_size = vocab_size
54
+ self.max_position_embeddings = max_position_embeddings
55
+ self.hidden_size = hidden_size
56
+ self.intermediate_size = intermediate_size
57
+ self.num_hidden_layers = num_hidden_layers
58
+ self.num_attention_heads = num_attention_heads
59
+ self.use_sliding_window = use_sliding_window
60
+ self.sliding_window = sliding_window if use_sliding_window else None
61
+ self.max_window_layers = max_window_layers
62
+
63
+ # for backward compatibility
64
+ if num_key_value_heads is None:
65
+ num_key_value_heads = num_attention_heads
66
+
67
+ self.num_key_value_heads = num_key_value_heads
68
+ self.hidden_act = hidden_act
69
+ self.initializer_range = initializer_range
70
+ self.rms_norm_eps = rms_norm_eps
71
+ self.use_cache = use_cache
72
+ self.rope_theta = rope_theta
73
+ self.rope_scaling = rope_scaling
74
+ self.attention_dropout = attention_dropout
75
+ # Validate the correctness of rotary position embeddings parameters
76
+ # BC: if there is a 'type' field, move it to 'rope_type'.
77
+ if self.rope_scaling is not None and "type" in self.rope_scaling:
78
+ self.rope_scaling["rope_type"] = self.rope_scaling["type"]
79
+ rope_config_validation(self)
80
+
81
+ super().__init__(
82
+ tie_word_embeddings=tie_word_embeddings,
83
+ **kwargs,
84
+ )
85
+ self.mask_token_id = mask_token_id
86
+ self.pad_token_id = pad_token_id
omni_diffusion/models/dream/generation_utils.py ADDED
@@ -0,0 +1,623 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2024 The Dream team, HKUNLP Group and the HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ import warnings
17
+ import copy
18
+ from dataclasses import dataclass
19
+ from typing import Any, Dict, Optional, Tuple, Union
20
+
21
+ import torch
22
+ import torch.distributions as dists
23
+ from torch.nn import functional as F
24
+ from transformers import __version__
25
+ from transformers.generation.configuration_utils import (
26
+ GenerationConfig
27
+ )
28
+ from transformers.utils import (
29
+ ModelOutput,
30
+ is_torchdynamo_compiling,
31
+ logging,
32
+ )
33
+ logger = logging.get_logger(__name__)
34
+ from tqdm import tqdm
35
+
36
+
37
+ def top_p_logits(logits, top_p=None):
38
+ sorted_logits, sorted_indices = torch.sort(logits, descending=True)
39
+ cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
40
+ sorted_indices_to_remove = cumulative_probs > top_p
41
+ # Shift the indices to the right to keep the first token above the threshold
42
+ sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
43
+ sorted_indices_to_remove[..., 0] = 0
44
+
45
+ mask = torch.zeros_like(logits, dtype=torch.bool, device=logits.device)
46
+ mask = mask.scatter_(-1, sorted_indices, sorted_indices_to_remove)
47
+ logits = logits.masked_fill(mask, torch.finfo(logits.dtype).min)
48
+ return logits
49
+
50
+ def top_k_logits(logits, top_k=None):
51
+ top_k = min(top_k, logits.size(-1)) # Safety check
52
+ # Remove all tokens with a probability less than the last token of the top-k
53
+ indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]
54
+ logits = logits.masked_fill(indices_to_remove, torch.finfo(logits.dtype).min)
55
+ return logits
56
+
57
+
58
+ def sample_tokens(logits, temperature=0.0, top_p=None, top_k=None, margin_confidence=False, neg_entropy=False,
59
+ repeat_penalty=1.0, max_position_penalty=1.0, past_x=None, mask_id=None,):
60
+ if temperature > 0:
61
+ logits = logits / temperature
62
+
63
+ if top_p is not None and top_p < 1:
64
+ logits = top_p_logits(logits, top_p)
65
+ if top_k is not None:
66
+ logits = top_k_logits(logits, top_k)
67
+
68
+ if repeat_penalty != 1.0:
69
+ select_mask = torch.logical_and((past_x != 0), (past_x != mask_id))
70
+ generated_tokens = set(past_x[select_mask].tolist())
71
+ for token in set(generated_tokens):
72
+ logits[:, token][logits[:, token] < 0] *= repeat_penalty
73
+ logits[:, token][logits[:, token] >= 0] /= repeat_penalty
74
+
75
+ if max_position_penalty != 1.0:
76
+ token_length = logits.shape[-2]
77
+ if token_length > 100:
78
+ penalty_map = [i / (token_length - 100) * (max_position_penalty - 1.0) + 1.0
79
+ for i in range(token_length - 100)]
80
+ penalty_map = torch.tensor(penalty_map).unsqueeze(-1).to(logits.device).to(logits.dtype)
81
+ penalty_map = torch.cat([torch.ones_like(logits[:100, :1]), penalty_map], dim=0)
82
+ penalty_map = penalty_map.repeat(1, logits.shape[-1])
83
+
84
+ logits[logits < 0] *= penalty_map[logits < 0]
85
+ logits[logits >= 0] /= penalty_map[logits >= 0]
86
+
87
+ probs = torch.softmax(logits, dim=-1)
88
+ if temperature > 0:
89
+ try:
90
+ x0 = dists.Categorical(probs=probs).sample()
91
+ confidence = torch.gather(probs, -1, x0.unsqueeze(-1)).squeeze(-1)
92
+ except:
93
+ confidence, x0 = probs.max(dim=-1)
94
+ else:
95
+ confidence, x0 = probs.max(dim=-1)
96
+
97
+ if margin_confidence:
98
+ sorted_probs, _ = torch.sort(probs, dim=-1, descending=True)
99
+ top1_probs = sorted_probs[:, 0]
100
+ top2_probs = sorted_probs[:, 1]
101
+ confidence = top1_probs - top2_probs
102
+
103
+ if neg_entropy:
104
+ epsilon = 1e-10
105
+ log_probs = torch.log(probs + epsilon)
106
+ confidence = torch.sum(probs * log_probs, dim=-1)
107
+
108
+ return confidence, x0
109
+
110
+
111
+ @dataclass
112
+ class DreamModelOutput(ModelOutput):
113
+ sequences: torch.LongTensor = None
114
+ history: Optional[Tuple[torch.FloatTensor]] = None
115
+
116
+
117
+ class DreamGenerationConfig(GenerationConfig):
118
+ def __init__(self, **kwargs):
119
+ self.temperature: float = kwargs.pop("temperature", 0.0)
120
+ self.top_p: Optional[float] = kwargs.pop("top_p", None)
121
+ self.top_k: Optional[int] = kwargs.pop("top_k", None)
122
+ self.max_length = kwargs.pop("max_length", 20)
123
+ self.max_new_tokens = kwargs.pop("max_new_tokens", None)
124
+ # diffusion specific params
125
+ self.eps: float = kwargs.pop("eps", 1e-3)
126
+ self.steps: int = kwargs.pop("steps", 512)
127
+ self.alg: str = kwargs.pop("alg", 'origin')
128
+ self.alg_temp: Optional[float] = kwargs.pop("alg_temp", None)
129
+
130
+ # Parameters that define the output variables of `generate`
131
+ self.num_return_sequences: int = kwargs.pop("num_return_sequences", 1)
132
+ self.return_dict_in_generate: bool = kwargs.pop("return_dict_in_generate", False)
133
+ self.output_history: bool = kwargs.pop("output_history", False)
134
+
135
+ # Special tokens that can be used at generation time
136
+ self.mask_token_id = kwargs.pop("mask_token_id", None)
137
+ self.pad_token_id = kwargs.pop("pad_token_id", None)
138
+ self.bos_token_id = kwargs.pop("bos_token_id", None)
139
+ self.eos_token_id = kwargs.pop("eos_token_id", None)
140
+
141
+ # Wild card
142
+ self.generation_kwargs = kwargs.pop("generation_kwargs", {})
143
+
144
+ # The remaining attributes do not parametrize `.generate()`, but are informative and/or used by the hub
145
+ # interface.
146
+ self._from_model_config = kwargs.pop("_from_model_config", False)
147
+ self._commit_hash = kwargs.pop("_commit_hash", None)
148
+ self.transformers_version = kwargs.pop("transformers_version", __version__)
149
+
150
+ # Additional attributes without default values
151
+ if not self._from_model_config:
152
+ # we don't want to copy values from the model config if we're initializing a `GenerationConfig` from a
153
+ # model's default configuration file
154
+ for key, value in kwargs.items():
155
+ try:
156
+ setattr(self, key, value)
157
+ except AttributeError as err:
158
+ logger.error(f"Can't set {key} with value {value} for {self}")
159
+ raise err
160
+
161
+ # Validate the values of the attributes
162
+ self.validate(is_init=True)
163
+
164
+ def validate(self, is_init=False):
165
+ pass
166
+
167
+ class DreamGenerationMixin:
168
+ @staticmethod
169
+ def _expand_inputs_for_generation(
170
+ expand_size: int = 1,
171
+ input_ids: Optional[torch.LongTensor] = None,
172
+ attention_mask: Optional[torch.LongTensor] = None
173
+ ) -> Tuple[torch.LongTensor, Dict[str, Any]]:
174
+ """Expands tensors from [batch_size, ...] to [batch_size * expand_size, ...]"""
175
+ # Do not call torch.repeat_interleave if expand_size is 1 because it clones
176
+ # the input tensor and thus requires more memory although no change is applied
177
+ if expand_size == 1:
178
+ return input_ids, attention_mask
179
+ if input_ids is not None:
180
+ input_ids = input_ids.repeat_interleave(expand_size, dim=0)
181
+ if attention_mask is not None:
182
+ attention_mask = attention_mask.repeat_interleave(expand_size, dim=0)
183
+ return input_ids, attention_mask
184
+
185
+ def _validate_generated_length(self, generation_config, input_ids_length, has_default_max_length):
186
+ """Performs validation related to the resulting generated length"""
187
+
188
+ # Can't throw warnings/exceptions during compilation
189
+ if is_torchdynamo_compiling():
190
+ return
191
+
192
+ # 1. Max length warnings related to poor parameterization
193
+ if has_default_max_length and generation_config.max_new_tokens is None and generation_config.max_length == 20:
194
+ # 20 is the default max_length of the generation config
195
+ warnings.warn(
196
+ f"Using the model-agnostic default `max_length` (={generation_config.max_length}) to control the "
197
+ "generation length. We recommend setting `max_new_tokens` to control the maximum length of the "
198
+ "generation.",
199
+ UserWarning,
200
+ )
201
+ if input_ids_length >= generation_config.max_length:
202
+ input_ids_string = "input_ids"
203
+ raise ValueError(
204
+ f"Input length of {input_ids_string} is {input_ids_length}, but `max_length` is set to"
205
+ f" {generation_config.max_length}. This can lead to unexpected behavior. You should consider"
206
+ " increasing `max_length` or, better yet, setting `max_new_tokens`."
207
+ )
208
+
209
+ def _prepare_generated_length(
210
+ self,
211
+ generation_config,
212
+ has_default_max_length,
213
+ input_ids_length,
214
+ ):
215
+ """Prepared max and min length in generation configs to avoid clashes between similar attributes"""
216
+
217
+ if generation_config.max_new_tokens is not None:
218
+ if not has_default_max_length and generation_config.max_length is not None:
219
+ logger.warning(
220
+ f"Both `max_new_tokens` (={generation_config.max_new_tokens}) and `max_length`(="
221
+ f"{generation_config.max_length}) seem to have been set. `max_new_tokens` will take precedence. "
222
+ "Please refer to the documentation for more information. "
223
+ "(https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)"
224
+ )
225
+ generation_config.max_length = generation_config.max_new_tokens + input_ids_length
226
+
227
+ elif has_default_max_length:
228
+ if generation_config.max_length == DreamGenerationConfig().max_length:
229
+ generation_config.max_length = generation_config.max_length + input_ids_length
230
+ max_position_embeddings = getattr(self.config, "max_position_embeddings", None)
231
+ if max_position_embeddings is not None:
232
+ generation_config.max_length = min(generation_config.max_length, max_position_embeddings)
233
+
234
+ return generation_config
235
+
236
+ def _prepare_generation_config(
237
+ self, generation_config: Optional[DreamGenerationConfig], **kwargs: Dict
238
+ ) -> DreamGenerationConfig:
239
+ """
240
+ Prepares the base generation config, then applies any generation configuration options from kwargs. This
241
+ function handles retrocompatibility with respect to configuration files.
242
+ """
243
+ # priority: `generation_config` argument > `model.generation_config` (the default generation config)
244
+ using_model_generation_config = False
245
+ if generation_config is None:
246
+ generation_config = DreamGenerationConfig.from_model_config(self.config)
247
+ using_model_generation_config = True
248
+
249
+ # `torch.compile` can't compile `copy.deepcopy`, arguments in `kwargs` that are part of `generation_config`
250
+ # will mutate the object with `.update`. As such, passing these arguments through `kwargs` is disabled -- an
251
+ # exception will be raised in `_validate_model_kwargs`
252
+ if not is_torchdynamo_compiling():
253
+ generation_config = copy.deepcopy(generation_config)
254
+ _kwargs = generation_config.update(**kwargs)
255
+ # If `generation_config` is provided, let's fallback ALL special tokens to the default values for the model
256
+ if not using_model_generation_config:
257
+ if generation_config.bos_token_id is None:
258
+ generation_config.bos_token_id = self.generation_config.bos_token_id
259
+ if generation_config.eos_token_id is None:
260
+ generation_config.eos_token_id = self.generation_config.eos_token_id
261
+ if generation_config.pad_token_id is None:
262
+ generation_config.pad_token_id = self.generation_config.pad_token_id
263
+ if generation_config.mask_token_id is None:
264
+ generation_config.mask_token_id = self.generation_config.mask_token_id
265
+
266
+ return generation_config
267
+
268
+ def _prepare_special_tokens(
269
+ self,
270
+ generation_config: DreamGenerationConfig,
271
+ device: Optional[Union[torch.device, str]] = None,
272
+ ):
273
+ """
274
+ Prepares the special tokens for generation, overwriting the generation config with their processed versions
275
+ converted to tensor.
276
+
277
+ Note that `generation_config` is changed in place and stops being serializable after this method is called.
278
+ That is no problem if called within `generate` (`generation_config` is a local copy that doesn't leave the
279
+ function). However, if called outside `generate`, consider creating a copy of `generation_config` first.
280
+ """
281
+
282
+ # Convert special tokens to tensors
283
+ def _tensor_or_none(token, device=None):
284
+ if token is None:
285
+ return token
286
+
287
+ device = device if device is not None else self.device
288
+ if isinstance(token, torch.Tensor):
289
+ return token.to(device)
290
+ return torch.tensor(token, device=device, dtype=torch.long)
291
+
292
+ bos_token_tensor = _tensor_or_none(generation_config.bos_token_id, device=device)
293
+ eos_token_tensor = _tensor_or_none(generation_config.eos_token_id, device=device)
294
+ pad_token_tensor = _tensor_or_none(generation_config.pad_token_id, device=device)
295
+ mask_token_tensor = _tensor_or_none(generation_config.mask_token_id, device=device)
296
+
297
+ # We can have more than one eos token. Always treat it as a 1D tensor (when it exists).
298
+ if eos_token_tensor is not None and eos_token_tensor.ndim == 0:
299
+ eos_token_tensor = eos_token_tensor.unsqueeze(0)
300
+
301
+ # Set pad token if unset (and there are conditions to do so)
302
+ if pad_token_tensor is None and eos_token_tensor is not None:
303
+ pad_token_tensor = eos_token_tensor[0]
304
+ logger.warning(f"Setting `pad_token_id` to `eos_token_id`:{pad_token_tensor} for open-end generation.")
305
+
306
+ # Update generation config with the updated special tokens tensors
307
+ # NOTE: this must be written into a different attribute name than the one holding the original special tokens
308
+ # (in their non-tensor form), in order to enable end-to-end compilation. See
309
+ # https://pytorch.org/docs/stable/torch.compiler_cudagraph_trees.html#limitations
310
+ generation_config._bos_token_tensor = bos_token_tensor
311
+ generation_config._eos_token_tensor = eos_token_tensor
312
+ generation_config._pad_token_tensor = pad_token_tensor
313
+ generation_config._mask_token_tensor = mask_token_tensor
314
+
315
+ @torch.no_grad()
316
+ def diffusion_generate(
317
+ self,
318
+ inputs: Optional[torch.Tensor] = None,
319
+ generation_config: Optional[DreamGenerationConfig] = None,
320
+ inputs_embeds=None,
321
+ prefix_lm=False,
322
+ alg=None,
323
+ block_size=-1,
324
+ cfg=0.0,
325
+ add_boa_token=False,
326
+ **kwargs,
327
+ ) -> Union[DreamModelOutput, torch.LongTensor]:
328
+ # 1. Handle `generation_config` and kwargs that might update it, and validate the `.generate()` call
329
+ generation_config = self._prepare_generation_config(generation_config, **kwargs)
330
+ generation_tokens_hook_func = kwargs.pop("generation_tokens_hook_func", lambda step, x, logits: x)
331
+ generation_logits_hook_func = kwargs.pop("generation_logits_hook_func", lambda step, x, logits: logits)
332
+ # breakpoint()
333
+ # 2. Define model inputs
334
+ if inputs is not None:
335
+ input_ids = inputs
336
+ device = input_ids.device
337
+ input_ids_length = input_ids.shape[-1]
338
+ else:
339
+ input_ids = None
340
+ device = inputs_embeds.device
341
+ input_ids_length = inputs_embeds.shape[1]
342
+ attention_mask = kwargs.pop("attention_mask", None)
343
+ self._prepare_special_tokens(generation_config, device=device)
344
+
345
+ # 3. Prepare `max_length`.
346
+
347
+ has_default_max_length = kwargs.get("max_length") is None and generation_config.max_length is not None
348
+ generation_config = self._prepare_generated_length(
349
+ generation_config=generation_config,
350
+ has_default_max_length=has_default_max_length,
351
+ input_ids_length=input_ids_length,
352
+ )
353
+
354
+ self._validate_generated_length(generation_config, input_ids_length, has_default_max_length)
355
+ # import pdb;pdb.set_trace()
356
+ # 4. Check input_ids
357
+ #if not is_torchdynamo_compiling() and self.device.type != input_ids.device.type:
358
+ if not is_torchdynamo_compiling() and self.device.type != device.type:
359
+ warnings.warn(
360
+ "You are calling .generate() with the `input_ids` being on a device type different"
361
+ f" than your model's device. `input_ids` is on {input_ids.device.type}, whereas the model"
362
+ f" is on {self.device.type}. You may experience unexpected behaviors or slower generation."
363
+ " Please make sure that you have put `input_ids` to the"
364
+ f" correct device by calling for example input_ids = input_ids.to('{self.device.type}') before"
365
+ " running `.generate()`.",
366
+ UserWarning,
367
+ )
368
+ # breakpoint()
369
+ if (
370
+ hasattr(generation_config, "pad_token_id") and
371
+ input_ids is not None and
372
+ torch.any(input_ids == generation_config.pad_token_id) and
373
+ attention_mask is None
374
+ ):
375
+ warnings.warn(
376
+ "Padding was detected but no attention mask is passed here. For correct "
377
+ "generation results, please set `attention_mask` when batch-padding inputs.",
378
+ UserWarning,
379
+ )
380
+ assert generation_config.num_return_sequences == 1, \
381
+ "Currently, we only support num_return_sequences = 1 for diffusion generation."
382
+ input_ids, attention_mask = self._expand_inputs_for_generation(
383
+ expand_size=generation_config.num_return_sequences,
384
+ input_ids=input_ids,
385
+ attention_mask=attention_mask
386
+ )
387
+ result = self._sample(
388
+ input_ids,
389
+ attention_mask=attention_mask,
390
+ generation_config=generation_config,
391
+ generation_tokens_hook_func=generation_tokens_hook_func,
392
+ generation_logits_hook_func=generation_logits_hook_func,
393
+ inputs_embeds=inputs_embeds,
394
+ device=device,
395
+ prefix_lm=prefix_lm,
396
+ alg=alg,
397
+ block_size=block_size,
398
+ cfg=cfg,
399
+ add_boa_token=add_boa_token,
400
+ **kwargs,
401
+ )
402
+ return result
403
+
404
+ def _sample(
405
+ self,
406
+ input_ids: torch.LongTensor,
407
+ attention_mask: Optional[torch.LongTensor],
408
+ generation_config: DreamGenerationConfig,
409
+ generation_tokens_hook_func,
410
+ generation_logits_hook_func,
411
+ inputs_embeds=None,
412
+ prefix_lm=False,
413
+ device=None,
414
+ step_ratio=None,
415
+ penalty=1.2,
416
+ alg=None,
417
+ block_size=None,
418
+ add_boa_token=False,
419
+ max_position_penalty=1.0,
420
+ repeat_penalty=1.0,
421
+ cfg=0.0,
422
+ **kwargs,
423
+ ) -> Union[DreamModelOutput, torch.LongTensor]:
424
+ output_history = True
425
+ return_dict_in_generate = generation_config.return_dict_in_generate
426
+ max_length = generation_config.max_length
427
+ mask_token_id = generation_config.mask_token_id
428
+ max_new_tokens = generation_config.max_new_tokens
429
+ steps = min(generation_config.steps, max_new_tokens)
430
+ eps = generation_config.eps
431
+ alg = generation_config.alg if alg is None else alg
432
+ print("denoise algorithm: " + alg)
433
+ alg_temp = generation_config.alg_temp
434
+ temperature = generation_config.temperature
435
+ top_p = generation_config.top_p
436
+ top_k = generation_config.top_k
437
+
438
+ histories = [] if (return_dict_in_generate and output_history) else None
439
+ all_logit = []
440
+ generated_tokens = []
441
+ block_size = max_new_tokens if block_size < 0 else block_size
442
+
443
+ if input_ids is None:
444
+ assert device is not None
445
+ assert inputs_embeds is not None
446
+ bsz, seq_len = inputs_embeds.shape[:2]
447
+ max_length = seq_len + max_new_tokens
448
+ input_ids = torch.full((bsz, seq_len), 0, dtype=torch.long).to(device)
449
+
450
+ tok_idx = None
451
+ past_key_values = None
452
+
453
+ x = F.pad(input_ids, (0, max_length - input_ids.shape[1]), value=mask_token_id)
454
+
455
+ timesteps = torch.linspace(1, eps, steps + 1, device=x.device)
456
+ x = generation_tokens_hook_func(None, x, None)
457
+
458
+ if step_ratio is not None:
459
+ steps = int(max_new_tokens * step_ratio)
460
+
461
+ if add_boa_token:
462
+ bos_index = int((x.shape[1] - (x == mask_token_id).sum()) + (x == mask_token_id).sum() * 0.2)
463
+ x[:, bos_index] = 151684 # <|begin_of_audio|>
464
+
465
+ input_x = x.clone()
466
+ total_steps = steps
467
+ block_num = (x == mask_token_id).sum() // block_size
468
+ if block_num * block_size < (x == mask_token_id).sum(): block_num += 1
469
+ input_length = input_ids.shape[-1]
470
+
471
+ task = None
472
+ if "task" in kwargs: task = kwargs['task']
473
+ if cfg > 0:
474
+ import random
475
+ empty_prompt = ""
476
+ if task == "S2I":
477
+ empty_prompt = "<|im_start|>system\nPlease generate an image based on the input audio.<|im_end|>\n"
478
+ empty_prompt += "<|im_start|>user\n<|im_end|>\n<|im_start|>assistant\n"
479
+ un_x = kwargs['tokenizer'].encode(empty_prompt)
480
+ elif task == "T2I":
481
+ empty_prompt = "<|im_start|>user\nGenerate an image based on the provided text description.\n"
482
+ empty_prompt += "<|im_end|>\n<|im_start|>assistant\n"
483
+ first_audio_token = kwargs['tokenizer'].encode("<|begin_of_audio|>")[0]
484
+ un_x_text = random.sample([_ for _ in range(first_audio_token)],
485
+ input_ids.shape[1] - len(kwargs['tokenizer'].encode(empty_prompt)))
486
+ un_x = kwargs['tokenizer'].encode("<|im_start|>user\nGenerate an image based on the provided \
487
+ text description.\n")
488
+ un_x = un_x + un_x_text + kwargs['tokenizer'].encode("<|im_end|>\n<|im_start|>assistant\n")
489
+
490
+ for block_idx in range(block_num):
491
+ block_mask = torch.zeros([x.shape[-1]]).to(torch.bool).to(x.device)
492
+ block_mask[input_length + block_idx * block_size: input_length + (block_idx + 1) * block_size] = True
493
+ steps = int(block_mask.sum() / (x.shape[-1] - input_length) * total_steps)
494
+ timesteps = torch.linspace(1, eps, steps + 1, device=x.device)
495
+ for i in tqdm(range(steps)):
496
+ mask_index = (x == mask_token_id)
497
+ if mask_index.sum() == 0: break
498
+ inputs_embeds_curr = self.model.embed_tokens(x)
499
+
500
+ if inputs_embeds is not None:
501
+ inputs_embeds_curr[:, :inputs_embeds.shape[1]] = inputs_embeds
502
+
503
+ if cfg > 0:
504
+ input_un_x = torch.tensor(un_x).unsqueeze(0).to(x.dtype).to(x.device)
505
+ input_un_x = torch.cat([input_un_x, x[:, input_ids.shape[1]:]], dim=1)
506
+ un_inpus_embeds = self.model.embed_tokens(input_un_x)
507
+
508
+ attention_mask_cond = torch.ones([1, inputs_embeds_curr.shape[1], inputs_embeds_curr.shape[1]])
509
+ attention_mask_cond = attention_mask_cond.to(torch.bool).to(inputs_embeds_curr.device)
510
+ attention_mask_uncond = torch.zeros([1, inputs_embeds_curr.shape[1], inputs_embeds_curr.shape[1]])
511
+ attention_mask_uncond[:, :un_inpus_embeds.shape[1], :un_inpus_embeds.shape[1]] = 1
512
+ attention_mask_uncond = attention_mask_uncond.to(torch.bool).to(inputs_embeds.device)
513
+ attention_mask = torch.cat([attention_mask_cond, attention_mask_uncond])
514
+ attention_mask = attention_mask.unsqueeze(1)
515
+
516
+ if inputs_embeds_curr.shape[1] != un_inpus_embeds.shape[1]:
517
+ un_inpus_embeds = torch.cat([un_inpus_embeds,
518
+ torch.zeros_like(inputs_embeds_curr[:, :inputs_embeds_curr.shape[1] -
519
+ un_inpus_embeds.shape[1], :])], dim=1)
520
+ input_inputs_embeds_curr = torch.cat([inputs_embeds_curr, un_inpus_embeds])
521
+
522
+ model_logits = self.forward_dream(None, attention_mask, tok_idx,
523
+ inputs_embeds=input_inputs_embeds_curr).logits
524
+ logits = model_logits[:1]; un_logits = model_logits[1:]
525
+ logits = un_logits + (cfg + 1) * (logits - un_logits)
526
+ logits = torch.cat([logits[:, :1], logits[:, :-1]], dim=1)
527
+
528
+ else:
529
+ logits = self.forward_dream(None, attention_mask, tok_idx,
530
+ inputs_embeds=inputs_embeds_curr).logits
531
+ logits = torch.cat([logits[:, :1], logits[:, :-1]], dim=1)
532
+
533
+ logits = generation_logits_hook_func(i, x, logits)
534
+
535
+ mask_logits = logits[mask_index]
536
+ if i == 0:
537
+ input_index = torch.where(mask_index[0]==True)[0][0]
538
+
539
+ t = timesteps[i]
540
+ s = timesteps[i + 1]
541
+
542
+ if alg == 'origin':
543
+ p_transfer = 1 - s / t if i < steps - 1 else 1
544
+ x0 = torch.zeros_like(x[mask_index], device=self.device, dtype=torch.long) + mask_token_id
545
+ transfer_index_t_s = torch.rand(*x0.shape, device=self.device) < p_transfer
546
+ _, x0[transfer_index_t_s] = sample_tokens(
547
+ mask_logits[transfer_index_t_s],
548
+ temperature=temperature,
549
+ top_p=top_p,
550
+ top_k=top_k,
551
+ max_position_penalty=max_position_penalty,
552
+ )
553
+ x[mask_index] = x0.clone()
554
+
555
+ else:
556
+ if alg == 'maskgit_plus':
557
+ confidence, x0 = sample_tokens(mask_logits, temperature=temperature, top_p=top_p, top_k=top_k,
558
+ max_position_penalty=max_position_penalty)
559
+ elif alg == 'topk_margin':
560
+ confidence, x0 = sample_tokens(
561
+ mask_logits,
562
+ temperature=temperature,
563
+ top_p=top_p,
564
+ top_k=top_k,
565
+ margin_confidence=True,
566
+ max_position_penalty=max_position_penalty,
567
+ )
568
+ elif alg == 'entropy':
569
+ confidence, x0 = sample_tokens(
570
+ mask_logits,
571
+ temperature,
572
+ top_p=top_p,
573
+ top_k=top_k,
574
+ neg_entropy=True,
575
+ max_position_penalty=max_position_penalty,
576
+ )
577
+ elif alg == "entropy-penalty":
578
+ confidence, x0 = sample_tokens(
579
+ mask_logits,
580
+ temperature,
581
+ top_p=top_p,
582
+ top_k=top_k,
583
+ neg_entropy=True,
584
+ repeat_penalty=repeat_penalty if len(histories) != 0 else 1.0,
585
+ past_x=histories[-1] if len(histories) != 0 else [],
586
+ mask_id=mask_token_id,
587
+ max_position_penalty=max_position_penalty,
588
+ )
589
+ else:
590
+ raise RuntimeError(f"Unknown alg: {alg}")
591
+
592
+ block_mask_1 = block_mask[mask_index[0]]
593
+ confidence = confidence + torch.where(block_mask_1, 0, -torch.inf).to(confidence.device)
594
+
595
+ num_mask_token = mask_index.sum()
596
+ num_mask_token = (x[:, block_mask] == mask_token_id).sum()
597
+ number_transfer_tokens = int(num_mask_token * (1 - s / t)) if i < steps - 1 else num_mask_token
598
+ number_transfer_tokens = max(number_transfer_tokens, 1)
599
+
600
+ if number_transfer_tokens > 0:
601
+ if alg_temp is None or alg_temp == 0:
602
+ _, transfer_index = torch.topk(confidence, number_transfer_tokens)
603
+ else:
604
+ confidence = confidence / alg_temp
605
+ confidence = F.softmax(confidence, dim=-1)
606
+ transfer_index = torch.multinomial(confidence, num_samples=number_transfer_tokens)
607
+
608
+ x0_ = torch.zeros_like(x0, device=self.device, dtype=torch.long) + mask_token_id
609
+ x0_[transfer_index] = x0[transfer_index].clone()
610
+ x[mask_index] = x0_
611
+
612
+ logit,indic = torch.max(torch.softmax(logits.clone(),dim=-1),-1)
613
+ logit = logit[0][x[0]!=0]
614
+ indic = indic[0][x[0]!=0]
615
+ temp_X = x[0][x[0]!=0]
616
+
617
+ x = generation_tokens_hook_func(i, x, logits)
618
+
619
+ if histories is not None:
620
+ histories.append(x.clone())
621
+ all_logit.append(torch.max(logits.clone(),-1)[-1])
622
+
623
+ return (x, histories)
omni_diffusion/models/dream/modeling_dream.py ADDED
@@ -0,0 +1,1136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2024 The Dream team, HKUNLP Group and the HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5
+ # and OPT and Qwen implementations in this library. It has been modified from its
6
+ # original forms to accommodate minor architectural differences compared
7
+ # to GPT-NeoX and OPT and Qwen used by the Meta AI and Qwen team that trained the model.
8
+ #
9
+ # Licensed under the Apache License, Version 2.0 (the "License");
10
+ # you may not use this file except in compliance with the License.
11
+ # You may obtain a copy of the License at
12
+ #
13
+ # http://www.apache.org/licenses/LICENSE-2.0
14
+ #
15
+ # Unless required by applicable law or agreed to in writing, software
16
+ # distributed under the License is distributed on an "AS IS" BASIS,
17
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
+ # See the License for the specific language governing permissions and
19
+ # limitations under the License.
20
+ """PyTorch Dream model."""
21
+
22
+
23
+
24
+ from .modeling_sensevoice import AudioEncoder
25
+ from .resampler_projector import ResamplerProjector
26
+ import random
27
+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
28
+
29
+ import math
30
+ from typing import List, Optional, Tuple, Union
31
+ import os
32
+ import torch
33
+ import torch.utils.checkpoint
34
+ from torch import nn
35
+
36
+ from transformers.activations import ACT2FN
37
+ from transformers.cache_utils import Cache, DynamicCache
38
+ from transformers.modeling_outputs import (
39
+ # BaseModelOutput,
40
+ MaskedLMOutput,
41
+ ModelOutput
42
+ )
43
+ from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
44
+ from transformers.modeling_utils import PreTrainedModel
45
+ from transformers.utils import (
46
+ add_start_docstrings,
47
+ add_start_docstrings_to_model_forward,
48
+ is_flash_attn_2_available,
49
+ is_flash_attn_greater_or_equal_2_10,
50
+ logging,
51
+ )
52
+ from transformers import PretrainedConfig
53
+ from .configuration_dream import DreamConfig
54
+ from .generation_utils import DreamGenerationMixin, DreamGenerationConfig
55
+ from dataclasses import dataclass
56
+ from typing import Any
57
+ if is_flash_attn_2_available():
58
+ from transformers.modeling_flash_attention_utils import _flash_attention_forward
59
+ from transformers.modeling_outputs import CausalLMOutputWithPast
60
+
61
+ from .modeling_sensevoice import AudioEncoder
62
+ from .resampler_projector import ResamplerProjector
63
+
64
+ import torch
65
+ import random
66
+
67
+
68
+ logger = logging.get_logger(__name__)
69
+
70
+
71
+ @dataclass
72
+ class MaskedLMOutput(ModelOutput):
73
+ """
74
+ Base class for masked language models outputs.
75
+
76
+ Args:
77
+ loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
78
+ Masked language modeling (MLM) loss.
79
+ logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
80
+ Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
81
+ hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed
82
+ or when `config.output_hidden_states=True`):
83
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has
84
+ an embedding layer, + one for the output of each layer) of shape `(batch_size,
85
+ sequence_length, hidden_size)`.
86
+
87
+ Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
88
+ attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed
89
+ or when `config.output_attentions=True`):
90
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
91
+ sequence_length)`.
92
+
93
+ Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
94
+ heads.
95
+ """
96
+
97
+ loss: Optional[torch.FloatTensor] = None
98
+ logits: torch.FloatTensor = None
99
+ hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
100
+ attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
101
+ past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
102
+
103
+ _CHECKPOINT_FOR_DOC = "Dream-7B"
104
+ _CONFIG_FOR_DOC = "DreamConfig"
105
+ import os
106
+ ENFORCE_NUM_ITEMIN_BATCH = os.environ.get("ENFORCE_NUM_ITEMIN_BATCH", False)
107
+
108
+ @dataclass
109
+ class BaseModelOutput(ModelOutput):
110
+ """
111
+ Base class for model's outputs, with potential hidden states and attentions.
112
+
113
+ Args:
114
+ last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
115
+ Sequence of hidden-states at the output of the last layer of the model.
116
+ hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed
117
+ or when `config.output_hidden_states=True`):
118
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
119
+ one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
120
+
121
+ Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
122
+ attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed
123
+ or when `config.output_attentions=True`):
124
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
125
+ sequence_length)`.
126
+
127
+ Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
128
+ heads.
129
+ """
130
+
131
+ last_hidden_state: torch.FloatTensor = None
132
+ hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
133
+ attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
134
+ past_key_values: Optional[Cache] = None
135
+ # Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->Dream
136
+
137
+
138
+ class DreamRMSNorm(nn.Module):
139
+ def __init__(self, hidden_size, eps=1e-6):
140
+ """
141
+ DreamRMSNorm is equivalent to T5LayerNorm
142
+ """
143
+ super().__init__()
144
+ self.weight = nn.Parameter(torch.ones(hidden_size))
145
+ self.variance_epsilon = eps
146
+
147
+ def forward(self, hidden_states):
148
+ input_dtype = hidden_states.dtype
149
+ hidden_states = hidden_states.to(torch.float32)
150
+ variance = hidden_states.pow(2).mean(-1, keepdim=True)
151
+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
152
+ return self.weight * hidden_states.to(input_dtype)
153
+
154
+ def extra_repr(self):
155
+ return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
156
+
157
+
158
+ # Copied from transformers.models.llama.modeling_llama.LlamaRotaryEmbedding with Llama->Dream
159
+ class DreamRotaryEmbedding(nn.Module):
160
+ def __init__(
161
+ self,
162
+ dim=None,
163
+ max_position_embeddings=2048,
164
+ base=10000,
165
+ device=None,
166
+ scaling_factor=1.0,
167
+ rope_type="default",
168
+ config: Optional[DreamConfig] = None,
169
+ ):
170
+ super().__init__()
171
+ # TODO (joao): remove the `if` below, only used for BC
172
+ self.rope_kwargs = {}
173
+ if config is None:
174
+ logger.warning_once(
175
+ "`DreamRotaryEmbedding` can now be fully parameterized by passing the model config through the "
176
+ "`config` argument. All other arguments will be removed in v4.46"
177
+ )
178
+ self.rope_kwargs = {
179
+ "rope_type": rope_type,
180
+ "factor": scaling_factor,
181
+ "dim": dim,
182
+ "base": base,
183
+ "max_position_embeddings": max_position_embeddings,
184
+ }
185
+ self.rope_type = rope_type
186
+ self.max_seq_len_cached = max_position_embeddings
187
+ self.original_max_seq_len = max_position_embeddings
188
+ else:
189
+ # BC: "rope_type" was originally "type"
190
+ if config.rope_scaling is not None:
191
+ self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type"))
192
+ else:
193
+ self.rope_type = "default"
194
+ self.max_seq_len_cached = config.max_position_embeddings
195
+ self.original_max_seq_len = config.max_position_embeddings
196
+
197
+ self.config = config
198
+ self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
199
+
200
+ inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device, **self.rope_kwargs)
201
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
202
+ self.original_inv_freq = self.inv_freq
203
+
204
+ def reset_parameters(self):
205
+ inv_freq, self.attention_scaling = self.rope_init_fn(self.config, self.inv_freq.device, **self.rope_kwargs)
206
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
207
+ self.original_inv_freq = self.inv_freq
208
+
209
+
210
+ def _dynamic_frequency_update(self, position_ids, device):
211
+ """
212
+ dynamic RoPE layers should recompute `inv_freq` in the following situations:
213
+ 1 - growing beyond the cached sequence length (allow scaling)
214
+ 2 - the current sequence length is in the original scale (avoid losing precision with small sequences)
215
+ """
216
+ seq_len = torch.max(position_ids) + 1
217
+ if seq_len > self.max_seq_len_cached: # growth
218
+ inv_freq, self.attention_scaling = self.rope_init_fn(
219
+ self.config, device, seq_len=seq_len, **self.rope_kwargs
220
+ )
221
+ self.register_buffer("inv_freq", inv_freq, persistent=False) # TODO joao: may break with compilation
222
+ self.max_seq_len_cached = seq_len
223
+
224
+ if seq_len < self.original_max_seq_len and self.max_seq_len_cached > self.original_max_seq_len: # reset
225
+ self.register_buffer("inv_freq", self.original_inv_freq, persistent=False)
226
+ self.max_seq_len_cached = self.original_max_seq_len
227
+
228
+ @torch.no_grad()
229
+ def forward(self, x, position_ids):
230
+ if "dynamic" in self.rope_type:
231
+ self._dynamic_frequency_update(position_ids, device=x.device)
232
+
233
+ # Core RoPE block
234
+ inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
235
+ position_ids_expanded = position_ids[:, None, :].float()
236
+ # Force float32 (see https://github.com/huggingface/transformers/pull/29285)
237
+ device_type = x.device.type
238
+ device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"
239
+ with torch.autocast(device_type=device_type, enabled=False):
240
+ freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
241
+ emb = torch.cat((freqs, freqs), dim=-1)
242
+ cos = emb.cos()
243
+ sin = emb.sin()
244
+
245
+ # Advanced RoPE types (e.g. yarn) apply a post-processing scaling factor, equivalent to scaling attention
246
+ cos = cos * self.attention_scaling
247
+ sin = sin * self.attention_scaling
248
+
249
+ return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
250
+
251
+
252
+ # Copied from transformers.models.llama.modeling_llama.rotate_half
253
+ def rotate_half(x):
254
+ """Rotates half the hidden dims of the input."""
255
+ x1 = x[..., : x.shape[-1] // 2]
256
+ x2 = x[..., x.shape[-1] // 2 :]
257
+ return torch.cat((-x2, x1), dim=-1)
258
+
259
+
260
+ # Copied from transformers.models.llama.modeling_llama.apply_rotary_pos_emb
261
+ def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
262
+ """Applies Rotary Position Embedding to the query and key tensors.
263
+
264
+ Args:
265
+ q (`torch.Tensor`): The query tensor.
266
+ k (`torch.Tensor`): The key tensor.
267
+ cos (`torch.Tensor`): The cosine part of the rotary embedding.
268
+ sin (`torch.Tensor`): The sine part of the rotary embedding.
269
+ position_ids (`torch.Tensor`, *optional*):
270
+ Deprecated and unused.
271
+ unsqueeze_dim (`int`, *optional*, defaults to 1):
272
+ The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
273
+ sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
274
+ that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
275
+ k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
276
+ cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
277
+ the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
278
+ Returns:
279
+ `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
280
+ """
281
+ cos = cos.unsqueeze(unsqueeze_dim)
282
+ sin = sin.unsqueeze(unsqueeze_dim)
283
+ q_embed = (q * cos) + (rotate_half(q) * sin)
284
+ k_embed = (k * cos) + (rotate_half(k) * sin)
285
+ return q_embed, k_embed
286
+
287
+
288
+ # Copied from transformers.models.mistral.modeling_mistral.MistralMLP with Mistral->Dream
289
+ class DreamMLP(nn.Module):
290
+ def __init__(self, config):
291
+ super().__init__()
292
+ self.hidden_size = config.hidden_size
293
+ self.intermediate_size = config.intermediate_size
294
+ self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
295
+ self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
296
+ self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
297
+ self.act_fn = ACT2FN[config.hidden_act]
298
+
299
+ def forward(self, hidden_state):
300
+ return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state))
301
+
302
+
303
+ # Copied from transformers.models.llama.modeling_llama.repeat_kv
304
+ def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
305
+ """
306
+ This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
307
+ num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
308
+ """
309
+ batch, num_key_value_heads, slen, head_dim = hidden_states.shape
310
+ if n_rep == 1:
311
+ return hidden_states
312
+ hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
313
+ return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
314
+
315
+
316
+ class DreamAttention(nn.Module):
317
+ """
318
+ Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer
319
+ and "Generating Long Sequences with Sparse Transformers".
320
+ """
321
+
322
+ def __init__(self, config: DreamConfig, layer_idx: Optional[int] = None):
323
+ super().__init__()
324
+ self.config = config
325
+ self.layer_idx = layer_idx
326
+ if layer_idx is None:
327
+ logger.warning_once(
328
+ f"Instantiating {self.__class__.__name__} without passing `layer_idx` is not recommended and will "
329
+ "to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` "
330
+ "when creating this class."
331
+ )
332
+
333
+ self.hidden_size = config.hidden_size
334
+ self.num_heads = config.num_attention_heads
335
+ self.head_dim = self.hidden_size // self.num_heads
336
+ self.num_key_value_heads = config.num_key_value_heads
337
+ self.num_key_value_groups = self.num_heads // self.num_key_value_heads
338
+ self.max_position_embeddings = config.max_position_embeddings
339
+ self.rope_theta = config.rope_theta
340
+ self.is_causal = False
341
+ self.attention_dropout = config.attention_dropout
342
+
343
+ if (self.head_dim * self.num_heads) != self.hidden_size:
344
+ raise ValueError(
345
+ f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
346
+ f" and `num_heads`: {self.num_heads})."
347
+ )
348
+ self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=True)
349
+ self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True)
350
+ self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True)
351
+ self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
352
+
353
+ self.rotary_emb = DreamRotaryEmbedding(config=self.config)
354
+
355
+ def forward(
356
+ self,
357
+ hidden_states: torch.Tensor,
358
+ attention_mask: Optional[torch.Tensor] = None,
359
+ position_ids: Optional[torch.LongTensor] = None,
360
+ past_key_value: Optional[Cache] = None,
361
+ output_attentions: bool = False,
362
+ use_cache: bool = False,
363
+ cache_position: Optional[torch.LongTensor] = None,
364
+ position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46
365
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
366
+ bsz, q_len, _ = hidden_states.size()
367
+
368
+ query_states = self.q_proj(hidden_states)
369
+ key_states = self.k_proj(hidden_states)
370
+ value_states = self.v_proj(hidden_states)
371
+
372
+ query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
373
+ key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
374
+ value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
375
+
376
+ if position_embeddings is None:
377
+ logger.warning_once(
378
+ "The attention layers in this model are transitioning from computing the RoPE embeddings internally "
379
+ "through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed "
380
+ "`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be "
381
+ "removed and `position_embeddings` will be mandatory."
382
+ )
383
+ cos, sin = self.rotary_emb(value_states, position_ids)
384
+ else:
385
+ cos, sin = position_embeddings
386
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
387
+
388
+ if past_key_value is not None:
389
+ cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models
390
+ key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
391
+
392
+ # repeat k/v heads if n_kv_heads < n_heads
393
+ key_states = repeat_kv(key_states, self.num_key_value_groups)
394
+ value_states = repeat_kv(value_states, self.num_key_value_groups)
395
+
396
+ attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
397
+ if attention_mask is not None: # no matter the length, we just slice it
398
+ causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
399
+ attn_weights = attn_weights + causal_mask
400
+
401
+ # upcast attention to fp32
402
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
403
+ attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training)
404
+ attn_output = torch.matmul(attn_weights, value_states)
405
+
406
+ if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
407
+ raise ValueError(
408
+ f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
409
+ f" {attn_output.size()}"
410
+ )
411
+
412
+ attn_output = attn_output.transpose(1, 2).contiguous()
413
+ attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
414
+
415
+ attn_output = self.o_proj(attn_output)
416
+
417
+ if not output_attentions:
418
+ attn_weights = None
419
+
420
+ return attn_output, attn_weights, past_key_value
421
+
422
+
423
+ class DreamSdpaAttention(DreamAttention):
424
+ """
425
+ Dream attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from
426
+ `DreamAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to
427
+ SDPA API.
428
+ """
429
+
430
+ # Adapted from DreamAttention.forward
431
+ def forward(
432
+ self,
433
+ hidden_states: torch.Tensor,
434
+ attention_mask: Optional[torch.Tensor] = None,
435
+ position_ids: Optional[torch.LongTensor] = None,
436
+ past_key_value: Optional[Cache] = None,
437
+ use_cache: bool = False,
438
+ output_attentions: bool = False,
439
+ cache_position: Optional[torch.LongTensor] = None,
440
+ position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46
441
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
442
+ if output_attentions:
443
+ # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"`
444
+ # once this is implemented.
445
+ logger.warning_once(
446
+ "DreamModel is using DreamSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` " \
447
+ " does not support" \
448
+ " `output_attentions=True`. Falling back to the manual attention implementation, "
449
+ 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. ' \
450
+ 'This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'
451
+ )
452
+ return super().forward(
453
+ hidden_states=hidden_states,
454
+ attention_mask=attention_mask,
455
+ position_ids=position_ids,
456
+ past_key_value=past_key_value,
457
+ output_attentions=output_attentions,
458
+ use_cache=use_cache,
459
+ )
460
+ bsz, q_len, _ = hidden_states.size()
461
+
462
+ query_states = self.q_proj(hidden_states)
463
+ key_states = self.k_proj(hidden_states)
464
+ value_states = self.v_proj(hidden_states)
465
+
466
+ query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
467
+ key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
468
+ value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
469
+
470
+ if position_embeddings is None:
471
+ logger.warning_once(
472
+ "The attention layers in this model are transitioning from computing the RoPE embeddings internally "
473
+ "through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed "
474
+ "`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be "
475
+ "removed and `position_embeddings` will be mandatory."
476
+ )
477
+ cos, sin = self.rotary_emb(value_states, position_ids)
478
+ else:
479
+ cos, sin = position_embeddings
480
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
481
+
482
+ if past_key_value is not None:
483
+ cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models
484
+ key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
485
+
486
+ key_states = repeat_kv(key_states, self.num_key_value_groups)
487
+ value_states = repeat_kv(value_states, self.num_key_value_groups)
488
+
489
+ # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs
490
+ # with custom attn_mask,
491
+ # Reference: https://github.com/pytorch/pytorch/issues/112577.
492
+ if query_states.device.type == "cuda" and attention_mask is not None:
493
+ query_states = query_states.contiguous()
494
+ key_states = key_states.contiguous()
495
+ value_states = value_states.contiguous()
496
+
497
+ # We dispatch to SDPA's Flash Attention or Efficient kernels via this `is_causal` if statement
498
+ # instead of an inline conditional assignment in SDPA to support both torch.compile's dynamic shapes
499
+ # and full graph options. An inline conditional prevents dynamic shapes from compiling.
500
+ # The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create
501
+ # a causal mask in case q_len == 1.
502
+
503
+ if attention_mask == None:
504
+ attention_mask = torch.ones([1, 1,
505
+ query_states.shape[2], query_states.shape[2]]).to(torch.bool).to(query_states.device)
506
+ bool_mask = attention_mask.to(torch.bool)
507
+
508
+ attn_output = torch.nn.functional.scaled_dot_product_attention(
509
+ query_states,
510
+ key_states,
511
+ value_states,
512
+ attn_mask=bool_mask ,
513
+ dropout_p=self.attention_dropout if self.training else 0.0,
514
+ is_causal=False, # hard coded
515
+ )
516
+
517
+ attn_output = attn_output.transpose(1, 2).contiguous()
518
+ attn_output = attn_output.view(bsz, q_len, self.hidden_size)
519
+
520
+ attn_output = self.o_proj(attn_output)
521
+
522
+ return attn_output, None, past_key_value
523
+
524
+ class DreamDecoderLayer(nn.Module):
525
+ def __init__(self, config: DreamConfig, layer_idx: int):
526
+ super().__init__()
527
+ self.hidden_size = config.hidden_size
528
+
529
+ if config.sliding_window and config._attn_implementation != "flash_attention_2":
530
+ logger.warning_once(
531
+ f"Sliding Window Attention is enabled but not implemented for "
532
+ f"`{config._attn_implementation}`; "
533
+ "unexpected results may be encountered."
534
+ )
535
+
536
+ # self.self_attn = Dream_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx)
537
+ self.self_attn = DreamSdpaAttention(config, layer_idx)
538
+
539
+ self.mlp = DreamMLP(config)
540
+ self.input_layernorm = DreamRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
541
+ self.post_attention_layernorm = DreamRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
542
+
543
+ # @torch.compile
544
+ def forward(
545
+ self,
546
+ hidden_states: torch.Tensor,
547
+ attention_mask: Optional[torch.Tensor] = None,
548
+ position_ids: Optional[torch.LongTensor] = None,
549
+ past_key_value: Optional[Tuple[torch.Tensor]] = None,
550
+ output_attentions: Optional[bool] = False,
551
+ use_cache: Optional[bool] = False,
552
+ cache_position: Optional[torch.LongTensor] = None,
553
+ position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
554
+ **kwargs,
555
+ ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
556
+ """
557
+ Args:
558
+ hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
559
+ attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
560
+ `(batch, sequence_length)` where padding elements are indicated by 0.
561
+ output_attentions (`bool`, *optional*):
562
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
563
+ returned tensors for more detail.
564
+ use_cache (`bool`, *optional*):
565
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
566
+ (see `past_key_values`).
567
+ past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
568
+ cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
569
+ Indices depicting the position of the input sequence tokens in the sequence.
570
+ position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
571
+ Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
572
+ with `head_dim` being the embedding dimension of each attention head.
573
+ kwargs (`dict`, *optional*):
574
+ Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
575
+ into the model
576
+ """
577
+
578
+ residual = hidden_states
579
+
580
+ hidden_states = self.input_layernorm(hidden_states)
581
+
582
+ hidden_states, self_attn_weights, present_key_value = self.self_attn(
583
+ hidden_states=hidden_states,
584
+ attention_mask=attention_mask,
585
+ position_ids=position_ids,
586
+ past_key_value=past_key_value,
587
+ output_attentions=output_attentions,
588
+ use_cache=use_cache,
589
+ cache_position=cache_position,
590
+ position_embeddings=position_embeddings,
591
+ )
592
+ hidden_states = residual + hidden_states
593
+
594
+ # Fully Connected
595
+ residual = hidden_states
596
+ hidden_states = self.post_attention_layernorm(hidden_states)
597
+ hidden_states = self.mlp(hidden_states)
598
+ hidden_states = residual + hidden_states
599
+
600
+ outputs = (hidden_states,)
601
+
602
+ if output_attentions:
603
+ outputs += (self_attn_weights,)
604
+
605
+ if use_cache:
606
+ outputs += (present_key_value,)
607
+
608
+ return outputs
609
+
610
+ class DreamPreTrainedModel(PreTrainedModel):
611
+ config_class = DreamConfig
612
+ base_model_prefix = "model"
613
+ supports_gradient_checkpointing = True
614
+ _no_split_modules = ["DreamDecoderLayer"]
615
+ _skip_keys_device_placement = "past_key_values"
616
+ _supports_flash_attn_2 = True
617
+ _supports_sdpa = True
618
+ _supports_cache_class = True
619
+ _supports_quantized_cache = True
620
+ _supports_static_cache = True
621
+
622
+ def _init_weights(self, module):
623
+ std = self.config.initializer_range
624
+ if isinstance(module, nn.Linear):
625
+ module.weight.data.normal_(mean=0.0, std=std)
626
+ if module.bias is not None:
627
+ module.bias.data.zero_()
628
+ elif isinstance(module, nn.Embedding):
629
+ module.weight.data.normal_(mean=0.0, std=std)
630
+ if module.padding_idx is not None:
631
+ module.weight.data[module.padding_idx].zero_()
632
+
633
+ @classmethod
634
+ def from_pretrained(
635
+ cls,
636
+ pretrained_model_name_or_path: Optional[Union[str, os.PathLike]],
637
+ *model_args,
638
+ config: Optional[Union[PretrainedConfig, str, os.PathLike]] = None,
639
+ cache_dir: Optional[Union[str, os.PathLike]] = None,
640
+ ignore_mismatched_sizes: bool = False,
641
+ force_download: bool = False,
642
+ local_files_only: bool = False,
643
+ token: Optional[Union[str, bool]] = None,
644
+ revision: str = "main",
645
+ use_safetensors: Optional[bool] = None,
646
+ weights_only: bool = True,
647
+ **kwargs,
648
+ ):
649
+ _ = None
650
+ try:
651
+ _model,_ = super().from_pretrained(
652
+ pretrained_model_name_or_path,
653
+ *model_args,
654
+ config=config,
655
+ cache_dir=cache_dir,
656
+ ignore_mismatched_sizes=ignore_mismatched_sizes,
657
+ force_download=force_download,
658
+ local_files_only=local_files_only,
659
+ token=token,
660
+ revision=revision,
661
+ use_safetensors=use_safetensors,
662
+ weights_only=weights_only,
663
+ **kwargs,
664
+ )
665
+ except Exception as e:
666
+ _model = super().from_pretrained(
667
+ pretrained_model_name_or_path,
668
+ *model_args,
669
+ config=config,
670
+ cache_dir=cache_dir,
671
+ ignore_mismatched_sizes=ignore_mismatched_sizes,
672
+ force_download=force_download,
673
+ local_files_only=local_files_only,
674
+ token=token,
675
+ revision=revision,
676
+ use_safetensors=use_safetensors,
677
+ weights_only=weights_only,
678
+ **kwargs,
679
+ )
680
+
681
+ resume_download = kwargs.get("resume_download", None)
682
+ proxies = kwargs.get("proxies", None)
683
+ subfolder = kwargs.get("subfolder", "")
684
+ from_auto_class = kwargs.get("_from_auto", False)
685
+ from_pipeline = kwargs.get("_from_pipeline", None)
686
+ _model.generation_config= DreamGenerationConfig.from_pretrained(
687
+ pretrained_model_name_or_path,
688
+ cache_dir=cache_dir,
689
+ force_download=force_download,
690
+ resume_download=resume_download,
691
+ proxies=proxies,
692
+ local_files_only=local_files_only,
693
+ token=token,
694
+ revision=revision,
695
+ subfolder=subfolder,
696
+ _from_auto=from_auto_class,
697
+ _from_pipeline=from_pipeline,
698
+ )
699
+ if _ is not None:
700
+ return _model,_
701
+ return _model
702
+
703
+ class DreamPrefixLMCache(Cache):
704
+
705
+ def __init__(self):
706
+ super().__init__()
707
+ self.past_key_values = {}
708
+ # this will not be updated beyond the prefilling phase
709
+
710
+ def update(
711
+ self,
712
+ key_states: torch.Tensor,
713
+ value_states: torch.Tensor,
714
+ layer_idx: int,
715
+ cache_kwargs = None,
716
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
717
+ if layer_idx in self.past_key_values:
718
+ past_key, past_value = self.past_key_values[layer_idx]
719
+ key_states = torch.cat((past_key, key_states), dim=-2)
720
+ value_states = torch.cat((past_value, value_states), dim=-2)
721
+ return key_states,value_states
722
+ else:
723
+ self.past_key_values[layer_idx] = (key_states, value_states)
724
+ return key_states, value_states
725
+
726
+ def get_seq_length(self, layer_idx: Optional[int] = 0) -> int:
727
+ """Returns the sequence length of the cached states. A layer index can be optionally passed."""
728
+ # TODO: deprecate this function in favor of `cache_position`
729
+ if len(self.past_key_values) == 0:
730
+ return 0
731
+ else:
732
+ return self.past_key_values[0][0].shape[-2]
733
+
734
+ def get_max_cache_shape(self) -> Optional[int]:
735
+ return None
736
+
737
+
738
+ import deepspeed
739
+ class DreamBaseModel(DreamPreTrainedModel):#
740
+ """
741
+ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`DreamDecoderLayer`]
742
+
743
+ Args:
744
+ config: DreamConfig
745
+ """
746
+
747
+ def __init__(self, config: DreamConfig):
748
+ super().__init__(config)
749
+ self.padding_idx = config.pad_token_id
750
+ self.vocab_size = config.vocab_size
751
+
752
+ self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
753
+ self.layers = nn.ModuleList(
754
+ [DreamDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
755
+ )
756
+ self._attn_implementation = config._attn_implementation
757
+ self.norm = DreamRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
758
+ self.rotary_emb = DreamRotaryEmbedding(config=config)
759
+
760
+ self.gradient_checkpointing = False
761
+ # Initialize weights and apply final processing
762
+
763
+ self.audio_model = AudioEncoder()
764
+ self.audio_projection = ResamplerProjector(512, config.hidden_size)
765
+
766
+ self.post_init()
767
+
768
+ def get_input_embeddings(self):
769
+ return self.embed_tokens
770
+
771
+ def set_input_embeddings(self, value):
772
+ self.embed_tokens = value
773
+
774
+ def forward(
775
+ self,
776
+ input_ids: torch.LongTensor = None,
777
+ attention_mask: Optional[torch.Tensor] = None,
778
+ audios: Optional[torch.FloatTensor] = None,
779
+ audio_indices: Optional[torch.LongTensor] = None,
780
+ position_ids: Optional[torch.LongTensor] = None,
781
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
782
+ inputs_embeds: Optional[torch.FloatTensor] = None,
783
+ use_cache: Optional[bool] = None,
784
+ output_attentions: Optional[bool] = None,
785
+ output_hidden_states: Optional[bool] = None,
786
+ return_dict: Optional[bool] = None,
787
+ cache_position: Optional[torch.LongTensor] = None,
788
+ ) -> Union[Tuple, BaseModelOutput]:
789
+
790
+ if (past_key_values is None or len(past_key_values) == 0) and audios is not None:
791
+ audio_embeds, audio_lengths = self.audio_model(audios)
792
+ assert audio_embeds.shape[0] == len(audios)
793
+ fake_audios = None
794
+
795
+ audio_embeds = self.audio_projection(audio_embeds)
796
+
797
+ elif self.training:
798
+ device = self.get_input_embeddings().weight.data.device
799
+ dtype = self.get_input_embeddings().weight.data.dtype
800
+ fake_audios = torch.ones((1, 1, 560), dtype=dtype, device=device)
801
+ audio_embeds, audio_lengths = self.audio_model(fake_audios)
802
+ audio_embeds = self.audio_projection(audio_embeds)
803
+
804
+ else:
805
+ fake_audios = None
806
+ audio_embeds = None
807
+
808
+
809
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
810
+ output_hidden_states = (
811
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
812
+ )
813
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
814
+
815
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
816
+
817
+ if (input_ids is None) ^ (inputs_embeds is not None):
818
+ raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
819
+
820
+ if self.gradient_checkpointing and self.training:
821
+ if use_cache:
822
+ logger.warning_once(
823
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
824
+ )
825
+ use_cache = False
826
+
827
+ if inputs_embeds is None:
828
+ inputs_embeds = self.embed_tokens(input_ids)
829
+
830
+
831
+
832
+ if fake_audios is not None:
833
+ inputs_embeds = inputs_embeds + audio_embeds.mean() * 0.0
834
+ elif audio_embeds is not None:
835
+ inputs_embeds = inputs_embeds.clone()
836
+ for audio_embeds_, audio_lengths_, audio_indices_ in zip(audio_embeds, audio_lengths, audio_indices,):
837
+ # print(f"{audio_embeds_.size()=} {audio_lengths_=} {audio_indices_.size()=}")
838
+ audio_embeds_ = audio_embeds_[:audio_lengths_, ...]
839
+ audio_embeds_ = audio_embeds_.to(inputs_embeds.device)
840
+ indices_b, indices_s = audio_indices_.to(inputs_embeds.device).unbind(dim=0)
841
+ inputs_embeds[indices_b.view(-1), indices_s.view(-1)] = audio_embeds_.view(-1, audio_embeds_.shape[-1])
842
+
843
+
844
+ if use_cache and past_key_values is None:
845
+ past_key_values = DreamPrefixLMCache()
846
+
847
+ if cache_position is None:
848
+ past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
849
+ cache_position = torch.arange(
850
+ past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
851
+ )
852
+
853
+ if position_ids is None:
854
+ position_ids = cache_position.unsqueeze(0)
855
+
856
+ hidden_states = inputs_embeds
857
+
858
+ # create position embeddings to be shared across the decoder layers
859
+ position_embeddings = self.rotary_emb(hidden_states, position_ids)
860
+
861
+ # decoder layers
862
+ all_hidden_states = () if output_hidden_states else None
863
+ all_self_attns = () if output_attentions else None
864
+
865
+ for decoder_layer in self.layers:
866
+ if output_hidden_states:
867
+ all_hidden_states += (hidden_states,)
868
+
869
+ if self.gradient_checkpointing and self.training:
870
+ layer_outputs = deepspeed.checkpointing.checkpoint(
871
+ decoder_layer,
872
+ hidden_states,
873
+ attention_mask,
874
+ position_ids,
875
+ past_key_values,
876
+ output_attentions,
877
+ use_cache,
878
+ cache_position,
879
+ position_embeddings,
880
+ )
881
+ else:
882
+ layer_outputs = decoder_layer(
883
+ hidden_states,
884
+ attention_mask=attention_mask,
885
+ position_ids=position_ids,
886
+ past_key_value=past_key_values,
887
+ output_attentions=output_attentions,
888
+ use_cache=use_cache,
889
+ cache_position=cache_position,
890
+ position_embeddings=position_embeddings,
891
+ )
892
+
893
+ # breakpoint()
894
+ if isinstance(layer_outputs,torch.Tensor):
895
+ layer_outputs = (layer_outputs,None)
896
+ hidden_states = layer_outputs[0]
897
+
898
+ if output_attentions:
899
+ all_self_attns += (layer_outputs[1],)
900
+
901
+ hidden_states = self.norm(hidden_states)
902
+
903
+ # add hidden states from the last decoder layer
904
+ if output_hidden_states:
905
+ all_hidden_states += (hidden_states,)
906
+
907
+ if not return_dict:
908
+ return tuple(v for v in [hidden_states, all_hidden_states, all_self_attns] if v is not None)
909
+ return BaseModelOutput(
910
+ last_hidden_state=hidden_states,
911
+ hidden_states=all_hidden_states,
912
+ attentions=all_self_attns,
913
+ past_key_values=past_key_values,
914
+ )
915
+
916
+
917
+ class DreamModel(DreamGenerationMixin, DreamPreTrainedModel):
918
+ _tied_weights_keys = ["lm_head.weight"]
919
+
920
+ def __init__(self, config):
921
+ super().__init__(config)
922
+ self.model = DreamBaseModel(config)
923
+ self.vocab_size = config.vocab_size
924
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
925
+
926
+ # Initialize weights and apply final processing
927
+ self.tokenizer = None
928
+ self.post_init()
929
+
930
+ def reset_rope_parameters(self):
931
+ self.model.rotary_emb.reset_parameters()
932
+ for layer in self.model.layers:
933
+ layer.self_attn.rotary_emb.reset_parameters()
934
+
935
+ def get_input_embeddings(self):
936
+ return self.model.embed_tokens
937
+
938
+ def set_input_embeddings(self, value):
939
+ self.model.embed_tokens = value
940
+
941
+ def get_output_embeddings(self):
942
+ return self.lm_head
943
+
944
+ def set_output_embeddings(self, new_embeddings):
945
+ self.lm_head = new_embeddings
946
+
947
+ def set_decoder(self, decoder):
948
+ self.model = decoder
949
+
950
+ def get_decoder(self):
951
+ return self.model
952
+
953
+ def forward(
954
+ self,
955
+ input_ids: torch.LongTensor = None,
956
+ attention_mask: Optional[torch.Tensor] = None,
957
+ audios: Optional[torch.FloatTensor] = None,
958
+ audio_indices: Optional[torch.LongTensor] = None,
959
+ position_ids: Optional[torch.LongTensor] = None,
960
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
961
+ inputs_embeds: Optional[torch.FloatTensor] = None,
962
+ labels: Optional[torch.LongTensor] = None,
963
+ use_cache: Optional[bool] = None,
964
+ output_attentions: Optional[bool] = None,
965
+ output_hidden_states: Optional[bool] = None,
966
+ return_dict: Optional[bool] = None,
967
+ cache_position: Optional[torch.LongTensor] = None,
968
+ num_logits_to_keep: int = 0,
969
+ num_items_in_batch: int = None,
970
+ **loss_kwargs,
971
+ ) -> Union[Tuple, MaskedLMOutput]:
972
+
973
+ num_items_in_batch = None
974
+ if ENFORCE_NUM_ITEMIN_BATCH:
975
+ num_items_in_batch = labels.ne(-100).sum()
976
+ num_items_in_batch = torch.distributed.reduce(num_items_in_batch)
977
+
978
+ is_new = position_ids == 0
979
+ # is_new[0] = True
980
+ segment_id = torch.cumsum(is_new.long(), dim=1) - 1
981
+ new_attention_mask = (segment_id.unsqueeze(1) == segment_id.unsqueeze(2)).long()
982
+ mask = attention_mask.unsqueeze(-1) # [bs, len, 1]
983
+ new_attention_mask = new_attention_mask * mask
984
+
985
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
986
+ output_hidden_states = (
987
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
988
+ )
989
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
990
+
991
+ outputs = self.model(
992
+ input_ids=input_ids,
993
+ attention_mask=new_attention_mask,
994
+ audios=audios,
995
+ audio_indices=audio_indices,
996
+ position_ids=position_ids,
997
+ past_key_values=past_key_values,
998
+ inputs_embeds=inputs_embeds,
999
+ use_cache=use_cache,
1000
+ output_attentions=output_attentions,
1001
+ output_hidden_states=output_hidden_states,
1002
+ return_dict=return_dict,
1003
+ cache_position=cache_position,
1004
+ )
1005
+ hidden_states = outputs[0]
1006
+ # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
1007
+ logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :])
1008
+
1009
+ loss = None
1010
+ if labels is not None:
1011
+ if ENFORCE_NUM_ITEMIN_BATCH:
1012
+ assert num_items_in_batch is not None, \
1013
+ "num_items_in_batch must be provided if ENFORCE_NUM_ITEMIN_BATCH is True"
1014
+ loss = self.loss_function(logits, labels,
1015
+ self.vocab_size,num_items_in_batch=num_items_in_batch, **loss_kwargs)
1016
+
1017
+ if not return_dict:
1018
+ output = (logits,) + outputs[1:]
1019
+ return (loss,) + output if loss is not None else output
1020
+
1021
+ return MaskedLMOutput(
1022
+ loss=loss,
1023
+ logits=logits,
1024
+ hidden_states=outputs.hidden_states,
1025
+ attentions=outputs.attentions,
1026
+ past_key_values=outputs.past_key_values
1027
+ )
1028
+
1029
+ def forward_dream(
1030
+ self,
1031
+ input_ids: torch.LongTensor = None,
1032
+ attention_mask: Optional[torch.Tensor] = None,
1033
+ position_ids: Optional[torch.LongTensor] = None,
1034
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
1035
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1036
+ labels: Optional[torch.LongTensor] = None,
1037
+ use_cache: Optional[bool] = None,
1038
+ output_attentions: Optional[bool] = None,
1039
+ output_hidden_states: Optional[bool] = None,
1040
+ return_dict: Optional[bool] = None,
1041
+ cache_position: Optional[torch.LongTensor] = None,
1042
+ num_logits_to_keep: int = 0,
1043
+ **loss_kwargs,
1044
+ ) -> Union[Tuple, MaskedLMOutput]:
1045
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
1046
+ output_hidden_states = (
1047
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
1048
+ )
1049
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1050
+ attention_mask = None
1051
+ # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
1052
+ outputs = self.model(
1053
+ input_ids=input_ids,
1054
+ attention_mask=attention_mask,
1055
+ position_ids=position_ids,
1056
+ past_key_values=past_key_values,
1057
+ inputs_embeds=inputs_embeds,
1058
+ use_cache=use_cache,
1059
+ output_attentions=output_attentions,
1060
+ output_hidden_states=output_hidden_states,
1061
+ return_dict=return_dict,
1062
+ cache_position=cache_position,
1063
+ )
1064
+
1065
+ hidden_states = outputs[0]
1066
+ # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
1067
+ logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :])
1068
+
1069
+ loss = None
1070
+ if labels is not None:
1071
+ loss = self.loss_function(logits, labels, self.vocab_size, **loss_kwargs)
1072
+
1073
+ if not return_dict:
1074
+ output = (logits,) + outputs[1:]
1075
+ return (loss,) + output if loss is not None else output
1076
+
1077
+ return MaskedLMOutput(
1078
+ loss=loss,
1079
+ logits=logits,
1080
+ hidden_states=outputs.hidden_states,
1081
+ attentions=outputs.attentions,
1082
+ past_key_values=outputs.past_key_values,
1083
+ )
1084
+
1085
+ @torch.no_grad()
1086
+ def generate(
1087
+ self,
1088
+ input_ids: Optional[torch.Tensor] = None,
1089
+ audios: Optional[torch.FloatTensor] = None,
1090
+ audio_indices: Optional[torch.LongTensor] = None,
1091
+ max_new_tokens=512,
1092
+ steps=512,
1093
+ temperature=0.2,
1094
+ top_p=0.95,
1095
+ alg_temp=0.,
1096
+ alg="entropy",
1097
+ output_history=False,
1098
+ **kwargs,
1099
+ ):
1100
+ # modalities = kwargs.pop("modalities", None) if "modalities" in kwargs and modalities is None else modalities
1101
+ position_ids = kwargs.pop("position_ids", None)
1102
+ attention_mask = kwargs.pop("attention_mask", None)
1103
+ if "inputs_embeds" in kwargs:
1104
+ raise NotImplementedError("`inputs_embeds` is not supported")
1105
+
1106
+ if audios is not None:
1107
+ audio_embeds, audio_lengths = self.model.audio_model(audios)
1108
+ assert audio_embeds.shape[0] == len(audios)
1109
+ audio_embeds = self.model.audio_projection(audio_embeds)
1110
+ else:
1111
+ audio_embeds = None
1112
+
1113
+ inputs_embeds = self.model.embed_tokens(input_ids)
1114
+
1115
+ if audio_embeds is not None:
1116
+ inputs_embeds = inputs_embeds.clone()
1117
+ for audio_embeds_, audio_lengths_, audio_indices_ in zip(audio_embeds, audio_lengths, audio_indices,):
1118
+ # print(f"{audio_embeds_.size()=} {audio_lengths_=} {audio_indices_.size()=}")
1119
+ audio_embeds_ = audio_embeds_[:audio_lengths_, ...]
1120
+ audio_embeds_ = audio_embeds_.to(inputs_embeds.device)
1121
+ indices_b, indices_s = audio_indices_.to(inputs_embeds.device).unbind(dim=0)
1122
+ inputs_embeds[indices_b.view(-1), indices_s.view(-1)] = audio_embeds_.view(-1, audio_embeds_.shape[-1])
1123
+
1124
+ return self.diffusion_generate(
1125
+ None,
1126
+ inputs_embeds=inputs_embeds,
1127
+ max_new_tokens=max_new_tokens,
1128
+ output_history=output_history,
1129
+ return_dict_in_generate=True,
1130
+ steps=steps,
1131
+ temperature=temperature,
1132
+ top_p=top_p,
1133
+ alg=alg,
1134
+ alg_temp=alg_temp,
1135
+ **kwargs
1136
+ )
omni_diffusion/models/dream/modeling_sensevoice.py ADDED
@@ -0,0 +1,1258 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ import time
3
+ import torch
4
+ from torch import nn
5
+ import torch.nn.functional as F
6
+ from typing import Iterable, Optional
7
+
8
+ from funasr.register import tables
9
+ from funasr.models.ctc.ctc import CTC
10
+ from funasr.utils.datadir_writer import DatadirWriter
11
+ from funasr.models.paraformer.search import Hypothesis
12
+ from funasr.train_utils.device_funcs import force_gatherable
13
+ from funasr.losses.label_smoothing_loss import LabelSmoothingLoss
14
+ from funasr.metrics.compute_acc import compute_accuracy, th_accuracy
15
+ from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank
16
+ # from utils.ctc_alignment import ctc_forced_align
17
+
18
+ def ctc_forced_align(
19
+ log_probs: torch.Tensor,
20
+ targets: torch.Tensor,
21
+ input_lengths: torch.Tensor,
22
+ target_lengths: torch.Tensor,
23
+ blank: int = 0,
24
+ ignore_id: int = -1,
25
+ ) -> torch.Tensor:
26
+ """Align a CTC label sequence to an emission.
27
+
28
+ Args:
29
+ log_probs (Tensor): log probability of CTC emission output.
30
+ Tensor of shape `(B, T, C)`. where `B` is the batch size, `T` is the input length,
31
+ `C` is the number of characters in alphabet including blank.
32
+ targets (Tensor): Target sequence. Tensor of shape `(B, L)`,
33
+ where `L` is the target length.
34
+ input_lengths (Tensor):
35
+ Lengths of the inputs (max value must each be <= `T`). 1-D Tensor of shape `(B,)`.
36
+ target_lengths (Tensor):
37
+ Lengths of the targets. 1-D Tensor of shape `(B,)`.
38
+ blank_id (int, optional): The index of blank symbol in CTC emission. (Default: 0)
39
+ ignore_id (int, optional): The index of ignore symbol in CTC emission. (Default: -1)
40
+ """
41
+ targets[targets == ignore_id] = blank
42
+
43
+ batch_size, input_time_size, _ = log_probs.size()
44
+ bsz_indices = torch.arange(batch_size, device=input_lengths.device)
45
+
46
+ _t_a_r_g_e_t_s_ = torch.cat(
47
+ (
48
+ torch.stack((torch.full_like(targets, blank), targets), dim=-1).flatten(start_dim=1),
49
+ torch.full_like(targets[:, :1], blank),
50
+ ),
51
+ dim=-1,
52
+ )
53
+ diff_labels = torch.cat(
54
+ (
55
+ torch.as_tensor([[False, False]], device=targets.device).expand(batch_size, -1),
56
+ _t_a_r_g_e_t_s_[:, 2:] != _t_a_r_g_e_t_s_[:, :-2],
57
+ ),
58
+ dim=1,
59
+ )
60
+
61
+ neg_inf = torch.tensor(float("-inf"), device=log_probs.device, dtype=log_probs.dtype)
62
+ padding_num = 2
63
+ padded_t = padding_num + _t_a_r_g_e_t_s_.size(-1)
64
+ best_score = torch.full((batch_size, padded_t), neg_inf, device=log_probs.device, dtype=log_probs.dtype)
65
+ best_score[:, padding_num + 0] = log_probs[:, 0, blank]
66
+ best_score[:, padding_num + 1] = log_probs[bsz_indices, 0, _t_a_r_g_e_t_s_[:, 1]]
67
+
68
+ backpointers = torch.zeros((batch_size, input_time_size, padded_t), device=log_probs.device, dtype=targets.dtype)
69
+
70
+ for t in range(1, input_time_size):
71
+ prev = torch.stack(
72
+ (best_score[:, 2:], best_score[:, 1:-1], torch.where(diff_labels, best_score[:, :-2], neg_inf))
73
+ )
74
+ prev_max_value, prev_max_idx = prev.max(dim=0)
75
+ best_score[:, padding_num:] = log_probs[:, t].gather(-1, _t_a_r_g_e_t_s_) + prev_max_value
76
+ backpointers[:, t, padding_num:] = prev_max_idx
77
+
78
+ l1l2 = best_score.gather(
79
+ -1, torch.stack((padding_num + target_lengths * 2 - 1, padding_num + target_lengths * 2), dim=-1)
80
+ )
81
+
82
+ path = torch.zeros((batch_size, input_time_size), device=best_score.device, dtype=torch.long)
83
+ path[bsz_indices, input_lengths - 1] = padding_num + target_lengths * 2 - 1 + l1l2.argmax(dim=-1)
84
+
85
+ for t in range(input_time_size - 1, 0, -1):
86
+ target_indices = path[:, t]
87
+ prev_max_idx = backpointers[bsz_indices, t, target_indices]
88
+ path[:, t - 1] += target_indices - prev_max_idx
89
+
90
+ alignments = _t_a_r_g_e_t_s_.gather(dim=-1, index=(path - padding_num).clamp(min=0))
91
+ return alignments
92
+
93
+ class SinusoidalPositionEncoder(torch.nn.Module):
94
+ """ """
95
+
96
+ def __int__(self, d_model=80, dropout_rate=0.1):
97
+ pass
98
+
99
+ def encode(
100
+ self, positions: torch.Tensor = None, depth: int = None, dtype: torch.dtype = torch.float32
101
+ ):
102
+ batch_size = positions.size(0)
103
+ positions = positions.type(dtype)
104
+ device = positions.device
105
+ log_timescale_increment = torch.log(torch.tensor([10000], dtype=dtype, device=device)) / (
106
+ depth / 2 - 1
107
+ )
108
+ inv_timescales = torch.exp(
109
+ torch.arange(depth / 2, device=device).type(dtype) * (-log_timescale_increment)
110
+ )
111
+ inv_timescales = torch.reshape(inv_timescales, [batch_size, -1])
112
+ scaled_time = torch.reshape(positions, [1, -1, 1]) * torch.reshape(
113
+ inv_timescales, [1, 1, -1]
114
+ )
115
+ encoding = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=2)
116
+ return encoding.type(dtype)
117
+
118
+ def forward(self, x):
119
+ batch_size, timesteps, input_dim = x.size()
120
+ positions = torch.arange(1, timesteps + 1, device=x.device)[None, :]
121
+ position_encoding = self.encode(positions, input_dim, x.dtype).to(x.device)
122
+
123
+ return x + position_encoding
124
+
125
+
126
+ class PositionwiseFeedForward(torch.nn.Module):
127
+ """Positionwise feed forward layer.
128
+
129
+ Args:
130
+ idim (int): Input dimenstion.
131
+ hidden_units (int): The number of hidden units.
132
+ dropout_rate (float): Dropout rate.
133
+
134
+ """
135
+
136
+ def __init__(self, idim, hidden_units, dropout_rate, activation=torch.nn.ReLU()):
137
+ """Construct an PositionwiseFeedForward object."""
138
+ super(PositionwiseFeedForward, self).__init__()
139
+ self.w_1 = torch.nn.Linear(idim, hidden_units)
140
+ self.w_2 = torch.nn.Linear(hidden_units, idim)
141
+ self.dropout = torch.nn.Dropout(dropout_rate)
142
+ self.activation = activation
143
+
144
+ def forward(self, x):
145
+ """Forward function."""
146
+ return self.w_2(self.dropout(self.activation(self.w_1(x))))
147
+
148
+
149
+ class MultiHeadedAttentionSANM(nn.Module):
150
+ """Multi-Head Attention layer.
151
+
152
+ Args:
153
+ n_head (int): The number of heads.
154
+ n_feat (int): The number of features.
155
+ dropout_rate (float): Dropout rate.
156
+
157
+ """
158
+
159
+ def __init__(
160
+ self,
161
+ n_head,
162
+ in_feat,
163
+ n_feat,
164
+ dropout_rate,
165
+ kernel_size,
166
+ sanm_shfit=0,
167
+ lora_list=None,
168
+ lora_rank=8,
169
+ lora_alpha=16,
170
+ lora_dropout=0.1,
171
+ ):
172
+ """Construct an MultiHeadedAttention object."""
173
+ super().__init__()
174
+ assert n_feat % n_head == 0
175
+ # We assume d_v always equals d_k
176
+ self.d_k = n_feat // n_head
177
+ self.h = n_head
178
+ # self.linear_q = nn.Linear(n_feat, n_feat)
179
+ # self.linear_k = nn.Linear(n_feat, n_feat)
180
+ # self.linear_v = nn.Linear(n_feat, n_feat)
181
+
182
+ self.linear_out = nn.Linear(n_feat, n_feat)
183
+ self.linear_q_k_v = nn.Linear(in_feat, n_feat * 3)
184
+ self.attn = None
185
+ self.dropout = nn.Dropout(p=dropout_rate)
186
+
187
+ self.fsmn_block = nn.Conv1d(
188
+ n_feat, n_feat, kernel_size, stride=1, padding=0, groups=n_feat, bias=False
189
+ )
190
+ # padding
191
+ left_padding = (kernel_size - 1) // 2
192
+ if sanm_shfit > 0:
193
+ left_padding = left_padding + sanm_shfit
194
+ right_padding = kernel_size - 1 - left_padding
195
+ self.pad_fn = nn.ConstantPad1d((left_padding, right_padding), 0.0)
196
+
197
+ def forward_fsmn(self, inputs, mask, mask_shfit_chunk=None):
198
+ b, t, d = inputs.size()
199
+ if mask is not None:
200
+ mask = torch.reshape(mask, (b, -1, 1))
201
+ if mask_shfit_chunk is not None:
202
+ mask = mask * mask_shfit_chunk
203
+ inputs = inputs * mask
204
+
205
+ x = inputs.transpose(1, 2)
206
+ x = self.pad_fn(x)
207
+ x = self.fsmn_block(x)
208
+ x = x.transpose(1, 2)
209
+ x += inputs
210
+ x = self.dropout(x)
211
+ if mask is not None:
212
+ x = x * mask
213
+ return x
214
+
215
+ def forward_qkv(self, x):
216
+ """Transform query, key and value.
217
+
218
+ Args:
219
+ query (torch.Tensor): Query tensor (#batch, time1, size).
220
+ key (torch.Tensor): Key tensor (#batch, time2, size).
221
+ value (torch.Tensor): Value tensor (#batch, time2, size).
222
+
223
+ Returns:
224
+ torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k).
225
+ torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k).
226
+ torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k).
227
+
228
+ """
229
+ b, t, d = x.size()
230
+ q_k_v = self.linear_q_k_v(x)
231
+ q, k, v = torch.split(q_k_v, int(self.h * self.d_k), dim=-1)
232
+ q_h = torch.reshape(q, (b, t, self.h, self.d_k)).transpose(
233
+ 1, 2
234
+ ) # (batch, head, time1, d_k)
235
+ k_h = torch.reshape(k, (b, t, self.h, self.d_k)).transpose(
236
+ 1, 2
237
+ ) # (batch, head, time2, d_k)
238
+ v_h = torch.reshape(v, (b, t, self.h, self.d_k)).transpose(
239
+ 1, 2
240
+ ) # (batch, head, time2, d_k)
241
+
242
+ return q_h, k_h, v_h, v
243
+
244
+ def forward_attention(self, value, scores, mask, mask_att_chunk_encoder=None):
245
+ """Compute attention context vector.
246
+
247
+ Args:
248
+ value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k).
249
+ scores (torch.Tensor): Attention score (#batch, n_head, time1, time2).
250
+ mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2).
251
+
252
+ Returns:
253
+ torch.Tensor: Transformed value (#batch, time1, d_model)
254
+ weighted by the attention score (#batch, time1, time2).
255
+
256
+ """
257
+ n_batch = value.size(0)
258
+ if mask is not None:
259
+ if mask_att_chunk_encoder is not None:
260
+ mask = mask * mask_att_chunk_encoder
261
+
262
+ mask = mask.unsqueeze(1).eq(0) # (batch, 1, *, time2)
263
+
264
+ min_value = -float(
265
+ "inf"
266
+ ) # float(numpy.finfo(torch.tensor(0, dtype=scores.dtype).numpy().dtype).min)
267
+ scores = scores.masked_fill(mask, min_value)
268
+ attn = torch.softmax(scores, dim=-1).masked_fill(
269
+ mask, 0.0
270
+ ) # (batch, head, time1, time2)
271
+ else:
272
+ attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2)
273
+
274
+ p_attn = self.dropout(attn)
275
+ x = torch.matmul(p_attn, value) # (batch, head, time1, d_k)
276
+ x = (
277
+ x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k)
278
+ ) # (batch, time1, d_model)
279
+
280
+ return self.linear_out(x) # (batch, time1, d_model)
281
+
282
+ def forward(self, x, mask, mask_shfit_chunk=None, mask_att_chunk_encoder=None):
283
+ """Compute scaled dot product attention.
284
+
285
+ Args:
286
+ query (torch.Tensor): Query tensor (#batch, time1, size).
287
+ key (torch.Tensor): Key tensor (#batch, time2, size).
288
+ value (torch.Tensor): Value tensor (#batch, time2, size).
289
+ mask (torch.Tensor): Mask tensor (#batch, 1, time2) or
290
+ (#batch, time1, time2).
291
+
292
+ Returns:
293
+ torch.Tensor: Output tensor (#batch, time1, d_model).
294
+
295
+ """
296
+ q_h, k_h, v_h, v = self.forward_qkv(x)
297
+ fsmn_memory = self.forward_fsmn(v, mask, mask_shfit_chunk)
298
+ q_h = q_h * self.d_k ** (-0.5)
299
+ scores = torch.matmul(q_h, k_h.transpose(-2, -1))
300
+ att_outs = self.forward_attention(v_h, scores, mask, mask_att_chunk_encoder)
301
+ return att_outs + fsmn_memory
302
+
303
+ def forward_chunk(self, x, cache=None, chunk_size=None, look_back=0):
304
+ """Compute scaled dot product attention.
305
+
306
+ Args:
307
+ query (torch.Tensor): Query tensor (#batch, time1, size).
308
+ key (torch.Tensor): Key tensor (#batch, time2, size).
309
+ value (torch.Tensor): Value tensor (#batch, time2, size).
310
+ mask (torch.Tensor): Mask tensor (#batch, 1, time2) or
311
+ (#batch, time1, time2).
312
+
313
+ Returns:
314
+ torch.Tensor: Output tensor (#batch, time1, d_model).
315
+
316
+ """
317
+ q_h, k_h, v_h, v = self.forward_qkv(x)
318
+ if chunk_size is not None and look_back > 0 or look_back == -1:
319
+ if cache is not None:
320
+ k_h_stride = k_h[:, :, : -(chunk_size[2]), :]
321
+ v_h_stride = v_h[:, :, : -(chunk_size[2]), :]
322
+ k_h = torch.cat((cache["k"], k_h), dim=2)
323
+ v_h = torch.cat((cache["v"], v_h), dim=2)
324
+
325
+ cache["k"] = torch.cat((cache["k"], k_h_stride), dim=2)
326
+ cache["v"] = torch.cat((cache["v"], v_h_stride), dim=2)
327
+ if look_back != -1:
328
+ cache["k"] = cache["k"][:, :, -(look_back * chunk_size[1]) :, :]
329
+ cache["v"] = cache["v"][:, :, -(look_back * chunk_size[1]) :, :]
330
+ else:
331
+ cache_tmp = {
332
+ "k": k_h[:, :, : -(chunk_size[2]), :],
333
+ "v": v_h[:, :, : -(chunk_size[2]), :],
334
+ }
335
+ cache = cache_tmp
336
+ fsmn_memory = self.forward_fsmn(v, None)
337
+ q_h = q_h * self.d_k ** (-0.5)
338
+ scores = torch.matmul(q_h, k_h.transpose(-2, -1))
339
+ att_outs = self.forward_attention(v_h, scores, None)
340
+ return att_outs + fsmn_memory, cache
341
+
342
+
343
+ class LayerNorm(nn.LayerNorm):
344
+ def __init__(self, *args, **kwargs):
345
+ super().__init__(*args, **kwargs)
346
+
347
+ def forward(self, input):
348
+ output = F.layer_norm(
349
+ input.float(),
350
+ self.normalized_shape,
351
+ self.weight.float() if self.weight is not None else None,
352
+ self.bias.float() if self.bias is not None else None,
353
+ self.eps,
354
+ )
355
+ return output.type_as(input)
356
+
357
+
358
+ def sequence_mask(lengths, maxlen=None, dtype=torch.float32, device=None):
359
+ if maxlen is None:
360
+ maxlen = lengths.max()
361
+ row_vector = torch.arange(0, maxlen, 1).to(lengths.device)
362
+ matrix = torch.unsqueeze(lengths, dim=-1)
363
+ mask = row_vector < matrix
364
+ mask = mask.detach()
365
+
366
+ return mask.to(dtype).to(device) if device is not None else mask.to(dtype)
367
+ # return mask.type(dtype).to(device) if device is not None else mask.type(dtype)
368
+
369
+
370
+ class EncoderLayerSANM(nn.Module):
371
+ def __init__(
372
+ self,
373
+ in_size,
374
+ size,
375
+ self_attn,
376
+ feed_forward,
377
+ dropout_rate,
378
+ normalize_before=True,
379
+ concat_after=False,
380
+ stochastic_depth_rate=0.0,
381
+ ):
382
+ """Construct an EncoderLayer object."""
383
+ super(EncoderLayerSANM, self).__init__()
384
+ self.self_attn = self_attn
385
+ self.feed_forward = feed_forward
386
+ self.norm1 = LayerNorm(in_size)
387
+ self.norm2 = LayerNorm(size)
388
+ self.dropout = nn.Dropout(dropout_rate)
389
+ self.in_size = in_size
390
+ self.size = size
391
+ self.normalize_before = normalize_before
392
+ self.concat_after = concat_after
393
+ if self.concat_after:
394
+ self.concat_linear = nn.Linear(size + size, size)
395
+ self.stochastic_depth_rate = stochastic_depth_rate
396
+ self.dropout_rate = dropout_rate
397
+
398
+ def forward(self, x, mask, cache=None, mask_shfit_chunk=None, mask_att_chunk_encoder=None):
399
+ """Compute encoded features.
400
+
401
+ Args:
402
+ x_input (torch.Tensor): Input tensor (#batch, time, size).
403
+ mask (torch.Tensor): Mask tensor for the input (#batch, time).
404
+ cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size).
405
+
406
+ Returns:
407
+ torch.Tensor: Output tensor (#batch, time, size).
408
+ torch.Tensor: Mask tensor (#batch, time).
409
+
410
+ """
411
+ skip_layer = False
412
+ # with stochastic depth, residual connection `x + f(x)` becomes
413
+ # `x <- x + 1 / (1 - p) * f(x)` at training time.
414
+ stoch_layer_coeff = 1.0
415
+ if self.training and self.stochastic_depth_rate > 0:
416
+ skip_layer = torch.rand(1).item() < self.stochastic_depth_rate
417
+ stoch_layer_coeff = 1.0 / (1 - self.stochastic_depth_rate)
418
+
419
+ if skip_layer:
420
+ if cache is not None:
421
+ x = torch.cat([cache, x], dim=1)
422
+ return x, mask
423
+
424
+ residual = x
425
+ if self.normalize_before:
426
+ x = self.norm1(x)
427
+
428
+ if self.concat_after:
429
+ x_concat = torch.cat(
430
+ (
431
+ x,
432
+ self.self_attn(
433
+ x,
434
+ mask,
435
+ mask_shfit_chunk=mask_shfit_chunk,
436
+ mask_att_chunk_encoder=mask_att_chunk_encoder,
437
+ ),
438
+ ),
439
+ dim=-1,
440
+ )
441
+ if self.in_size == self.size:
442
+ x = residual + stoch_layer_coeff * self.concat_linear(x_concat)
443
+ else:
444
+ x = stoch_layer_coeff * self.concat_linear(x_concat)
445
+ else:
446
+ if self.in_size == self.size:
447
+ x = residual + stoch_layer_coeff * self.dropout(
448
+ self.self_attn(
449
+ x,
450
+ mask,
451
+ mask_shfit_chunk=mask_shfit_chunk,
452
+ mask_att_chunk_encoder=mask_att_chunk_encoder,
453
+ )
454
+ )
455
+ else:
456
+ x = stoch_layer_coeff * self.dropout(
457
+ self.self_attn(
458
+ x,
459
+ mask,
460
+ mask_shfit_chunk=mask_shfit_chunk,
461
+ mask_att_chunk_encoder=mask_att_chunk_encoder,
462
+ )
463
+ )
464
+ if not self.normalize_before:
465
+ x = self.norm1(x)
466
+
467
+ residual = x
468
+ if self.normalize_before:
469
+ x = self.norm2(x)
470
+ x = residual + stoch_layer_coeff * self.dropout(self.feed_forward(x))
471
+ if not self.normalize_before:
472
+ x = self.norm2(x)
473
+
474
+ return x, mask, cache, mask_shfit_chunk, mask_att_chunk_encoder
475
+
476
+ def forward_chunk(self, x, cache=None, chunk_size=None, look_back=0):
477
+ """Compute encoded features.
478
+
479
+ Args:
480
+ x_input (torch.Tensor): Input tensor (#batch, time, size).
481
+ mask (torch.Tensor): Mask tensor for the input (#batch, time).
482
+ cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size).
483
+
484
+ Returns:
485
+ torch.Tensor: Output tensor (#batch, time, size).
486
+ torch.Tensor: Mask tensor (#batch, time).
487
+
488
+ """
489
+
490
+ residual = x
491
+ if self.normalize_before:
492
+ x = self.norm1(x)
493
+
494
+ if self.in_size == self.size:
495
+ attn, cache = self.self_attn.forward_chunk(x, cache, chunk_size, look_back)
496
+ x = residual + attn
497
+ else:
498
+ x, cache = self.self_attn.forward_chunk(x, cache, chunk_size, look_back)
499
+
500
+ if not self.normalize_before:
501
+ x = self.norm1(x)
502
+
503
+ residual = x
504
+ if self.normalize_before:
505
+ x = self.norm2(x)
506
+ x = residual + self.feed_forward(x)
507
+ if not self.normalize_before:
508
+ x = self.norm2(x)
509
+
510
+ return x, cache
511
+
512
+
513
+ @tables.register("encoder_classes", "SenseVoiceEncoderSmall")
514
+ class SenseVoiceEncoderSmall(nn.Module):
515
+ """
516
+ Author: Speech Lab of DAMO Academy, Alibaba Group
517
+ SCAMA: Streaming chunk-aware multihead attention for online end-to-end speech recognition
518
+ https://arxiv.org/abs/2006.01713
519
+ """
520
+
521
+ def __init__(
522
+ self,
523
+ input_size: int,
524
+ output_size: int = 256,
525
+ attention_heads: int = 4,
526
+ linear_units: int = 2048,
527
+ num_blocks: int = 6,
528
+ tp_blocks: int = 0,
529
+ dropout_rate: float = 0.1,
530
+ positional_dropout_rate: float = 0.1,
531
+ attention_dropout_rate: float = 0.0,
532
+ stochastic_depth_rate: float = 0.0,
533
+ input_layer: Optional[str] = "conv2d",
534
+ pos_enc_class=SinusoidalPositionEncoder,
535
+ normalize_before: bool = True,
536
+ concat_after: bool = False,
537
+ positionwise_layer_type: str = "linear",
538
+ positionwise_conv_kernel_size: int = 1,
539
+ padding_idx: int = -1,
540
+ kernel_size: int = 11,
541
+ sanm_shfit: int = 0,
542
+ selfattention_layer_type: str = "sanm",
543
+ **kwargs,
544
+ ):
545
+ super().__init__()
546
+ self._output_size = output_size
547
+
548
+ self.embed = SinusoidalPositionEncoder()
549
+
550
+ self.normalize_before = normalize_before
551
+
552
+ positionwise_layer = PositionwiseFeedForward
553
+ positionwise_layer_args = (
554
+ output_size,
555
+ linear_units,
556
+ dropout_rate,
557
+ )
558
+
559
+ encoder_selfattn_layer = MultiHeadedAttentionSANM
560
+ encoder_selfattn_layer_args0 = (
561
+ attention_heads,
562
+ input_size,
563
+ output_size,
564
+ attention_dropout_rate,
565
+ kernel_size,
566
+ sanm_shfit,
567
+ )
568
+ encoder_selfattn_layer_args = (
569
+ attention_heads,
570
+ output_size,
571
+ output_size,
572
+ attention_dropout_rate,
573
+ kernel_size,
574
+ sanm_shfit,
575
+ )
576
+
577
+ self.encoders0 = nn.ModuleList(
578
+ [
579
+ EncoderLayerSANM(
580
+ input_size,
581
+ output_size,
582
+ encoder_selfattn_layer(*encoder_selfattn_layer_args0),
583
+ positionwise_layer(*positionwise_layer_args),
584
+ dropout_rate,
585
+ )
586
+ for i in range(1)
587
+ ]
588
+ )
589
+ self.encoders = nn.ModuleList(
590
+ [
591
+ EncoderLayerSANM(
592
+ output_size,
593
+ output_size,
594
+ encoder_selfattn_layer(*encoder_selfattn_layer_args),
595
+ positionwise_layer(*positionwise_layer_args),
596
+ dropout_rate,
597
+ )
598
+ for i in range(num_blocks - 1)
599
+ ]
600
+ )
601
+
602
+ self.tp_encoders = nn.ModuleList(
603
+ [
604
+ EncoderLayerSANM(
605
+ output_size,
606
+ output_size,
607
+ encoder_selfattn_layer(*encoder_selfattn_layer_args),
608
+ positionwise_layer(*positionwise_layer_args),
609
+ dropout_rate,
610
+ )
611
+ for i in range(tp_blocks)
612
+ ]
613
+ )
614
+
615
+ self.after_norm = LayerNorm(output_size)
616
+
617
+ self.tp_norm = LayerNorm(output_size)
618
+
619
+ def output_size(self) -> int:
620
+ return self._output_size
621
+
622
+ def forward(
623
+ self,
624
+ xs_pad: torch.Tensor,
625
+ ilens: torch.Tensor,
626
+ ):
627
+ """Embed positions in tensor."""
628
+ masks = sequence_mask(ilens, dtype=torch.bfloat16, device=ilens.device)[:, None, :]
629
+ # print(f"{masks=}")
630
+ # print(f"{ilens=}")
631
+ # print(f"{(masks>0.5).squeeze(1).sum(1).int()=}")
632
+
633
+ xs_pad *= self.output_size() ** 0.5
634
+
635
+ xs_pad = self.embed(xs_pad)
636
+
637
+ # forward encoder1
638
+ for layer_idx, encoder_layer in enumerate(self.encoders0):
639
+ encoder_outs = encoder_layer(xs_pad, masks)
640
+ xs_pad, masks = encoder_outs[0], encoder_outs[1]
641
+
642
+ for layer_idx, encoder_layer in enumerate(self.encoders):
643
+ encoder_outs = encoder_layer(xs_pad, masks)
644
+ xs_pad, masks = encoder_outs[0], encoder_outs[1]
645
+
646
+ xs_pad = self.after_norm(xs_pad)
647
+
648
+ # forward encoder2
649
+ # olens = masks.squeeze(1).sum(1).int()
650
+ olens = (masks > 0.5).squeeze(1).sum(1).int()
651
+
652
+ for layer_idx, encoder_layer in enumerate(self.tp_encoders):
653
+ encoder_outs = encoder_layer(xs_pad, masks)
654
+ xs_pad, masks = encoder_outs[0], encoder_outs[1]
655
+
656
+ xs_pad = self.tp_norm(xs_pad)
657
+ return xs_pad, olens
658
+
659
+
660
+ @tables.register("model_classes", "SenseVoiceSmall")
661
+ class SenseVoiceSmall(nn.Module):
662
+ """CTC-attention hybrid Encoder-Decoder model"""
663
+
664
+ def __init__(
665
+ self,
666
+ specaug: str = None,
667
+ specaug_conf: dict = None,
668
+ normalize: str = None,
669
+ normalize_conf: dict = None,
670
+ encoder: str = None,
671
+ encoder_conf: dict = None,
672
+ ctc_conf: dict = None,
673
+ input_size: int = 80,
674
+ vocab_size: int = -1,
675
+ ignore_id: int = -1,
676
+ blank_id: int = 0,
677
+ sos: int = 1,
678
+ eos: int = 2,
679
+ length_normalized_loss: bool = False,
680
+ **kwargs,
681
+ ):
682
+
683
+ super().__init__()
684
+
685
+ if specaug is not None:
686
+ specaug_class = tables.specaug_classes.get(specaug)
687
+ specaug = specaug_class(**specaug_conf)
688
+ if normalize is not None:
689
+ normalize_class = tables.normalize_classes.get(normalize)
690
+ normalize = normalize_class(**normalize_conf)
691
+ encoder_class = tables.encoder_classes.get(encoder)
692
+ encoder = encoder_class(input_size=input_size, **encoder_conf)
693
+ encoder_output_size = encoder.output_size()
694
+
695
+ if ctc_conf is None:
696
+ ctc_conf = {}
697
+ ctc = CTC(odim=vocab_size, encoder_output_size=encoder_output_size, **ctc_conf)
698
+
699
+ self.blank_id = blank_id
700
+ self.sos = sos if sos is not None else vocab_size - 1
701
+ self.eos = eos if eos is not None else vocab_size - 1
702
+ self.vocab_size = vocab_size
703
+ self.ignore_id = ignore_id
704
+ self.specaug = specaug
705
+ self.normalize = normalize
706
+ self.encoder = encoder
707
+ self.error_calculator = None
708
+
709
+ self.ctc = ctc
710
+
711
+ self.length_normalized_loss = length_normalized_loss
712
+ self.encoder_output_size = encoder_output_size
713
+
714
+ self.lid_dict = {"auto": 0, "zh": 3, "en": 4, "yue": 7, "ja": 11, "ko": 12, "nospeech": 13}
715
+ self.lid_int_dict = {24884: 3, 24885: 4, 24888: 7, 24892: 11, 24896: 12, 24992: 13}
716
+ self.textnorm_dict = {"withitn": 14, "woitn": 15}
717
+ self.textnorm_int_dict = {25016: 14, 25017: 15}
718
+ self.embed = torch.nn.Embedding(7 + len(self.lid_dict) + len(self.textnorm_dict), input_size)
719
+ self.emo_dict = {"unk": 25009, "happy": 25001, "sad": 25002, "angry": 25003, "neutral": 25004}
720
+
721
+ self.criterion_att = LabelSmoothingLoss(
722
+ size=self.vocab_size,
723
+ padding_idx=self.ignore_id,
724
+ smoothing=kwargs.get("lsm_weight", 0.0),
725
+ normalize_length=self.length_normalized_loss,
726
+ )
727
+
728
+ @staticmethod
729
+ def from_pretrained(model:str=None, **kwargs):
730
+ from funasr import AutoModel
731
+ model, kwargs = AutoModel.build_model(model=model, trust_remote_code=True, **kwargs)
732
+
733
+ return model, kwargs
734
+
735
+ def forward(
736
+ self,
737
+ speech: torch.Tensor,
738
+ speech_lengths: torch.Tensor,
739
+ text: torch.Tensor,
740
+ text_lengths: torch.Tensor,
741
+ **kwargs,
742
+ ):
743
+ """Encoder + Decoder + Calc loss
744
+ Args:
745
+ speech: (Batch, Length, ...)
746
+ speech_lengths: (Batch, )
747
+ text: (Batch, Length)
748
+ text_lengths: (Batch,)
749
+ """
750
+ # import pdb;
751
+ # pdb.set_trace()
752
+ if len(text_lengths.size()) > 1:
753
+ text_lengths = text_lengths[:, 0]
754
+ if len(speech_lengths.size()) > 1:
755
+ speech_lengths = speech_lengths[:, 0]
756
+
757
+ batch_size = speech.shape[0]
758
+
759
+ # 1. Encoder
760
+ encoder_out, encoder_out_lens = self.encode(speech, speech_lengths, text)
761
+
762
+ loss_ctc, cer_ctc = None, None
763
+ loss_rich, acc_rich = None, None
764
+ stats = dict()
765
+
766
+ loss_ctc, cer_ctc = self._calc_ctc_loss(
767
+ encoder_out[:, 4:, :], encoder_out_lens - 4, text[:, 4:], text_lengths - 4
768
+ )
769
+
770
+ loss_rich, acc_rich = self._calc_rich_ce_loss(
771
+ encoder_out[:, :4, :], text[:, :4]
772
+ )
773
+
774
+ loss = loss_ctc + loss_rich
775
+ # Collect total loss stats
776
+ stats["loss_ctc"] = torch.clone(loss_ctc.detach()) if loss_ctc is not None else None
777
+ stats["loss_rich"] = torch.clone(loss_rich.detach()) if loss_rich is not None else None
778
+ stats["loss"] = torch.clone(loss.detach()) if loss is not None else None
779
+ stats["acc_rich"] = acc_rich
780
+
781
+ # force_gatherable: to-device and to-tensor if scalar for DataParallel
782
+ if self.length_normalized_loss:
783
+ batch_size = int((text_lengths + 1).sum())
784
+ loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device)
785
+ return loss, stats, weight
786
+
787
+ def encode(
788
+ self,
789
+ speech: torch.Tensor,
790
+ speech_lengths: torch.Tensor,
791
+ text: torch.Tensor,
792
+ **kwargs,
793
+ ):
794
+ """Frontend + Encoder. Note that this method is used by asr_inference.py
795
+ Args:
796
+ speech: (Batch, Length, ...)
797
+ speech_lengths: (Batch, )
798
+ ind: int
799
+ """
800
+
801
+ # Data augmentation
802
+ if self.specaug is not None and self.training:
803
+ speech, speech_lengths = self.specaug(speech, speech_lengths)
804
+
805
+ # Normalization for feature: e.g. Global-CMVN, Utterance-CMVN
806
+ if self.normalize is not None:
807
+ speech, speech_lengths = self.normalize(speech, speech_lengths)
808
+
809
+
810
+ lids = torch.LongTensor([[self.lid_int_dict[int(lid)]
811
+ if torch.rand(1) > 0.2 and int(lid) in self.lid_int_dict else 0 ]
812
+ for lid in text[:, 0]]).to(speech.device)
813
+ language_query = self.embed(lids)
814
+
815
+ styles = torch.LongTensor([[self.textnorm_int_dict[int(style)]] for style in text[:, 3]]).to(speech.device)
816
+ style_query = self.embed(styles)
817
+ speech = torch.cat((style_query, speech), dim=1)
818
+ speech_lengths += 1
819
+
820
+ event_emo_query = self.embed(torch.LongTensor([[1, 2]]).to(speech.device)).repeat(speech.size(0), 1, 1)
821
+ input_query = torch.cat((language_query, event_emo_query), dim=1)
822
+ speech = torch.cat((input_query, speech), dim=1)
823
+ speech_lengths += 3
824
+
825
+ encoder_out, encoder_out_lens = self.encoder(speech, speech_lengths)
826
+
827
+ return encoder_out, encoder_out_lens
828
+
829
+ def _calc_ctc_loss(
830
+ self,
831
+ encoder_out: torch.Tensor,
832
+ encoder_out_lens: torch.Tensor,
833
+ ys_pad: torch.Tensor,
834
+ ys_pad_lens: torch.Tensor,
835
+ ):
836
+ # Calc CTC loss
837
+ loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
838
+
839
+ # Calc CER using CTC
840
+ cer_ctc = None
841
+ if not self.training and self.error_calculator is not None:
842
+ ys_hat = self.ctc.argmax(encoder_out).data
843
+ cer_ctc = self.error_calculator(ys_hat.cpu(), ys_pad.cpu(), is_ctc=True)
844
+ return loss_ctc, cer_ctc
845
+
846
+ def _calc_rich_ce_loss(
847
+ self,
848
+ encoder_out: torch.Tensor,
849
+ ys_pad: torch.Tensor,
850
+ ):
851
+ decoder_out = self.ctc.ctc_lo(encoder_out)
852
+ # 2. Compute attention loss
853
+ loss_rich = self.criterion_att(decoder_out, ys_pad.contiguous())
854
+ acc_rich = th_accuracy(
855
+ decoder_out.view(-1, self.vocab_size),
856
+ ys_pad.contiguous(),
857
+ ignore_label=self.ignore_id,
858
+ )
859
+
860
+ return loss_rich, acc_rich
861
+
862
+
863
+ def inference(
864
+ self,
865
+ data_in,
866
+ data_lengths=None,
867
+ key: list = ["wav_file_tmp_name"],
868
+ tokenizer=None,
869
+ frontend=None,
870
+ **kwargs,
871
+ ):
872
+
873
+
874
+ meta_data = {}
875
+ if (
876
+ isinstance(data_in, torch.Tensor) and kwargs.get("data_type", "sound") == "fbank"
877
+ ): # fbank
878
+ speech, speech_lengths = data_in, data_lengths
879
+ if len(speech.shape) < 3:
880
+ speech = speech[None, :, :]
881
+ if speech_lengths is None:
882
+ speech_lengths = speech.shape[1]
883
+ else:
884
+ # extract fbank feats
885
+ time1 = time.perf_counter()
886
+ audio_sample_list = load_audio_text_image_video(
887
+ data_in,
888
+ fs=frontend.fs,
889
+ audio_fs=kwargs.get("fs", 16000),
890
+ data_type=kwargs.get("data_type", "sound"),
891
+ tokenizer=tokenizer,
892
+ )
893
+ time2 = time.perf_counter()
894
+ meta_data["load_data"] = f"{time2 - time1:0.3f}"
895
+ speech, speech_lengths = extract_fbank(
896
+ audio_sample_list, data_type=kwargs.get("data_type", "sound"), frontend=frontend
897
+ )
898
+ time3 = time.perf_counter()
899
+ meta_data["extract_feat"] = f"{time3 - time2:0.3f}"
900
+ meta_data["batch_data_time"] = (
901
+ speech_lengths.sum().item() * frontend.frame_shift * frontend.lfr_n / 1000
902
+ )
903
+
904
+ speech = speech.to(device=kwargs["device"])
905
+ speech_lengths = speech_lengths.to(device=kwargs["device"])
906
+
907
+ language = kwargs.get("language", "auto")
908
+ language_query = self.embed(
909
+ torch.LongTensor(
910
+ [[self.lid_dict[language] if language in self.lid_dict else 0]]
911
+ ).to(speech.device)
912
+ ).repeat(speech.size(0), 1, 1)
913
+
914
+ use_itn = kwargs.get("use_itn", False)
915
+ output_timestamp = kwargs.get("output_timestamp", False)
916
+
917
+ textnorm = kwargs.get("text_norm", None)
918
+ if textnorm is None:
919
+ textnorm = "withitn" if use_itn else "woitn"
920
+ textnorm_query = self.embed(
921
+ torch.LongTensor([[self.textnorm_dict[textnorm]]]).to(speech.device)
922
+ ).repeat(speech.size(0), 1, 1)
923
+ speech = torch.cat((textnorm_query, speech), dim=1)
924
+ speech_lengths += 1
925
+
926
+ event_emo_query = self.embed(torch.LongTensor([[1, 2]]).to(speech.device)).repeat(
927
+ speech.size(0), 1, 1
928
+ )
929
+ input_query = torch.cat((language_query, event_emo_query), dim=1)
930
+ speech = torch.cat((input_query, speech), dim=1)
931
+ speech_lengths += 3
932
+
933
+ # Encoder
934
+ encoder_out, encoder_out_lens = self.encoder(speech, speech_lengths)
935
+ if isinstance(encoder_out, tuple):
936
+ encoder_out = encoder_out[0]
937
+
938
+ # c. Passed the encoder result and the beam search
939
+ ctc_logits = self.ctc.log_softmax(encoder_out)
940
+ if kwargs.get("ban_emo_unk", False):
941
+ ctc_logits[:, :, self.emo_dict["unk"]] = -float("inf")
942
+
943
+ results = []
944
+ b, n, d = encoder_out.size()
945
+ if isinstance(key[0], (list, tuple)):
946
+ key = key[0]
947
+ if len(key) < b:
948
+ key = key * b
949
+ for i in range(b):
950
+ x = ctc_logits[i, : encoder_out_lens[i].item(), :]
951
+ yseq = x.argmax(dim=-1)
952
+ yseq = torch.unique_consecutive(yseq, dim=-1)
953
+
954
+ ibest_writer = None
955
+ if kwargs.get("output_dir") is not None:
956
+ if not hasattr(self, "writer"):
957
+ self.writer = DatadirWriter(kwargs.get("output_dir"))
958
+ ibest_writer = self.writer[f"1best_recog"]
959
+
960
+ mask = yseq != self.blank_id
961
+ token_int = yseq[mask].tolist()
962
+
963
+ # Change integer-ids to tokens
964
+ text = tokenizer.decode(token_int)
965
+ if ibest_writer is not None:
966
+ ibest_writer["text"][key[i]] = text
967
+
968
+ if output_timestamp:
969
+ from itertools import groupby
970
+ timestamp = []
971
+ tokens = tokenizer.text2tokens(text)[4:]
972
+
973
+ logits_speech = self.ctc.softmax(encoder_out)[i, 4:encoder_out_lens[i].item(), :]
974
+
975
+ pred = logits_speech.argmax(-1).cpu()
976
+ logits_speech[pred==self.blank_id, self.blank_id] = 0
977
+
978
+ align = ctc_forced_align(
979
+ logits_speech.unsqueeze(0).float(),
980
+ torch.Tensor(token_int[4:]).unsqueeze(0).long().to(logits_speech.device),
981
+ (encoder_out_lens-4).long(),
982
+ torch.tensor(len(token_int)-4).unsqueeze(0).long().to(logits_speech.device),
983
+ ignore_id=self.ignore_id,
984
+ )
985
+
986
+ pred = groupby(align[0, :encoder_out_lens[0]])
987
+ _start = 0
988
+ token_id = 0
989
+ ts_max = encoder_out_lens[i] - 4
990
+ for pred_token, pred_frame in pred:
991
+ _end = _start + len(list(pred_frame))
992
+ if pred_token != 0:
993
+ ts_left = max((_start*60-30)/1000, 0)
994
+ ts_right = min((_end*60-30)/1000, (ts_max*60-30)/1000)
995
+ timestamp.append([tokens[token_id], ts_left, ts_right])
996
+ token_id += 1
997
+ _start = _end
998
+
999
+ result_i = {"key": key[i], "text": text, "timestamp": timestamp}
1000
+ results.append(result_i)
1001
+ else:
1002
+ result_i = {"key": key[i], "text": text}
1003
+ results.append(result_i)
1004
+ return results, meta_data
1005
+
1006
+
1007
+ def inference_encode(
1008
+ self,
1009
+ data_in,
1010
+ data_lengths=None,
1011
+ key: list = ["wav_file_tmp_name"],
1012
+ tokenizer=None,
1013
+ frontend=None,
1014
+ **kwargs,
1015
+ ):
1016
+
1017
+ # fbank
1018
+ speech, speech_lengths = data_in, data_lengths
1019
+ if len(speech.shape) < 3:
1020
+ speech = speech[None, :, :]
1021
+ if speech_lengths is None:
1022
+ speech_lengths = speech.shape[1]
1023
+
1024
+ speech = speech.to(device=kwargs["device"])
1025
+ speech_lengths = speech_lengths.to(device=kwargs["device"])
1026
+
1027
+ language = kwargs.get("language", "auto")
1028
+ language_query = self.embed(
1029
+ torch.LongTensor(
1030
+ [[self.lid_dict[language] if language in self.lid_dict else 0]]
1031
+ ).to(speech.device)
1032
+ ).repeat(speech.size(0), 1, 1)
1033
+
1034
+ use_itn = kwargs.get("use_itn", False)
1035
+ output_timestamp = kwargs.get("output_timestamp", False)
1036
+
1037
+ textnorm = kwargs.get("text_norm", None)
1038
+ if textnorm is None:
1039
+ textnorm = "withitn" if use_itn else "woitn"
1040
+ textnorm_query = self.embed(
1041
+ torch.LongTensor([[self.textnorm_dict[textnorm]]]).to(speech.device)
1042
+ ).repeat(speech.size(0), 1, 1)
1043
+ speech = torch.cat((textnorm_query, speech), dim=1)
1044
+ speech_lengths += 1
1045
+
1046
+ event_emo_query = self.embed(torch.LongTensor([[1, 2]]).to(speech.device)).repeat(
1047
+ speech.size(0), 1, 1
1048
+ )
1049
+ input_query = torch.cat((language_query, event_emo_query), dim=1)
1050
+ speech = torch.cat((input_query, speech), dim=1)
1051
+ speech_lengths += 3
1052
+
1053
+ # Encoder
1054
+ encoder_out, encoder_out_lens = self.encoder(speech, speech_lengths)
1055
+ if isinstance(encoder_out, tuple):
1056
+ encoder_out = encoder_out[0]
1057
+
1058
+ return encoder_out, encoder_out_lens
1059
+
1060
+ def export(self, **kwargs):
1061
+ from export_meta import export_rebuild_model
1062
+
1063
+ if "max_seq_len" not in kwargs:
1064
+ kwargs["max_seq_len"] = 512
1065
+ models = export_rebuild_model(model=self, **kwargs)
1066
+ return models
1067
+
1068
+
1069
+
1070
+ class AudioEncoder(nn.Module):
1071
+
1072
+ def __init__(
1073
+ self,
1074
+ model_dir=None
1075
+ ):
1076
+
1077
+ super().__init__()
1078
+
1079
+ # model_dir = "SenceVoiceSmall"
1080
+ from huggingface_hub import snapshot_download
1081
+ model_dir = snapshot_download(repo_id="FunAudioLLM/SenseVoiceSmall")
1082
+ self.model, self.kwargs = self.build_model(model=model_dir, trust_remote_code=True,)
1083
+
1084
+
1085
+ def forward(
1086
+ self,
1087
+ audios,
1088
+ ):
1089
+
1090
+ # print(f"{[x for x in audios]=}")
1091
+ # print(f"{[x.sum() for x in audios]=}")
1092
+ # print(f"{[x.size() for x in audios]=}")
1093
+
1094
+ from torch.nn.utils.rnn import pad_sequence
1095
+ feats_pad = pad_sequence(audios, batch_first=True, padding_value=0.0)
1096
+ # feats_lens = torch.as_tensor([len(x) + 4 for x in audios])
1097
+ feats_lens = torch.as_tensor([len(x) for x in audios])
1098
+
1099
+ feats_pad = feats_pad.to(torch.bfloat16)
1100
+
1101
+ # print(f"{feats_pad.sum()=}")
1102
+ # print(f"{feats_pad.size()=}")
1103
+ # print(f"{feats_lens=}")
1104
+
1105
+ encoder_out, encoder_out_lens = self.model.inference_encode(
1106
+ feats_pad,
1107
+ data_lengths=feats_lens,
1108
+ language="auto", # "zh", "en", "yue", "ja", "ko", "nospeech"
1109
+ use_itn=False,
1110
+ ban_emo_unk=False,
1111
+ **self.kwargs,
1112
+ )
1113
+
1114
+ # print(f"{encoder_out.sum()=}")
1115
+ # print(f"{encoder_out.size()=}")
1116
+ # print(f"{encoder_out_lens=}")
1117
+ return encoder_out, encoder_out_lens
1118
+
1119
+ audio_embeds = []
1120
+ for x, y in zip(encoder_out, encoder_out_lens):
1121
+ audio_embeds.append(x[:y, ...])
1122
+
1123
+ audio_embeds = torch.stack(audio_embeds, dim=0)
1124
+
1125
+ return audio_embeds
1126
+
1127
+ # https://github.com/modelscope/FunASR/blob/main/funasr/auto/auto_model.py
1128
+ @staticmethod
1129
+ def build_model(**kwargs):
1130
+ from omegaconf import DictConfig, ListConfig
1131
+ import os
1132
+
1133
+ from funasr.download.download_model_from_hub import download_model
1134
+ from funasr.train_utils.set_all_random_seed import set_all_random_seed
1135
+ from funasr.register import tables
1136
+ from funasr.train_utils.load_pretrained_model import load_pretrained_model
1137
+ from funasr.utils.misc import deep_update
1138
+
1139
+ import logging
1140
+
1141
+ assert "model" in kwargs
1142
+ if "model_conf" not in kwargs:
1143
+ logging.info("download models from model hub: {}".format(kwargs.get("hub", "ms")))
1144
+ kwargs = download_model(**kwargs)
1145
+
1146
+ set_all_random_seed(kwargs.get("seed", 0))
1147
+
1148
+ device = kwargs.get("device", "cuda")
1149
+ if not torch.cuda.is_available() or kwargs.get("ngpu", 1) == 0:
1150
+ device = "cpu"
1151
+ kwargs["batch_size"] = 1
1152
+ kwargs["device"] = device
1153
+
1154
+ torch.set_num_threads(kwargs.get("ncpu", 4))
1155
+
1156
+ # build tokenizer
1157
+ tokenizer = kwargs.get("tokenizer", None)
1158
+ kwargs["tokenizer"] = tokenizer
1159
+ kwargs["vocab_size"] = -1
1160
+
1161
+ if tokenizer is not None:
1162
+ tokenizers = (
1163
+ tokenizer.split(",") if isinstance(tokenizer, str) else tokenizer
1164
+ ) # type of tokenizers is list!!!
1165
+ tokenizers_conf = kwargs.get("tokenizer_conf", {})
1166
+ tokenizers_build = []
1167
+ vocab_sizes = []
1168
+ token_lists = []
1169
+
1170
+ ### === only for kws ===
1171
+ token_list_files = kwargs.get("token_lists", [])
1172
+ seg_dicts = kwargs.get("seg_dicts", [])
1173
+ ### === only for kws ===
1174
+
1175
+ if not isinstance(tokenizers_conf, (list, tuple, ListConfig)):
1176
+ tokenizers_conf = [tokenizers_conf] * len(tokenizers)
1177
+
1178
+ for i, tokenizer in enumerate(tokenizers):
1179
+ tokenizer_class = tables.tokenizer_classes.get(tokenizer)
1180
+ tokenizer_conf = tokenizers_conf[i]
1181
+
1182
+ ### === only for kws ===
1183
+ if len(token_list_files) > 1:
1184
+ tokenizer_conf["token_list"] = token_list_files[i]
1185
+ if len(seg_dicts) > 1:
1186
+ tokenizer_conf["seg_dict"] = seg_dicts[i]
1187
+ ### === only for kws ===
1188
+
1189
+ tokenizer = tokenizer_class(**tokenizer_conf)
1190
+ tokenizers_build.append(tokenizer)
1191
+ token_list = tokenizer.token_list if hasattr(tokenizer, "token_list") else None
1192
+ token_list = (
1193
+ tokenizer.get_vocab() if hasattr(tokenizer, "get_vocab") else token_list
1194
+ )
1195
+ vocab_size = -1
1196
+ if token_list is not None:
1197
+ vocab_size = len(token_list)
1198
+
1199
+ if vocab_size == -1 and hasattr(tokenizer, "get_vocab_size"):
1200
+ vocab_size = tokenizer.get_vocab_size()
1201
+ token_lists.append(token_list)
1202
+ vocab_sizes.append(vocab_size)
1203
+
1204
+ if len(tokenizers_build) <= 1:
1205
+ tokenizers_build = tokenizers_build[0]
1206
+ token_lists = token_lists[0]
1207
+ vocab_sizes = vocab_sizes[0]
1208
+
1209
+ kwargs["tokenizer"] = tokenizers_build
1210
+ kwargs["vocab_size"] = vocab_sizes
1211
+ kwargs["token_list"] = token_lists
1212
+
1213
+ # build frontend
1214
+ frontend = kwargs.get("frontend", None)
1215
+ kwargs["input_size"] = None
1216
+ if frontend is not None:
1217
+ frontend_class = tables.frontend_classes.get(frontend)
1218
+ frontend = frontend_class(**kwargs.get("frontend_conf", {}))
1219
+ kwargs["input_size"] = (
1220
+ frontend.output_size() if hasattr(frontend, "output_size") else None
1221
+ )
1222
+ kwargs["frontend"] = frontend
1223
+ # build model
1224
+ model_class = tables.model_classes.get(kwargs["model"])
1225
+ assert model_class is not None, f'{kwargs["model"]} is not registered'
1226
+ model_conf = {}
1227
+ deep_update(model_conf, kwargs.get("model_conf", {}))
1228
+ deep_update(model_conf, kwargs)
1229
+ model = model_class(**model_conf)
1230
+
1231
+ # init_param
1232
+ init_param = kwargs.get("init_param", None)
1233
+ if init_param is not None:
1234
+ if os.path.exists(init_param):
1235
+ logging.info(f"Loading pretrained params from {init_param}")
1236
+ load_pretrained_model(
1237
+ model=model,
1238
+ path=init_param,
1239
+ ignore_init_mismatch=kwargs.get("ignore_init_mismatch", True),
1240
+ oss_bucket=kwargs.get("oss_bucket", None),
1241
+ scope_map=kwargs.get("scope_map", []),
1242
+ excludes=kwargs.get("excludes", None),
1243
+ )
1244
+ else:
1245
+ print(f"error, init_param does not exist!: {init_param}")
1246
+
1247
+ # fp16
1248
+ if kwargs.get("fp16", False):
1249
+ model.to(torch.float16)
1250
+ elif kwargs.get("bf16", False):
1251
+ model.to(torch.bfloat16)
1252
+ # model.to(device)
1253
+
1254
+ if not kwargs.get("disable_log", True):
1255
+ tables.print()
1256
+
1257
+ return model, kwargs
1258
+
omni_diffusion/models/dream/resampler_projector.py ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+
4
+ import math
5
+
6
+ class ResamplerProjector(nn.Module):
7
+ """
8
+ A projection module designed to map input features (e.g., from a vision encoder)
9
+ to a target hidden size. It flattens spatial dimensions into a sequence
10
+ and applies an MLP.
11
+ """
12
+ def __init__(self, proj_input_size, hidden_size):
13
+ super().__init__()
14
+
15
+ self.pre_proj_layernorm = torch.nn.LayerNorm(proj_input_size)
16
+
17
+ self.mlp = nn.Sequential(
18
+ nn.Linear(proj_input_size, hidden_size, bias=False),
19
+ nn.GELU(),
20
+ nn.Linear(hidden_size, hidden_size, bias=False),
21
+ )
22
+ self.mlp.apply(init_weights)
23
+ self.pre_proj_layernorm.apply(init_weights)
24
+
25
+ def forward(self, x, *args, **kwargs):
26
+ x = x.reshape(x.shape[0], -1, x.shape[-1])
27
+ x = self.pre_proj_layernorm(x)
28
+ x = self.mlp(x)
29
+ return x
30
+
31
+ def init_weights(m):
32
+ if isinstance(m, nn.Linear):
33
+ torch.nn.init.normal_(m.weight, mean=0.0, std=0.02)
34
+ if m.bias is not None:
35
+ torch.nn.init.zeros_(m.bias)
36
+
37
+ if isinstance(m, nn.LayerNorm):
38
+ torch.nn.init.ones_(m.weight)
39
+ torch.nn.init.zeros_(m.bias)
40
+
omni_diffusion/models/dream/tokenization_dream.py ADDED
@@ -0,0 +1,351 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2024 The Dream team, HKUNLP Group and The HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # This code is based on Qwen's implementations in this library.
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ """Tokenization classes for Dream."""
17
+
18
+ import json
19
+ import os
20
+ import unicodedata
21
+ from functools import lru_cache
22
+ from typing import Optional, Tuple
23
+
24
+ import regex as re
25
+
26
+ from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
27
+ from transformers.utils import logging
28
+
29
+
30
+ logger = logging.get_logger(__name__)
31
+
32
+ VOCAB_FILES_NAMES = {
33
+ "vocab_file": "vocab.json",
34
+ "merges_file": "merges.txt",
35
+ }
36
+
37
+
38
+ MAX_MODEL_INPUT_SIZES = {"dream/dream-tokenizer": 32768}
39
+
40
+ regex_parts = [
41
+ r"(?i:'s|'t|'re|'ve|'m|'ll|'d)", # Contractions (case-insensitive)
42
+ r"[^\r\n\p{L}\p{N}]?\p{L}+", # Words (with optional leading symbol)
43
+ r"\p{N}", # Numbers
44
+ r" ?[^\s\p{L}\p{N}]+[\r\n]*", # Punctuation and symbols
45
+ r"\s*[\r\n]+", # Newlines (handling indentation)
46
+ r"\s+(?!\S)", # Trailing whitespace
47
+ r"\s+", # General whitespace
48
+ ]
49
+ PRETOKENIZE_REGEX = "|".join(regex_parts)
50
+
51
+
52
+ @lru_cache()
53
+ # Copied from transformers.models.gpt2.tokenization_gpt2.bytes_to_unicode
54
+ def bytes_to_unicode():
55
+ """
56
+ Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
57
+ characters the bpe code barfs on.
58
+
59
+ The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
60
+ if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
61
+ decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
62
+ tables between utf-8 bytes and unicode strings.
63
+ """
64
+ bs = (
65
+ list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
66
+ )
67
+ cs = bs[:]
68
+ n = 0
69
+ for b in range(2**8):
70
+ if b not in bs:
71
+ bs.append(b)
72
+ cs.append(2**8 + n)
73
+ n += 1
74
+ cs = [chr(n) for n in cs]
75
+ return dict(zip(bs, cs))
76
+
77
+
78
+ # Copied from transformers.models.gpt2.tokenization_gpt2.get_pairs
79
+ def get_pairs(word):
80
+ """
81
+ Return set of symbol pairs in a word.
82
+
83
+ Word is represented as tuple of symbols (symbols being variable-length strings).
84
+ """
85
+ pairs = set()
86
+ prev_char = word[0]
87
+ for char in word[1:]:
88
+ pairs.add((prev_char, char))
89
+ prev_char = char
90
+ return pairs
91
+
92
+
93
+ class DreamTokenizer(PreTrainedTokenizer):
94
+ """
95
+ Construct a Dream tokenizer. Based on byte-level Byte-Pair-Encoding.
96
+
97
+ Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
98
+ be encoded differently whether it is at the beginning of the sentence (without space) or not:
99
+
100
+ ```python
101
+ >>> from transformers import AutoTokenizer
102
+
103
+ >>> tokenizer = AutoTokenizer.from_pretrained("Dream-org/Dream-v0-Base-7B", trust_remote_code=True)
104
+ >>> tokenizer("Hello world")["input_ids"]
105
+ [9707, 1879]
106
+
107
+ >>> tokenizer(" Hello world")["input_ids"]
108
+ [21927, 1879]
109
+ ```
110
+ This is expected.
111
+
112
+ You should not use GPT2Tokenizer instead, because of the different pretokenization rules.
113
+
114
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
115
+ this superclass for more information regarding those methods.
116
+
117
+ Args:
118
+ vocab_file (`str`):
119
+ Path to the vocabulary file.
120
+ merges_file (`str`):
121
+ Path to the merges file.
122
+ errors (`str`, *optional*, defaults to `"replace"`):
123
+ Paradigm to follow when decoding bytes to UTF-8. See
124
+ [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
125
+ unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
126
+ The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
127
+ token instead.
128
+ bos_token (`str`, *optional*):
129
+ The beginning of sequence token. Not applicable for this tokenizer.
130
+ eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
131
+ The end of sequence token.
132
+ pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
133
+ The token used for padding, for example when batching sequences of different lengths.
134
+ clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
135
+ Whether or not the model should cleanup the spaces that were added when splitting the input text during the
136
+ tokenization process. Not applicable to this tokenizer, since tokenization does not add spaces.
137
+ split_special_tokens (`bool`, *optional*, defaults to `False`):
138
+ Whether or not the special tokens should be split during the tokenization process. The default behavior is
139
+ to not split special tokens. This means that if `<|endoftext|>` is the `eos_token`, then
140
+ `tokenizer.tokenize("<|endoftext|>") = ['<|endoftext|>`]. Otherwise, if `split_special_tokens=True`,
141
+ then `tokenizer.tokenize("<|endoftext|>")` will be give `['<', '|', 'endo', 'ft', 'ext', '|', '>']`.
142
+ This argument is only supported for `slow` tokenizers for the moment.
143
+ """
144
+
145
+ vocab_files_names = VOCAB_FILES_NAMES
146
+ model_input_names = ["input_ids", "attention_mask"]
147
+
148
+ def __init__(
149
+ self,
150
+ vocab_file,
151
+ merges_file,
152
+ errors="replace",
153
+ unk_token="<|endoftext|>",
154
+ bos_token=None,
155
+ eos_token="<|endoftext|>",
156
+ pad_token="<|endoftext|>",
157
+ clean_up_tokenization_spaces=False,
158
+ split_special_tokens=False,
159
+ **kwargs,
160
+ ):
161
+ # Dream vocab does not contain control tokens; added tokens need to be special
162
+ bos_token = (
163
+ AddedToken(bos_token, lstrip=False, rstrip=False, special=True, normalized=False)
164
+ if isinstance(bos_token, str)
165
+ else bos_token
166
+ )
167
+ eos_token = (
168
+ AddedToken(eos_token, lstrip=False, rstrip=False, special=True, normalized=False)
169
+ if isinstance(eos_token, str)
170
+ else eos_token
171
+ )
172
+ unk_token = (
173
+ AddedToken(unk_token, lstrip=False, rstrip=False, special=True, normalized=False)
174
+ if isinstance(unk_token, str)
175
+ else unk_token
176
+ )
177
+ pad_token = (
178
+ AddedToken(pad_token, lstrip=False, rstrip=False, special=True, normalized=False)
179
+ if isinstance(pad_token, str)
180
+ else pad_token
181
+ )
182
+
183
+ with open(vocab_file, encoding="utf-8") as vocab_handle:
184
+ self.encoder = json.load(vocab_handle)
185
+ self.decoder = {v: k for k, v in self.encoder.items()}
186
+ self.errors = errors # how to handle errors in decoding
187
+ self.byte_encoder = bytes_to_unicode()
188
+ self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
189
+ bpe_merges = []
190
+ with open(merges_file, encoding="utf-8") as merges_handle:
191
+ for i, line in enumerate(merges_handle):
192
+ line = line.strip()
193
+ if (i == 0 and line.startswith("#version:")) or not line:
194
+ continue
195
+ bpe_merges.append(tuple(line.split()))
196
+ self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
197
+ # NOTE: the cache can grow without bound and will get really large for long running processes
198
+ # (esp. for texts of language that do not use space between word, e.g. Chinese); technically
199
+ # not a memory leak but appears as one.
200
+ # GPT2Tokenizer has the same problem, so let's be consistent.
201
+ self.cache = {}
202
+
203
+ self.pat = re.compile(PRETOKENIZE_REGEX)
204
+
205
+ if kwargs.get("add_prefix_space", False):
206
+ logger.warning_once(
207
+ f"{self.__class__.__name} does not support `add_prefix_space`, setting it to True has no effect."
208
+ )
209
+
210
+ super().__init__(
211
+ errors=errors,
212
+ bos_token=bos_token,
213
+ eos_token=eos_token,
214
+ pad_token=pad_token,
215
+ unk_token=unk_token,
216
+ clean_up_tokenization_spaces=clean_up_tokenization_spaces,
217
+ split_special_tokens=split_special_tokens,
218
+ **kwargs,
219
+ )
220
+
221
+ @property
222
+ def vocab_size(self) -> int:
223
+ return len(self.encoder)
224
+
225
+ # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.get_vocab
226
+ def get_vocab(self):
227
+ return dict(self.encoder, **self.added_tokens_encoder)
228
+
229
+ # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.bpe
230
+ def bpe(self, token):
231
+ if token in self.cache:
232
+ return self.cache[token]
233
+ word = tuple(token)
234
+ pairs = get_pairs(word)
235
+
236
+ if not pairs:
237
+ return token
238
+
239
+ while True:
240
+ bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
241
+ if bigram not in self.bpe_ranks:
242
+ break
243
+ first, second = bigram
244
+ new_word = []
245
+ i = 0
246
+ while i < len(word):
247
+ try:
248
+ j = word.index(first, i)
249
+ except ValueError:
250
+ new_word.extend(word[i:])
251
+ break
252
+ else:
253
+ new_word.extend(word[i:j])
254
+ i = j
255
+
256
+ if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
257
+ new_word.append(first + second)
258
+ i += 2
259
+ else:
260
+ new_word.append(word[i])
261
+ i += 1
262
+ new_word = tuple(new_word)
263
+ word = new_word
264
+ if len(word) == 1:
265
+ break
266
+ else:
267
+ pairs = get_pairs(word)
268
+ word = " ".join(word)
269
+ self.cache[token] = word
270
+ return word
271
+
272
+ # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._tokenize
273
+ def _tokenize(self, text):
274
+ """Tokenize a string."""
275
+ bpe_tokens = []
276
+ for token in re.findall(self.pat, text):
277
+ token = "".join(
278
+ self.byte_encoder[b] for b in token.encode("utf-8")
279
+ ) # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
280
+ bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" "))
281
+ return bpe_tokens
282
+
283
+ # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._convert_token_to_id
284
+ def _convert_token_to_id(self, token):
285
+ """Converts a token (str) in an id using the vocab."""
286
+ return self.encoder.get(token, self.encoder.get(self.unk_token))
287
+
288
+ # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._convert_id_to_token
289
+ def _convert_id_to_token(self, index):
290
+ """Converts an index (integer) in a token (str) using the vocab."""
291
+ return self.decoder.get(index)
292
+
293
+ # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.convert_tokens_to_string
294
+ def convert_tokens_to_string(self, tokens):
295
+ """Converts a sequence of tokens (string) in a single string."""
296
+ text = "".join(tokens)
297
+ text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors)
298
+ return text
299
+
300
+ def decode(
301
+ self,
302
+ token_ids,
303
+ skip_special_tokens: bool = False,
304
+ clean_up_tokenization_spaces: Optional[bool] = False,
305
+ spaces_between_special_tokens: bool = False,
306
+ **kwargs,
307
+ ) -> str:
308
+ # `spaces_between_special_tokens` defaults to True for _decode in slow tokenizers
309
+ # and cannot be configured elsewhere, but it should default to False for DreamTokenizer
310
+ return super().decode(
311
+ token_ids,
312
+ skip_special_tokens=skip_special_tokens,
313
+ clean_up_tokenization_spaces=clean_up_tokenization_spaces,
314
+ spaces_between_special_tokens=spaces_between_special_tokens,
315
+ **kwargs,
316
+ )
317
+
318
+ # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.save_vocabulary
319
+ def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
320
+ if not os.path.isdir(save_directory):
321
+ logger.error(f"Vocabulary path ({save_directory}) should be a directory")
322
+ return
323
+ vocab_file = os.path.join(
324
+ save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
325
+ )
326
+ merge_file = os.path.join(
327
+ save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
328
+ )
329
+
330
+ with open(vocab_file, "w", encoding="utf-8") as f:
331
+ f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")
332
+
333
+ index = 0
334
+ with open(merge_file, "w", encoding="utf-8") as writer:
335
+ writer.write("#version: 0.2\n")
336
+ for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
337
+ if index != token_index:
338
+ logger.warning(
339
+ f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
340
+ " Please check that the tokenizer is not corrupted!"
341
+ )
342
+ index = token_index
343
+ writer.write(" ".join(bpe_tokens) + "\n")
344
+ index += 1
345
+
346
+ return vocab_file, merge_file
347
+
348
+ def prepare_for_tokenization(self, text, **kwargs):
349
+ text = unicodedata.normalize("NFC", text)
350
+ return (text, kwargs)
351
+
omni_diffusion/models/dream/tokenizer_config.json ADDED
@@ -0,0 +1,219 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_prefix_space": false,
4
+ "added_tokens_decoder": {
5
+ "151643": {
6
+ "content": "<|endoftext|>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "151644": {
14
+ "content": "<|im_start|>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "151645": {
22
+ "content": "<|im_end|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "151646": {
30
+ "content": "<|object_ref_start|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "151647": {
38
+ "content": "<|object_ref_end|>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ },
45
+ "151648": {
46
+ "content": "<|box_start|>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": true
52
+ },
53
+ "151649": {
54
+ "content": "<|box_end|>",
55
+ "lstrip": false,
56
+ "normalized": false,
57
+ "rstrip": false,
58
+ "single_word": false,
59
+ "special": true
60
+ },
61
+ "151650": {
62
+ "content": "<|quad_start|>",
63
+ "lstrip": false,
64
+ "normalized": false,
65
+ "rstrip": false,
66
+ "single_word": false,
67
+ "special": true
68
+ },
69
+ "151651": {
70
+ "content": "<|quad_end|>",
71
+ "lstrip": false,
72
+ "normalized": false,
73
+ "rstrip": false,
74
+ "single_word": false,
75
+ "special": true
76
+ },
77
+ "151652": {
78
+ "content": "<|vision_start|>",
79
+ "lstrip": false,
80
+ "normalized": false,
81
+ "rstrip": false,
82
+ "single_word": false,
83
+ "special": true
84
+ },
85
+ "151653": {
86
+ "content": "<|vision_end|>",
87
+ "lstrip": false,
88
+ "normalized": false,
89
+ "rstrip": false,
90
+ "single_word": false,
91
+ "special": true
92
+ },
93
+ "151654": {
94
+ "content": "<|vision_pad|>",
95
+ "lstrip": false,
96
+ "normalized": false,
97
+ "rstrip": false,
98
+ "single_word": false,
99
+ "special": true
100
+ },
101
+ "151655": {
102
+ "content": "<|image_pad|>",
103
+ "lstrip": false,
104
+ "normalized": false,
105
+ "rstrip": false,
106
+ "single_word": false,
107
+ "special": true
108
+ },
109
+ "151656": {
110
+ "content": "<|video_pad|>",
111
+ "lstrip": false,
112
+ "normalized": false,
113
+ "rstrip": false,
114
+ "single_word": false,
115
+ "special": true
116
+ },
117
+ "151657": {
118
+ "content": "<tool_call>",
119
+ "lstrip": false,
120
+ "normalized": false,
121
+ "rstrip": false,
122
+ "single_word": false,
123
+ "special": false
124
+ },
125
+ "151658": {
126
+ "content": "</tool_call>",
127
+ "lstrip": false,
128
+ "normalized": false,
129
+ "rstrip": false,
130
+ "single_word": false,
131
+ "special": false
132
+ },
133
+ "151659": {
134
+ "content": "<|fim_prefix|>",
135
+ "lstrip": false,
136
+ "normalized": false,
137
+ "rstrip": false,
138
+ "single_word": false,
139
+ "special": false
140
+ },
141
+ "151660": {
142
+ "content": "<|fim_middle|>",
143
+ "lstrip": false,
144
+ "normalized": false,
145
+ "rstrip": false,
146
+ "single_word": false,
147
+ "special": false
148
+ },
149
+ "151661": {
150
+ "content": "<|fim_suffix|>",
151
+ "lstrip": false,
152
+ "normalized": false,
153
+ "rstrip": false,
154
+ "single_word": false,
155
+ "special": false
156
+ },
157
+ "151662": {
158
+ "content": "<|fim_pad|>",
159
+ "lstrip": false,
160
+ "normalized": false,
161
+ "rstrip": false,
162
+ "single_word": false,
163
+ "special": false
164
+ },
165
+ "151663": {
166
+ "content": "<|repo_name|>",
167
+ "lstrip": false,
168
+ "normalized": false,
169
+ "rstrip": false,
170
+ "single_word": false,
171
+ "special": false
172
+ },
173
+ "151664": {
174
+ "content": "<|file_sep|>",
175
+ "lstrip": false,
176
+ "normalized": false,
177
+ "rstrip": false,
178
+ "single_word": false,
179
+ "special": false
180
+ },
181
+ "151665": {
182
+ "content": "<|beginoftext|>",
183
+ "lstrip": false,
184
+ "normalized": false,
185
+ "rstrip": false,
186
+ "single_word": false,
187
+ "special": true
188
+ },
189
+ "151666": {
190
+ "content": "<|mask|>",
191
+ "lstrip": false,
192
+ "normalized": false,
193
+ "rstrip": false,
194
+ "single_word": false,
195
+ "special": true
196
+ }
197
+ },
198
+ "additional_special_tokens": [
199
+ "<|beginoftext|>",
200
+ "<|mask|>"
201
+ ],
202
+ "auto_map": {
203
+ "AutoTokenizer": [
204
+ "tokenization_dream.DreamTokenizer",
205
+ null
206
+ ]
207
+ },
208
+ "bos_token": "<|beginoftext|>",
209
+ "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0]['role'] == 'system' %}\n {{- messages[0]['content'] }}\n {%- else %}\n {{- 'You are a helpful assistant.' }}\n {%- endif %}\n {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>system\\nYou are a helpful assistant.<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role }}\n {%- if message.content %}\n {{- '\\n' + message.content }}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- message.content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
210
+ "clean_up_tokenization_spaces": false,
211
+ "eos_token": "<|endoftext|>",
212
+ "errors": "replace",
213
+ "mask_token": "<|mask|>",
214
+ "model_max_length": 131072,
215
+ "pad_token": "<|endoftext|>",
216
+ "split_special_tokens": false,
217
+ "tokenizer_class": "DreamTokenizer",
218
+ "unk_token": null
219
+ }
omni_diffusion/models/magvit/common_modules.py ADDED
@@ -0,0 +1,393 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Modified from https://github.com/CompVis/taming-transformers/blob/master/taming/modules/diffusionmodules/model.py#L34
3
+ """
4
+
5
+ import math
6
+ from typing import Tuple, Union
7
+
8
+ import numpy as np
9
+ import torch
10
+ import torch.nn as nn
11
+ import torch.nn.functional as F
12
+ from einops import rearrange, repeat
13
+ from einops.layers.torch import Rearrange
14
+
15
+
16
+ def nonlinearity(x):
17
+ # swish
18
+ return x * torch.sigmoid(x)
19
+
20
+
21
+ def Normalize(in_channels):
22
+ return torch.nn.GroupNorm(
23
+ num_groups=32, num_channels=in_channels, eps=1e-6, affine=True
24
+ )
25
+
26
+
27
+ class Upsample(nn.Module):
28
+ """
29
+ Spatial upsampling layer.
30
+ Upsamples by a factor of 2 using nearest neighbor interpolation,
31
+ optionally followed by a convolution.
32
+ """
33
+ def __init__(self, in_channels, with_conv):
34
+ super().__init__()
35
+ self.with_conv = with_conv
36
+ if self.with_conv:
37
+ self.conv = torch.nn.Conv2d(
38
+ in_channels, in_channels, kernel_size=3, stride=1, padding=1
39
+ )
40
+
41
+ def forward(self, x):
42
+ x = torch.nn.functional.interpolate(x, scale_factor=2.0, mode="nearest")
43
+ if self.with_conv:
44
+ x = self.conv(x)
45
+ return x
46
+
47
+
48
+ class DepthToSpaceUpsample(nn.Module):
49
+ """
50
+ Upsamples spatially by moving channel depth to spatial dimensions (PixelShuffle).
51
+ Expands channels by 4, then rearranges (c * 2 * 2) -> (h * 2, w * 2).
52
+ """
53
+ def __init__(
54
+ self,
55
+ in_channels,
56
+ ):
57
+ super().__init__()
58
+ conv = nn.Conv2d(in_channels, in_channels * 4, 1)
59
+
60
+ self.net = nn.Sequential(
61
+ conv,
62
+ nn.SiLU(),
63
+ Rearrange("b (c p1 p2) h w -> b c (h p1) (w p2)", p1=2, p2=2),
64
+ )
65
+
66
+ self.init_conv_(conv)
67
+
68
+ def init_conv_(self, conv):
69
+ o, i, h, w = conv.weight.shape
70
+ conv_weight = torch.empty(o // 4, i, h, w)
71
+ nn.init.kaiming_uniform_(conv_weight)
72
+ conv_weight = repeat(conv_weight, "o ... -> (o 4) ...")
73
+
74
+ conv.weight.data.copy_(conv_weight)
75
+ nn.init.zeros_(conv.bias.data)
76
+
77
+ def forward(self, x):
78
+ out = self.net(x)
79
+ return out
80
+
81
+
82
+ class Downsample(nn.Module):
83
+ """
84
+ Spatial downsampling layer.
85
+ Uses either a strided convolution (with padding) or average pooling.
86
+ """
87
+ def __init__(self, in_channels, with_conv):
88
+ super().__init__()
89
+ self.with_conv = with_conv
90
+ if self.with_conv:
91
+ # no asymmetric padding in torch conv, must do it ourselves
92
+ self.conv = torch.nn.Conv2d(
93
+ in_channels, in_channels, kernel_size=3, stride=2, padding=0
94
+ )
95
+
96
+ def forward(self, x):
97
+ if self.with_conv:
98
+ pad = (0, 1, 0, 1)
99
+ x = torch.nn.functional.pad(x, pad, mode="constant", value=0)
100
+ x = self.conv(x)
101
+ else:
102
+ x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2)
103
+ return x
104
+
105
+
106
+ def unpack_time(t, batch):
107
+ _, c, w, h = t.size()
108
+ out = torch.reshape(t, [batch, -1, c, w, h])
109
+ out = rearrange(out, "b t c h w -> b c t h w")
110
+ return out
111
+
112
+
113
+ def pack_time(t):
114
+ out = rearrange(t, "b c t h w -> b t c h w")
115
+ _, _, c, w, h = out.size()
116
+ return torch.reshape(out, [-1, c, w, h])
117
+
118
+
119
+ class TimeDownsample2x(nn.Module):
120
+ """
121
+ Temporal downsampling layer.
122
+ Reduces the time dimension (T) by half using 1D convolution with causal padding.
123
+ """
124
+ def __init__(
125
+ self,
126
+ dim,
127
+ dim_out=None,
128
+ kernel_size=3,
129
+ ):
130
+ super().__init__()
131
+ if dim_out is None:
132
+ dim_out = dim
133
+ self.time_causal_padding = (kernel_size - 1, 0)
134
+ self.conv = nn.Conv1d(dim, dim_out, kernel_size, stride=2)
135
+
136
+ def forward(self, x):
137
+ x = rearrange(x, "b c t h w -> b h w c t")
138
+ b, h, w, c, t = x.size()
139
+ x = torch.reshape(x, [-1, c, t])
140
+
141
+ x = F.pad(x, self.time_causal_padding)
142
+ out = self.conv(x)
143
+
144
+ out = torch.reshape(out, [b, h, w, c, t])
145
+ out = rearrange(out, "b h w c t -> b c t h w")
146
+ out = rearrange(out, "b h w c t -> b c t h w")
147
+ return out
148
+
149
+
150
+ class TimeUpsample2x(nn.Module):
151
+ """
152
+ Temporal upsampling layer.
153
+ Doubles the time dimension (T) using a PixelShuffle-like approach on the time axis.
154
+ """
155
+ def __init__(self, dim, dim_out=None):
156
+ super().__init__()
157
+ if dim_out is None:
158
+ dim_out = dim
159
+ conv = nn.Conv1d(dim, dim_out * 2, 1)
160
+
161
+ self.net = nn.Sequential(
162
+ nn.SiLU(), conv, Rearrange("b (c p) t -> b c (t p)", p=2)
163
+ )
164
+
165
+ self.init_conv_(conv)
166
+
167
+ def init_conv_(self, conv):
168
+ o, i, t = conv.weight.shape
169
+ conv_weight = torch.empty(o // 2, i, t)
170
+ nn.init.kaiming_uniform_(conv_weight)
171
+ conv_weight = repeat(conv_weight, "o ... -> (o 2) ...")
172
+
173
+ conv.weight.data.copy_(conv_weight)
174
+ nn.init.zeros_(conv.bias.data)
175
+
176
+ def forward(self, x):
177
+ x = rearrange(x, "b c t h w -> b h w c t")
178
+ b, h, w, c, t = x.size()
179
+ x = torch.reshape(x, [-1, c, t])
180
+
181
+ out = self.net(x)
182
+ out = out[:, :, 1:].contiguous()
183
+
184
+ out = torch.reshape(out, [b, h, w, c, t])
185
+ out = rearrange(out, "b h w c t -> b c t h w")
186
+ return out
187
+
188
+
189
+ class AttnBlock(nn.Module):
190
+ """
191
+ Standard Self-Attention Block for spatial dimensions.
192
+ """
193
+ def __init__(self, in_channels):
194
+ super().__init__()
195
+ self.in_channels = in_channels
196
+
197
+ self.norm = Normalize(in_channels)
198
+ self.q = torch.nn.Conv2d(
199
+ in_channels, in_channels, kernel_size=1, stride=1, padding=0
200
+ )
201
+ self.k = torch.nn.Conv2d(
202
+ in_channels, in_channels, kernel_size=1, stride=1, padding=0
203
+ )
204
+ self.v = torch.nn.Conv2d(
205
+ in_channels, in_channels, kernel_size=1, stride=1, padding=0
206
+ )
207
+ self.proj_out = torch.nn.Conv2d(
208
+ in_channels, in_channels, kernel_size=1, stride=1, padding=0
209
+ )
210
+
211
+ def forward(self, x):
212
+ h_ = x
213
+ h_ = self.norm(h_)
214
+ q = self.q(h_)
215
+ k = self.k(h_)
216
+ v = self.v(h_)
217
+
218
+ # compute attention
219
+ b, c, h, w = q.shape
220
+ q = q.reshape(b, c, h * w)
221
+ q = q.permute(0, 2, 1) # b,hw,c
222
+ k = k.reshape(b, c, h * w) # b,c,hw
223
+ w_ = torch.bmm(q, k) # b,hw,hw w[b,i,j]=sum_c q[b,i,c]k[b,c,j]
224
+ w_ = w_ * (int(c) ** (-0.5))
225
+ w_ = torch.nn.functional.softmax(w_, dim=2)
226
+
227
+ # attend to values
228
+ v = v.reshape(b, c, h * w)
229
+ w_ = w_.permute(0, 2, 1) # b,hw,hw (first hw of k, second of q)
230
+ h_ = torch.bmm(v, w_) # b, c,hw (hw of q) h_[b,c,j] = sum_i v[b,c,i] w_[b,i,j]
231
+ h_ = h_.reshape(b, c, h, w)
232
+
233
+ h_ = self.proj_out(h_)
234
+
235
+ return x + h_
236
+
237
+
238
+ class TimeAttention(AttnBlock):
239
+ """
240
+ Applies the AttnBlock over the temporal dimension (T).
241
+ It treats spatial dimensions (H, W) as part of the batch.
242
+ """
243
+ def forward(self, x, *args, **kwargs):
244
+ x = rearrange(x, "b c t h w -> b h w t c")
245
+ b, h, w, t, c = x.size()
246
+ x = torch.reshape(x, (-1, t, c))
247
+
248
+ x = super().forward(x, *args, **kwargs)
249
+
250
+ x = torch.reshape(x, [b, h, w, t, c])
251
+ return rearrange(x, "b h w t c -> b c t h w")
252
+
253
+
254
+ class Residual(nn.Module):
255
+ def __init__(self, fn: nn.Module):
256
+ super().__init__()
257
+ self.fn = fn
258
+
259
+ def forward(self, x, **kwargs):
260
+ return self.fn(x, **kwargs) + x
261
+
262
+
263
+ def cast_tuple(t, length=1):
264
+ return t if isinstance(t, tuple) else ((t,) * length)
265
+
266
+
267
+ class CausalConv3d(nn.Module):
268
+ """
269
+ 3D Convolution with Causal Padding on the time axis.
270
+ Ensures that the convolution at time T only depends on frames <= T.
271
+ """
272
+ def __init__(
273
+ self,
274
+ chan_in,
275
+ chan_out,
276
+ kernel_size: Union[int, Tuple[int, int, int]],
277
+ pad_mode="constant",
278
+ **kwargs
279
+ ):
280
+ super().__init__()
281
+ kernel_size = cast_tuple(kernel_size, 3)
282
+
283
+ time_kernel_size, height_kernel_size, width_kernel_size = kernel_size
284
+
285
+ dilation = kwargs.pop("dilation", 1)
286
+ stride = kwargs.pop("stride", 1)
287
+
288
+ self.pad_mode = pad_mode
289
+ time_pad = dilation * (time_kernel_size - 1) + (1 - stride)
290
+ height_pad = height_kernel_size // 2
291
+ width_pad = width_kernel_size // 2
292
+
293
+ self.time_pad = time_pad
294
+ self.time_causal_padding = (
295
+ width_pad,
296
+ width_pad,
297
+ height_pad,
298
+ height_pad,
299
+ time_pad,
300
+ 0,
301
+ )
302
+
303
+ stride = (stride, 1, 1)
304
+ dilation = (dilation, 1, 1)
305
+ self.conv = nn.Conv3d(
306
+ chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs
307
+ )
308
+
309
+ def forward(self, x):
310
+ pad_mode = self.pad_mode if self.time_pad < x.shape[2] else "constant"
311
+
312
+ x = F.pad(x, self.time_causal_padding, mode=pad_mode)
313
+ return self.conv(x)
314
+
315
+
316
+ def ResnetBlockCausal3D(
317
+ dim, kernel_size: Union[int, Tuple[int, int, int]], pad_mode: str = "constant"
318
+ ):
319
+ net = nn.Sequential(
320
+ Normalize(dim),
321
+ nn.SiLU(),
322
+ CausalConv3d(dim, dim, kernel_size, pad_mode),
323
+ Normalize(dim),
324
+ nn.SiLU(),
325
+ CausalConv3d(dim, dim, kernel_size, pad_mode),
326
+ )
327
+ return Residual(net)
328
+
329
+
330
+ class ResnetBlock(nn.Module):
331
+ """
332
+ Standard ResNet Block commonly used in Diffusion Models.
333
+ Includes support for Time Embeddings (temb) injection.
334
+ """
335
+ def __init__(
336
+ self,
337
+ *,
338
+ in_channels,
339
+ out_channels=None,
340
+ conv_shortcut=False,
341
+ dropout,
342
+ temb_channels=512
343
+ ):
344
+ super().__init__()
345
+ self.in_channels = in_channels
346
+ out_channels = in_channels if out_channels is None else out_channels
347
+ self.out_channels = out_channels
348
+ self.use_conv_shortcut = conv_shortcut
349
+
350
+ self.norm1 = Normalize(in_channels)
351
+ self.conv1 = torch.nn.Conv2d(
352
+ in_channels, out_channels, kernel_size=3, stride=1, padding=1
353
+ )
354
+ if temb_channels > 0:
355
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
356
+ else:
357
+ self.temb_proj = None
358
+ self.norm2 = Normalize(out_channels)
359
+ self.dropout = torch.nn.Dropout(dropout)
360
+ self.conv2 = torch.nn.Conv2d(
361
+ out_channels, out_channels, kernel_size=3, stride=1, padding=1
362
+ )
363
+ if self.in_channels != self.out_channels:
364
+ if self.use_conv_shortcut:
365
+ self.conv_shortcut = torch.nn.Conv2d(
366
+ in_channels, out_channels, kernel_size=3, stride=1, padding=1
367
+ )
368
+ else:
369
+ self.nin_shortcut = torch.nn.Conv2d(
370
+ in_channels, out_channels, kernel_size=1, stride=1, padding=0
371
+ )
372
+
373
+ def forward(self, x, temb):
374
+ h = x
375
+ h = self.norm1(h)
376
+ h = nonlinearity(h)
377
+ h = self.conv1(h)
378
+
379
+ if temb is not None:
380
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
381
+
382
+ h = self.norm2(h)
383
+ h = nonlinearity(h)
384
+ h = self.dropout(h)
385
+ h = self.conv2(h)
386
+
387
+ if self.in_channels != self.out_channels:
388
+ if self.use_conv_shortcut:
389
+ x = self.conv_shortcut(x)
390
+ else:
391
+ x = self.nin_shortcut(x)
392
+
393
+ return x + h
omni_diffusion/models/magvit/misc.py ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # copy from MMaDA
2
+
3
+ from omegaconf import OmegaConf
4
+ import torch
5
+ from typing import (
6
+ Any,
7
+ Callable,
8
+ Dict,
9
+ Iterable,
10
+ List,
11
+ NamedTuple,
12
+ NewType,
13
+ Optional,
14
+ Sized,
15
+ Tuple,
16
+ Type,
17
+ TypeVar,
18
+ Union,
19
+ )
20
+ try:
21
+ from typing import Literal
22
+ except ImportError:
23
+ from typing_extensions import Literal
24
+
25
+ # Tensor dtype
26
+ # for jaxtyping usage, see https://github.com/google/jaxtyping/blob/main/API.md
27
+ from jaxtyping import Bool, Complex, Float, Inexact, Int, Integer, Num, Shaped, UInt
28
+
29
+ # Config type
30
+ from omegaconf import DictConfig
31
+
32
+ # PyTorch Tensor type
33
+ from torch import Tensor
34
+
35
+ # Runtime type checking decorator
36
+ from typeguard import typechecked as typechecker
37
+
38
+
39
+ def broadcast(tensor, src=0):
40
+ if not _distributed_available():
41
+ return tensor
42
+ else:
43
+ torch.distributed.broadcast(tensor, src=src)
44
+ return tensor
45
+
46
+ def _distributed_available():
47
+ return torch.distributed.is_available() and torch.distributed.is_initialized()
48
+
49
+ def parse_structured(fields: Any, cfg: Optional[Union[dict, DictConfig]] = None) -> Any:
50
+ # added by Xavier -- delete '--local-rank' in multi-nodes training, don't know why there is such a keyword
51
+ if '--local-rank' in cfg:
52
+ del cfg['--local-rank']
53
+ # added by Xavier -- delete '--local-rank' in multi-nodes training, don't know why there is such a keyword
54
+ scfg = OmegaConf.structured(fields(**cfg))
55
+ return scfg
omni_diffusion/models/magvit/modeling_magvitv2.py ADDED
@@ -0,0 +1,445 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Modified from https://github.com/Gen-Verse/MMaDA/blob/main/models/modeling_magvitv2.py
3
+ """
4
+
5
+ from dataclasses import dataclass, field
6
+ import numpy as np
7
+ import torch
8
+ import torch.nn as nn
9
+ from .common_modules import *
10
+ from .modeling_utils import ConfigMixin, ModelMixin, register_to_config
11
+ from .misc import *
12
+ import math
13
+
14
+ class Updateable:
15
+ def do_update_step(
16
+ self, epoch: int, global_step: int, on_load_weights: bool = False
17
+ ):
18
+ for attr in self.__dir__():
19
+ if attr.startswith("_"):
20
+ continue
21
+ try:
22
+ module = getattr(self, attr)
23
+ except:
24
+ continue # ignore attributes like property, which can't be retrived using getattr?
25
+ if isinstance(module, Updateable):
26
+ module.do_update_step(
27
+ epoch, global_step, on_load_weights=on_load_weights
28
+ )
29
+ self.update_step(epoch, global_step, on_load_weights=on_load_weights)
30
+
31
+ def do_update_step_end(self, epoch: int, global_step: int):
32
+ for attr in self.__dir__():
33
+ if attr.startswith("_"):
34
+ continue
35
+ try:
36
+ module = getattr(self, attr)
37
+ except:
38
+ continue # ignore attributes like property, which can't be retrived using getattr?
39
+ if isinstance(module, Updateable):
40
+ module.do_update_step_end(epoch, global_step)
41
+ self.update_step_end(epoch, global_step)
42
+
43
+ def update_step(self, epoch: int, global_step: int, on_load_weights: bool = False):
44
+ # override this method to implement custom update logic
45
+ # if on_load_weights is True, you should be careful doing things related to model evaluations,
46
+ # as the models and tensors are not guarenteed to be on the same device
47
+ pass
48
+
49
+ def update_step_end(self, epoch: int, global_step: int):
50
+ pass
51
+
52
+ class VQGANEncoder(ModelMixin, ConfigMixin):
53
+ @dataclass
54
+ class Config:
55
+ ch: int = 128
56
+ ch_mult: List[int] = field(default_factory=lambda: [1, 2, 2, 4, 4])
57
+ num_res_blocks: List[int] = field(default_factory=lambda: [4, 3, 4, 3, 4])
58
+ attn_resolutions: List[int] = field(default_factory=lambda: [5])
59
+ dropout: float = 0.0
60
+ in_ch: int = 3
61
+ out_ch: int = 3
62
+ resolution: int = 256
63
+ z_channels: int = 13
64
+ double_z: bool = False
65
+
66
+ def __init__(self,
67
+ ch: int = 128,
68
+ ch_mult: List[int] = [1, 2, 2, 4, 4],
69
+ num_res_blocks: List[int] = [4, 3, 4, 3, 4],
70
+ attn_resolutions: List[int] = [5],
71
+ dropout: float = 0.0,
72
+ in_ch: int = 3,
73
+ out_ch: int = 3,
74
+ resolution: int = 256,
75
+ z_channels: int = 13,
76
+ double_z: bool = False):
77
+ super().__init__()
78
+ self.ch = ch
79
+ self.temb_ch = 0
80
+ self.num_resolutions = len(ch_mult)
81
+ self.num_res_blocks = num_res_blocks
82
+ self.resolution = resolution
83
+ self.in_ch = in_ch
84
+ # downsampling
85
+ self.conv_in = torch.nn.Conv2d(
86
+ self.in_ch, self.ch, kernel_size=3, stride=1, padding=1
87
+ )
88
+
89
+ curr_res = self.resolution
90
+ in_ch_mult = (1,) + tuple(ch_mult)
91
+ self.down = nn.ModuleList()
92
+ for i_level in range(self.num_resolutions):
93
+ block = nn.ModuleList()
94
+ attn = nn.ModuleList()
95
+ block_in = self.ch * in_ch_mult[i_level]
96
+ block_out = self.ch * ch_mult[i_level]
97
+ for i_block in range(self.num_res_blocks[i_level]):
98
+ block.append(
99
+ ResnetBlock(
100
+ in_channels=block_in,
101
+ out_channels=block_out,
102
+ temb_channels=self.temb_ch,
103
+ dropout=dropout,
104
+ )
105
+ )
106
+ block_in = block_out
107
+ if curr_res in attn_resolutions:
108
+ attn.append(AttnBlock(block_in))
109
+ down = nn.Module()
110
+ down.block = block
111
+ down.attn = attn
112
+ if i_level != self.num_resolutions - 1:
113
+ down.downsample = Downsample(block_in, True)
114
+ curr_res = curr_res // 2
115
+ self.down.append(down)
116
+
117
+ # middle
118
+ self.mid = nn.Module()
119
+ self.mid.block_1 = ResnetBlock(
120
+ in_channels=block_in,
121
+ out_channels=block_in,
122
+ temb_channels=self.temb_ch,
123
+ dropout=dropout,
124
+ )
125
+ self.mid.attn_1 = AttnBlock(block_in)
126
+ self.mid.block_2 = ResnetBlock(
127
+ in_channels=block_in,
128
+ out_channels=block_in,
129
+ temb_channels=self.temb_ch,
130
+ dropout=dropout,
131
+ )
132
+
133
+
134
+ self.norm_out = Normalize(block_in)
135
+ self.conv_out = torch.nn.Conv2d(
136
+ block_in,
137
+ 2 * z_channels if double_z else z_channels,
138
+ kernel_size=3,
139
+ stride=1,
140
+ padding=1,
141
+ )
142
+
143
+ self.quant_conv = torch.nn.Conv2d(z_channels, z_channels, 1)
144
+ # for param in self.parameters():
145
+ # broadcast(param, src=0)
146
+
147
+ def forward(self, x):
148
+ # timestep embedding
149
+ temb = None
150
+
151
+ # downsampling
152
+ hs = [self.conv_in(x)]
153
+ for i_level in range(self.num_resolutions):
154
+ for i_block in range(self.num_res_blocks[i_level]):
155
+ h = self.down[i_level].block[i_block](hs[-1], temb)
156
+ if len(self.down[i_level].attn) > 0:
157
+ h = self.down[i_level].attn[i_block](h)
158
+ hs.append(h)
159
+ if i_level != self.num_resolutions - 1:
160
+ hs.append(self.down[i_level].downsample(hs[-1]))
161
+
162
+ # middle
163
+ h = hs[-1]
164
+ h = self.mid.block_1(h, temb)
165
+ h = self.mid.attn_1(h)
166
+ h = self.mid.block_2(h, temb)
167
+
168
+ # end
169
+ h = self.norm_out(h)
170
+ h = nonlinearity(h)
171
+ h = self.conv_out(h)
172
+ h = self.quant_conv(h)
173
+ return h
174
+
175
+
176
+ class LFQuantizer(nn.Module):
177
+ def __init__(self, num_codebook_entry: int = -1,
178
+ codebook_dim: int = 13,
179
+ beta: float = 0.25,
180
+ entropy_multiplier: float = 0.1,
181
+ commit_loss_multiplier: float = 0.1, ):
182
+ super().__init__()
183
+ self.codebook_size = 2 ** codebook_dim
184
+ print(
185
+ f"Look-up free quantizer with codebook size: {self.codebook_size}"
186
+ )
187
+ self.e_dim = codebook_dim
188
+ self.beta = beta
189
+
190
+ indices = torch.arange(self.codebook_size)
191
+
192
+ binary = (
193
+ indices.unsqueeze(1)
194
+ >> torch.arange(codebook_dim - 1, -1, -1, dtype=torch.long)
195
+ ) & 1
196
+
197
+ embedding = binary.float() * 2 - 1
198
+ self.register_buffer("embedding", embedding)
199
+ self.register_buffer(
200
+ "power_vals", 2 ** torch.arange(codebook_dim - 1, -1, -1)
201
+ )
202
+ self.commit_loss_multiplier = commit_loss_multiplier
203
+ self.entropy_multiplier = entropy_multiplier
204
+
205
+ def get_indices(self, z_q):
206
+ return (
207
+ (self.power_vals.reshape(1, -1, 1, 1) * (z_q > 0).float())
208
+ .sum(1, keepdim=True)
209
+ .long()
210
+ )
211
+
212
+ def get_codebook_entry(self, indices, shape=None):
213
+ if shape is None:
214
+ h, w = int(math.sqrt(indices.shape[-1])), int(math.sqrt(indices.shape[-1]))
215
+ else:
216
+ h, w = shape
217
+ b, _ = indices.shape
218
+ indices = indices.reshape(-1)
219
+ z_q = self.embedding[indices]
220
+ z_q = z_q.view(b, h, w, -1)
221
+
222
+ # reshape back to match original input shape
223
+ z_q = z_q.permute(0, 3, 1, 2).contiguous()
224
+
225
+ return z_q
226
+
227
+ def forward(self, z, get_code=False):
228
+ """
229
+ Inputs the output of the encoder network z and maps it to a discrete
230
+ one-hot vector that is the index of the closest embedding vector e_j
231
+ z (continuous) -> z_q (discrete)
232
+ z.shape = (batch, channel, height, width)
233
+ quantization pipeline:
234
+ 1. get encoder input (B,C,H,W)
235
+ 2. flatten input to (B*H*W,C)
236
+ """
237
+ if get_code:
238
+ return self.get_codebook_entry(z)
239
+
240
+ # reshape z -> (batch, height, width, channel) and flatten
241
+ z = z.permute(0, 2, 3, 1).contiguous()
242
+ z_flattened = z.view(-1, self.e_dim)
243
+ ge_zero = (z_flattened > 0).float()
244
+ ones = torch.ones_like(z_flattened)
245
+ z_q = ones * ge_zero + -ones * (1 - ge_zero)
246
+
247
+ # preserve gradients
248
+ z_q = z_flattened + (z_q - z_flattened).detach()
249
+
250
+ # compute entropy loss
251
+ CatDist = torch.distributions.categorical.Categorical
252
+ logit = torch.stack(
253
+ [
254
+ -(z_flattened - torch.ones_like(z_q)).pow(2),
255
+ -(z_flattened - torch.ones_like(z_q) * -1).pow(2),
256
+ ],
257
+ dim=-1,
258
+ )
259
+ cat_dist = CatDist(logits=logit)
260
+ entropy = cat_dist.entropy().mean()
261
+ mean_prob = cat_dist.probs.mean(0)
262
+ mean_entropy = CatDist(probs=mean_prob).entropy().mean()
263
+
264
+ # compute loss for embedding
265
+ commit_loss = torch.mean(
266
+ (z_q.detach() - z_flattened) ** 2
267
+ ) + self.beta * torch.mean((z_q - z_flattened.detach()) ** 2)
268
+
269
+ # reshape back to match original input shape
270
+ z_q = z_q.view(z.shape)
271
+ z_q = z_q.permute(0, 3, 1, 2).contiguous()
272
+
273
+ return {
274
+ "z": z_q,
275
+ "quantizer_loss": commit_loss * self.commit_loss_multiplier,
276
+ "entropy_loss": (entropy - mean_entropy) * self.entropy_multiplier,
277
+ "indices": self.get_indices(z_q),
278
+ }
279
+
280
+
281
+ class VQGANDecoder(ModelMixin, ConfigMixin):
282
+ def __init__(self, ch: int = 128,
283
+ ch_mult: List[int] = [1, 1, 2, 2, 4],
284
+ num_res_blocks: List[int] = [4, 4, 3, 4, 3],
285
+ attn_resolutions: List[int] = [5],
286
+ dropout: float = 0.0,
287
+ in_ch: int = 3,
288
+ out_ch: int = 3,
289
+ resolution: int = 256,
290
+ z_channels: int = 13,
291
+ double_z: bool = False):
292
+ super().__init__()
293
+ self.ch = ch
294
+ self.temb_ch = 0
295
+ self.num_resolutions = len(ch_mult)
296
+ self.num_res_blocks = num_res_blocks
297
+ self.resolution = resolution
298
+ self.in_ch = in_ch
299
+ self.give_pre_end = False
300
+
301
+ self.z_channels = z_channels
302
+ # compute in_ch_mult, block_in and curr_res at lowest res
303
+ in_ch_mult = (1,) + tuple(ch_mult)
304
+ block_in = ch * ch_mult[self.num_resolutions - 1]
305
+ curr_res = self.resolution // 2 ** (self.num_resolutions - 1)
306
+ self.z_shape = (1, z_channels, curr_res, curr_res)
307
+ print(
308
+ "Working with z of shape {} = {} dimensions.".format(
309
+ self.z_shape, np.prod(self.z_shape)
310
+ )
311
+ )
312
+
313
+ # z to block_in
314
+ self.conv_in = torch.nn.Conv2d(
315
+ z_channels, block_in, kernel_size=3, stride=1, padding=1
316
+ )
317
+
318
+ # middle
319
+ self.mid = nn.Module()
320
+ self.mid.block_1 = ResnetBlock(
321
+ in_channels=block_in,
322
+ out_channels=block_in,
323
+ temb_channels=self.temb_ch,
324
+ dropout=dropout,
325
+ )
326
+ self.mid.attn_1 = AttnBlock(block_in)
327
+ self.mid.block_2 = ResnetBlock(
328
+ in_channels=block_in,
329
+ out_channels=block_in,
330
+ temb_channels=self.temb_ch,
331
+ dropout=dropout,
332
+ )
333
+
334
+ # upsampling
335
+ self.up = nn.ModuleList()
336
+ for i_level in reversed(range(self.num_resolutions)):
337
+ block = nn.ModuleList()
338
+ attn = nn.ModuleList()
339
+ block_out = ch * ch_mult[i_level]
340
+ for i_block in range(self.num_res_blocks[i_level]):
341
+ block.append(
342
+ ResnetBlock(
343
+ in_channels=block_in,
344
+ out_channels=block_out,
345
+ temb_channels=self.temb_ch,
346
+ dropout=dropout,
347
+ )
348
+ )
349
+ block_in = block_out
350
+ if curr_res in attn_resolutions:
351
+ attn.append(AttnBlock(block_in))
352
+ up = nn.Module()
353
+ up.block = block
354
+ up.attn = attn
355
+ if i_level != 0:
356
+ up.upsample = Upsample(block_in, True)
357
+ curr_res = curr_res * 2
358
+ self.up.insert(0, up) # prepend to get consistent order
359
+
360
+ self.norm_out = Normalize(block_in)
361
+ self.conv_out = torch.nn.Conv2d(
362
+ block_in, out_ch, kernel_size=3, stride=1, padding=1
363
+ )
364
+ self.post_quant_conv = torch.nn.Conv2d(
365
+ z_channels, z_channels, 1
366
+ )
367
+
368
+
369
+ def forward(self, z):
370
+ # assert z.shape[1:] == self.z_shape[1:]
371
+ self.last_z_shape = z.shape
372
+ # timestep embedding
373
+ temb = None
374
+ output = dict()
375
+ z = self.post_quant_conv(z)
376
+
377
+ # z to block_in
378
+ h = self.conv_in(z)
379
+
380
+ # middle
381
+ h = self.mid.block_1(h, temb)
382
+ h = self.mid.attn_1(h)
383
+ h = self.mid.block_2(h, temb)
384
+
385
+ # upsampling
386
+ for i_level in reversed(range(self.num_resolutions)):
387
+ for i_block in range(self.num_res_blocks[i_level]):
388
+ h = self.up[i_level].block[i_block](h, temb)
389
+ if len(self.up[i_level].attn) > 0:
390
+ h = self.up[i_level].attn[i_block](h)
391
+ if i_level != 0:
392
+ h = self.up[i_level].upsample(h)
393
+
394
+ # end
395
+ output["output"] = h
396
+ if self.give_pre_end:
397
+ return output
398
+
399
+ h = self.norm_out(h)
400
+ h = nonlinearity(h)
401
+ h = self.conv_out(h)
402
+ output["output"] = h
403
+ return output
404
+
405
+
406
+ class MAGVITv2(ModelMixin, ConfigMixin):
407
+ @register_to_config
408
+ def __init__(
409
+ self,
410
+ ):
411
+ super().__init__()
412
+
413
+ self.encoder = VQGANEncoder()
414
+ self.decoder = VQGANDecoder()
415
+ self.quantize = LFQuantizer()
416
+
417
+ def forward(self, pixel_values, return_loss=False):
418
+ pass
419
+
420
+ def encode(self, pixel_values, return_loss=False):
421
+ hidden_states = self.encoder(pixel_values)
422
+ quantized_states = self.quantize(hidden_states)['z']
423
+ codebook_indices = self.quantize.get_indices(quantized_states).reshape(pixel_values.shape[0], -1)
424
+ output = (quantized_states, codebook_indices)
425
+ return output
426
+
427
+ def get_code(self, pixel_values):
428
+ hidden_states = self.encoder(pixel_values)
429
+ codebook_indices = self.quantize.get_indices(self.quantize(hidden_states)['z'])
430
+ codebook_indices = codebook_indices.reshape(pixel_values.shape[0], -1)
431
+
432
+ return codebook_indices
433
+
434
+ def decode_code(self, codebook_indices, shape=None):
435
+ z_q = self.quantize.get_codebook_entry(codebook_indices, shape=shape)
436
+
437
+ reconstructed_pixel_values = self.decoder(z_q)["output"]
438
+ return reconstructed_pixel_values
439
+
440
+
441
+ if __name__ == '__main__':
442
+ encoder = VQGANEncoder()
443
+ import ipdb
444
+ ipdb.set_trace()
445
+ print()
omni_diffusion/models/magvit/modeling_utils.py ADDED
@@ -0,0 +1,1184 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Modified from https://github.com/Gen-Verse/MMaDA/blob/main/models/modeling_utils.py
3
+ """
4
+
5
+ # coding=utf-8
6
+ # Copyright 2024 The HuggingFace Inc. team.
7
+ # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
8
+ #
9
+ # Licensed under the Apache License, Version 2.0 (the "License");
10
+ # you may not use this file except in compliance with the License.
11
+ # You may obtain a copy of the License at
12
+ #
13
+ # http://www.apache.org/licenses/LICENSE-2.0
14
+ #
15
+ # Unless required by applicable law or agreed to in writing, software
16
+ # distributed under the License is distributed on an "AS IS" BASIS,
17
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
+ # See the License for the specific language governing permissions and
19
+ # limitations under the License.
20
+
21
+ import inspect
22
+ import itertools
23
+ import json
24
+ import os
25
+ import re
26
+ from collections import OrderedDict
27
+ from functools import partial
28
+ from pathlib import Path
29
+ from typing import Any, Callable, List, Optional, Tuple, Union
30
+
31
+ import safetensors
32
+ import torch
33
+ from huggingface_hub import create_repo, split_torch_state_dict_into_shards
34
+ from huggingface_hub.utils import validate_hf_hub_args
35
+ from torch import Tensor, nn
36
+
37
+ from diffusers import __version__
38
+ from diffusers.utils import (
39
+ FLAX_WEIGHTS_NAME,
40
+ SAFE_WEIGHTS_INDEX_NAME,
41
+ WEIGHTS_INDEX_NAME,
42
+ _add_variant,
43
+ _get_checkpoint_shard_files,
44
+ _get_model_file,
45
+ deprecate,
46
+ is_accelerate_available,
47
+ is_torch_version,
48
+ logging,
49
+ )
50
+
51
+ CONFIG_NAME = "config.json"
52
+ WEIGHTS_NAME = "pytorch_model.bin"
53
+ SAFETENSORS_WEIGHTS_NAME = "pytorch_model.safetensors"
54
+ HUGGINGFACE_CO_RESOLVE_ENDPOINT = "https://huggingface.co"
55
+
56
+ from diffusers.utils.hub_utils import (
57
+ PushToHubMixin,
58
+ load_or_create_model_card,
59
+ populate_model_card,
60
+ )
61
+ from diffusers.models.model_loading_utils import (
62
+ _determine_device_map,
63
+ _fetch_index_file,
64
+ _load_state_dict_into_model,
65
+ load_model_dict_into_meta,
66
+ )
67
+
68
+ from diffusers.configuration_utils import ConfigMixin, register_to_config
69
+ import diffusers
70
+
71
+ logger = logging.get_logger(__name__)
72
+
73
+ _REGEX_SHARD = re.compile(r"(.*?)-\d{5}-of-\d{5}")
74
+
75
+
76
+ if is_torch_version(">=", "1.9.0"):
77
+ _LOW_CPU_MEM_USAGE_DEFAULT = True
78
+ else:
79
+ _LOW_CPU_MEM_USAGE_DEFAULT = False
80
+
81
+
82
+ if is_accelerate_available():
83
+ import accelerate
84
+
85
+
86
+ def get_parameter_device(parameter: torch.nn.Module) -> torch.device:
87
+ try:
88
+ parameters_and_buffers = itertools.chain(parameter.parameters(), parameter.buffers())
89
+ return next(parameters_and_buffers).device
90
+ except StopIteration:
91
+ # For torch.nn.DataParallel compatibility in PyTorch 1.5
92
+
93
+ def find_tensor_attributes(module: torch.nn.Module) -> List[Tuple[str, Tensor]]:
94
+ tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)]
95
+ return tuples
96
+
97
+ gen = parameter._named_members(get_members_fn=find_tensor_attributes)
98
+ first_tuple = next(gen)
99
+ return first_tuple[1].device
100
+
101
+
102
+ def get_parameter_dtype(parameter: torch.nn.Module) -> torch.dtype:
103
+ try:
104
+ params = tuple(parameter.parameters())
105
+ if len(params) > 0:
106
+ return params[0].dtype
107
+
108
+ buffers = tuple(parameter.buffers())
109
+ if len(buffers) > 0:
110
+ return buffers[0].dtype
111
+
112
+ except StopIteration:
113
+ # For torch.nn.DataParallel compatibility in PyTorch 1.5
114
+
115
+ def find_tensor_attributes(module: torch.nn.Module) -> List[Tuple[str, Tensor]]:
116
+ tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)]
117
+ return tuples
118
+
119
+ gen = parameter._named_members(get_members_fn=find_tensor_attributes)
120
+ first_tuple = next(gen)
121
+ return first_tuple[1].dtype
122
+
123
+
124
+ class ModelMixin(torch.nn.Module, PushToHubMixin):
125
+ r"""
126
+ Base class for all models.
127
+
128
+ [`ModelMixin`] takes care of storing the model configuration and provides methods for loading, downloading and
129
+ saving models.
130
+
131
+ - **config_name** ([`str`]) -- Filename to save a model to when calling [`~models.ModelMixin.save_pretrained`].
132
+ """
133
+
134
+ config_name = CONFIG_NAME
135
+ _automatically_saved_args = ["_diffusers_version", "_class_name", "_name_or_path"]
136
+ _supports_gradient_checkpointing = False
137
+ _keys_to_ignore_on_load_unexpected = None
138
+ _no_split_modules = None
139
+
140
+ def __init__(self):
141
+ super().__init__()
142
+
143
+ def __getattr__(self, name: str) -> Any:
144
+ """The only reason we overwrite `getattr` here is to gracefully deprecate accessing
145
+ config attributes directly. See https://github.com/huggingface/diffusers/pull/3129 We need to overwrite
146
+ __getattr__ here in addition so that we don't trigger `torch.nn.Module`'s __getattr__':
147
+ https://pytorch.org/docs/stable/_modules/torch/nn/modules/module.html#Module
148
+ """
149
+
150
+ is_in_config = "_internal_dict" in self.__dict__ and hasattr(self.__dict__["_internal_dict"], name)
151
+ is_attribute = name in self.__dict__
152
+
153
+ if is_in_config and not is_attribute:
154
+ deprecation_message = f"Accessing config attribute `{name}` directly via \
155
+ '{type(self).__name__}' object attribute is deprecated. \
156
+ Please access '{name}' over '{type(self).__name__}'s config object instead, \
157
+ e.g. 'unet.config.{name}'."
158
+ deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False, stacklevel=3)
159
+ return self._internal_dict[name]
160
+
161
+ # call PyTorch's https://pytorch.org/docs/stable/_modules/torch/nn/modules/module.html#Module
162
+ return super().__getattr__(name)
163
+
164
+ @property
165
+ def is_gradient_checkpointing(self) -> bool:
166
+ """
167
+ Whether gradient checkpointing is activated for this model or not.
168
+ """
169
+ return any(hasattr(m, "gradient_checkpointing") and m.gradient_checkpointing for m in self.modules())
170
+
171
+ def enable_gradient_checkpointing(self) -> None:
172
+ """
173
+ Activates gradient checkpointing for the current model (may be referred to as *activation checkpointing* or
174
+ *checkpoint activations* in other frameworks).
175
+ """
176
+ if not self._supports_gradient_checkpointing:
177
+ raise ValueError(f"{self.__class__.__name__} does not support gradient checkpointing.")
178
+ self.apply(partial(self._set_gradient_checkpointing, value=True))
179
+
180
+ def disable_gradient_checkpointing(self) -> None:
181
+ """
182
+ Deactivates gradient checkpointing for the current model (may be referred to as *activation checkpointing* or
183
+ *checkpoint activations* in other frameworks).
184
+ """
185
+ if self._supports_gradient_checkpointing:
186
+ self.apply(partial(self._set_gradient_checkpointing, value=False))
187
+
188
+ # def set_use_npu_flash_attention(self, valid: bool) -> None:
189
+ # r"""
190
+ # Set the switch for the npu flash attention.
191
+ # """
192
+
193
+ # def fn_recursive_set_npu_flash_attention(module: torch.nn.Module):
194
+ # if hasattr(module, "set_use_npu_flash_attention"):
195
+ # module.set_use_npu_flash_attention(valid)
196
+
197
+ # for child in module.children():
198
+ # fn_recursive_set_npu_flash_attention(child)
199
+
200
+ # for module in self.children():
201
+ # if isinstance(module, torch.nn.Module):
202
+ # fn_recursive_set_npu_flash_attention(module)
203
+
204
+ # def enable_npu_flash_attention(self) -> None:
205
+ # r"""
206
+ # Enable npu flash attention from torch_npu
207
+
208
+ # """
209
+ # self.set_use_npu_flash_attention(True)
210
+
211
+ # def disable_npu_flash_attention(self) -> None:
212
+ # r"""
213
+ # disable npu flash attention from torch_npu
214
+
215
+ # """
216
+ # self.set_use_npu_flash_attention(False)
217
+
218
+ # def set_use_memory_efficient_attention_xformers(
219
+ # self, valid: bool, attention_op: Optional[Callable] = None
220
+ # ) -> None:
221
+ # # Recursively walk through all the children.
222
+ # # Any children which exposes the set_use_memory_efficient_attention_xformers method
223
+ # # gets the message
224
+ # def fn_recursive_set_mem_eff(module: torch.nn.Module):
225
+ # if hasattr(module, "set_use_memory_efficient_attention_xformers"):
226
+ # module.set_use_memory_efficient_attention_xformers(valid, attention_op)
227
+
228
+ # for child in module.children():
229
+ # fn_recursive_set_mem_eff(child)
230
+
231
+ # for module in self.children():
232
+ # if isinstance(module, torch.nn.Module):
233
+ # fn_recursive_set_mem_eff(module)
234
+
235
+ # def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None) -> None:
236
+ # r"""
237
+ # Enable memory efficient attention from [xFormers](https://facebookresearch.github.io/xformers/).
238
+
239
+ # When this option is enabled, you should observe lower GPU memory usage and a potential speed up during
240
+ # inference. Speed up during training is not guaranteed.
241
+
242
+ # <Tip warning={true}>
243
+
244
+ # ⚠️ When memory efficient attention and sliced attention are both enabled, memory efficient attention takes
245
+ # precedent.
246
+
247
+ # </Tip>
248
+
249
+ # Parameters:
250
+ # attention_op (`Callable`, *optional*):
251
+ # Override the default `None` operator for use as `op` argument to the
252
+ # [`memory_efficient_attention()`](https://facebookresearch.github.io/xformers/components/ops.html#x
253
+ # formers.ops.memory_efficient_attention)
254
+ # function of xFormers.
255
+
256
+ # Examples:
257
+
258
+ # ```py
259
+ # >>> import torch
260
+ # >>> from diffusers import UNet2DConditionModel
261
+ # >>> from xformers.ops import MemoryEfficientAttentionFlashAttentionOp
262
+
263
+ # >>> model = UNet2DConditionModel.from_pretrained(
264
+ # ... "stabilityai/stable-diffusion-2-1", subfolder="unet", torch_dtype=torch.float16
265
+ # ... )
266
+ # >>> model = model.to("cuda")
267
+ # >>> model.enable_xformers_memory_efficient_attention(attention_op=MemoryEfficientAttentionFlashAttentionOp)
268
+ # ```
269
+ # """
270
+ # self.set_use_memory_efficient_attention_xformers(True, attention_op)
271
+
272
+ # def disable_xformers_memory_efficient_attention(self) -> None:
273
+ # r"""
274
+ # Disable memory efficient attention from [xFormers](https://facebookresearch.github.io/xformers/).
275
+ # """
276
+ # self.set_use_memory_efficient_attention_xformers(False)
277
+
278
+ def save_pretrained(
279
+ self,
280
+ save_directory: Union[str, os.PathLike],
281
+ is_main_process: bool = True,
282
+ save_function: Optional[Callable] = None,
283
+ safe_serialization: bool = True,
284
+ variant: Optional[str] = None,
285
+ max_shard_size: Union[int, str] = "10GB",
286
+ push_to_hub: bool = False,
287
+ **kwargs,
288
+ ):
289
+ """
290
+ Save a model and its configuration file to a directory so that it can be reloaded using the
291
+ [`~models.ModelMixin.from_pretrained`] class method.
292
+
293
+ Arguments:
294
+ save_directory (`str` or `os.PathLike`):
295
+ Directory to save a model and its configuration file to. Will be created if it doesn't exist.
296
+ is_main_process (`bool`, *optional*, defaults to `True`):
297
+ Whether the process calling this is the main process or not. Useful during distributed training and you
298
+ need to call this function on all processes. In this case, set `is_main_process=True` only on the main
299
+ process to avoid race conditions.
300
+ save_function (`Callable`):
301
+ The function to use to save the state dictionary. Useful during distributed training when you need to
302
+ replace `torch.save` with another method. Can be configured with the environment variable
303
+ `DIFFUSERS_SAVE_MODE`.
304
+ safe_serialization (`bool`, *optional*, defaults to `True`):
305
+ Whether to save the model using `safetensors` or the traditional PyTorch way with `pickle`.
306
+ variant (`str`, *optional*):
307
+ If specified, weights are saved in the format `pytorch_model.<variant>.bin`.
308
+ max_shard_size (`int` or `str`, defaults to `"10GB"`):
309
+ The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size
310
+ lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5GB"`).
311
+ If expressed as an integer, the unit is bytes. Note that this limit will be decreased after a certain
312
+ period of time (starting from Oct 2024) to allow users to upgrade to the latest version of `diffusers`.
313
+ This is to establish a common default size for this argument across different libraries in the Hugging
314
+ Face ecosystem (`transformers`, and `accelerate`, for example).
315
+ push_to_hub (`bool`, *optional*, defaults to `False`):
316
+ Whether or not to push your model to the Hugging Face Hub after saving it. You can specify the
317
+ repository you want to push to with `repo_id` (will default to the name of `save_directory` in your
318
+ namespace).
319
+ kwargs (`Dict[str, Any]`, *optional*):
320
+ Additional keyword arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method.
321
+ """
322
+ if os.path.isfile(save_directory):
323
+ logger.error(f"Provided path ({save_directory}) should be a directory, not a file")
324
+ return
325
+
326
+ weights_name = SAFETENSORS_WEIGHTS_NAME if safe_serialization else WEIGHTS_NAME
327
+ weights_name = _add_variant(weights_name, variant)
328
+ weight_name_split = weights_name.split(".")
329
+ if len(weight_name_split) in [2, 3]:
330
+ weights_name_pattern = weight_name_split[0] + "{suffix}." + ".".join(weight_name_split[1:])
331
+ else:
332
+ raise ValueError(f"Invalid {weights_name} provided.")
333
+
334
+ os.makedirs(save_directory, exist_ok=True)
335
+
336
+ if push_to_hub:
337
+ commit_message = kwargs.pop("commit_message", None)
338
+ private = kwargs.pop("private", False)
339
+ create_pr = kwargs.pop("create_pr", False)
340
+ token = kwargs.pop("token", None)
341
+ repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1])
342
+ repo_id = create_repo(repo_id, exist_ok=True, private=private, token=token).repo_id
343
+
344
+ # Only save the model itself if we are using distributed training
345
+ model_to_save = self
346
+
347
+ # Attach architecture to the config
348
+ # Save the config
349
+ if is_main_process:
350
+ model_to_save.save_config(save_directory)
351
+
352
+ # Save the model
353
+ state_dict = model_to_save.state_dict()
354
+
355
+ # Save the model
356
+ state_dict_split = split_torch_state_dict_into_shards(
357
+ state_dict, max_shard_size=max_shard_size, filename_pattern=weights_name_pattern
358
+ )
359
+
360
+ # Clean the folder from a previous save
361
+ if is_main_process:
362
+ for filename in os.listdir(save_directory):
363
+ if filename in state_dict_split.filename_to_tensors.keys():
364
+ continue
365
+ full_filename = os.path.join(save_directory, filename)
366
+ if not os.path.isfile(full_filename):
367
+ continue
368
+ weights_without_ext = weights_name_pattern.replace(".bin", "").replace(".safetensors", "")
369
+ weights_without_ext = weights_without_ext.replace("{suffix}", "")
370
+ filename_without_ext = filename.replace(".bin", "").replace(".safetensors", "")
371
+ # make sure that file to be deleted matches format of sharded file, e.g. pytorch_model-00001-of-00005
372
+ if (
373
+ filename.startswith(weights_without_ext)
374
+ and _REGEX_SHARD.fullmatch(filename_without_ext) is not None
375
+ ):
376
+ os.remove(full_filename)
377
+
378
+ for filename, tensors in state_dict_split.filename_to_tensors.items():
379
+ shard = {tensor: state_dict[tensor] for tensor in tensors}
380
+ filepath = os.path.join(save_directory, filename)
381
+ if safe_serialization:
382
+ # At some point we will need to deal better with save_function (used for TPU and other distributed
383
+ # joyfulness), but for now this enough.
384
+ safetensors.torch.save_file(shard, filepath, metadata={"format": "pt"})
385
+ else:
386
+ torch.save(shard, filepath)
387
+
388
+ if state_dict_split.is_sharded:
389
+ index = {
390
+ "metadata": state_dict_split.metadata,
391
+ "weight_map": state_dict_split.tensor_to_filename,
392
+ }
393
+ save_index_file = SAFE_WEIGHTS_INDEX_NAME if safe_serialization else WEIGHTS_INDEX_NAME
394
+ save_index_file = os.path.join(save_directory, _add_variant(save_index_file, variant))
395
+ # Save the index as well
396
+ with open(save_index_file, "w", encoding="utf-8") as f:
397
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
398
+ f.write(content)
399
+ logger.info(
400
+ f"The model is bigger than the maximum size per checkpoint ({max_shard_size}) and is going to be "
401
+ f"split in {len(state_dict_split.filename_to_tensors)} checkpoint shards. You can find where each "
402
+ f"parameters has been saved in the index located at {save_index_file}."
403
+ )
404
+ else:
405
+ path_to_weights = os.path.join(save_directory, weights_name)
406
+ logger.info(f"Model weights saved in {path_to_weights}")
407
+
408
+ if push_to_hub:
409
+ # Create a new empty model card and eventually tag it
410
+ model_card = load_or_create_model_card(repo_id, token=token)
411
+ model_card = populate_model_card(model_card)
412
+ model_card.save(Path(save_directory, "README.md").as_posix())
413
+
414
+ self._upload_folder(
415
+ save_directory,
416
+ repo_id,
417
+ token=token,
418
+ commit_message=commit_message,
419
+ create_pr=create_pr,
420
+ )
421
+
422
+ @classmethod
423
+ @validate_hf_hub_args
424
+ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs):
425
+ r"""
426
+ Instantiate a pretrained PyTorch model from a pretrained model configuration.
427
+
428
+ The model is set in evaluation mode - `model.eval()` - by default, and dropout modules are deactivated. To
429
+ train the model, set it back in training mode with `model.train()`.
430
+
431
+ Parameters:
432
+ pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*):
433
+ Can be either:
434
+
435
+ - A string, the *model id* (for example `google/ddpm-celebahq-256`) of a pretrained model hosted on
436
+ the Hub.
437
+ - A path to a *directory* (for example `./my_model_directory`) containing the model weights saved
438
+ with [`~ModelMixin.save_pretrained`].
439
+
440
+ cache_dir (`Union[str, os.PathLike]`, *optional*):
441
+ Path to a directory where a downloaded pretrained model configuration is cached if the standard cache
442
+ is not used.
443
+ torch_dtype (`str` or `torch.dtype`, *optional*):
444
+ Override the default `torch.dtype` and load the model with another dtype. If `"auto"` is passed, the
445
+ dtype is automatically derived from the model's weights.
446
+ force_download (`bool`, *optional*, defaults to `False`):
447
+ Whether or not to force the (re-)download of the model weights and configuration files, overriding the
448
+ cached versions if they exist.
449
+ proxies (`Dict[str, str]`, *optional*):
450
+ A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128',
451
+ 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
452
+ output_loading_info (`bool`, *optional*, defaults to `False`):
453
+ Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages.
454
+ local_files_only(`bool`, *optional*, defaults to `False`):
455
+ Whether to only load local model weights and configuration files or not. If set to `True`, the model
456
+ won't be downloaded from the Hub.
457
+ token (`str` or *bool*, *optional*):
458
+ The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from
459
+ `diffusers-cli login` (stored in `~/.huggingface`) is used.
460
+ revision (`str`, *optional*, defaults to `"main"`):
461
+ The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier
462
+ allowed by Git.
463
+ from_flax (`bool`, *optional*, defaults to `False`):
464
+ Load the model weights from a Flax checkpoint save file.
465
+ subfolder (`str`, *optional*, defaults to `""`):
466
+ The subfolder location of a model file within a larger model repository on the Hub or locally.
467
+ mirror (`str`, *optional*):
468
+ Mirror source to resolve accessibility issues if you're downloading a model in China. We do not
469
+ guarantee the timeliness or safety of the source, and you should refer to the mirror site for more
470
+ information.
471
+ device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*):
472
+ A map that specifies where each submodule should go. It doesn't need to be defined for each
473
+ parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the
474
+ same device. Defaults to `None`, meaning that the model will be loaded on CPU.
475
+
476
+ Set `device_map="auto"` to have 🤗 Accelerate automatically compute the most optimized `device_map`. For
477
+ more information about each option see [designing a device
478
+ map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map).
479
+ max_memory (`Dict`, *optional*):
480
+ A dictionary device identifier for the maximum memory. Will default to the maximum memory available for
481
+ each GPU and the available CPU RAM if unset.
482
+ offload_folder (`str` or `os.PathLike`, *optional*):
483
+ The path to offload weights if `device_map` contains the value `"disk"`.
484
+ offload_state_dict (`bool`, *optional*):
485
+ If `True`, temporarily offloads the CPU state dict to the hard drive to avoid running out of CPU RAM if
486
+ the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to `True`
487
+ when there is some disk offload.
488
+ low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`):
489
+ Speed up model loading only loading the pretrained weights and not initializing the weights. This also
490
+ tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model.
491
+ Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this
492
+ argument to `True` will raise an error.
493
+ variant (`str`, *optional*):
494
+ Load weights from a specified `variant` filename such as `"fp16"` or `"ema"`. This is ignored when
495
+ loading `from_flax`.
496
+ use_safetensors (`bool`, *optional*, defaults to `None`):
497
+ If set to `None`, the `safetensors` weights are downloaded if they're available **and** if the
498
+ `safetensors` library is installed. If set to `True`, the model is forcibly loaded from `safetensors`
499
+ weights. If set to `False`, `safetensors` weights are not loaded.
500
+
501
+ <Tip>
502
+
503
+ To use private or [gated models](https://huggingface.co/docs/hub/models-gated#gated-models), log-in with
504
+ `huggingface-cli login`. You can also activate the special
505
+ ["offline-mode"](https://huggingface.co/diffusers/installation.html#offline-mode) to use this method in a
506
+ firewalled environment.
507
+
508
+ </Tip>
509
+
510
+ Example:
511
+
512
+ ```py
513
+ from diffusers import UNet2DConditionModel
514
+
515
+ unet = UNet2DConditionModel.from_pretrained("runwayml/stable-diffusion-v1-5", subfolder="unet")
516
+ ```
517
+
518
+ If you get the error message below, you need to finetune the weights for your downstream task:
519
+
520
+ ```bash
521
+ Some weights of UNet2DConditionModel were not initialized from the model checkpoint at
522
+ runwayml/stable-diffusion-v1-5 and are newly initialized because the shapes did not match:
523
+ - conv_in.weight: found shape torch.Size([320, 4, 3, 3]) in the checkpoint and torch.Size([320, 9, 3, 3])
524
+ in the model instantiated
525
+ You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
526
+ ```
527
+ """
528
+ cache_dir = kwargs.pop("cache_dir", None)
529
+ ignore_mismatched_sizes = kwargs.pop("ignore_mismatched_sizes", False)
530
+ force_download = kwargs.pop("force_download", False)
531
+ from_flax = kwargs.pop("from_flax", False)
532
+ proxies = kwargs.pop("proxies", None)
533
+ output_loading_info = kwargs.pop("output_loading_info", False)
534
+ local_files_only = kwargs.pop("local_files_only", None)
535
+ token = kwargs.pop("token", None)
536
+ revision = kwargs.pop("revision", None)
537
+ torch_dtype = kwargs.pop("torch_dtype", None)
538
+ subfolder = kwargs.pop("subfolder", None)
539
+ device_map = kwargs.pop("device_map", None)
540
+ max_memory = kwargs.pop("max_memory", None)
541
+ offload_folder = kwargs.pop("offload_folder", None)
542
+ offload_state_dict = kwargs.pop("offload_state_dict", False)
543
+ low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", _LOW_CPU_MEM_USAGE_DEFAULT)
544
+ variant = kwargs.pop("variant", None)
545
+ use_safetensors = kwargs.pop("use_safetensors", None)
546
+
547
+ allow_pickle = False
548
+ if use_safetensors is None:
549
+ use_safetensors = True
550
+ allow_pickle = True
551
+
552
+ def _check_input(low_cpu_mem_usage, device_map):
553
+ if low_cpu_mem_usage and not is_accelerate_available():
554
+ low_cpu_mem_usage = False
555
+ logger.warning(
556
+ "Cannot initialize model with low cpu memory usage because `accelerate` was not found in the"
557
+ " environment. Defaulting to `low_cpu_mem_usage=False`. It is strongly recommended to install"
558
+ " `accelerate` for faster and less memory-intense model loading. You can do so with: \n```\npip"
559
+ " install accelerate\n```\n."
560
+ )
561
+
562
+ if device_map is not None and not is_accelerate_available():
563
+ raise NotImplementedError(
564
+ "Loading and dispatching requires `accelerate`. Please make sure to install accelerate or set"
565
+ " `device_map=None`. You can install accelerate with `pip install accelerate`."
566
+ )
567
+
568
+ # Check if we can handle device_map and dispatching the weights
569
+ if device_map is not None and not is_torch_version(">=", "1.9.0"):
570
+ raise NotImplementedError(
571
+ "Loading and dispatching requires torch >= 1.9.0. Please either update your PyTorch version or set"
572
+ " `device_map=None`."
573
+ )
574
+
575
+ if low_cpu_mem_usage is True and not is_torch_version(">=", "1.9.0"):
576
+ raise NotImplementedError(
577
+ "Low memory initialization requires torch >= 1.9.0. Please either update your PyTorch version or "
578
+ "set `low_cpu_mem_usage=False`."
579
+ )
580
+
581
+ if low_cpu_mem_usage is False and device_map is not None:
582
+ raise ValueError(
583
+ f"You cannot set `low_cpu_mem_usage` to `False` while using device_map={device_map} for loading and"
584
+ " dispatching. Please make sure to set `low_cpu_mem_usage=True`."
585
+ )
586
+ _check_input(low_cpu_mem_usage, device_map)
587
+
588
+ # change device_map into a map if we passed an int, a str or a torch.device
589
+ if isinstance(device_map, torch.device):
590
+ device_map = {"": device_map}
591
+ elif isinstance(device_map, str) and device_map not in ["auto", "balanced", "balanced_low_0", "sequential"]:
592
+ try:
593
+ device_map = {"": torch.device(device_map)}
594
+ except RuntimeError:
595
+ raise ValueError(
596
+ "When passing device_map as a string, the value needs to be a device name (e.g. cpu, cuda:0) or "
597
+ f"'auto', 'balanced', 'balanced_low_0', 'sequential' but found {device_map}."
598
+ )
599
+ elif isinstance(device_map, int):
600
+ if device_map < 0:
601
+ raise ValueError(
602
+ "You can't pass device_map as a negative int. " \
603
+ "If you want to put the model on the cpu, pass device_map = 'cpu' "
604
+ )
605
+ else:
606
+ device_map = {"": device_map}
607
+
608
+ if device_map is not None:
609
+ if low_cpu_mem_usage is None:
610
+ low_cpu_mem_usage = True
611
+ elif not low_cpu_mem_usage:
612
+ raise ValueError("Passing along a `device_map` requires `low_cpu_mem_usage=True`")
613
+
614
+ if low_cpu_mem_usage:
615
+ if device_map is not None and not is_torch_version(">=", "1.10"):
616
+ # The max memory utils require PyTorch >= 1.10 to have torch.cuda.mem_get_info.
617
+ raise ValueError("`low_cpu_mem_usage` and `device_map` require PyTorch >= 1.10.")
618
+
619
+ # Load config if we don't provide a configuration
620
+ config_path = pretrained_model_name_or_path
621
+
622
+ user_agent = {
623
+ "diffusers": __version__,
624
+ "file_type": "model",
625
+ "framework": "pytorch",
626
+ }
627
+
628
+ # load config
629
+ def _load_config(cls, config_path, **kwargs):
630
+ config, unused_kwargs, commit_hash = cls.load_config(
631
+ config_path,
632
+ cache_dir=cache_dir,
633
+ return_unused_kwargs=True,
634
+ return_commit_hash=True,
635
+ force_download=force_download,
636
+ proxies=proxies,
637
+ local_files_only=local_files_only,
638
+ token=token,
639
+ revision=revision,
640
+ subfolder=subfolder,
641
+ user_agent=user_agent,
642
+ **kwargs,
643
+ )
644
+
645
+ # Determine if we're loading from a directory of sharded checkpoints.
646
+ is_sharded = False
647
+ index_file = None
648
+ is_local = os.path.isdir(pretrained_model_name_or_path)
649
+ index_file = _fetch_index_file(
650
+ is_local=is_local,
651
+ pretrained_model_name_or_path=pretrained_model_name_or_path,
652
+ subfolder=subfolder or "",
653
+ use_safetensors=use_safetensors,
654
+ cache_dir=cache_dir,
655
+ variant=variant,
656
+ force_download=force_download,
657
+ proxies=proxies,
658
+ local_files_only=local_files_only,
659
+ token=token,
660
+ revision=revision,
661
+ user_agent=user_agent,
662
+ commit_hash=commit_hash,
663
+ )
664
+ if index_file is not None and index_file.is_file():
665
+ is_sharded = True
666
+
667
+ if is_sharded and from_flax:
668
+ raise ValueError("Loading of sharded checkpoints is not supported when `from_flax=True`.")
669
+ return config, unused_kwargs, commit_hash, is_sharded, index_file
670
+
671
+ config, unused_kwargs, commit_hash, is_sharded, index_file = _load_config(cls, config_path, **kwargs)
672
+
673
+ # load model
674
+ model_file = None
675
+ if from_flax:
676
+ model_file = _get_model_file(
677
+ pretrained_model_name_or_path,
678
+ weights_name=FLAX_WEIGHTS_NAME,
679
+ cache_dir=cache_dir,
680
+ force_download=force_download,
681
+ proxies=proxies,
682
+ local_files_only=local_files_only,
683
+ token=token,
684
+ revision=revision,
685
+ subfolder=subfolder,
686
+ user_agent=user_agent,
687
+ commit_hash=commit_hash,
688
+ )
689
+ model = cls.from_config(config, **unused_kwargs)
690
+
691
+ # Convert the weights
692
+ from .modeling_pytorch_flax_utils import load_flax_checkpoint_in_pytorch_model
693
+
694
+ model = load_flax_checkpoint_in_pytorch_model(model, model_file)
695
+ else:
696
+ if is_sharded:
697
+ sharded_ckpt_cached_folder, sharded_metadata = _get_checkpoint_shard_files(
698
+ pretrained_model_name_or_path,
699
+ index_file,
700
+ cache_dir=cache_dir,
701
+ proxies=proxies,
702
+ local_files_only=local_files_only,
703
+ token=token,
704
+ user_agent=user_agent,
705
+ revision=revision,
706
+ subfolder=subfolder or "",
707
+ )
708
+
709
+ elif use_safetensors and not is_sharded:
710
+ try:
711
+ model_file = _get_model_file(
712
+ pretrained_model_name_or_path,
713
+ weights_name=_add_variant(SAFETENSORS_WEIGHTS_NAME, variant),
714
+ cache_dir=cache_dir,
715
+ force_download=force_download,
716
+ proxies=proxies,
717
+ local_files_only=local_files_only,
718
+ token=token,
719
+ revision=revision,
720
+ subfolder=subfolder,
721
+ user_agent=user_agent,
722
+ commit_hash=commit_hash,
723
+ )
724
+
725
+ except IOError as e:
726
+ logger.error(f"An error occurred while trying to fetch {pretrained_model_name_or_path}: {e}")
727
+ if not allow_pickle:
728
+ raise
729
+ logger.warning(
730
+ "Defaulting to unsafe serialization. Pass `allow_pickle=False` to raise an error instead."
731
+ )
732
+
733
+ if model_file is None and not is_sharded:
734
+ model_file = _get_model_file(
735
+ pretrained_model_name_or_path,
736
+ weights_name=_add_variant(WEIGHTS_NAME, variant),
737
+ cache_dir=cache_dir,
738
+ force_download=force_download,
739
+ proxies=proxies,
740
+ local_files_only=local_files_only,
741
+ token=token,
742
+ revision=revision,
743
+ subfolder=subfolder,
744
+ user_agent=user_agent,
745
+ commit_hash=commit_hash,
746
+ )
747
+
748
+ if low_cpu_mem_usage:
749
+ # Instantiate model with empty weights
750
+ def load_model_in_low_cpu_mem_usage(config, device_map, unused_kwargs, commit_hash, is_sharded,
751
+ index_file):
752
+ with accelerate.init_empty_weights():
753
+ model = cls.from_config(config, **unused_kwargs)
754
+
755
+ # if device_map is None, load the state dict and move the params from meta device to the cpu
756
+ if device_map is None and not is_sharded:
757
+ param_device = "cpu"
758
+ state_dict = diffusers.models.model_loading_utils.load_state_dict(model_file, variant=variant)
759
+ model._convert_deprecated_attn(state_dict)
760
+ # move the params from meta device to cpu
761
+ missing_keys = set(model.state_dict().keys()) - set(state_dict.keys())
762
+ if len(missing_keys) > 0:
763
+ raise ValueError(
764
+ f"Cannot load {cls} from {pretrained_model_name_or_path} because the following keys are"
765
+ f" missing: \n {', '.join(missing_keys)}. \n Please make sure to pass"
766
+ " `low_cpu_mem_usage=False` and `device_map=None` if you want to randomly initialize"
767
+ " those weights or else make sure your checkpoint file is correct."
768
+ )
769
+
770
+ unexpected_keys = load_model_dict_into_meta(
771
+ model,
772
+ state_dict,
773
+ device=param_device,
774
+ dtype=torch_dtype,
775
+ model_name_or_path=pretrained_model_name_or_path,
776
+ )
777
+
778
+ if cls._keys_to_ignore_on_load_unexpected is not None:
779
+ for pat in cls._keys_to_ignore_on_load_unexpected:
780
+ unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None]
781
+
782
+ if len(unexpected_keys) > 0:
783
+ logger.warning(
784
+ f"Some weights of the model checkpoint were not used when initializing "
785
+ f" {cls.__name__}: \n {[', '.join(unexpected_keys)]}"
786
+ )
787
+
788
+ else: # else let accelerate handle loading and dispatching.
789
+ # Load weights and dispatch according to the device_map
790
+ # by default the device_map is None and the weights are loaded on the CPU
791
+ force_hook = True
792
+ device_map = _determine_device_map(model, device_map, max_memory, torch_dtype)
793
+ if device_map is None and is_sharded:
794
+ # we load the parameters on the cpu
795
+ device_map = {"": "cpu"}
796
+ force_hook = False
797
+ try:
798
+ accelerate.load_checkpoint_and_dispatch(
799
+ model,
800
+ model_file if not is_sharded else index_file,
801
+ device_map,
802
+ max_memory=max_memory,
803
+ offload_folder=offload_folder,
804
+ offload_state_dict=offload_state_dict,
805
+ dtype=torch_dtype,
806
+ force_hooks=force_hook,
807
+ strict=True,
808
+ )
809
+ except AttributeError as e:
810
+ # When using accelerate loading, we do not have the ability to load the state
811
+ # dict and rename the weight names manually. Additionally, accelerate skips
812
+ # torch loading conventions and directly writes into `module.{_buffers, _parameters}`
813
+ # (which look like they should be private variables?), so we can't use the standard hooks
814
+ # to rename parameters on load. We need to mimic the original weight names so the correct
815
+ # attributes are available. After we have loaded the weights, we convert the deprecated
816
+ # names to the new non-deprecated names. Then we _greatly encourage_ the user to convert
817
+ # the weights so we don't have to do this again.
818
+
819
+ if "'Attention' object has no attribute" in str(e):
820
+ logger.warning(
821
+ f"Taking `{str(e)}` while using `accelerate.load_checkpoint_and_dispatch` to mean "
822
+ f"{pretrained_model_name_or_path} was saved with deprecated attention block weight "
823
+ f"names. We will load it with the deprecated attention block names and convert them"
824
+ f" on the fly to the new attention block format. Please re-save the model after "
825
+ f"this conversion so we don't have to do the on the fly renaming in the future. "
826
+ f"If the model is from a hub checkpoint, please also re-upload it or open a PR on"
827
+ f" the original repository."
828
+ )
829
+ model._temp_convert_self_to_deprecated_attention_blocks()
830
+ accelerate.load_checkpoint_and_dispatch(
831
+ model,
832
+ model_file if not is_sharded else index_file,
833
+ device_map,
834
+ max_memory=max_memory,
835
+ offload_folder=offload_folder,
836
+ offload_state_dict=offload_state_dict,
837
+ dtype=torch_dtype,
838
+ force_hooks=force_hook,
839
+ strict=True,
840
+ )
841
+ model._undo_temp_convert_self_to_deprecated_attention_blocks()
842
+ else:
843
+ raise e
844
+
845
+ loading_info = {
846
+ "missing_keys": [],
847
+ "unexpected_keys": [],
848
+ "mismatched_keys": [],
849
+ "error_msgs": [],
850
+ }
851
+ return model, loading_info
852
+ model, loading_info = load_model_in_low_cpu_mem_usage(config, device_map, unused_kwargs, commit_hash,
853
+ is_sharded, index_file)
854
+
855
+ else:
856
+ model = cls.from_config(config, **unused_kwargs)
857
+
858
+ state_dict = load_state_dict(model_file, variant=variant)
859
+ model._convert_deprecated_attn(state_dict)
860
+
861
+ model, missing_keys, unexpected_keys, mismatched_keys, error_msgs = cls._load_pretrained_model(
862
+ model,
863
+ state_dict,
864
+ model_file,
865
+ pretrained_model_name_or_path,
866
+ ignore_mismatched_sizes=ignore_mismatched_sizes,
867
+ )
868
+
869
+ loading_info = {
870
+ "missing_keys": missing_keys,
871
+ "unexpected_keys": unexpected_keys,
872
+ "mismatched_keys": mismatched_keys,
873
+ "error_msgs": error_msgs,
874
+ }
875
+
876
+ if torch_dtype is not None and not isinstance(torch_dtype, torch.dtype):
877
+ raise ValueError(
878
+ f"{torch_dtype} needs to be of type `torch.dtype`, e.g. `torch.float16`, but is {type(torch_dtype)}."
879
+ )
880
+ elif torch_dtype is not None:
881
+ model = model.to(torch_dtype)
882
+
883
+ model.register_to_config(_name_or_path=pretrained_model_name_or_path)
884
+
885
+ # Set model in evaluation mode to deactivate DropOut modules by default
886
+ model.eval()
887
+ if output_loading_info:
888
+ return model, loading_info
889
+
890
+ return model
891
+
892
+ @classmethod
893
+ def _load_pretrained_model(
894
+ cls,
895
+ model,
896
+ state_dict: OrderedDict,
897
+ resolved_archive_file,
898
+ pretrained_model_name_or_path: Union[str, os.PathLike],
899
+ ignore_mismatched_sizes: bool = False,
900
+ ):
901
+ # Retrieve missing & unexpected_keys
902
+ model_state_dict = model.state_dict()
903
+ loaded_keys = list(state_dict.keys())
904
+
905
+ expected_keys = list(model_state_dict.keys())
906
+
907
+ original_loaded_keys = loaded_keys
908
+
909
+ missing_keys = list(set(expected_keys) - set(loaded_keys))
910
+ unexpected_keys = list(set(loaded_keys) - set(expected_keys))
911
+
912
+ # Make sure we are able to load base models as well as derived models (with heads)
913
+ model_to_load = model
914
+
915
+ def _find_mismatched_keys(
916
+ state_dict,
917
+ model_state_dict,
918
+ loaded_keys,
919
+ ignore_mismatched_sizes,
920
+ ):
921
+ mismatched_keys = []
922
+ if ignore_mismatched_sizes:
923
+ for checkpoint_key in loaded_keys:
924
+ model_key = checkpoint_key
925
+
926
+ if (
927
+ model_key in model_state_dict
928
+ and state_dict[checkpoint_key].shape != model_state_dict[model_key].shape
929
+ ):
930
+ mismatched_keys.append(
931
+ (checkpoint_key, state_dict[checkpoint_key].shape, model_state_dict[model_key].shape)
932
+ )
933
+ del state_dict[checkpoint_key]
934
+ return mismatched_keys
935
+
936
+ if state_dict is not None:
937
+ # Whole checkpoint
938
+ mismatched_keys = _find_mismatched_keys(
939
+ state_dict,
940
+ model_state_dict,
941
+ original_loaded_keys,
942
+ ignore_mismatched_sizes,
943
+ )
944
+ error_msgs = _load_state_dict_into_model(model_to_load, state_dict)
945
+
946
+ if len(error_msgs) > 0:
947
+ error_msg = "\n\t".join(error_msgs)
948
+ if "size mismatch" in error_msg:
949
+ error_msg += (
950
+ "\n\tYou may consider adding `ignore_mismatched_sizes=True` in the model `from_pretrained` method."
951
+ )
952
+ raise RuntimeError(f"Error(s) in loading state_dict for {model.__class__.__name__}:\n\t{error_msg}")
953
+
954
+ if len(unexpected_keys) > 0:
955
+ logger.warning(
956
+ f"Some weights of the model checkpoint at {pretrained_model_name_or_path} were not used when"
957
+ f" initializing {model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are"
958
+ f" initializing {model.__class__.__name__} from the checkpoint of a model trained on another task"
959
+ " or with another architecture (e.g. initializing a BertForSequenceClassification model from a"
960
+ " BertForPreTraining model).\n- This IS NOT expected if you are initializing"
961
+ f" {model.__class__.__name__} from the checkpoint of a model that you expect to be exactly"
962
+ " identical (initializing a BertForSequenceClassification model from a"
963
+ " BertForSequenceClassification model)."
964
+ )
965
+ else:
966
+ logger.info(f"All model checkpoint weights were used when initializing {model.__class__.__name__}.\n")
967
+ if len(missing_keys) > 0:
968
+ logger.warning(
969
+ f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at"
970
+ f" {pretrained_model_name_or_path} and are newly initialized: {missing_keys}\nYou should probably"
971
+ " TRAIN this model on a down-stream task to be able to use it for predictions and inference."
972
+ )
973
+ elif len(mismatched_keys) == 0:
974
+ logger.info(
975
+ f"All the weights of {model.__class__.__name__} were initialized from the model checkpoint at"
976
+ f" {pretrained_model_name_or_path}.\nIf your task is similar to the task the model of the"
977
+ f" checkpoint was trained on, you can already use {model.__class__.__name__} for predictions"
978
+ " without further training."
979
+ )
980
+ if len(mismatched_keys) > 0:
981
+ mismatched_warning = "\n".join(
982
+ [
983
+ f"- {key}: found shape {shape1} in the checkpoint and {shape2} in the model instantiated"
984
+ for key, shape1, shape2 in mismatched_keys
985
+ ]
986
+ )
987
+ logger.warning(
988
+ f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at"
989
+ f" {pretrained_model_name_or_path} and are newly initialized because the shapes did not"
990
+ f" match:\n{mismatched_warning}\nYou should probably TRAIN this model on a down-stream task to be"
991
+ " able to use it for predictions and inference."
992
+ )
993
+
994
+ return model, missing_keys, unexpected_keys, mismatched_keys, error_msgs
995
+
996
+ @classmethod
997
+ def _get_signature_keys(cls, obj):
998
+ parameters = inspect.signature(obj.__init__).parameters
999
+ required_parameters = {k: v for k, v in parameters.items() if v.default == inspect._empty}
1000
+ optional_parameters = set({k for k, v in parameters.items() if v.default != inspect._empty})
1001
+ expected_modules = set(required_parameters.keys()) - {"self"}
1002
+
1003
+ return expected_modules, optional_parameters
1004
+
1005
+ # Adapted from `transformers` modeling_utils.py
1006
+ def _get_no_split_modules(self, device_map: str):
1007
+ """
1008
+ Get the modules of the model that should not be spit when using device_map. We iterate through the modules to
1009
+ get the underlying `_no_split_modules`.
1010
+
1011
+ Args:
1012
+ device_map (`str`):
1013
+ The device map value. Options are ["auto", "balanced", "balanced_low_0", "sequential"]
1014
+
1015
+ Returns:
1016
+ `List[str]`: List of modules that should not be split
1017
+ """
1018
+ _no_split_modules = set()
1019
+ modules_to_check = [self]
1020
+ while len(modules_to_check) > 0:
1021
+ module = modules_to_check.pop(-1)
1022
+ # if the module does not appear in _no_split_modules, we also check the children
1023
+ if module.__class__.__name__ not in _no_split_modules:
1024
+ if isinstance(module, ModelMixin):
1025
+ if module._no_split_modules is None:
1026
+ raise ValueError(
1027
+ f"{module.__class__.__name__} does not support `device_map='{device_map}'`. To implement "
1028
+ f"support, the model class needs to implement the `_no_split_modules` attribute."
1029
+ )
1030
+ else:
1031
+ _no_split_modules = _no_split_modules | set(module._no_split_modules)
1032
+ modules_to_check += list(module.children())
1033
+ return list(_no_split_modules)
1034
+
1035
+ @property
1036
+ def device(self) -> torch.device:
1037
+ """
1038
+ `torch.device`: The device on which the module is (assuming that all the module parameters are on the same
1039
+ device).
1040
+ """
1041
+ return get_parameter_device(self)
1042
+
1043
+ @property
1044
+ def dtype(self) -> torch.dtype:
1045
+ """
1046
+ `torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype).
1047
+ """
1048
+ return get_parameter_dtype(self)
1049
+
1050
+ def num_parameters(self, only_trainable: bool = False, exclude_embeddings: bool = False) -> int:
1051
+ """
1052
+ Get number of (trainable or non-embedding) parameters in the module.
1053
+
1054
+ Args:
1055
+ only_trainable (`bool`, *optional*, defaults to `False`):
1056
+ Whether or not to return only the number of trainable parameters.
1057
+ exclude_embeddings (`bool`, *optional*, defaults to `False`):
1058
+ Whether or not to return only the number of non-embedding parameters.
1059
+
1060
+ Returns:
1061
+ `int`: The number of parameters.
1062
+
1063
+ Example:
1064
+
1065
+ ```py
1066
+ from diffusers import UNet2DConditionModel
1067
+
1068
+ model_id = "runwayml/stable-diffusion-v1-5"
1069
+ unet = UNet2DConditionModel.from_pretrained(model_id, subfolder="unet")
1070
+ unet.num_parameters(only_trainable=True)
1071
+ 859520964
1072
+ ```
1073
+ """
1074
+
1075
+ if exclude_embeddings:
1076
+ embedding_param_names = [
1077
+ f"{name}.weight"
1078
+ for name, module_type in self.named_modules()
1079
+ if isinstance(module_type, torch.nn.Embedding)
1080
+ ]
1081
+ non_embedding_parameters = [
1082
+ parameter for name, parameter in self.named_parameters() if name not in embedding_param_names
1083
+ ]
1084
+ return sum(p.numel() for p in non_embedding_parameters if p.requires_grad or not only_trainable)
1085
+ else:
1086
+ return sum(p.numel() for p in self.parameters() if p.requires_grad or not only_trainable)
1087
+
1088
+ def _convert_deprecated_attn(self, state_dict: OrderedDict) -> None:
1089
+ # original function name: _convert_deprecated_attention_blocks
1090
+ deprecated_attention_block_paths = []
1091
+
1092
+ def recursive_find_attn_block(name, module):
1093
+ if hasattr(module, "_from_deprecated_attn_block") and module._from_deprecated_attn_block:
1094
+ deprecated_attention_block_paths.append(name)
1095
+
1096
+ for sub_name, sub_module in module.named_children():
1097
+ sub_name = sub_name if name == "" else f"{name}.{sub_name}"
1098
+ recursive_find_attn_block(sub_name, sub_module)
1099
+
1100
+ recursive_find_attn_block("", self)
1101
+
1102
+ # NOTE: we have to check if the deprecated parameters are in the state dict
1103
+ # because it is possible we are loading from a state dict that was already
1104
+ # converted
1105
+
1106
+ for path in deprecated_attention_block_paths:
1107
+ # group_norm path stays the same
1108
+
1109
+ # query -> to_q
1110
+ if f"{path}.query.weight" in state_dict:
1111
+ state_dict[f"{path}.to_q.weight"] = state_dict.pop(f"{path}.query.weight")
1112
+ if f"{path}.query.bias" in state_dict:
1113
+ state_dict[f"{path}.to_q.bias"] = state_dict.pop(f"{path}.query.bias")
1114
+
1115
+ # key -> to_k
1116
+ if f"{path}.key.weight" in state_dict:
1117
+ state_dict[f"{path}.to_k.weight"] = state_dict.pop(f"{path}.key.weight")
1118
+ if f"{path}.key.bias" in state_dict:
1119
+ state_dict[f"{path}.to_k.bias"] = state_dict.pop(f"{path}.key.bias")
1120
+
1121
+ # value -> to_v
1122
+ if f"{path}.value.weight" in state_dict:
1123
+ state_dict[f"{path}.to_v.weight"] = state_dict.pop(f"{path}.value.weight")
1124
+ if f"{path}.value.bias" in state_dict:
1125
+ state_dict[f"{path}.to_v.bias"] = state_dict.pop(f"{path}.value.bias")
1126
+
1127
+ # proj_attn -> to_out.0
1128
+ if f"{path}.proj_attn.weight" in state_dict:
1129
+ state_dict[f"{path}.to_out.0.weight"] = state_dict.pop(f"{path}.proj_attn.weight")
1130
+ if f"{path}.proj_attn.bias" in state_dict:
1131
+ state_dict[f"{path}.to_out.0.bias"] = state_dict.pop(f"{path}.proj_attn.bias")
1132
+
1133
+
1134
+ class LegacyModelMixin(ModelMixin):
1135
+ r"""
1136
+ A subclass of `ModelMixin` to resolve class mapping from legacy classes (like `Transformer2DModel`) to more
1137
+ pipeline-specific classes (like `DiTTransformer2DModel`).
1138
+ """
1139
+
1140
+ @classmethod
1141
+ @validate_hf_hub_args
1142
+ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs):
1143
+ # To prevent dependency import problem.
1144
+ from diffusers.models.model_loading_utils import _fetch_remapped_cls_from_config
1145
+
1146
+ # Create a copy of the kwargs so that we don't mess with the keyword arguments in the downstream calls.
1147
+ kwargs_copy = kwargs.copy()
1148
+
1149
+ cache_dir = kwargs.pop("cache_dir", None)
1150
+ force_download = kwargs.pop("force_download", False)
1151
+ proxies = kwargs.pop("proxies", None)
1152
+ local_files_only = kwargs.pop("local_files_only", None)
1153
+ token = kwargs.pop("token", None)
1154
+ revision = kwargs.pop("revision", None)
1155
+ subfolder = kwargs.pop("subfolder", None)
1156
+
1157
+ # Load config if we don't provide a configuration
1158
+ config_path = pretrained_model_name_or_path
1159
+
1160
+ user_agent = {
1161
+ "diffusers": __version__,
1162
+ "file_type": "model",
1163
+ "framework": "pytorch",
1164
+ }
1165
+
1166
+ # load config
1167
+ config, _, _ = cls.load_config(
1168
+ config_path,
1169
+ cache_dir=cache_dir,
1170
+ return_unused_kwargs=True,
1171
+ return_commit_hash=True,
1172
+ force_download=force_download,
1173
+ proxies=proxies,
1174
+ local_files_only=local_files_only,
1175
+ token=token,
1176
+ revision=revision,
1177
+ subfolder=subfolder,
1178
+ user_agent=user_agent,
1179
+ **kwargs,
1180
+ )
1181
+ # resolve remapping
1182
+ remapped_class = _fetch_remapped_cls_from_config(config, cls)
1183
+
1184
+ return remapped_class.from_pretrained(pretrained_model_name_or_path, **kwargs_copy)
omni_diffusion/tokenizer.py ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from .constants import (
2
+ BOX_END_TOKEN,
3
+ BOX_START_TOKEN,
4
+ IMG_CONTEXT_TOKEN,
5
+ IMG_END_TOKEN,
6
+ IMG_START_TOKEN,
7
+ IMG_TAG_TOKEN,
8
+ PATCH_CONTEXT_TOKEN,
9
+ PATCH_END_TOKEN,
10
+ PATCH_START_TOKEN,
11
+ QUAD_END_TOKEN,
12
+ QUAD_START_TOKEN,
13
+ REF_END_TOKEN,
14
+ REF_START_TOKEN,
15
+ VID_CONTEXT_TOKEN,
16
+ VID_END_TOKEN,
17
+ VID_START_TOKEN,
18
+ VID_TAG_TOKEN,
19
+ )
20
+
21
+
22
+ def update_tokenizer(tokenizer, audio_tokenizer_type=None):
23
+ """
24
+ Adds special tokens related to multimodal tasks (Image, Video, Patch, Bounding Box, etc.)
25
+ to the tokenizer vocabulary.
26
+ Also handles specific tokenizer updates for audio models like SenseVoice/GLM4Voice.
27
+ """
28
+ token_list = [
29
+ IMG_START_TOKEN,
30
+ IMG_END_TOKEN,
31
+ IMG_CONTEXT_TOKEN,
32
+ VID_START_TOKEN,
33
+ VID_END_TOKEN,
34
+ VID_CONTEXT_TOKEN,
35
+ PATCH_START_TOKEN,
36
+ PATCH_END_TOKEN,
37
+ PATCH_CONTEXT_TOKEN,
38
+ QUAD_START_TOKEN,
39
+ QUAD_END_TOKEN,
40
+ REF_START_TOKEN,
41
+ REF_END_TOKEN,
42
+ BOX_START_TOKEN,
43
+ BOX_END_TOKEN,
44
+ IMG_TAG_TOKEN,
45
+ VID_TAG_TOKEN,
46
+ ]
47
+ num_new_tokens = tokenizer.add_tokens(token_list, special_tokens=True)
48
+
49
+ if audio_tokenizer_type == "sensevoice_glm4voice":
50
+ from .tokenizer_sensevoice_glm4voice import (
51
+ update_tokenizer_for_sensevoice_glm4voice,
52
+ SenseVoiceGLM4VoiceTokenizer,
53
+ )
54
+ return update_tokenizer_for_sensevoice_glm4voice(tokenizer)
55
+
56
+ raise NotImplementedError
57
+
58
+
59
+ def get_audio_tokenizer(model_name_or_path, audio_tokenizer_type, flow_path=None, rank=None):
60
+ """
61
+ Factory function to initialize and return the appropriate audio tokenizer instance
62
+ based on the provided type (e.g., 'sensevoice_glm4voice').
63
+ """
64
+ if audio_tokenizer_type is None:
65
+ return None
66
+
67
+ if audio_tokenizer_type == "sensevoice_glm4voice":
68
+ from .tokenizer_sensevoice_glm4voice import (
69
+ update_tokenizer_for_sensevoice_glm4voice,
70
+ SenseVoiceGLM4VoiceTokenizer,
71
+ )
72
+ return SenseVoiceGLM4VoiceTokenizer(model_name_or_path, flow_path=flow_path, rank=rank)
73
+
74
+ raise NotImplementedError
omni_diffusion/tokenizer_magvitv2.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import glob
2
+ import io
3
+ import logging
4
+ import math
5
+ import os
6
+ import tarfile
7
+ import uuid
8
+
9
+ import safetensors
10
+ import torch
11
+ from transformers import WhisperFeatureExtractor, WhisperTokenizerFast
12
+
13
+ import torchaudio
14
+
15
+ from transformers import WhisperFeatureExtractor
16
+ from speech_tokenizer.modeling_whisper import WhisperVQEncoder
17
+ from flow_inference import AudioDecoder
18
+ from omni_diffusion.models.magvit.modeling_magvitv2 import MAGVITv2
19
+
20
+ from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank
21
+ from funasr.models.sense_voice.model import SenseVoiceSmall
22
+
23
+ logger = logging.getLogger(__name__)
24
+ logger.setLevel(logging.INFO)
25
+
26
+
27
+ class MagVITV2Tokenizer:
28
+ def __init__(self, model_path=None, rank=None):
29
+ if rank is None and torch.distributed.is_initialized():
30
+ rank = torch.distributed.get_rank()
31
+ self.rank = rank % 8
32
+ else:
33
+ self.rank = rank
34
+ logger.info(f"{self.rank=}")
35
+ self.model_path = model_path
36
+
37
+ def load_model(self):
38
+
39
+ if hasattr(self, "image_tokenizer"):
40
+ return
41
+
42
+ if self.rank is not None:
43
+ self.device = f"cuda:{self.rank}"
44
+ torch.cuda.set_device(self.rank)
45
+ else:
46
+ self.device = "cpu"
47
+
48
+ self.image_tokenizer = MAGVITv2()
49
+ logger.info(f"{self.device=} Loading MAGVITv2")
50
+ self.image_tokenizer = self.image_tokenizer.from_pretrained(self.model_path).to(self.device)
51
+ self.image_tokenizer.eval()
52
+ self.image_tokenizer.requires_grad_(False)
53
+ logger.info(f"{self.device=} Loading MAGVITv2 done")
54
+
55
+ def encode(self, image):
56
+ image = image.to(self.device)
57
+ image_tokens = self.image_tokenizer.get_code(image)
58
+ return image_tokens
59
+
60
+ def decode(self, image_tokens):
61
+ images = self.image_tokenizer.decode_code(image_tokens)
62
+ return images
63
+
64
+ def apply_to_role(self, role, **kwargs):
65
+ return True
omni_diffusion/tokenizer_sensevoice_glm4voice.py ADDED
@@ -0,0 +1,280 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import glob
2
+ import io
3
+ import logging
4
+ import math
5
+ import os
6
+ import tarfile
7
+ import uuid
8
+
9
+ import safetensors
10
+ import torch
11
+ from transformers import WhisperFeatureExtractor, WhisperTokenizerFast
12
+
13
+ import torchaudio
14
+
15
+ from transformers import WhisperFeatureExtractor
16
+ from speech_tokenizer.modeling_whisper import WhisperVQEncoder
17
+ from flow_inference import AudioDecoder
18
+
19
+ from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank
20
+ from funasr.models.sense_voice.model import SenseVoiceSmall
21
+ import sys
22
+
23
+ from .constants import (
24
+ AUD_CONTEXT_TOKEN,
25
+ AUD_END_TOKEN,
26
+ AUD_START_TOKEN,
27
+ AUD_TAG_TOKEN,
28
+ BOX_END_TOKEN,
29
+ BOX_START_TOKEN,
30
+ IMG_CONTEXT_TOKEN,
31
+ IMG_END_TOKEN,
32
+ IMG_START_TOKEN,
33
+ IMG_TAG_TOKEN,
34
+ PATCH_CONTEXT_TOKEN,
35
+ PATCH_END_TOKEN,
36
+ PATCH_START_TOKEN,
37
+ QUAD_END_TOKEN,
38
+ QUAD_START_TOKEN,
39
+ REF_END_TOKEN,
40
+ REF_START_TOKEN,
41
+ VID_CONTEXT_TOKEN,
42
+ VID_END_TOKEN,
43
+ VID_START_TOKEN,
44
+ VID_TAG_TOKEN,
45
+ )
46
+
47
+ logger = logging.getLogger(__name__)
48
+ logger.setLevel(logging.INFO)
49
+
50
+
51
+ def update_tokenizer_for_sensevoice_glm4voice(tokenizer):
52
+ token_list = [
53
+ IMG_START_TOKEN,
54
+ IMG_END_TOKEN,
55
+ IMG_CONTEXT_TOKEN,
56
+ VID_START_TOKEN,
57
+ VID_END_TOKEN,
58
+ VID_CONTEXT_TOKEN,
59
+ PATCH_START_TOKEN,
60
+ PATCH_END_TOKEN,
61
+ PATCH_CONTEXT_TOKEN,
62
+ AUD_START_TOKEN,
63
+ AUD_END_TOKEN,
64
+ AUD_CONTEXT_TOKEN,
65
+ QUAD_START_TOKEN,
66
+ QUAD_END_TOKEN,
67
+ REF_START_TOKEN,
68
+ REF_END_TOKEN,
69
+ BOX_START_TOKEN,
70
+ BOX_END_TOKEN,
71
+ IMG_TAG_TOKEN,
72
+ VID_TAG_TOKEN,
73
+ AUD_TAG_TOKEN,
74
+ ]
75
+ num_new_tokens = tokenizer.add_tokens(token_list, special_tokens=True)
76
+
77
+ token_list = [f"<|audio_{i}|>" for i in range(16384)]
78
+ num_new_tokens = tokenizer.add_tokens(token_list, special_tokens=False)
79
+
80
+ return tokenizer
81
+
82
+
83
+ class SenseVoiceGLM4VoiceTokenizer:
84
+ def __init__(self, model_name_or_path, flow_path=None, rank=None):
85
+ self.model_name_or_path = model_name_or_path
86
+ self.flow_path = flow_path
87
+
88
+ if rank is None and torch.distributed.is_initialized():
89
+ rank = torch.distributed.get_rank()
90
+ self.rank = rank % 8
91
+ else:
92
+ self.rank = rank
93
+ logger.info(f"{self.rank=}")
94
+
95
+ self.sample_rate = 16000
96
+
97
+ self.is_discrete = True
98
+ self.is_contiguous = True
99
+
100
+ self._resample_buffer: dict[int, torchaudio.transforms.Resample] = {}
101
+ for sample_rate in [8000, 22050, 24000, 32000, 44100, 48000]:
102
+ print(f"torchaudio.transforms.Resample {sample_rate=} {self.sample_rate=}")
103
+ self._resample_buffer[sample_rate] = torchaudio.transforms.Resample(
104
+ orig_freq=sample_rate, new_freq=self.sample_rate
105
+ )
106
+
107
+ def load_model(self):
108
+
109
+ if hasattr(self, "whisper_model"):
110
+ return
111
+ if self.rank is not None:
112
+ self.device = f"cuda:{self.rank}"
113
+ torch.cuda.set_device(self.rank)
114
+ else:
115
+ self.device = "cpu"
116
+
117
+ logger.info(f"{self.device=} Loading SenseVoiceSmall")
118
+ from huggingface_hub import snapshot_download
119
+ model_dir = snapshot_download(repo_id="FunAudioLLM/SenseVoiceSmall")
120
+
121
+ _, self.kwargs = SenseVoiceSmall.from_pretrained(model=model_dir, device=self.device)
122
+ logger.info(f"{self.device=} Loading SenseVoiceSmall Done")
123
+
124
+ logger.info(f"{self.device=} Loading GLM4VoiceTokenizer")
125
+ self.whisper_model = (
126
+ WhisperVQEncoder.from_pretrained(self.model_name_or_path).eval().to(self.device)
127
+ )
128
+ self.feature_extractor = WhisperFeatureExtractor.from_pretrained(self.model_name_or_path)
129
+ print(hasattr(self, "whisper_model"),'whisper_model')
130
+
131
+ if self.flow_path is not None:
132
+ flow_config = os.path.join(self.flow_path, "config.yaml")
133
+ flow_checkpoint = os.path.join(self.flow_path, "flow.pt")
134
+ hift_checkpoint = os.path.join(self.flow_path, "hift.pt")
135
+
136
+ # Flow & Hift
137
+ self.audio_decoder = AudioDecoder(
138
+ config_path=flow_config,
139
+ flow_ckpt_path=flow_checkpoint,
140
+ hift_ckpt_path=hift_checkpoint,
141
+ device=self.device,
142
+ )
143
+ logger.info(f"{self.device=} Loading GLM4VoiceTokenizer Done")
144
+
145
+ def encode(self, audio_path, is_discrete=False, is_contiguous=True, **kwargs):
146
+
147
+ import faulthandler
148
+ faulthandler.enable()
149
+
150
+ if not hasattr(self, "whisper_model"):
151
+ self.load_model()
152
+
153
+ assert not (is_discrete and is_contiguous)
154
+ assert is_discrete or is_contiguous
155
+
156
+ if is_discrete:
157
+ audio_tokens = self.extract_speech_token(
158
+ self.whisper_model, self.feature_extractor, [audio_path], device=self.device
159
+ )[0]
160
+ return audio_tokens
161
+
162
+ if is_contiguous:
163
+
164
+ audio, sample_rate = torchaudio.load(audio_path)
165
+ audio = audio.mean(0)
166
+ if sample_rate != self.sample_rate:
167
+ if sample_rate not in self._resample_buffer:
168
+ print(f"torchaudio.transforms.Resample {sample_rate=} {self.sample_rate=} {self.device=}", flush=True)
169
+ self._resample_buffer[sample_rate] = torchaudio.transforms.Resample(
170
+ orig_freq=sample_rate, new_freq=self.sample_rate
171
+ ).to(self.device)
172
+ audio = audio.to(self.device)
173
+ self._resample_buffer[sample_rate].to(self.device)
174
+ audio = self._resample_buffer[sample_rate](audio[None, :])[0, :]
175
+ audio = audio.cpu()
176
+
177
+ frontend = self.kwargs["frontend"]
178
+
179
+ speech, speech_lengths = extract_fbank(audio, data_type="sound", frontend=frontend)
180
+
181
+ speech = speech[0]
182
+
183
+ return speech
184
+
185
+ def decode(self, audio_tokens, option_steps=10, **kwargs):
186
+ if not hasattr(self, "whisper_model"):
187
+ self.load_model()
188
+
189
+ this_uuid = str(uuid.uuid4())
190
+ this_uuid = "abc"
191
+
192
+ tts_token = torch.tensor(audio_tokens, device=self.device).unsqueeze(0)
193
+
194
+ flow_prompt_speech_token = torch.zeros(1, 0, dtype=torch.int64).to(self.device)
195
+ prompt_speech_feat = torch.zeros(1, 0, 80).to(self.device)
196
+
197
+ tts_speech, tts_mel = self.audio_decoder.token2wav(
198
+ tts_token,
199
+ uuid=this_uuid,
200
+ prompt_token=flow_prompt_speech_token.to(self.device),
201
+ prompt_feat=prompt_speech_feat.to(self.device),
202
+ finalize=True,
203
+ option_steps=option_steps,
204
+ )
205
+ tts_speechs = []
206
+ tts_speechs.append(tts_speech.squeeze())
207
+ tts_speech = torch.cat(tts_speechs, dim=-1).cpu()
208
+
209
+ return tts_speech
210
+
211
+ def apply_to_role(self, role, **kwargs):
212
+ is_discrete = kwargs.get("is_discrete", False)
213
+ if is_discrete and role in ["assistant", "gpt"]:
214
+ return True
215
+
216
+ is_contiguous = kwargs.get("is_contiguous", False)
217
+ if is_contiguous and role in ["user", "human"]:
218
+ return True
219
+
220
+ return False
221
+
222
+ def extract_speech_token(self, model, feature_extractor, utts, device="cuda"):
223
+ with torch.no_grad():
224
+ audios, indices = [], []
225
+ for idx, utt in enumerate(utts):
226
+ if isinstance(utt, tuple):
227
+ audio, sample_rate = utt
228
+ else:
229
+ audio, sample_rate = torchaudio.load(utt)
230
+ audio = audio.to(device)
231
+ if sample_rate != 16000:
232
+ if sample_rate not in self._resample_buffer:
233
+ print(f"torchaudio.transforms.Resample {sample_rate=} {self.sample_rate=} {device=}")
234
+ self._resample_buffer[sample_rate] = torchaudio.transforms.Resample(
235
+ orig_freq=sample_rate, new_freq=16000
236
+ ).to(device)
237
+ self._resample_buffer[sample_rate].to(device)
238
+ audio = self._resample_buffer[sample_rate](audio)
239
+
240
+ audio = audio[0]
241
+ audio = audio.cpu().numpy()
242
+ time_step = 0
243
+ while time_step * 16000 < audio.shape[0]:
244
+ audio_segment = audio[time_step * 16000 : (time_step + 30) * 16000]
245
+ audios.append(audio_segment)
246
+ indices.append(idx)
247
+ time_step += 30
248
+ pooling_kernel_size = model.config.pooling_kernel_size or 1
249
+ stride = (
250
+ model.conv1.stride[0]
251
+ * model.conv2.stride[0]
252
+ * pooling_kernel_size
253
+ * feature_extractor.hop_length
254
+ )
255
+ all_speech_tokens = [[] for _ in range(len(utts))]
256
+ batch_size = 128
257
+ for start in range(0, len(audios), batch_size):
258
+ features = feature_extractor(
259
+ audios[start : start + batch_size],
260
+ sampling_rate=16000,
261
+ return_attention_mask=True,
262
+ return_tensors="pt",
263
+ device=device,
264
+ padding="longest",
265
+ pad_to_multiple_of=stride,
266
+ )
267
+ features = features.to(device=device)
268
+ outputs = model(**features)
269
+ speech_tokens = outputs.quantized_token_ids
270
+ attention_mask = features.attention_mask[
271
+ :, :: model.conv1.stride[0] * model.conv2.stride[0]
272
+ ]
273
+ attention_mask = attention_mask[:, :: model.config.pooling_kernel_size]
274
+ assert attention_mask.shape == speech_tokens.shape
275
+ for i in range(len(speech_tokens)):
276
+ idx = indices[start + i]
277
+ speech_token = speech_tokens[i][attention_mask[i].bool()].tolist()
278
+ all_speech_tokens[idx].extend(speech_token)
279
+
280
+ return all_speech_tokens
requirements.txt ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ transformers==4.51.3
2
+ accelerate>=1.1.1
3
+ sentencepiece
4
+ einops
5
+ safetensors
6
+ pillow
7
+ numpy
8
+ opencv-python-headless
9
+ torchaudio
10
+ torchvision
11
+ timm
12
+ decord
13
+ natsort
14
+ tiktoken
15
+ funasr
16
+ hyperpyyaml
17
+ https://huggingface.co/datasets/multimodalart/zerogpu-blackwell-wheels/resolve/main/wheels/pt210-cu128-cp310/flash_attn-2.8.3-cp310-cp310-linux_x86_64.whl
s2i_0.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b8bd63844e968ffd84e083d34411ea6ccfcbdf101df3d8f0853cb557428be2f
3
+ size 593358
svqa_0.jpg ADDED

Git LFS Details

  • SHA256: 885b9b03f88681745a5078087a2811e2d2fef8b492e33ffaaf70b5e290cd3dea
  • Pointer size: 131 Bytes
  • Size of remote file: 185 kB
svqa_0.wav ADDED
Binary file (48.1 kB). View file
 
third_party/GLM-4-Voice/.gitignore ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ *venv
2
+ *.DS_Store
3
+ *.idea/
4
+ test*
third_party/GLM-4-Voice/LICENSE ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright 2024 GLM-4-Voice Model Team @ Zhipu AI
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
third_party/GLM-4-Voice/README.md ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # GLM-4-Voice
2
+ <p align="center">
3
+ 📄<a href="https://arxiv.org/abs/2412.02612" target="_blank"> Report </a> • 🤗 <a href="https://huggingface.co/THUDM/glm-4-voice-9b" target="_blank">HF Repo</a> • 🤖 <a href="https://modelscope.cn/studios/ZhipuAI/GLM-4-Voice-Demo" target="_blank">Demo</a> • 🐦 <a href="https://twitter.com/thukeg" target="_blank">Twitter</a>
4
+ </p>
5
+
6
+ Read this in [English](./README_en.md)
7
+
8
+ GLM-4-Voice 是智谱 AI 推出的端到端语音模型。GLM-4-Voice 能够直接理解和生成中英文语音,进行实时语音对话,并且能够遵循用户的指令要求改变语音的情感、语调、语速、方言等属性。
9
+
10
+ ## Model Architecture
11
+ ![Model Architecture](./resources/architecture.jpeg)
12
+
13
+ GLM-4-Voice 由三个部分组成:
14
+ * GLM-4-Voice-Tokenizer: 通过在 [Whisper](https://github.com/openai/whisper) 的 Encoder 部分增加 Vector Quantization 并在 ASR 数据上有监督训练,将连续的语音输入转化为离散的 token。每秒音频平均只需要用 12.5 个离散 token 表示。
15
+ * GLM-4-Voice-Decoder: 基于 [CosyVoice](https://github.com/FunAudioLLM/CosyVoice) 的 Flow Matching 模型结构训练的支持流式推理的语音解码器,将离散化的语音 token 转化为连续的语音输出。最少只需要 10 个语音 token 即可开始生成,降低端到端对话延迟。
16
+ * GLM-4-Voice-9B: 在 [GLM-4-9B](https://github.com/THUDM/GLM-4) 的基础上进行语音模态的预训练和对齐,从而能够理解和生成离散化的语音 token。
17
+
18
+ 预训练方面,为了攻克模型在语音模态下的智商和合成表现力两个难关,我们将 Speech2Speech 任务解耦合为“根据用户音频做出文本回复”和“根据文本回复和用户语音合成回复语音”两个任务,并设计两种预训练目标,分别基于文本预训练数据和无监督音频数据合成语音-文本交错数据以适配这两种任务形式。GLM-4-Voice-9B 在 GLM-4-9B 的基座模型基础之上,经过了数百万小时音频和数千亿 token 的音频文本交错数据预训练,拥有很强的音频理解和建模能力。
19
+
20
+ 对齐方面,为了支持高质量的语音对话,我们设计了一套流式思考架构:根据用户语音,GLM-4-Voice 可以流式交替输出文本和语音两个模态的内容,其中语音模态以文本作为参照保证回复内容的高质量,并根据用户的语音指令要求做出相应的声音变化,在最大程度保留语言模型智商的情况下仍然具有端到端建模的能力,同时具备低延迟性,最低只需要输出 20 个 token 便可以合成语音。
21
+
22
+ ## Model List
23
+
24
+ | Model | Type | Download |
25
+ |:---------------------:|:----------------:|:------------------------------------------------------------------------------------------------------------------------------------------------:|
26
+ | GLM-4-Voice-Tokenizer | Speech Tokenizer | [🤗 Huggingface](https://huggingface.co/THUDM/glm-4-voice-tokenizer) [🤖 ModelScope](https://modelscope.cn/models/ZhipuAI/glm-4-voice-tokenizer) |
27
+ | GLM-4-Voice-9B | Chat Model | [🤗 Huggingface](https://huggingface.co/THUDM/glm-4-voice-9b) [🤖 ModelScope](https://modelscope.cn/models/ZhipuAI/glm-4-voice-9b) |
28
+ | GLM-4-Voice-Decoder | Speech Decoder | [🤗 Huggingface](https://huggingface.co/THUDM/glm-4-voice-decoder) [🤖 ModelScope](https://modelscope.cn/models/ZhipuAI/glm-4-voice-decoder) |
29
+
30
+ ## Usage
31
+ 我们提供了可以直接启动的 Web Demo。用户可以输入语音或文本,模型会同时给出语音和文字回复。
32
+
33
+ ![](resources/web_demo.png)
34
+
35
+ ### Preparation
36
+
37
+ 首先下载仓库
38
+ ```shell
39
+ git clone --recurse-submodules https://github.com/THUDM/GLM-4-Voice
40
+ cd GLM-4-Voice
41
+ ```
42
+ 然后安装依赖。也可以使用我们提供的镜像 `zhipuai/glm-4-voice:0.1` 以跳过这一步。
43
+ ```shell
44
+ pip install -r requirements.txt
45
+ ```
46
+ 由于 Decoder 模型不支持通过 `transformers` 初始化,因此 checkpoint 需要单独下载。
47
+
48
+ ```shell
49
+ # git 模型下载,请确保已安装 git-lfs
50
+ git lfs install
51
+ git clone https://huggingface.co/THUDM/glm-4-voice-decoder
52
+ ```
53
+
54
+ ### Launch Web Demo
55
+
56
+ 1. 启动模型服务
57
+
58
+ ```shell
59
+ python model_server.py --host localhost --model-path THUDM/glm-4-voice-9b --port 10000 --dtype bfloat16 --device cuda:0
60
+ ```
61
+
62
+ 如果你需要使用 Int4 精度启动,请运行
63
+
64
+ ```shell
65
+ python model_server.py --host localhost --model-path THUDM/glm-4-voice-9b --port 10000 --dtype int4 --device cuda:0
66
+ ```
67
+
68
+ 此命令会自动下载 `glm-4-voice-9b`。如果网络条件不好,也手动下载之后通过 `--model-path` 指定本地的路径。
69
+
70
+ 2. 启动 web 服务
71
+
72
+ ```shell
73
+ python web_demo.py --tokenizer-path THUDM/glm-4-voice-tokenizer --model-path THUDM/glm-4-voice-9b --flow-path ./glm-4-voice-decoder
74
+ ```
75
+
76
+ 即可在 http://127.0.0.1:8888 访问 web demo。
77
+
78
+ 此命令会自动下载 `glm-4-voice-tokenizer` 和 `glm-4-voice-9b`。 请注意,`glm-4-voice-decoder` 需要手动下载。
79
+
80
+ 如果网络条件不好,可以手动下载这三个模型之后通过 `--tokenizer-path`, `--flow-path` 和 `--model-path` 指定本地的路径。
81
+
82
+ ### Known Issues
83
+
84
+ * Gradio 的流式音频播放效果不稳定。在生成完成后点击对话框中的音频质量会更高。
85
+
86
+ ## Cases
87
+
88
+ 我们提供了 GLM-4-Voice 的部分对话案例,包括控制情绪、改变语速、生成方言等。
89
+
90
+ * 用轻柔的声音引导我放松
91
+
92
+ https://github.com/user-attachments/assets/4e3d9200-076d-4c28-a641-99df3af38eb0
93
+
94
+ * 用激动的声音解说足球比赛
95
+
96
+ https://github.com/user-attachments/assets/0163de2d-e876-4999-b1bc-bbfa364b799b
97
+
98
+ * 用哀怨的声音讲一个鬼故事
99
+
100
+ https://github.com/user-attachments/assets/a75b2087-d7bc-49fa-a0c5-e8c99935b39a
101
+
102
+ * 用东北话介绍一下冬天有多冷
103
+
104
+ https://github.com/user-attachments/assets/91ba54a1-8f5c-4cfe-8e87-16ed1ecf4037
105
+
106
+ * 用重庆话念“吃葡萄不吐葡萄皮”
107
+
108
+ https://github.com/user-attachments/assets/7eb72461-9e84-4d8e-9c58-1809cf6a8a9b
109
+
110
+ * 用北京话念一句绕口令
111
+
112
+ https://github.com/user-attachments/assets/a9bb223e-9c0a-440d-8537-0a7f16e31651
113
+
114
+ * 加快语速
115
+
116
+ https://github.com/user-attachments/assets/c98a4604-366b-4304-917f-3c850a82fe9f
117
+
118
+ * 再快一点
119
+
120
+ https://github.com/user-attachments/assets/d5ff0815-74f8-4738-b0f1-477cfc8dcc2d
121
+
122
+ ## Acknowledgements
123
+
124
+ 本项目的部分代码来自:
125
+ * [CosyVoice](https://github.com/FunAudioLLM/CosyVoice)
126
+ * [transformers](https://github.com/huggingface/transformers)
127
+ * [GLM-4](https://github.com/THUDM/GLM-4)
128
+
129
+ ## 协议
130
+
131
+ + GLM-4 模型的权重的使用则需要遵循 [模型协议](https://huggingface.co/THUDM/glm-4-voice-9b/blob/main/LICENSE)。
132
+
133
+ + 本开源仓库的代码则遵循 [Apache 2.0](LICENSE) 协议。
134
+
135
+ ## 引用
136
+
137
+ ```
138
+ @misc{zeng2024glm4,
139
+ title={GLM-4-Voice: Towards Intelligent and Human-Like End-to-End Spoken Chatbot},
140
+ author={Aohan Zeng and Zhengxiao Du and Mingdao Liu and Kedong Wang and Shengmin Jiang and Lei Zhao and Yuxiao Dong and Jie Tang},
141
+ year={2024},
142
+ eprint={2412.02612},
143
+ archivePrefix={arXiv},
144
+ primaryClass={cs.CL},
145
+ url={https://arxiv.org/abs/2412.02612},
146
+ }
147
+ ```
148
+
149
+ ```
150
+ @misc{zeng2024scaling,
151
+ title={Scaling Speech-Text Pre-training with Synthetic Interleaved Data},
152
+ author={Aohan Zeng and Zhengxiao Du and Mingdao Liu and Lei Zhang and Shengmin Jiang and Yuxiao Dong and Jie Tang},
153
+ year={2024},
154
+ eprint={2411.17607},
155
+ archivePrefix={arXiv},
156
+ primaryClass={cs.CL},
157
+ url={https://arxiv.org/abs/2411.17607},
158
+ }
159
+ ```
third_party/GLM-4-Voice/README_en.md ADDED
@@ -0,0 +1,148 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # GLM-4-Voice
2
+ <p align="center">
3
+ 📄<a href="https://arxiv.org/abs/2412.02612" target="_blank"> Report </a> • 🤗 <a href="https://huggingface.co/THUDM/glm-4-voice-9b" target="_blank">HF Repo</a> • 🤖 <a href="https://modelscope.cn/studios/ZhipuAI/GLM-4-Voice-Demo" target="_blank">Demo</a> • 🐦 <a href="https://twitter.com/thukeg" target="_blank">Twitter</a>
4
+ </p>
5
+
6
+ GLM-4-Voice is an end-to-end voice model launched by Zhipu AI. GLM-4-Voice can directly understand and generate Chinese and English speech, engage in real-time voice conversations, and change attributes such as emotion, intonation, speech rate, and dialect based on user instructions.
7
+
8
+ ## Model Architecture
9
+
10
+ ![Model Architecture](./resources/architecture.jpeg)
11
+ We provide the three components of GLM-4-Voice:
12
+ * GLM-4-Voice-Tokenizer: Trained by adding vector quantization to the encoder part of [Whisper](https://github.com/openai/whisper), converting continuous speech input into discrete tokens. Each second of audio is converted into 12.5 discrete tokens.
13
+ * GLM-4-Voice-9B: Pre-trained and aligned on speech modality based on [GLM-4-9B](https://github.com/THUDM/GLM-4), enabling understanding and generation of discretized speech.
14
+ * GLM-4-Voice-Decoder: A speech decoder supporting streaming inference, retrained based on [CosyVoice](https://github.com/FunAudioLLM/CosyVoice), converting discrete speech tokens into continuous speech output. Generation can start with as few as 10 audio tokens, reducing conversation latency.
15
+
16
+ ## Model List
17
+
18
+ | Model | Type | Download |
19
+ |:---------------------:|:----------------:|:--------------------------------------------------------------------:|
20
+ | GLM-4-Voice-Tokenizer | Speech Tokenizer | [🤗 Huggingface](https://huggingface.co/THUDM/glm-4-voice-tokenizer) |
21
+ | GLM-4-Voice-9B | Chat Model | [🤗 Huggingface](https://huggingface.co/THUDM/glm-4-voice-9b) |
22
+ | GLM-4-Voice-Decoder | Speech Decoder | [🤗 Huggingface](https://huggingface.co/THUDM/glm-4-voice-decoder) |
23
+
24
+ ## Usage
25
+ We provide a Web Demo that can be launched directly. Users can input speech or text, and the model will respond with both speech and text.
26
+
27
+ ![](resources/web_demo.png)
28
+
29
+ ### Preparation
30
+
31
+ First, download the repository
32
+ ```shell
33
+ git clone --recurse-submodules https://github.com/THUDM/GLM-4-Voice
34
+ cd GLM-4-Voice
35
+ ```
36
+ Then, install the dependencies. You can also use our pre-built docker image `zhipuai/glm-4-voice:0.1` to skip the step.
37
+ ```shell
38
+ pip install -r requirements.txt
39
+ ```
40
+ Since the Decoder model does not support initialization via `transformers`, the checkpoint needs to be downloaded separately.
41
+
42
+ ```shell
43
+ # Git model download, please ensure git-lfs is installed
44
+ git clone https://huggingface.co/THUDM/glm-4-voice-decoder
45
+ ```
46
+
47
+ ### Launch Web Demo
48
+
49
+ 1. Start the model server
50
+
51
+ ```shell
52
+ python model_server.py --host localhost --model-path THUDM/glm-4-voice-9b --port 10000 --dtype bfloat16 --device cuda:0
53
+ ```
54
+
55
+ If you need to launch with Int4 precision, run
56
+
57
+ ```shell
58
+ python model_server.py --host localhost --model-path THUDM/glm-4-voice-9b --port 10000 --dtype int4 --device cuda:0
59
+ ```
60
+
61
+ This command will automatically download `glm-4-voice-9b`. If network conditions are poor, you can manually download it and specify the local path using `--model-path`.
62
+
63
+ 2. Start the web service
64
+
65
+ ```shell
66
+ python web_demo.py --tokenizer-path THUDM/glm-4-voice-tokenizer --model-path THUDM/glm-4-voice-9b --flow-path ./glm-4-voice-decoder
67
+ ```
68
+
69
+ You can access the web demo at [http://127.0.0.1:8888](http://127.0.0.1:8888).
70
+ This command will automatically download `glm-4-voice-tokenizer` and `glm-4-voice-9b`. Please note that `glm-4-voice-decoder` needs to be downloaded manually.
71
+ If the network connection is poor, you can manually download these three models and specify the local paths using `--tokenizer-path`, `--flow-path`, and `--model-path`.
72
+
73
+ ### Known Issues
74
+ * Gradio’s streaming audio playback can be unstable. The audio quality will be higher when clicking on the audio in the dialogue box after generation is complete.
75
+
76
+ ## Examples
77
+ We provide some dialogue cases for GLM-4-Voice, including emotion control, speech rate alteration, dialect generation, etc. (The examples are in Chinese.)
78
+
79
+ * Use a gentle voice to guide me to relax
80
+
81
+ https://github.com/user-attachments/assets/4e3d9200-076d-4c28-a641-99df3af38eb0
82
+
83
+ * Use an excited voice to commentate a football match
84
+
85
+ https://github.com/user-attachments/assets/0163de2d-e876-4999-b1bc-bbfa364b799b
86
+
87
+ * Tell a ghost story with a mournful voice
88
+
89
+ https://github.com/user-attachments/assets/a75b2087-d7bc-49fa-a0c5-e8c99935b39a
90
+
91
+ * Introduce how cold winter is with a Northeastern dialect
92
+
93
+ https://github.com/user-attachments/assets/91ba54a1-8f5c-4cfe-8e87-16ed1ecf4037
94
+
95
+ * Say "Eat grapes without spitting out the skins" in Chongqing dialect
96
+
97
+ https://github.com/user-attachments/assets/7eb72461-9e84-4d8e-9c58-1809cf6a8a9b
98
+
99
+ * Recite a tongue twister with a Beijing accent
100
+
101
+ https://github.com/user-attachments/assets/a9bb223e-9c0a-440d-8537-0a7f16e31651
102
+
103
+ * Increase the speech rate
104
+
105
+ https://github.com/user-attachments/assets/c98a4604-366b-4304-917f-3c850a82fe9f
106
+
107
+ * Even faster
108
+
109
+ https://github.com/user-attachments/assets/d5ff0815-74f8-4738-b0f1-477cfc8dcc2d
110
+
111
+ ## Acknowledgements
112
+
113
+ Some code in this project is from:
114
+ * [CosyVoice](https://github.com/FunAudioLLM/CosyVoice)
115
+ * [transformers](https://github.com/huggingface/transformers)
116
+ * [GLM-4](https://github.com/THUDM/GLM-4)
117
+
118
+ ## License Agreement
119
+
120
+ + The use of GLM-4 model weights must follow the [Model License Agreement](https://huggingface.co/THUDM/glm-4-voice-9b/blob/main/LICENSE).
121
+
122
+ + The code in this open-source repository is licensed under the [Apache 2.0](LICENSE) License.
123
+
124
+ ## Citation
125
+
126
+ ```
127
+ @misc{zeng2024glm4,
128
+ title={GLM-4-Voice: Towards Intelligent and Human-Like End-to-End Spoken Chatbot},
129
+ author={Aohan Zeng and Zhengxiao Du and Mingdao Liu and Kedong Wang and Shengmin Jiang and Lei Zhao and Yuxiao Dong and Jie Tang},
130
+ year={2024},
131
+ eprint={2412.02612},
132
+ archivePrefix={arXiv},
133
+ primaryClass={cs.CL},
134
+ url={https://arxiv.org/abs/2412.02612},
135
+ }
136
+ ```
137
+
138
+ ```
139
+ @misc{zeng2024scaling,
140
+ title={Scaling Speech-Text Pre-training with Synthetic Interleaved Data},
141
+ author={Aohan Zeng and Zhengxiao Du and Mingdao Liu and Lei Zhang and Shengmin Jiang and Yuxiao Dong and Jie Tang},
142
+ year={2024},
143
+ eprint={2411.17607},
144
+ archivePrefix={arXiv},
145
+ primaryClass={cs.CL},
146
+ url={https://arxiv.org/abs/2411.17607},
147
+ }
148
+ ```
third_party/GLM-4-Voice/audio_process.py ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import librosa
3
+ import soundfile as sf
4
+ import numpy as np
5
+ from pathlib import Path
6
+ import io
7
+
8
+ # Split audio stream at silence points to prevent playback stuttering issues
9
+ # caused by AAC encoder frame padding when streaming audio through Gradio audio components.
10
+ class AudioStreamProcessor:
11
+ def __init__(self, sr=22050, min_silence_duration=0.1, threshold_db=-40):
12
+ self.sr = sr
13
+ self.min_silence_duration = min_silence_duration
14
+ self.threshold_db = threshold_db
15
+ self.buffer = np.array([])
16
+
17
+
18
+ def process(self, audio_data, last=False):
19
+ """
20
+ Add audio data and process it
21
+ params:
22
+ audio_data: audio data in numpy array
23
+ last: whether this is the last chunk of data
24
+ returns:
25
+ Processed audio data, returns None if no split point is found
26
+ """
27
+
28
+ # Add new data to buffer
29
+ self.buffer = np.concatenate([self.buffer, audio_data]) if len(self.buffer) > 0 else audio_data
30
+
31
+ if last:
32
+ result = self.buffer
33
+ self.buffer = np.array([])
34
+ return self._to_wav_bytes(result)
35
+
36
+ # Find silence boundary
37
+ split_point = self._find_silence_boundary(self.buffer)
38
+
39
+ if split_point is not None:
40
+ # Modified: Extend split point to the end of silence
41
+ silence_end = self._find_silence_end(split_point)
42
+ result = self.buffer[:silence_end]
43
+ self.buffer = self.buffer[silence_end:]
44
+ return self._to_wav_bytes(result)
45
+
46
+ return None
47
+
48
+ def _find_silence_boundary(self, audio):
49
+ """
50
+ Find the starting point of silence boundary in audio
51
+ """
52
+ # Convert audio to decibels
53
+ db = librosa.amplitude_to_db(np.abs(audio), ref=np.max)
54
+
55
+ # Find points below threshold
56
+ silence_points = np.where(db < self.threshold_db)[0]
57
+
58
+ if len(silence_points) == 0:
59
+ return None
60
+
61
+ # Calculate minimum silence samples
62
+ min_silence_samples = int(self.min_silence_duration * self.sr)
63
+
64
+ # Search backwards for continuous silence segment starting point
65
+ for i in range(len(silence_points) - min_silence_samples, -1, -1):
66
+ if i < 0:
67
+ break
68
+ if np.all(np.diff(silence_points[i:i+min_silence_samples]) == 1):
69
+ return silence_points[i]
70
+
71
+ return None
72
+
73
+ def _find_silence_end(self, start_point):
74
+ """
75
+ Find the end point of silence segment
76
+ """
77
+ db = librosa.amplitude_to_db(np.abs(self.buffer[start_point:]), ref=np.max)
78
+ silence_points = np.where(db >= self.threshold_db)[0]
79
+
80
+ if len(silence_points) == 0:
81
+ return len(self.buffer)
82
+
83
+ return start_point + silence_points[0]
84
+
85
+ def _to_wav_bytes(self, audio_data):
86
+ """
87
+ trans_to_wav_bytes
88
+ """
89
+ wav_buffer = io.BytesIO()
90
+ sf.write(wav_buffer, audio_data, self.sr, format='WAV')
91
+ return wav_buffer.getvalue()
92
+
93
+
third_party/GLM-4-Voice/cosyvoice/__init__.py ADDED
File without changes
third_party/GLM-4-Voice/cosyvoice/bin/inference.py ADDED
@@ -0,0 +1,114 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu)
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from __future__ import print_function
16
+
17
+ import argparse
18
+ import logging
19
+ logging.getLogger('matplotlib').setLevel(logging.WARNING)
20
+ import os
21
+
22
+ import torch
23
+ from torch.utils.data import DataLoader
24
+ import torchaudio
25
+ from hyperpyyaml import load_hyperpyyaml
26
+ from tqdm import tqdm
27
+ from cosyvoice.cli.model import CosyVoiceModel
28
+
29
+ from cosyvoice.dataset.dataset import Dataset
30
+
31
+ def get_args():
32
+ parser = argparse.ArgumentParser(description='inference with your model')
33
+ parser.add_argument('--config', required=True, help='config file')
34
+ parser.add_argument('--prompt_data', required=True, help='prompt data file')
35
+ parser.add_argument('--prompt_utt2data', required=True, help='prompt data file')
36
+ parser.add_argument('--tts_text', required=True, help='tts input file')
37
+ parser.add_argument('--llm_model', required=True, help='llm model file')
38
+ parser.add_argument('--flow_model', required=True, help='flow model file')
39
+ parser.add_argument('--hifigan_model', required=True, help='hifigan model file')
40
+ parser.add_argument('--gpu',
41
+ type=int,
42
+ default=-1,
43
+ help='gpu id for this rank, -1 for cpu')
44
+ parser.add_argument('--mode',
45
+ default='sft',
46
+ choices=['sft', 'zero_shot'],
47
+ help='inference mode')
48
+ parser.add_argument('--result_dir', required=True, help='asr result file')
49
+ args = parser.parse_args()
50
+ print(args)
51
+ return args
52
+
53
+
54
+ def main():
55
+ args = get_args()
56
+ logging.basicConfig(level=logging.DEBUG,
57
+ format='%(asctime)s %(levelname)s %(message)s')
58
+ os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu)
59
+
60
+ # Init cosyvoice models from configs
61
+ use_cuda = args.gpu >= 0 and torch.cuda.is_available()
62
+ device = torch.device('cuda' if use_cuda else 'cpu')
63
+ with open(args.config, 'r') as f:
64
+ configs = load_hyperpyyaml(f)
65
+
66
+ model = CosyVoiceModel(configs['llm'], configs['flow'], configs['hift'])
67
+ model.load(args.llm_model, args.flow_model, args.hifigan_model)
68
+
69
+ test_dataset = Dataset(args.prompt_data, data_pipeline=configs['data_pipeline'], mode='inference', shuffle=False, partition=False, tts_file=args.tts_text, prompt_utt2data=args.prompt_utt2data)
70
+ test_data_loader = DataLoader(test_dataset, batch_size=None, num_workers=0)
71
+
72
+ del configs
73
+ os.makedirs(args.result_dir, exist_ok=True)
74
+ fn = os.path.join(args.result_dir, 'wav.scp')
75
+ f = open(fn, 'w')
76
+ with torch.no_grad():
77
+ for batch_idx, batch in tqdm(enumerate(test_data_loader)):
78
+ utts = batch["utts"]
79
+ assert len(utts) == 1, "inference mode only support batchsize 1"
80
+ text = batch["text"]
81
+ text_token = batch["text_token"].to(device)
82
+ text_token_len = batch["text_token_len"].to(device)
83
+ tts_text = batch["tts_text"]
84
+ tts_index = batch["tts_index"]
85
+ tts_text_token = batch["tts_text_token"].to(device)
86
+ tts_text_token_len = batch["tts_text_token_len"].to(device)
87
+ speech_token = batch["speech_token"].to(device)
88
+ speech_token_len = batch["speech_token_len"].to(device)
89
+ speech_feat = batch["speech_feat"].to(device)
90
+ speech_feat_len = batch["speech_feat_len"].to(device)
91
+ utt_embedding = batch["utt_embedding"].to(device)
92
+ spk_embedding = batch["spk_embedding"].to(device)
93
+ if args.mode == 'sft':
94
+ model_input = {'text': tts_text_token, 'text_len': tts_text_token_len,
95
+ 'llm_embedding': spk_embedding, 'flow_embedding': spk_embedding}
96
+ else:
97
+ model_input = {'text': tts_text_token, 'text_len': tts_text_token_len,
98
+ 'prompt_text': text_token, 'prompt_text_len': text_token_len,
99
+ 'llm_prompt_speech_token': speech_token, 'llm_prompt_speech_token_len': speech_token_len,
100
+ 'flow_prompt_speech_token': speech_token, 'flow_prompt_speech_token_len': speech_token_len,
101
+ 'prompt_speech_feat': speech_feat, 'prompt_speech_feat_len': speech_feat_len,
102
+ 'llm_embedding': utt_embedding, 'flow_embedding': utt_embedding}
103
+ model_output = model.inference(**model_input)
104
+ tts_key = '{}_{}'.format(utts[0], tts_index[0])
105
+ tts_fn = os.path.join(args.result_dir, '{}.wav'.format(tts_key))
106
+ torchaudio.save(tts_fn, model_output['tts_speech'], sample_rate=22050)
107
+ f.write('{} {}\n'.format(tts_key, tts_fn))
108
+ f.flush()
109
+ f.close()
110
+ logging.info('Result wav.scp saved in {}'.format(fn))
111
+
112
+
113
+ if __name__ == '__main__':
114
+ main()
third_party/GLM-4-Voice/cosyvoice/bin/train.py ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu)
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from __future__ import print_function
16
+ import argparse
17
+ import datetime
18
+ import logging
19
+ logging.getLogger('matplotlib').setLevel(logging.WARNING)
20
+ from copy import deepcopy
21
+ import torch
22
+ import torch.distributed as dist
23
+ # import deepspeed
24
+ import pdb
25
+ from hyperpyyaml import load_hyperpyyaml
26
+
27
+ from torch.distributed.elastic.multiprocessing.errors import record
28
+
29
+ from cosyvoice.utils.executor import Executor
30
+ from cosyvoice.utils.train_utils import (
31
+ init_distributed,
32
+ init_dataset_and_dataloader,
33
+ init_optimizer_and_scheduler,
34
+ init_summarywriter, save_model,
35
+ wrap_cuda_model, check_modify_and_save_config)
36
+
37
+
38
+ def get_args():
39
+ parser = argparse.ArgumentParser(description='training your network')
40
+ parser.add_argument('--train_engine',
41
+ default='torch_ddp',
42
+ choices=['torch_ddp', 'deepspeed'],
43
+ help='Engine for paralleled training')
44
+ parser.add_argument('--model', required=True, help='model which will be trained')
45
+ parser.add_argument('--config', required=True, help='config file')
46
+ parser.add_argument('--train_data', required=True, help='train data file')
47
+ parser.add_argument('--cv_data', required=True, help='cv data file')
48
+ parser.add_argument('--checkpoint', help='checkpoint model')
49
+ parser.add_argument('--model_dir', required=True, help='save model dir')
50
+ parser.add_argument('--tensorboard_dir',
51
+ default='tensorboard',
52
+ help='tensorboard log dir')
53
+ parser.add_argument('--ddp.dist_backend',
54
+ dest='dist_backend',
55
+ default='nccl',
56
+ choices=['nccl', 'gloo'],
57
+ help='distributed backend')
58
+ parser.add_argument('--num_workers',
59
+ default=0,
60
+ type=int,
61
+ help='num of subprocess workers for reading')
62
+ parser.add_argument('--prefetch',
63
+ default=100,
64
+ type=int,
65
+ help='prefetch number')
66
+ parser.add_argument('--pin_memory',
67
+ action='store_true',
68
+ default=False,
69
+ help='Use pinned memory buffers used for reading')
70
+ parser.add_argument('--deepspeed.save_states',
71
+ dest='save_states',
72
+ default='model_only',
73
+ choices=['model_only', 'model+optimizer'],
74
+ help='save model/optimizer states')
75
+ parser.add_argument('--timeout',
76
+ default=30,
77
+ type=int,
78
+ help='timeout (in seconds) of cosyvoice_join.')
79
+ # parser = deepspeed.add_config_arguments(parser)
80
+ args = parser.parse_args()
81
+ return args
82
+
83
+
84
+ @record
85
+ def main():
86
+ args = get_args()
87
+ logging.basicConfig(level=logging.DEBUG,
88
+ format='%(asctime)s %(levelname)s %(message)s')
89
+
90
+ override_dict = {k: None for k in ['llm', 'flow', 'hift'] if k != args.model}
91
+ with open(args.config, 'r') as f:
92
+ configs = load_hyperpyyaml(f, overrides=override_dict)
93
+ configs['train_conf'].update(vars(args))
94
+
95
+ # Init env for ddp
96
+ init_distributed(args)
97
+
98
+ # Get dataset & dataloader
99
+ train_dataset, cv_dataset, train_data_loader, cv_data_loader = \
100
+ init_dataset_and_dataloader(args, configs)
101
+
102
+ # Do some sanity checks and save config to arsg.model_dir
103
+ configs = check_modify_and_save_config(args, configs)
104
+
105
+ # Tensorboard summary
106
+ writer = init_summarywriter(args)
107
+
108
+ # load checkpoint
109
+ model = configs[args.model]
110
+ if args.checkpoint is not None:
111
+ model.load_state_dict(torch.load(args.checkpoint, map_location='cpu'))
112
+
113
+ # Dispatch model from cpu to gpu
114
+ model = wrap_cuda_model(args, model)
115
+
116
+ # Get optimizer & scheduler
117
+ model, optimizer, scheduler = init_optimizer_and_scheduler(args, configs, model)
118
+ # pdb.set_trace()
119
+ # Save init checkpoints
120
+ info_dict = deepcopy(configs['train_conf'])
121
+ save_model(model, 'init', info_dict)
122
+
123
+ # Get executor
124
+ executor = Executor()
125
+
126
+ # Start training loop
127
+ for epoch in range(info_dict['max_epoch']):
128
+ executor.epoch = epoch
129
+ train_dataset.set_epoch(epoch)
130
+ dist.barrier()
131
+ # try:
132
+ # dist.barrier()
133
+ # except RuntimeError as e:
134
+ # logging.info('except RuntimeError as e: {}'.format(e))
135
+ group_join = dist.new_group(backend="gloo", timeout=datetime.timedelta(seconds=args.timeout))
136
+ executor.train_one_epoc(model, optimizer, scheduler, train_data_loader, cv_data_loader, writer, info_dict, group_join)
137
+ dist.destroy_process_group(group_join)
138
+
139
+ if __name__ == '__main__':
140
+ main()
third_party/GLM-4-Voice/cosyvoice/cli/__init__.py ADDED
File without changes
third_party/GLM-4-Voice/cosyvoice/cli/cosyvoice.py ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu)
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ import os
15
+ import torch
16
+ from hyperpyyaml import load_hyperpyyaml
17
+ from modelscope import snapshot_download
18
+ from cosyvoice.cli.frontend import CosyVoiceFrontEnd
19
+ from cosyvoice.cli.model import CosyVoiceModel
20
+
21
+ class CosyVoice:
22
+
23
+ def __init__(self, model_dir):
24
+ instruct = True if '-Instruct' in model_dir else False
25
+ self.model_dir = model_dir
26
+ if not os.path.exists(model_dir):
27
+ model_dir = snapshot_download(model_dir)
28
+ with open('{}/cosyvoice.yaml'.format(model_dir), 'r') as f:
29
+ configs = load_hyperpyyaml(f)
30
+ self.frontend = CosyVoiceFrontEnd(configs['get_tokenizer'],
31
+ configs['feat_extractor'],
32
+ '{}/campplus.onnx'.format(model_dir),
33
+ '{}/speech_tokenizer_v1.onnx'.format(model_dir),
34
+ '{}/spk2info.pt'.format(model_dir),
35
+ instruct,
36
+ configs['allowed_special'])
37
+ self.model = CosyVoiceModel(configs['llm'], configs['flow'], configs['hift'])
38
+ self.model.load('{}/llm.pt'.format(model_dir),
39
+ '{}/flow.pt'.format(model_dir),
40
+ '{}/hift.pt'.format(model_dir))
41
+ del configs
42
+
43
+ def list_avaliable_spks(self):
44
+ spks = list(self.frontend.spk2info.keys())
45
+ return spks
46
+
47
+ def inference_sft(self, tts_text, spk_id):
48
+ tts_speeches = []
49
+ for i in self.frontend.text_normalize(tts_text, split=True):
50
+ model_input = self.frontend.frontend_sft(i, spk_id)
51
+ model_output = self.model.inference(**model_input)
52
+ tts_speeches.append(model_output['tts_speech'])
53
+ return {'tts_speech': torch.concat(tts_speeches, dim=1)}
54
+
55
+ def inference_zero_shot(self, tts_text, prompt_text, prompt_speech_16k):
56
+ prompt_text = self.frontend.text_normalize(prompt_text, split=False)
57
+ tts_speeches = []
58
+ for i in self.frontend.text_normalize(tts_text, split=True):
59
+ model_input = self.frontend.frontend_zero_shot(i, prompt_text, prompt_speech_16k)
60
+ model_output = self.model.inference(**model_input)
61
+ tts_speeches.append(model_output['tts_speech'])
62
+ return {'tts_speech': torch.concat(tts_speeches, dim=1)}
63
+
64
+ def inference_cross_lingual(self, tts_text, prompt_speech_16k):
65
+ if self.frontend.instruct is True:
66
+ raise ValueError('{} do not support cross_lingual inference'.format(self.model_dir))
67
+ tts_speeches = []
68
+ for i in self.frontend.text_normalize(tts_text, split=True):
69
+ model_input = self.frontend.frontend_cross_lingual(i, prompt_speech_16k)
70
+ model_output = self.model.inference(**model_input)
71
+ tts_speeches.append(model_output['tts_speech'])
72
+ return {'tts_speech': torch.concat(tts_speeches, dim=1)}
73
+
74
+ def inference_instruct(self, tts_text, spk_id, instruct_text):
75
+ if self.frontend.instruct is False:
76
+ raise ValueError('{} do not support instruct inference'.format(self.model_dir))
77
+ instruct_text = self.frontend.text_normalize(instruct_text, split=False)
78
+ tts_speeches = []
79
+ for i in self.frontend.text_normalize(tts_text, split=True):
80
+ model_input = self.frontend.frontend_instruct(i, spk_id, instruct_text)
81
+ model_output = self.model.inference(**model_input)
82
+ tts_speeches.append(model_output['tts_speech'])
83
+ return {'tts_speech': torch.concat(tts_speeches, dim=1)}
third_party/GLM-4-Voice/cosyvoice/cli/frontend.py ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu)
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ from functools import partial
15
+ import onnxruntime
16
+ import torch
17
+ import numpy as np
18
+ import whisper
19
+ from typing import Callable
20
+ import torchaudio.compliance.kaldi as kaldi
21
+ import torchaudio
22
+ import os
23
+ import re
24
+ import inflect
25
+ try:
26
+ import ttsfrd
27
+ use_ttsfrd = True
28
+ except ImportError:
29
+ print("failed to import ttsfrd, use WeTextProcessing instead")
30
+ from tn.chinese.normalizer import Normalizer as ZhNormalizer
31
+ from tn.english.normalizer import Normalizer as EnNormalizer
32
+ use_ttsfrd = False
33
+ from cosyvoice.utils.frontend_utils import contains_chinese, replace_blank, replace_corner_mark, remove_bracket, spell_out_number, split_paragraph
34
+
35
+
36
+ class CosyVoiceFrontEnd:
37
+
38
+ def __init__(self,
39
+ get_tokenizer: Callable,
40
+ feat_extractor: Callable,
41
+ campplus_model: str,
42
+ speech_tokenizer_model: str,
43
+ spk2info: str = '',
44
+ instruct: bool = False,
45
+ allowed_special: str = 'all'):
46
+ self.tokenizer = get_tokenizer()
47
+ self.feat_extractor = feat_extractor
48
+ self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
49
+ option = onnxruntime.SessionOptions()
50
+ option.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
51
+ option.intra_op_num_threads = 1
52
+ self.campplus_session = onnxruntime.InferenceSession(campplus_model, sess_options=option, providers=["CPUExecutionProvider"])
53
+ self.speech_tokenizer_session = onnxruntime.InferenceSession(speech_tokenizer_model, sess_options=option, providers=["CUDAExecutionProvider"if torch.cuda.is_available() else "CPUExecutionProvider"])
54
+ if os.path.exists(spk2info):
55
+ self.spk2info = torch.load(spk2info, map_location=self.device)
56
+ self.instruct = instruct
57
+ self.allowed_special = allowed_special
58
+ self.inflect_parser = inflect.engine()
59
+ self.use_ttsfrd = use_ttsfrd
60
+ if self.use_ttsfrd:
61
+ self.frd = ttsfrd.TtsFrontendEngine()
62
+ ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
63
+ assert self.frd.initialize('{}/../../pretrained_models/CosyVoice-ttsfrd/resource'.format(ROOT_DIR)) is True, 'failed to initialize ttsfrd resource'
64
+ self.frd.set_lang_type('pinyin')
65
+ self.frd.enable_pinyin_mix(True)
66
+ self.frd.set_breakmodel_index(1)
67
+ else:
68
+ self.zh_tn_model = ZhNormalizer(remove_erhua=False, full_to_half=False)
69
+ self.en_tn_model = EnNormalizer()
70
+
71
+ def _extract_text_token(self, text):
72
+ text_token = self.tokenizer.encode(text, allowed_special=self.allowed_special)
73
+ text_token = torch.tensor([text_token], dtype=torch.int32).to(self.device)
74
+ text_token_len = torch.tensor([text_token.shape[1]], dtype=torch.int32).to(self.device)
75
+ return text_token, text_token_len
76
+
77
+ def _extract_speech_token(self, speech):
78
+ feat = whisper.log_mel_spectrogram(speech, n_mels=128)
79
+ speech_token = self.speech_tokenizer_session.run(None, {self.speech_tokenizer_session.get_inputs()[0].name: feat.detach().cpu().numpy(),
80
+ self.speech_tokenizer_session.get_inputs()[1].name: np.array([feat.shape[2]], dtype=np.int32)})[0].flatten().tolist()
81
+ speech_token = torch.tensor([speech_token], dtype=torch.int32).to(self.device)
82
+ speech_token_len = torch.tensor([speech_token.shape[1]], dtype=torch.int32).to(self.device)
83
+ return speech_token, speech_token_len
84
+
85
+ def _extract_spk_embedding(self, speech):
86
+ feat = kaldi.fbank(speech,
87
+ num_mel_bins=80,
88
+ dither=0,
89
+ sample_frequency=16000)
90
+ feat = feat - feat.mean(dim=0, keepdim=True)
91
+ embedding = self.campplus_session.run(None, {self.campplus_session.get_inputs()[0].name: feat.unsqueeze(dim=0).cpu().numpy()})[0].flatten().tolist()
92
+ embedding = torch.tensor([embedding]).to(self.device)
93
+ return embedding
94
+
95
+ def _extract_speech_feat(self, speech):
96
+ speech_feat = self.feat_extractor(speech).squeeze(dim=0).transpose(0, 1).to(self.device)
97
+ speech_feat = speech_feat.unsqueeze(dim=0)
98
+ speech_feat_len = torch.tensor([speech_feat.shape[1]], dtype=torch.int32).to(self.device)
99
+ return speech_feat, speech_feat_len
100
+
101
+ def text_normalize(self, text, split=True):
102
+ text = text.strip()
103
+ if contains_chinese(text):
104
+ if self.use_ttsfrd:
105
+ text = self.frd.get_frd_extra_info(text, 'input')
106
+ else:
107
+ text = self.zh_tn_model.normalize(text)
108
+ text = text.replace("\n", "")
109
+ text = replace_blank(text)
110
+ text = replace_corner_mark(text)
111
+ text = text.replace(".", "、")
112
+ text = text.replace(" - ", ",")
113
+ text = remove_bracket(text)
114
+ text = re.sub(r'[,,]+$', '。', text)
115
+ texts = [i for i in split_paragraph(text, partial(self.tokenizer.encode, allowed_special=self.allowed_special), "zh", token_max_n=80,
116
+ token_min_n=60, merge_len=20,
117
+ comma_split=False)]
118
+ else:
119
+ if self.use_ttsfrd:
120
+ text = self.frd.get_frd_extra_info(text, 'input')
121
+ else:
122
+ text = self.en_tn_model.normalize(text)
123
+ text = spell_out_number(text, self.inflect_parser)
124
+ texts = [i for i in split_paragraph(text, partial(self.tokenizer.encode, allowed_special=self.allowed_special), "en", token_max_n=80,
125
+ token_min_n=60, merge_len=20,
126
+ comma_split=False)]
127
+ if split is False:
128
+ return text
129
+ return texts
130
+
131
+ def frontend_sft(self, tts_text, spk_id):
132
+ tts_text_token, tts_text_token_len = self._extract_text_token(tts_text)
133
+ embedding = self.spk2info[spk_id]['embedding']
134
+ model_input = {'text': tts_text_token, 'text_len': tts_text_token_len, 'llm_embedding': embedding, 'flow_embedding': embedding}
135
+ return model_input
136
+
137
+ def frontend_zero_shot(self, tts_text, prompt_text, prompt_speech_16k):
138
+ tts_text_token, tts_text_token_len = self._extract_text_token(tts_text)
139
+ prompt_text_token, prompt_text_token_len = self._extract_text_token(prompt_text)
140
+ prompt_speech_22050 = torchaudio.transforms.Resample(orig_freq=16000, new_freq=22050)(prompt_speech_16k)
141
+ speech_feat, speech_feat_len = self._extract_speech_feat(prompt_speech_22050)
142
+ speech_token, speech_token_len = self._extract_speech_token(prompt_speech_16k)
143
+ embedding = self._extract_spk_embedding(prompt_speech_16k)
144
+ model_input = {'text': tts_text_token, 'text_len': tts_text_token_len,
145
+ 'prompt_text': prompt_text_token, 'prompt_text_len': prompt_text_token_len,
146
+ 'llm_prompt_speech_token': speech_token, 'llm_prompt_speech_token_len': speech_token_len,
147
+ 'flow_prompt_speech_token': speech_token, 'flow_prompt_speech_token_len': speech_token_len,
148
+ 'prompt_speech_feat': speech_feat, 'prompt_speech_feat_len': speech_feat_len,
149
+ 'llm_embedding': embedding, 'flow_embedding': embedding}
150
+ return model_input
151
+
152
+ def frontend_cross_lingual(self, tts_text, prompt_speech_16k):
153
+ model_input = self.frontend_zero_shot(tts_text, '', prompt_speech_16k)
154
+ # in cross lingual mode, we remove prompt in llm
155
+ del model_input['prompt_text']
156
+ del model_input['prompt_text_len']
157
+ del model_input['llm_prompt_speech_token']
158
+ del model_input['llm_prompt_speech_token_len']
159
+ return model_input
160
+
161
+ def frontend_instruct(self, tts_text, spk_id, instruct_text):
162
+ model_input = self.frontend_sft(tts_text, spk_id)
163
+ # in instruct mode, we remove spk_embedding in llm due to information leakage
164
+ del model_input['llm_embedding']
165
+ instruct_text_token, instruct_text_token_len = self._extract_text_token(instruct_text + '<endofprompt>')
166
+ model_input['prompt_text'] = instruct_text_token
167
+ model_input['prompt_text_len'] = instruct_text_token_len
168
+ return model_input