Spaces:
Running on Zero
Running on Zero
Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +6 -0
- README.md +14 -8
- app.py +532 -0
- asr_0.wav +3 -0
- chat_template.txt +2 -0
- omni_diffusion/__init__.py +2 -0
- omni_diffusion/constants.py +105 -0
- omni_diffusion/data/__init__.py +3 -0
- omni_diffusion/data/build.py +110 -0
- omni_diffusion/data/data_collator.py +122 -0
- omni_diffusion/data/dataset_base.py +424 -0
- omni_diffusion/data/dataset_qwen2.py +981 -0
- omni_diffusion/data/processor/__init__.py +0 -0
- omni_diffusion/data/processor/audio_processor.py +142 -0
- omni_diffusion/data/processor/image_processor.py +358 -0
- omni_diffusion/data/utils.py +212 -0
- omni_diffusion/models/__init__.py +3 -0
- omni_diffusion/models/dream/__init__.py +12 -0
- omni_diffusion/models/dream/config.yaml +98 -0
- omni_diffusion/models/dream/config_dream_resume.json +37 -0
- omni_diffusion/models/dream/configuration.json +14 -0
- omni_diffusion/models/dream/configuration_dream.py +86 -0
- omni_diffusion/models/dream/generation_utils.py +623 -0
- omni_diffusion/models/dream/modeling_dream.py +1136 -0
- omni_diffusion/models/dream/modeling_sensevoice.py +1258 -0
- omni_diffusion/models/dream/resampler_projector.py +40 -0
- omni_diffusion/models/dream/tokenization_dream.py +351 -0
- omni_diffusion/models/dream/tokenizer_config.json +219 -0
- omni_diffusion/models/magvit/common_modules.py +393 -0
- omni_diffusion/models/magvit/misc.py +55 -0
- omni_diffusion/models/magvit/modeling_magvitv2.py +445 -0
- omni_diffusion/models/magvit/modeling_utils.py +1184 -0
- omni_diffusion/tokenizer.py +74 -0
- omni_diffusion/tokenizer_magvitv2.py +65 -0
- omni_diffusion/tokenizer_sensevoice_glm4voice.py +280 -0
- requirements.txt +17 -0
- s2i_0.wav +3 -0
- svqa_0.jpg +3 -0
- svqa_0.wav +0 -0
- third_party/GLM-4-Voice/.gitignore +4 -0
- third_party/GLM-4-Voice/LICENSE +201 -0
- third_party/GLM-4-Voice/README.md +159 -0
- third_party/GLM-4-Voice/README_en.md +148 -0
- third_party/GLM-4-Voice/audio_process.py +93 -0
- third_party/GLM-4-Voice/cosyvoice/__init__.py +0 -0
- third_party/GLM-4-Voice/cosyvoice/bin/inference.py +114 -0
- third_party/GLM-4-Voice/cosyvoice/bin/train.py +140 -0
- third_party/GLM-4-Voice/cosyvoice/cli/__init__.py +0 -0
- third_party/GLM-4-Voice/cosyvoice/cli/cosyvoice.py +83 -0
- third_party/GLM-4-Voice/cosyvoice/cli/frontend.py +168 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
asr_0.wav filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
s2i_0.wav filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
svqa_0.jpg filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
third_party/GLM-4-Voice/resources/architecture.jpeg filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
third_party/GLM-4-Voice/resources/web_demo.png filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
vqa_0.png filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
|
@@ -1,13 +1,19 @@
|
|
| 1 |
---
|
| 2 |
-
title: Omni
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version: 6.
|
| 8 |
-
python_version: '3.12'
|
| 9 |
app_file: app.py
|
| 10 |
-
|
|
|
|
|
|
|
| 11 |
---
|
| 12 |
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: Omni-Diffusion
|
| 3 |
+
emoji: 🌐
|
| 4 |
+
colorFrom: yellow
|
| 5 |
+
colorTo: purple
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: 6.15.1
|
|
|
|
| 8 |
app_file: app.py
|
| 9 |
+
short_description: Any-to-any multimodal model with discrete diffusion
|
| 10 |
+
python_version: "3.10"
|
| 11 |
+
startup_duration_timeout: 1h
|
| 12 |
---
|
| 13 |
|
| 14 |
+
# Omni-Diffusion
|
| 15 |
+
|
| 16 |
+
Interactive demo for **Omni-Diffusion**, a unified any-to-any multimodal model based on masked discrete diffusion.
|
| 17 |
+
Supports text-to-image, speech-to-image, text-to-speech, ASR, visual question answering, and spoken VQA.
|
| 18 |
+
|
| 19 |
+
Paper: [Omni-Diffusion: Unified Multimodal Understanding and Generation with Masked Discrete Diffusion](https://arxiv.org/abs/2603.06577)
|
app.py
ADDED
|
@@ -0,0 +1,532 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
|
| 3 |
+
import spaces
|
| 4 |
+
import torch
|
| 5 |
+
import gradio as gr
|
| 6 |
+
import numpy as np
|
| 7 |
+
import re
|
| 8 |
+
import sys
|
| 9 |
+
import random
|
| 10 |
+
import tempfile
|
| 11 |
+
import torchaudio
|
| 12 |
+
import cv2
|
| 13 |
+
from transformers import AutoConfig, AutoModel, AutoTokenizer
|
| 14 |
+
from transformers.generation import GenerationConfig
|
| 15 |
+
|
| 16 |
+
# Add third_party paths for GLM-4-Voice
|
| 17 |
+
_HERE = os.path.dirname(os.path.abspath(__file__))
|
| 18 |
+
sys.path.insert(0, os.path.join(_HERE, "third_party", "GLM-4-Voice"))
|
| 19 |
+
sys.path.insert(0, os.path.join(_HERE, "third_party", "GLM-4-Voice", "cosyvoice"))
|
| 20 |
+
sys.path.insert(0, os.path.join(_HERE, "third_party", "GLM-4-Voice", "third_party", "Matcha-TTS"))
|
| 21 |
+
|
| 22 |
+
from omni_diffusion.data.processor.audio_processor import add_audio_input_contiguous
|
| 23 |
+
from omni_diffusion.tokenizer import get_audio_tokenizer
|
| 24 |
+
from omni_diffusion.data.processor.image_processor import ImageProcessor
|
| 25 |
+
|
| 26 |
+
MODEL_ID = "lijiang/Omni-Diffusion"
|
| 27 |
+
AUDIO_TOKENIZER_PATH = "THUDM/glm-4-voice-tokenizer"
|
| 28 |
+
AUDIO_DECODER_PATH = "THUDM/glm-4-voice-decoder"
|
| 29 |
+
IMAGE_TOKENIZER_PATH = "showlab/magvitv2"
|
| 30 |
+
AUDIO_TOKENIZER_TYPE = "sensevoice_glm4voice"
|
| 31 |
+
|
| 32 |
+
# Load chat template from file to avoid escaping issues
|
| 33 |
+
_TEMPLATE_PATH = os.path.join(_HERE, "chat_template.txt")
|
| 34 |
+
with open(_TEMPLATE_PATH, "r") as _f:
|
| 35 |
+
qwen2_chat_template = _f.read()
|
| 36 |
+
|
| 37 |
+
def set_seed(seed=42):
|
| 38 |
+
random.seed(seed)
|
| 39 |
+
np.random.seed(seed)
|
| 40 |
+
torch.manual_seed(seed)
|
| 41 |
+
if torch.cuda.is_available():
|
| 42 |
+
torch.cuda.manual_seed(seed)
|
| 43 |
+
torch.cuda.manual_seed_all(seed)
|
| 44 |
+
|
| 45 |
+
set_seed()
|
| 46 |
+
|
| 47 |
+
print("Loading model...", flush=True)
|
| 48 |
+
|
| 49 |
+
config = AutoConfig.from_pretrained(MODEL_ID, trust_remote_code=True)
|
| 50 |
+
|
| 51 |
+
tokenizer = AutoTokenizer.from_pretrained(
|
| 52 |
+
MODEL_ID, trust_remote_code=True, chat_template=qwen2_chat_template
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
model = AutoModel.from_pretrained(
|
| 56 |
+
MODEL_ID,
|
| 57 |
+
trust_remote_code=True,
|
| 58 |
+
torch_dtype=torch.bfloat16,
|
| 59 |
+
attn_implementation="sdpa",
|
| 60 |
+
).to("cuda").eval()
|
| 61 |
+
|
| 62 |
+
model.generation_config = GenerationConfig.from_pretrained(MODEL_ID, trust_remote_code=True)
|
| 63 |
+
model.generation_config.max_new_tokens = 8192
|
| 64 |
+
model.generation_config.chat_format = "chatml"
|
| 65 |
+
model.generation_config.max_window_size = 8192
|
| 66 |
+
model.generation_config.use_cache = True
|
| 67 |
+
model.generation_config.do_sample = False
|
| 68 |
+
model.generation_config.temperature = 1.0
|
| 69 |
+
model.generation_config.top_k = 50
|
| 70 |
+
model.generation_config.top_p = 1.0
|
| 71 |
+
model.generation_config.num_beams = 1
|
| 72 |
+
model.generation_config.pad_token_id = tokenizer.pad_token_id
|
| 73 |
+
|
| 74 |
+
print("Loading audio tokenizer...", flush=True)
|
| 75 |
+
audio_tokenizer = get_audio_tokenizer(
|
| 76 |
+
AUDIO_TOKENIZER_PATH,
|
| 77 |
+
AUDIO_TOKENIZER_TYPE,
|
| 78 |
+
flow_path=AUDIO_DECODER_PATH,
|
| 79 |
+
rank=0,
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
print("Loading image processor...", flush=True)
|
| 83 |
+
image_processor = ImageProcessor(
|
| 84 |
+
IMAGE_TOKENIZER_PATH,
|
| 85 |
+
"dynamic",
|
| 86 |
+
image_size=512,
|
| 87 |
+
normalize_type="imagenet",
|
| 88 |
+
min_patch_grid=1,
|
| 89 |
+
max_patch_grid=12,
|
| 90 |
+
)
|
| 91 |
+
image_processor.image_tokenizer.rank = 0
|
| 92 |
+
image_processor.load_model()
|
| 93 |
+
|
| 94 |
+
print("All models loaded successfully!", flush=True)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def find_audio_segments_regex(text):
|
| 98 |
+
pattern = re.compile(r"<\|begin_of_audio\|>(.*?)<\|end_of_audio\|>", re.DOTALL)
|
| 99 |
+
segments = pattern.findall(text)
|
| 100 |
+
return [segment.strip() for segment in segments]
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def run_inference(
|
| 104 |
+
task,
|
| 105 |
+
text_prompt="",
|
| 106 |
+
audio_path=None,
|
| 107 |
+
image_path=None,
|
| 108 |
+
max_tokens=128,
|
| 109 |
+
steps=128,
|
| 110 |
+
alg="entropy",
|
| 111 |
+
cfg=0.0,
|
| 112 |
+
max_position_penalty=1.0,
|
| 113 |
+
repeat_penalty=1.0,
|
| 114 |
+
add_boa_token=0,
|
| 115 |
+
output_text_only=False,
|
| 116 |
+
):
|
| 117 |
+
"""Run Omni-Diffusion inference for a given task.
|
| 118 |
+
|
| 119 |
+
Args:
|
| 120 |
+
task: One of T2I, S2I, TTS, ASR, VQA, SVQA.
|
| 121 |
+
text_prompt: Text input for the task.
|
| 122 |
+
audio_path: Path to input audio file (for ASR, S2I, SVQA).
|
| 123 |
+
image_path: Path to input image file (for VQA, SVQA).
|
| 124 |
+
max_tokens: Maximum number of new tokens to generate.
|
| 125 |
+
steps: Number of diffusion steps.
|
| 126 |
+
alg: Algorithm for generation ("entropy", "entropy-penalty", "origin").
|
| 127 |
+
cfg: Classifier-free guidance scale.
|
| 128 |
+
max_position_penalty: Position penalty for generation.
|
| 129 |
+
repeat_penalty: Repeat penalty for generation.
|
| 130 |
+
add_boa_token: Whether to add beginning-of-audio token.
|
| 131 |
+
output_text_only: Whether to output text only.
|
| 132 |
+
|
| 133 |
+
Returns:
|
| 134 |
+
Tuple of (text_output, audio_output, image_output).
|
| 135 |
+
"""
|
| 136 |
+
system_message = []
|
| 137 |
+
|
| 138 |
+
if task == "S2I":
|
| 139 |
+
system_message = [{"role": "system", "content": "Please generate an image based on the input audio."}]
|
| 140 |
+
elif task == "SQA":
|
| 141 |
+
system_message = [{"role": "system", "content": "Please response the input audio."}]
|
| 142 |
+
elif task == "SVQA":
|
| 143 |
+
system_message = [{"role": "system", "content": "Please response the input audio based on the given image."}]
|
| 144 |
+
|
| 145 |
+
if audio_path is not None and image_path is not None:
|
| 146 |
+
messages = system_message + [{"role": "user", "content": "<|audio|>\n<|image|>"}]
|
| 147 |
+
elif audio_path is not None:
|
| 148 |
+
messages = system_message + [{"role": "user", "content": text_prompt + "\n<|audio|>"}]
|
| 149 |
+
elif image_path is not None:
|
| 150 |
+
messages = system_message + [{"role": "user", "content": text_prompt + "\n<|image|>"}]
|
| 151 |
+
else:
|
| 152 |
+
messages = system_message + [{"role": "user", "content": text_prompt}]
|
| 153 |
+
|
| 154 |
+
if audio_path is not None and audio_tokenizer.apply_to_role("user", is_discrete=True):
|
| 155 |
+
audio_tokens = audio_tokenizer.encode(audio_path)
|
| 156 |
+
audio_tokens_str = "".join(f"<|audio_{i}|>" for i in audio_tokens)
|
| 157 |
+
messages[-1]["content"] = messages[-1]["content"].replace("<|audio|>", audio_tokens_str)
|
| 158 |
+
|
| 159 |
+
if image_path is not None:
|
| 160 |
+
image_tokens = image_processor.process_images_with_subpatch(image_path, 512)
|
| 161 |
+
image_tokens = image_processor.get_image_token(image_tokens)
|
| 162 |
+
image_tokens = image_tokens[0].tolist()
|
| 163 |
+
image_tokens_str = "".join(f"<|image_{i}|>" for i in image_tokens)
|
| 164 |
+
messages[-1]["content"] = messages[-1]["content"].replace("<|image|>", image_tokens_str)
|
| 165 |
+
|
| 166 |
+
input_ids = tokenizer.apply_chat_template(
|
| 167 |
+
messages, tokenize=True, add_generation_prompt=True
|
| 168 |
+
)
|
| 169 |
+
|
| 170 |
+
audios = None
|
| 171 |
+
audio_indices = None
|
| 172 |
+
if audio_path is not None and audio_tokenizer.apply_to_role("user", is_contiguous=True):
|
| 173 |
+
audio_paths = [audio_path]
|
| 174 |
+
input_ids, audios, audio_indices = add_audio_input_contiguous(
|
| 175 |
+
input_ids, audio_paths, tokenizer, audio_tokenizer
|
| 176 |
+
)
|
| 177 |
+
|
| 178 |
+
input_ids_tensor = torch.tensor([input_ids], dtype=torch.long).to("cuda")
|
| 179 |
+
|
| 180 |
+
with torch.no_grad():
|
| 181 |
+
outputs, histories = model.generate(
|
| 182 |
+
input_ids_tensor,
|
| 183 |
+
audios=audios,
|
| 184 |
+
audio_indices=audio_indices,
|
| 185 |
+
temperature=0.0,
|
| 186 |
+
top_p=0.9,
|
| 187 |
+
steps=steps,
|
| 188 |
+
max_new_tokens=max_tokens,
|
| 189 |
+
alg=alg,
|
| 190 |
+
cfg=cfg,
|
| 191 |
+
tokenizer=tokenizer,
|
| 192 |
+
add_boa_token=add_boa_token,
|
| 193 |
+
max_position_penalty=max_position_penalty,
|
| 194 |
+
repeat_penalty=repeat_penalty,
|
| 195 |
+
output_text_only=output_text_only,
|
| 196 |
+
task=task,
|
| 197 |
+
)
|
| 198 |
+
|
| 199 |
+
audio_offset = tokenizer.convert_tokens_to_ids("<|audio_0|>")
|
| 200 |
+
image_offset = tokenizer.convert_tokens_to_ids("<|image_0|>")
|
| 201 |
+
audio_tokens_out = []
|
| 202 |
+
image_tokens_out = []
|
| 203 |
+
text_tokens_out = []
|
| 204 |
+
|
| 205 |
+
for token_id in outputs[0][input_ids_tensor.shape[1]:]:
|
| 206 |
+
tid = token_id.item()
|
| 207 |
+
if tid >= audio_offset and tid < audio_offset + 16384:
|
| 208 |
+
audio_tokens_out.append(tid - audio_offset)
|
| 209 |
+
elif tid >= image_offset:
|
| 210 |
+
image_tokens_out.append(tid - image_offset)
|
| 211 |
+
else:
|
| 212 |
+
text_tokens_out.append(tid)
|
| 213 |
+
|
| 214 |
+
# Decode audio
|
| 215 |
+
tts_speech = None
|
| 216 |
+
if len(audio_tokens_out) > 0:
|
| 217 |
+
tts_speech = audio_tokenizer.decode(audio_tokens_out, source_speech_16k=None)
|
| 218 |
+
|
| 219 |
+
# Decode image
|
| 220 |
+
image_out = None
|
| 221 |
+
if len(image_tokens_out) < 256 and len(image_tokens_out) > 0:
|
| 222 |
+
image_tokens_out += [image_tokens_out[-1]] * (256 - len(image_tokens_out))
|
| 223 |
+
|
| 224 |
+
if len(image_tokens_out) > 0:
|
| 225 |
+
gen_token_ids = torch.stack([torch.tensor(t) for t in image_tokens_out], dim=0).unsqueeze(0)
|
| 226 |
+
gen_token_ids = torch.clamp(gen_token_ids, max=8192 - 1, min=0)
|
| 227 |
+
with torch.no_grad():
|
| 228 |
+
image_out = image_processor.image_tokenizer.image_tokenizer.decode_code(
|
| 229 |
+
gen_token_ids[:, :256]
|
| 230 |
+
)
|
| 231 |
+
image_out = torch.clamp((image_out + 1.0) / 2.0, min=0.0, max=1.0)
|
| 232 |
+
image_out *= 255.0
|
| 233 |
+
image_out = image_out.permute(0, 2, 3, 1).cpu().numpy().astype(np.uint8)
|
| 234 |
+
image_out = image_out[:, :, :, [2, 1, 0]][0]
|
| 235 |
+
|
| 236 |
+
# Decode text
|
| 237 |
+
text_out = tokenizer.decode(text_tokens_out, skip_special_tokens=True)
|
| 238 |
+
|
| 239 |
+
return text_out, tts_speech, image_out
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
@spaces.GPU(duration=300)
|
| 243 |
+
def generate_t2i(prompt, steps=260, alg="entropy-penalty", repeat_penalty=1.2):
|
| 244 |
+
"""Generate an image from a text prompt.
|
| 245 |
+
|
| 246 |
+
Args:
|
| 247 |
+
prompt: Text description of the image to generate.
|
| 248 |
+
steps: Number of diffusion denoising steps.
|
| 249 |
+
alg: Generation algorithm.
|
| 250 |
+
repeat_penalty: Penalty for repeated tokens.
|
| 251 |
+
"""
|
| 252 |
+
message = "Generate an image based on the provided text description.\n" + prompt
|
| 253 |
+
text_out, audio_out, image_out = run_inference(
|
| 254 |
+
task="T2I",
|
| 255 |
+
text_prompt=message,
|
| 256 |
+
max_tokens=260,
|
| 257 |
+
steps=steps,
|
| 258 |
+
alg=alg,
|
| 259 |
+
repeat_penalty=repeat_penalty,
|
| 260 |
+
max_position_penalty=2.0,
|
| 261 |
+
)
|
| 262 |
+
return image_out, text_out
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
@spaces.GPU(duration=300)
|
| 266 |
+
def generate_tts(text, steps=25, alg="entropy", repeat_penalty=1.0):
|
| 267 |
+
"""Convert text to speech.
|
| 268 |
+
|
| 269 |
+
Args:
|
| 270 |
+
text: Text to synthesize.
|
| 271 |
+
steps: Number of diffusion steps.
|
| 272 |
+
alg: Generation algorithm.
|
| 273 |
+
repeat_penalty: Penalty for repeated tokens.
|
| 274 |
+
"""
|
| 275 |
+
message = "Convert the text to speech.\n" + text
|
| 276 |
+
text_out, audio_out, image_out = run_inference(
|
| 277 |
+
task="TTS",
|
| 278 |
+
text_prompt=message,
|
| 279 |
+
max_tokens=50,
|
| 280 |
+
steps=steps,
|
| 281 |
+
alg=alg,
|
| 282 |
+
repeat_penalty=repeat_penalty,
|
| 283 |
+
)
|
| 284 |
+
if audio_out is not None:
|
| 285 |
+
audio_path = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name
|
| 286 |
+
torchaudio.save(audio_path, audio_out.unsqueeze(0), 22050, format="wav")
|
| 287 |
+
return audio_path, text_out
|
| 288 |
+
return None, text_out
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
@spaces.GPU(duration=300)
|
| 292 |
+
def generate_asr(audio_file, steps=25, alg="entropy", repeat_penalty=1.0):
|
| 293 |
+
"""Convert speech to text (ASR).
|
| 294 |
+
|
| 295 |
+
Args:
|
| 296 |
+
audio_file: Input audio file path.
|
| 297 |
+
steps: Number of diffusion steps.
|
| 298 |
+
alg: Generation algorithm.
|
| 299 |
+
repeat_penalty: Penalty for repeated tokens.
|
| 300 |
+
"""
|
| 301 |
+
text_out, audio_out, image_out = run_inference(
|
| 302 |
+
task="ASR",
|
| 303 |
+
text_prompt="Convert the speech to text.",
|
| 304 |
+
audio_path=audio_file,
|
| 305 |
+
max_tokens=50,
|
| 306 |
+
steps=steps,
|
| 307 |
+
alg=alg,
|
| 308 |
+
repeat_penalty=repeat_penalty,
|
| 309 |
+
)
|
| 310 |
+
return text_out
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
@spaces.GPU(duration=300)
|
| 314 |
+
def generate_vqa(image_file, question, steps=64, alg="entropy", repeat_penalty=1.0):
|
| 315 |
+
"""Answer a question about an image (Visual QA).
|
| 316 |
+
|
| 317 |
+
Args:
|
| 318 |
+
image_file: Input image file path.
|
| 319 |
+
question: Question about the image.
|
| 320 |
+
steps: Number of diffusion steps.
|
| 321 |
+
alg: Generation algorithm.
|
| 322 |
+
repeat_penalty: Penalty for repeated tokens.
|
| 323 |
+
"""
|
| 324 |
+
text_out, audio_out, image_out = run_inference(
|
| 325 |
+
task="VQA",
|
| 326 |
+
text_prompt=question,
|
| 327 |
+
image_path=image_file,
|
| 328 |
+
max_tokens=64,
|
| 329 |
+
steps=steps,
|
| 330 |
+
alg=alg,
|
| 331 |
+
repeat_penalty=repeat_penalty,
|
| 332 |
+
)
|
| 333 |
+
return text_out
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
@spaces.GPU(duration=300)
|
| 337 |
+
def generate_s2i(audio_file, steps=260, alg="entropy-penalty", repeat_penalty=1.5, cfg=2.0):
|
| 338 |
+
"""Generate an image from an audio input (Speech-to-Image).
|
| 339 |
+
|
| 340 |
+
Args:
|
| 341 |
+
audio_file: Input audio file path.
|
| 342 |
+
steps: Number of diffusion steps.
|
| 343 |
+
alg: Generation algorithm.
|
| 344 |
+
repeat_penalty: Penalty for repeated tokens.
|
| 345 |
+
cfg: Classifier-free guidance scale.
|
| 346 |
+
"""
|
| 347 |
+
text_out, audio_out, image_out = run_inference(
|
| 348 |
+
task="S2I",
|
| 349 |
+
audio_path=audio_file,
|
| 350 |
+
max_tokens=260,
|
| 351 |
+
steps=steps,
|
| 352 |
+
alg=alg,
|
| 353 |
+
repeat_penalty=repeat_penalty,
|
| 354 |
+
max_position_penalty=2.0,
|
| 355 |
+
cfg=cfg,
|
| 356 |
+
)
|
| 357 |
+
return image_out, text_out
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
@spaces.GPU(duration=300)
|
| 361 |
+
def generate_svqa(image_file, audio_file, steps=128, alg="entropy-penalty", repeat_penalty=1.2):
|
| 362 |
+
"""Answer a spoken question about an image (Spoken Visual QA).
|
| 363 |
+
|
| 364 |
+
Args:
|
| 365 |
+
image_file: Input image file path.
|
| 366 |
+
audio_file: Input audio (spoken question) file path.
|
| 367 |
+
steps: Number of diffusion steps.
|
| 368 |
+
alg: Generation algorithm.
|
| 369 |
+
repeat_penalty: Penalty for repeated tokens.
|
| 370 |
+
"""
|
| 371 |
+
text_out, audio_out, image_out = run_inference(
|
| 372 |
+
task="SVQA",
|
| 373 |
+
audio_path=audio_file,
|
| 374 |
+
image_path=image_file,
|
| 375 |
+
add_boa_token=1,
|
| 376 |
+
max_tokens=128,
|
| 377 |
+
steps=steps,
|
| 378 |
+
alg=alg,
|
| 379 |
+
repeat_penalty=repeat_penalty,
|
| 380 |
+
)
|
| 381 |
+
result_audio = None
|
| 382 |
+
if audio_out is not None:
|
| 383 |
+
result_audio = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name
|
| 384 |
+
torchaudio.save(result_audio, audio_out.unsqueeze(0), 22050, format="wav")
|
| 385 |
+
return text_out, result_audio
|
| 386 |
+
|
| 387 |
+
|
| 388 |
+
CSS = """
|
| 389 |
+
#col-container { max-width: 1100px; margin: 0 auto; }
|
| 390 |
+
.dark .gradio-container { color: var(--body-text-color); }
|
| 391 |
+
"""
|
| 392 |
+
|
| 393 |
+
with gr.Blocks(theme=gr.themes.Citrus(), css=CSS) as demo:
|
| 394 |
+
gr.Markdown("# Omni-Diffusion: Unified Multimodal Understanding and Generation")
|
| 395 |
+
gr.Markdown("Interactive demo for any-to-any multimodal generation with masked discrete diffusion. Supports text-to-image, speech-to-image, TTS, ASR, visual QA, and spoken visual QA.")
|
| 396 |
+
gr.Markdown("[Paper](https://arxiv.org/abs/2603.06577) | [Model](https://huggingface.co/lijiang/Omni-Diffusion) | [GitHub](https://github.com/VITA-MLLM/Omni-Diffusion)")
|
| 397 |
+
|
| 398 |
+
with gr.Tabs():
|
| 399 |
+
with gr.Tab("Text-to-Image"):
|
| 400 |
+
with gr.Row():
|
| 401 |
+
with gr.Column(scale=1):
|
| 402 |
+
t2i_prompt = gr.Textbox(label="Prompt", placeholder="Describe the image to generate...", lines=3)
|
| 403 |
+
t2i_btn = gr.Button("Generate Image", variant="primary")
|
| 404 |
+
with gr.Accordion("Advanced", open=False):
|
| 405 |
+
t2i_steps = gr.Slider(50, 512, value=260, step=10, label="Steps")
|
| 406 |
+
t2i_alg = gr.Dropdown(["entropy-penalty", "entropy", "origin"], value="entropy-penalty", label="Algorithm")
|
| 407 |
+
t2i_rp = gr.Slider(0.5, 3.0, value=1.2, step=0.1, label="Repeat Penalty")
|
| 408 |
+
with gr.Column(scale=1):
|
| 409 |
+
t2i_image = gr.Image(label="Generated Image", type="numpy")
|
| 410 |
+
t2i_text = gr.Textbox(label="Output Text", lines=2)
|
| 411 |
+
gr.Examples(
|
| 412 |
+
examples=[["The image shows a landscape background with double exposure glasses of wine, displaying a hyperealistic and detailed view of the subject."]],
|
| 413 |
+
inputs=[t2i_prompt],
|
| 414 |
+
outputs=[t2i_image, t2i_text],
|
| 415 |
+
fn=generate_t2i,
|
| 416 |
+
cache_examples=False,
|
| 417 |
+
run_on_click=True,
|
| 418 |
+
)
|
| 419 |
+
t2i_btn.click(generate_t2i, inputs=[t2i_prompt, t2i_steps, t2i_alg, t2i_rp], outputs=[t2i_image, t2i_text])
|
| 420 |
+
|
| 421 |
+
with gr.Tab("Text-to-Speech"):
|
| 422 |
+
with gr.Row():
|
| 423 |
+
with gr.Column(scale=1):
|
| 424 |
+
tts_text = gr.Textbox(label="Text", placeholder="Text to synthesize...", lines=3)
|
| 425 |
+
tts_btn = gr.Button("Generate Speech", variant="primary")
|
| 426 |
+
with gr.Accordion("Advanced", open=False):
|
| 427 |
+
tts_steps = gr.Slider(10, 100, value=25, step=5, label="Steps")
|
| 428 |
+
tts_alg = gr.Dropdown(["entropy", "entropy-penalty", "origin"], value="entropy", label="Algorithm")
|
| 429 |
+
tts_rp = gr.Slider(0.5, 3.0, value=1.0, step=0.1, label="Repeat Penalty")
|
| 430 |
+
with gr.Column(scale=1):
|
| 431 |
+
tts_audio = gr.Audio(label="Generated Audio")
|
| 432 |
+
tts_text_out = gr.Textbox(label="Output Text", lines=2)
|
| 433 |
+
gr.Examples(
|
| 434 |
+
examples=[["Get the trust fund to the bank early."], ["It is a nice walk though, with trees and a river and stuff."]],
|
| 435 |
+
inputs=[tts_text],
|
| 436 |
+
outputs=[tts_audio, tts_text_out],
|
| 437 |
+
fn=generate_tts,
|
| 438 |
+
cache_examples=False,
|
| 439 |
+
run_on_click=True,
|
| 440 |
+
)
|
| 441 |
+
tts_btn.click(generate_tts, inputs=[tts_text, tts_steps, tts_alg, tts_rp], outputs=[tts_audio, tts_text_out])
|
| 442 |
+
|
| 443 |
+
with gr.Tab("Speech-to-Text (ASR)"):
|
| 444 |
+
with gr.Row():
|
| 445 |
+
with gr.Column(scale=1):
|
| 446 |
+
asr_audio = gr.Audio(label="Input Audio", type="filepath")
|
| 447 |
+
asr_btn = gr.Button("Transcribe", variant="primary")
|
| 448 |
+
with gr.Accordion("Advanced", open=False):
|
| 449 |
+
asr_steps = gr.Slider(10, 100, value=25, step=5, label="Steps")
|
| 450 |
+
asr_alg = gr.Dropdown(["entropy", "entropy-penalty", "origin"], value="entropy", label="Algorithm")
|
| 451 |
+
asr_rp = gr.Slider(0.5, 3.0, value=1.0, step=0.1, label="Repeat Penalty")
|
| 452 |
+
with gr.Column(scale=1):
|
| 453 |
+
asr_text = gr.Textbox(label="Transcription", lines=5)
|
| 454 |
+
gr.Examples(
|
| 455 |
+
examples=[["asr_0.wav"]],
|
| 456 |
+
inputs=[asr_audio],
|
| 457 |
+
outputs=[asr_text],
|
| 458 |
+
fn=generate_asr,
|
| 459 |
+
cache_examples=False,
|
| 460 |
+
run_on_click=True,
|
| 461 |
+
)
|
| 462 |
+
asr_btn.click(generate_asr, inputs=[asr_audio, asr_steps, asr_alg, asr_rp], outputs=[asr_text])
|
| 463 |
+
|
| 464 |
+
with gr.Tab("Visual QA"):
|
| 465 |
+
with gr.Row():
|
| 466 |
+
with gr.Column(scale=1):
|
| 467 |
+
vqa_image = gr.Image(label="Input Image", type="filepath")
|
| 468 |
+
vqa_question = gr.Textbox(label="Question", placeholder="Ask about the image...", lines=2)
|
| 469 |
+
vqa_btn = gr.Button("Answer", variant="primary")
|
| 470 |
+
with gr.Accordion("Advanced", open=False):
|
| 471 |
+
vqa_steps = gr.Slider(20, 200, value=64, step=10, label="Steps")
|
| 472 |
+
vqa_alg = gr.Dropdown(["entropy", "entropy-penalty", "origin"], value="entropy", label="Algorithm")
|
| 473 |
+
vqa_rp = gr.Slider(0.5, 3.0, value=1.0, step=0.1, label="Repeat Penalty")
|
| 474 |
+
with gr.Column(scale=1):
|
| 475 |
+
vqa_answer = gr.Textbox(label="Answer", lines=5)
|
| 476 |
+
gr.Examples(
|
| 477 |
+
examples=[["vqa_0.png", "Is the glass of orange juice half empty or half full?"]],
|
| 478 |
+
inputs=[vqa_image, vqa_question],
|
| 479 |
+
outputs=[vqa_answer],
|
| 480 |
+
fn=generate_vqa,
|
| 481 |
+
cache_examples=False,
|
| 482 |
+
run_on_click=True,
|
| 483 |
+
)
|
| 484 |
+
vqa_btn.click(generate_vqa, inputs=[vqa_image, vqa_question, vqa_steps, vqa_alg, vqa_rp], outputs=[vqa_answer])
|
| 485 |
+
|
| 486 |
+
with gr.Tab("Speech-to-Image"):
|
| 487 |
+
with gr.Row():
|
| 488 |
+
with gr.Column(scale=1):
|
| 489 |
+
s2i_audio = gr.Audio(label="Input Audio", type="filepath")
|
| 490 |
+
s2i_btn = gr.Button("Generate Image", variant="primary")
|
| 491 |
+
with gr.Accordion("Advanced", open=False):
|
| 492 |
+
s2i_steps = gr.Slider(50, 512, value=260, step=10, label="Steps")
|
| 493 |
+
s2i_alg = gr.Dropdown(["entropy-penalty", "entropy", "origin"], value="entropy-penalty", label="Algorithm")
|
| 494 |
+
s2i_rp = gr.Slider(0.5, 3.0, value=1.5, step=0.1, label="Repeat Penalty")
|
| 495 |
+
s2i_cfg = gr.Slider(0.0, 5.0, value=2.0, step=0.5, label="CFG Scale")
|
| 496 |
+
with gr.Column(scale=1):
|
| 497 |
+
s2i_image = gr.Image(label="Generated Image", type="numpy")
|
| 498 |
+
s2i_text = gr.Textbox(label="Output Text", lines=2)
|
| 499 |
+
gr.Examples(
|
| 500 |
+
examples=[["s2i_0.wav"]],
|
| 501 |
+
inputs=[s2i_audio],
|
| 502 |
+
outputs=[s2i_image, s2i_text],
|
| 503 |
+
fn=generate_s2i,
|
| 504 |
+
cache_examples=False,
|
| 505 |
+
run_on_click=True,
|
| 506 |
+
)
|
| 507 |
+
s2i_btn.click(generate_s2i, inputs=[s2i_audio, s2i_steps, s2i_alg, s2i_rp, s2i_cfg], outputs=[s2i_image, s2i_text])
|
| 508 |
+
|
| 509 |
+
with gr.Tab("Spoken Visual QA"):
|
| 510 |
+
with gr.Row():
|
| 511 |
+
with gr.Column(scale=1):
|
| 512 |
+
svqa_image = gr.Image(label="Input Image", type="filepath")
|
| 513 |
+
svqa_audio = gr.Audio(label="Input Audio (spoken question)", type="filepath")
|
| 514 |
+
svqa_btn = gr.Button("Answer", variant="primary")
|
| 515 |
+
with gr.Accordion("Advanced", open=False):
|
| 516 |
+
svqa_steps = gr.Slider(20, 300, value=128, step=10, label="Steps")
|
| 517 |
+
svqa_alg = gr.Dropdown(["entropy-penalty", "entropy", "origin"], value="entropy-penalty", label="Algorithm")
|
| 518 |
+
svqa_rp = gr.Slider(0.5, 3.0, value=1.2, step=0.1, label="Repeat Penalty")
|
| 519 |
+
with gr.Column(scale=1):
|
| 520 |
+
svqa_text = gr.Textbox(label="Answer (Text)", lines=3)
|
| 521 |
+
svqa_audio_out = gr.Audio(label="Answer (Audio)")
|
| 522 |
+
gr.Examples(
|
| 523 |
+
examples=[["svqa_0.jpg", "svqa_0.wav"]],
|
| 524 |
+
inputs=[svqa_image, svqa_audio],
|
| 525 |
+
outputs=[svqa_text, svqa_audio_out],
|
| 526 |
+
fn=generate_svqa,
|
| 527 |
+
cache_examples=False,
|
| 528 |
+
run_on_click=True,
|
| 529 |
+
)
|
| 530 |
+
svqa_btn.click(generate_svqa, inputs=[svqa_image, svqa_audio, svqa_steps, svqa_alg, svqa_rp], outputs=[svqa_text, svqa_audio_out])
|
| 531 |
+
|
| 532 |
+
demo.launch(mcp_server=True)
|
asr_0.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8cc5488d43dadd87e8fa93f8c2a418b6771b7a9201c87ef39fff7b10e5650803
|
| 3 |
+
size 1108218
|
chat_template.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0]['role'] == 'system' %}\n {{- messages[0]['content'] }}\n {%- endif %}\n {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role }}\n {%- if message.content %}\n {{- '\\n' + message.content }}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- message.content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n
|
omni_diffusion/__init__.py
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# from .models import *
|
| 2 |
+
from .data import *
|
omni_diffusion/constants.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
|
| 3 |
+
logger = logging.getLogger(__name__)
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
if False:
|
| 7 |
+
IMG_TAG_TOKEN = "<image>"
|
| 8 |
+
IMG_CONTEXT_TOKEN = "<IMG_CONTEXT>"
|
| 9 |
+
IMG_START_TOKEN = "<img>"
|
| 10 |
+
IMG_END_TOKEN = "</img>"
|
| 11 |
+
|
| 12 |
+
VID_TAG_TOKEN = "<video>"
|
| 13 |
+
VID_CONTEXT_TOKEN = "<VID_CONTEXT>"
|
| 14 |
+
VID_START_TOKEN = "<vid>"
|
| 15 |
+
VID_END_TOKEN = "</vid>"
|
| 16 |
+
|
| 17 |
+
PATCH_CONTEXT_TOKEN = "<PATCH_CONTEXT>"
|
| 18 |
+
PATCH_START_TOKEN = "<patch>"
|
| 19 |
+
PATCH_END_TOKEN = "</patch>"
|
| 20 |
+
|
| 21 |
+
AUD_TAG_TOKEN = "<audio>"
|
| 22 |
+
AUD_START_TOKEN = "<|begin_of_audio|>"
|
| 23 |
+
AUD_END_TOKEN = "<|end_of_audio|>"
|
| 24 |
+
|
| 25 |
+
QUAD_START_TOKEN = "<quad>"
|
| 26 |
+
QUAD_END_TOKEN = "</quad>"
|
| 27 |
+
REF_START_TOKEN = "<ref>"
|
| 28 |
+
REF_END_TOKEN = "</ref>"
|
| 29 |
+
BOX_START_TOKEN = "<box>"
|
| 30 |
+
BOX_END_TOKEN = "</box>"
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
if True:
|
| 34 |
+
|
| 35 |
+
IMG_TAG_TOKEN = "<|image|>"
|
| 36 |
+
IMG_CONTEXT_TOKEN = "<|context_of_image|>"
|
| 37 |
+
IMG_START_TOKEN = "<|begin_of_image|>"
|
| 38 |
+
IMG_END_TOKEN = "<|end_of_image|>"
|
| 39 |
+
|
| 40 |
+
VID_TAG_TOKEN = "<|video|>"
|
| 41 |
+
VID_CONTEXT_TOKEN = "<|context_of_video|>"
|
| 42 |
+
VID_START_TOKEN = "<|begin_of_video|>"
|
| 43 |
+
VID_END_TOKEN = "<|end_of_video|>"
|
| 44 |
+
|
| 45 |
+
PATCH_CONTEXT_TOKEN = "<|context_of_patch|>"
|
| 46 |
+
PATCH_START_TOKEN = "<|begin_of_patch|>"
|
| 47 |
+
PATCH_END_TOKEN = "<|end_of_patch|>"
|
| 48 |
+
|
| 49 |
+
AUD_TAG_TOKEN = "<|audio|>"
|
| 50 |
+
AUD_CONTEXT_TOKEN = "<|context_of_audio|>"
|
| 51 |
+
AUD_START_TOKEN = "<|begin_of_audio|>"
|
| 52 |
+
AUD_END_TOKEN = "<|end_of_audio|>"
|
| 53 |
+
|
| 54 |
+
QUAD_START_TOKEN = "<|begin_of_quad|>"
|
| 55 |
+
QUAD_END_TOKEN = "<|end_of_quad|>"
|
| 56 |
+
REF_START_TOKEN = "<|begin_of_ref|>"
|
| 57 |
+
REF_END_TOKEN = "<|end_of_ref|>"
|
| 58 |
+
BOX_START_TOKEN = "<|begin_of_box|>"
|
| 59 |
+
BOX_END_TOKEN = "<|end_of_box|>"
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
logger.info(f"{IMG_TAG_TOKEN=}")
|
| 63 |
+
logger.info(f"{IMG_CONTEXT_TOKEN=}")
|
| 64 |
+
logger.info(f"{IMG_START_TOKEN=}")
|
| 65 |
+
logger.info(f"{IMG_END_TOKEN=}")
|
| 66 |
+
|
| 67 |
+
logger.info(f"{VID_TAG_TOKEN=}")
|
| 68 |
+
logger.info(f"{VID_CONTEXT_TOKEN=}")
|
| 69 |
+
logger.info(f"{VID_START_TOKEN=}")
|
| 70 |
+
logger.info(f"{VID_END_TOKEN=}")
|
| 71 |
+
|
| 72 |
+
logger.info(f"{PATCH_CONTEXT_TOKEN=}")
|
| 73 |
+
logger.info(f"{PATCH_START_TOKEN=}")
|
| 74 |
+
logger.info(f"{PATCH_END_TOKEN=}")
|
| 75 |
+
|
| 76 |
+
logger.info(f"{AUD_TAG_TOKEN=}")
|
| 77 |
+
logger.info(f"{AUD_CONTEXT_TOKEN=}")
|
| 78 |
+
logger.info(f"{AUD_START_TOKEN=}")
|
| 79 |
+
logger.info(f"{AUD_END_TOKEN=}")
|
| 80 |
+
|
| 81 |
+
# IMAGENET_MEAN = (0.485, 0.456, 0.406)
|
| 82 |
+
# IMAGENET_STD = (0.229, 0.224, 0.225)
|
| 83 |
+
|
| 84 |
+
# CLIP_MEAN = (0.4814546, 0.4578275, 0.40821073)
|
| 85 |
+
# CLIP_STD = (0.2686295, 0.2613025, 0.2757711)
|
| 86 |
+
|
| 87 |
+
# SIGLIP_MEAN = (0.5, 0.5, 0.5)
|
| 88 |
+
# SIGLIP_STD = (0.5, 0.5, 0.5)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
IMAGENET_DEFAULT_MEAN = [0.485, 0.456, 0.406]
|
| 92 |
+
IMAGENET_DEFAULT_STD = [0.229, 0.224, 0.225]
|
| 93 |
+
IMAGENET_STANDARD_MEAN = [0.5, 0.5, 0.5]
|
| 94 |
+
IMAGENET_STANDARD_STD = [0.5, 0.5, 0.5]
|
| 95 |
+
OPENAI_CLIP_MEAN = [0.48145466, 0.4578275, 0.40821073]
|
| 96 |
+
OPENAI_CLIP_STD = [0.26862954, 0.26130258, 0.27577711]
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
# Model Constants
|
| 100 |
+
IGNORE_INDEX = -100
|
| 101 |
+
IMAGE_TOKEN_INDEX = -200
|
| 102 |
+
DEFAULT_IMAGE_TOKEN = IMG_CONTEXT_TOKEN
|
| 103 |
+
DEFAULT_IMAGE_PATCH_TOKEN = PATCH_CONTEXT_TOKEN
|
| 104 |
+
DEFAULT_IM_START_TOKEN = IMG_START_TOKEN
|
| 105 |
+
DEFAULT_IM_END_TOKEN = IMG_END_TOKEN
|
omni_diffusion/data/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# data.__init__
|
| 2 |
+
|
| 3 |
+
from .build import build_supervised_dataset_deepspeed
|
omni_diffusion/data/build.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
import os
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
|
| 5 |
+
from datasets import concatenate_datasets, load_dataset
|
| 6 |
+
|
| 7 |
+
from .data_collator import DataCollatorForSupervisedDataset, collate_fn_deepspeed
|
| 8 |
+
from .dataset_qwen2 import Qwen2Dataset
|
| 9 |
+
|
| 10 |
+
logger = logging.getLogger(__name__)
|
| 11 |
+
logger.setLevel(logging.INFO)
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def build_supervised_dataset_deepspeed(
|
| 15 |
+
model_config,
|
| 16 |
+
model_args,
|
| 17 |
+
data_args,
|
| 18 |
+
training_args,
|
| 19 |
+
tokenizer,
|
| 20 |
+
create_position_ids=True,
|
| 21 |
+
create_loss_mask=False,
|
| 22 |
+
shift_token=False,
|
| 23 |
+
):
|
| 24 |
+
"""
|
| 25 |
+
Builds the supervised dataset and data collator specifically for DeepSpeed training.
|
| 26 |
+
|
| 27 |
+
Args:
|
| 28 |
+
model_config: Configuration object for the model.
|
| 29 |
+
model_args: Arguments related to model architecture and parameters.
|
| 30 |
+
data_args: Arguments related to data processing and loading.
|
| 31 |
+
training_args: Arguments related to the training loop (e.g., output dir, seed).
|
| 32 |
+
tokenizer: The tokenizer instance to process text.
|
| 33 |
+
create_position_ids (bool): Whether to generate position IDs.
|
| 34 |
+
create_loss_mask (bool): Whether to generate a mask for loss calculation.
|
| 35 |
+
shift_token (bool): Whether to shift tokens for causal language modeling.
|
| 36 |
+
|
| 37 |
+
Returns:
|
| 38 |
+
dict: A dictionary containing 'train', 'validation' datasets and the 'data_collator'.
|
| 39 |
+
"""
|
| 40 |
+
|
| 41 |
+
logging.info("building dataset...")
|
| 42 |
+
|
| 43 |
+
cfg_path = data_args.dataset_name
|
| 44 |
+
max_padding_length = model_args.model_max_length
|
| 45 |
+
output_dir = training_args.output_dir
|
| 46 |
+
|
| 47 |
+
create_attention_mask = data_args.create_attention_mask
|
| 48 |
+
create_attention_mask_2d = data_args.create_attention_mask_2d
|
| 49 |
+
|
| 50 |
+
image_size = model_args.image_size
|
| 51 |
+
image_token_length = model_args.image_token_length
|
| 52 |
+
|
| 53 |
+
max_num_frame = model_args.max_num_frame
|
| 54 |
+
max_fps = model_args.max_fps
|
| 55 |
+
|
| 56 |
+
reset_position_ids = data_args.reset_position_ids
|
| 57 |
+
reset_attention_mask = data_args.reset_attention_mask
|
| 58 |
+
variable_length = data_args.variable_length
|
| 59 |
+
|
| 60 |
+
min_patch_grid = model_args.min_patch_grid
|
| 61 |
+
max_patch_grid = model_args.max_patch_grid
|
| 62 |
+
process_type = model_args.vision_process_type
|
| 63 |
+
normalize_type = model_args.vision_normalize_type
|
| 64 |
+
|
| 65 |
+
audio_tokenizer_path = model_args.audio_tokenizer_path
|
| 66 |
+
audio_tokenizer_type = model_args.audio_tokenizer_type
|
| 67 |
+
image_tokenizer_path = model_args.image_tokenizer_path
|
| 68 |
+
|
| 69 |
+
seed = training_args.seed
|
| 70 |
+
cross_dataset_joint = data_args.cross_dataset_joint
|
| 71 |
+
dataset_joint = data_args.dataset_joint
|
| 72 |
+
|
| 73 |
+
TrainDataset = Qwen2Dataset
|
| 74 |
+
|
| 75 |
+
train_dataset = TrainDataset(
|
| 76 |
+
cfg_path,
|
| 77 |
+
tokenizer,
|
| 78 |
+
image_size=image_size,
|
| 79 |
+
image_token_length=image_token_length,
|
| 80 |
+
max_padding_length=max_padding_length,
|
| 81 |
+
variable_length=variable_length,
|
| 82 |
+
output_dir=output_dir,
|
| 83 |
+
training_args=None,
|
| 84 |
+
shift_token=shift_token,
|
| 85 |
+
create_position_ids=create_position_ids,
|
| 86 |
+
create_attention_mask=create_attention_mask,
|
| 87 |
+
create_attention_mask_2d=create_attention_mask_2d,
|
| 88 |
+
create_loss_mask=create_loss_mask,
|
| 89 |
+
max_num_frame=max_num_frame,
|
| 90 |
+
max_fps=max_fps,
|
| 91 |
+
reset_position_ids=reset_position_ids,
|
| 92 |
+
reset_attention_mask=reset_attention_mask,
|
| 93 |
+
min_patch_grid=min_patch_grid,
|
| 94 |
+
max_patch_grid=max_patch_grid,
|
| 95 |
+
process_type=process_type,
|
| 96 |
+
normalize_type=normalize_type,
|
| 97 |
+
seed=seed,
|
| 98 |
+
cross_dataset_joint=cross_dataset_joint,
|
| 99 |
+
dataset_joint=dataset_joint,
|
| 100 |
+
audio_tokenizer_type=audio_tokenizer_type,
|
| 101 |
+
audio_tokenizer_path=audio_tokenizer_path,
|
| 102 |
+
image_tokenizer_path=image_tokenizer_path,
|
| 103 |
+
use_megatron=False,
|
| 104 |
+
)
|
| 105 |
+
eval_dataset = None
|
| 106 |
+
|
| 107 |
+
data_collator = collate_fn_deepspeed
|
| 108 |
+
|
| 109 |
+
return dict(train=train_dataset, validation=eval_dataset, data_collator=data_collator)
|
| 110 |
+
|
omni_diffusion/data/data_collator.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import itertools
|
| 2 |
+
import json
|
| 3 |
+
import logging
|
| 4 |
+
import math
|
| 5 |
+
import os
|
| 6 |
+
import re
|
| 7 |
+
from dataclasses import dataclass, field
|
| 8 |
+
from typing import Dict, List, Optional, Sequence
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
import transformers
|
| 12 |
+
from torch.utils.data import default_collate
|
| 13 |
+
from transformers.trainer_pt_utils import LabelSmoother
|
| 14 |
+
|
| 15 |
+
IGNORE_TOKEN_ID = LabelSmoother.ignore_index
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
@dataclass
|
| 19 |
+
class DataCollatorForSupervisedDataset(object):
|
| 20 |
+
"""
|
| 21 |
+
Data collator for supervised fine-tuning.
|
| 22 |
+
Handles padding of input IDs and labels, creation of attention masks,
|
| 23 |
+
and concatenation of multimodal inputs (images).
|
| 24 |
+
"""
|
| 25 |
+
tokenizer: transformers.PreTrainedTokenizer
|
| 26 |
+
|
| 27 |
+
def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
|
| 28 |
+
input_ids = [instance["input_ids"] for instance in instances]
|
| 29 |
+
labels = [instance["labels"] for instance in instances]
|
| 30 |
+
|
| 31 |
+
input_ids = torch.nn.utils.rnn.pad_sequence(
|
| 32 |
+
input_ids,
|
| 33 |
+
batch_first=True,
|
| 34 |
+
padding_value=self.tokenizer.pad_token_id,
|
| 35 |
+
)
|
| 36 |
+
labels = torch.nn.utils.rnn.pad_sequence(
|
| 37 |
+
labels,
|
| 38 |
+
batch_first=True,
|
| 39 |
+
padding_value=IGNORE_TOKEN_ID,
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
input_ids = input_ids[:, : self.tokenizer.model_max_length]
|
| 43 |
+
labels = labels[:, : self.tokenizer.model_max_length]
|
| 44 |
+
|
| 45 |
+
batch = dict(
|
| 46 |
+
input_ids=input_ids,
|
| 47 |
+
labels=labels,
|
| 48 |
+
attention_mask=input_ids.ne(self.tokenizer.pad_token_id),
|
| 49 |
+
)
|
| 50 |
+
|
| 51 |
+
if "images" in instances[0]:
|
| 52 |
+
images = [instance["images"] for instance in instances]
|
| 53 |
+
batch["images"] = torch.cat(images, dim=0)
|
| 54 |
+
|
| 55 |
+
if "doclm_images" in instances[0]:
|
| 56 |
+
images = [instance["doclm_images"] for instance in instances]
|
| 57 |
+
batch["doclm_images"] = torch.cat(images, dim=0)
|
| 58 |
+
|
| 59 |
+
if "image_paths" in instances[0]:
|
| 60 |
+
image_paths = [instance["image_paths"] for instance in instances]
|
| 61 |
+
batch["image_paths"] = image_paths
|
| 62 |
+
|
| 63 |
+
if "pixel_values" in instances[0]:
|
| 64 |
+
pixel_values = torch.cat([instance["pixel_values"] for instance in instances])
|
| 65 |
+
batch["pixel_values"] = pixel_values
|
| 66 |
+
|
| 67 |
+
if "image_flags" in instances[0]:
|
| 68 |
+
image_flags = torch.cat([instance["image_flags"] for instance in instances])
|
| 69 |
+
batch["image_flags"] = image_flags
|
| 70 |
+
|
| 71 |
+
return batch
|
| 72 |
+
|
| 73 |
+
def collate_fn_deepspeed(batch):
|
| 74 |
+
"""
|
| 75 |
+
Custom collation function designed for DeepSpeed training.
|
| 76 |
+
Handles complex multimodal data (images/audio) and potentially packed sequences (cu_seq_lens).
|
| 77 |
+
"""
|
| 78 |
+
tmp_batch = [{} for _ in range(len(batch))]
|
| 79 |
+
if "cu_seq_lens" in batch[0]:
|
| 80 |
+
cu_seq_lens = [x["cu_seq_lens"] for x in batch]
|
| 81 |
+
max_seq_len = [x["max_seq_len"] for x in batch]
|
| 82 |
+
else:
|
| 83 |
+
cu_seq_lens = None
|
| 84 |
+
max_seq_len = None
|
| 85 |
+
|
| 86 |
+
if "images" in batch[0].keys():
|
| 87 |
+
for new_x, x in zip(tmp_batch, batch):
|
| 88 |
+
new_x["images"] = x.pop("images")
|
| 89 |
+
new_x["image_indices"] = x.pop("image_indices")
|
| 90 |
+
|
| 91 |
+
if "audios" in batch[0].keys():
|
| 92 |
+
for new_x, x in zip(tmp_batch, batch):
|
| 93 |
+
new_x["audios"] = x.pop("audios")
|
| 94 |
+
new_x["audio_indices"] = x.pop("audio_indices")
|
| 95 |
+
|
| 96 |
+
new_batch = default_collate(batch)
|
| 97 |
+
|
| 98 |
+
if "images" in tmp_batch[0].keys():
|
| 99 |
+
|
| 100 |
+
new_batch["images"] = torch.cat([x["images"] for x in tmp_batch], dim=0)
|
| 101 |
+
|
| 102 |
+
for sample_idx, sample in enumerate(tmp_batch):
|
| 103 |
+
sample["image_indices"][0, :, :] = sample_idx
|
| 104 |
+
|
| 105 |
+
new_batch["image_indices"] = torch.cat([x["image_indices"] for x in tmp_batch], dim=1)
|
| 106 |
+
|
| 107 |
+
if "audios" in tmp_batch[0].keys():
|
| 108 |
+
|
| 109 |
+
new_batch["audios"] = list(itertools.chain.from_iterable([x["audios"] for x in tmp_batch]))
|
| 110 |
+
# print(f"{[x.size() for x in sample['audios']]}")
|
| 111 |
+
|
| 112 |
+
for sample_idx, sample in enumerate(tmp_batch):
|
| 113 |
+
for j in range(len(sample["audio_indices"])):
|
| 114 |
+
sample["audio_indices"][j][0, :, :] = sample_idx
|
| 115 |
+
|
| 116 |
+
new_batch["audio_indices"] = list(
|
| 117 |
+
itertools.chain.from_iterable([x["audio_indices"] for x in tmp_batch])
|
| 118 |
+
)
|
| 119 |
+
# print(f"{[x.size() for x in sample['audio_indices']]}")
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
return new_batch
|
omni_diffusion/data/dataset_base.py
ADDED
|
@@ -0,0 +1,424 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import contextlib
|
| 2 |
+
import json
|
| 3 |
+
import logging
|
| 4 |
+
import os
|
| 5 |
+
import pdb
|
| 6 |
+
import re
|
| 7 |
+
import traceback
|
| 8 |
+
import uuid
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
import torch
|
| 12 |
+
import yaml
|
| 13 |
+
from PIL import Image
|
| 14 |
+
|
| 15 |
+
from torchvision import transforms
|
| 16 |
+
from torchvision.transforms import InterpolationMode
|
| 17 |
+
|
| 18 |
+
from .processor.audio_processor import AudioProcessor
|
| 19 |
+
from .processor.image_processor import ImageProcessor
|
| 20 |
+
from .utils import draw_data
|
| 21 |
+
|
| 22 |
+
logger = logging.getLogger(__name__)
|
| 23 |
+
logger.setLevel(logging.INFO)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class BaseDataset(torch.utils.data.Dataset):
|
| 27 |
+
def __init__(
|
| 28 |
+
self,
|
| 29 |
+
cfg_path,
|
| 30 |
+
tokenizer,
|
| 31 |
+
image_size=448,
|
| 32 |
+
image_token_length=1024,
|
| 33 |
+
max_padding_length=32768,
|
| 34 |
+
variable_length=False,
|
| 35 |
+
output_dir="",
|
| 36 |
+
add_task_symbol=True,
|
| 37 |
+
training_args=None,
|
| 38 |
+
shift_token=False,
|
| 39 |
+
create_position_ids=True,
|
| 40 |
+
create_attention_mask=True,
|
| 41 |
+
create_attention_mask_2d=False,
|
| 42 |
+
create_loss_mask=False,
|
| 43 |
+
max_num_frame=8,
|
| 44 |
+
max_fps=1,
|
| 45 |
+
reset_position_ids=False,
|
| 46 |
+
reset_attention_mask=False,
|
| 47 |
+
min_patch_grid=1,
|
| 48 |
+
max_patch_grid=6,
|
| 49 |
+
process_type="anyres",
|
| 50 |
+
normalize_type="imagenet",
|
| 51 |
+
seed=42,
|
| 52 |
+
cross_dataset_joint=False,
|
| 53 |
+
dataset_joint=True,
|
| 54 |
+
audio_tokenizer_type=None,
|
| 55 |
+
audio_tokenizer_path=None,
|
| 56 |
+
image_tokenizer_path=None,
|
| 57 |
+
text_audio_interval_ratio=None,
|
| 58 |
+
use_megatron=True,
|
| 59 |
+
skip_samples=0,
|
| 60 |
+
):
|
| 61 |
+
super(BaseDataset, self).__init__()
|
| 62 |
+
|
| 63 |
+
self.cfg_path = cfg_path
|
| 64 |
+
with open(self.cfg_path, "r", encoding="utf8") as cfg_file:
|
| 65 |
+
cfg_data = cfg_file.read()
|
| 66 |
+
|
| 67 |
+
self.cfg = yaml.load(cfg_data, Loader=yaml.CLoader)
|
| 68 |
+
logger.info(f"cfg {self.cfg}")
|
| 69 |
+
|
| 70 |
+
self.tokenizer = tokenizer
|
| 71 |
+
self.max_padding_length = max_padding_length
|
| 72 |
+
self.variable_length = variable_length
|
| 73 |
+
self.output_dir = output_dir
|
| 74 |
+
self.training_args = training_args
|
| 75 |
+
self.shift_token = shift_token
|
| 76 |
+
self.create_position_ids = create_position_ids
|
| 77 |
+
self.create_attention_mask = create_attention_mask
|
| 78 |
+
self.create_attention_mask_2d = create_attention_mask_2d
|
| 79 |
+
self.create_loss_mask = create_loss_mask
|
| 80 |
+
self.max_num_frame = max_num_frame
|
| 81 |
+
self.max_fps = max_fps
|
| 82 |
+
self.reset_position_ids = reset_position_ids
|
| 83 |
+
self.reset_attention_mask = reset_attention_mask
|
| 84 |
+
|
| 85 |
+
self.seed = seed
|
| 86 |
+
self.cross_dataset_joint = cross_dataset_joint
|
| 87 |
+
self.dataset_joint = dataset_joint
|
| 88 |
+
|
| 89 |
+
self.image_size = image_size
|
| 90 |
+
self.image_token_length = image_token_length
|
| 91 |
+
|
| 92 |
+
self.do_dataset_format = self.cfg.get("do_dataset_format", False)
|
| 93 |
+
self.do_dataset_cast = self.cfg.get("do_dataset_cast", False)
|
| 94 |
+
self.xlsx_sample_num = self.cfg.get("xlsx_sample_num", 5)
|
| 95 |
+
|
| 96 |
+
self.processor = {}
|
| 97 |
+
self.processor["image"] = ImageProcessor(
|
| 98 |
+
image_tokenizer_path,
|
| 99 |
+
process_type,
|
| 100 |
+
image_size=self.image_size,
|
| 101 |
+
normalize_type=normalize_type,
|
| 102 |
+
min_patch_grid=min_patch_grid,
|
| 103 |
+
max_patch_grid=max_patch_grid,
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
self.processor["audio"] = AudioProcessor(
|
| 107 |
+
audio_tokenizer_path=audio_tokenizer_path,
|
| 108 |
+
audio_tokenizer_type=audio_tokenizer_type,
|
| 109 |
+
text_audio_interval_ratio=text_audio_interval_ratio
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
self.total_samples = {}
|
| 113 |
+
self.processed_samples = {}
|
| 114 |
+
self.unjoint_samples = {}
|
| 115 |
+
self.joint_samples = {}
|
| 116 |
+
self.source2jsonpath = {}
|
| 117 |
+
self.source2prefixpath = {}
|
| 118 |
+
self.skip_samples = skip_samples
|
| 119 |
+
|
| 120 |
+
if use_megatron:
|
| 121 |
+
self.load_data()
|
| 122 |
+
else:
|
| 123 |
+
with main_process_first(local=False, desc="Loading data"):
|
| 124 |
+
self.load_data()
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def load_data(self):
|
| 128 |
+
from datasets import Dataset, DatasetDict, concatenate_datasets, load_dataset
|
| 129 |
+
|
| 130 |
+
raw_data = None
|
| 131 |
+
|
| 132 |
+
sampled_data = {}
|
| 133 |
+
source_idx = 0
|
| 134 |
+
for data_name, data_info in self.cfg["dataset"].items():
|
| 135 |
+
data_ratio = data_info.get("ratio", 1)
|
| 136 |
+
data_num = data_info.get("num", 999999999)
|
| 137 |
+
|
| 138 |
+
prefix_path = data_info.get("prefix_path", "")
|
| 139 |
+
|
| 140 |
+
if data_ratio == 0:
|
| 141 |
+
continue
|
| 142 |
+
|
| 143 |
+
if data_num == 0:
|
| 144 |
+
continue
|
| 145 |
+
|
| 146 |
+
for data_idx, data_path in enumerate(data_info["json_paths"]):
|
| 147 |
+
logger.info("-" * 100)
|
| 148 |
+
|
| 149 |
+
if not os.path.isfile(data_path) and not os.path.isdir(data_path):
|
| 150 |
+
logger.warning(f"Data file no found {data_path}")
|
| 151 |
+
continue
|
| 152 |
+
|
| 153 |
+
this_data = load_json(data_path, self.output_dir)
|
| 154 |
+
# this_data = load_data_one(data_path, self.outout_dir)
|
| 155 |
+
if this_data is None:
|
| 156 |
+
logger.warning(f"Failed to load {data_path}")
|
| 157 |
+
continue
|
| 158 |
+
# print(f"this_data {this_data}")
|
| 159 |
+
|
| 160 |
+
column_names = list(this_data.features)
|
| 161 |
+
if "id" in column_names:
|
| 162 |
+
this_data = this_data.remove_columns("id")
|
| 163 |
+
|
| 164 |
+
# sources = [data_path] * len(this_data)
|
| 165 |
+
sources = [source_idx] * len(this_data)
|
| 166 |
+
# sources = [data_name] * len(this_data)
|
| 167 |
+
this_data = this_data.add_column("source", sources)
|
| 168 |
+
|
| 169 |
+
self.source2jsonpath[source_idx] = data_path
|
| 170 |
+
self.source2prefixpath[source_idx] = prefix_path
|
| 171 |
+
self.total_samples[source_idx] = len(this_data)
|
| 172 |
+
self.processed_samples[source_idx] = 0
|
| 173 |
+
self.unjoint_samples[source_idx] = 0
|
| 174 |
+
self.joint_samples[source_idx] = 0
|
| 175 |
+
|
| 176 |
+
source_idx += 1
|
| 177 |
+
|
| 178 |
+
if "images" not in column_names:
|
| 179 |
+
# images = [[]] * len(this_data)
|
| 180 |
+
images = [None] * len(this_data)
|
| 181 |
+
this_data = this_data.add_column("images", images)
|
| 182 |
+
|
| 183 |
+
if "videos" not in column_names:
|
| 184 |
+
# videos = [[]] * len(this_data)
|
| 185 |
+
videos = [None] * len(this_data)
|
| 186 |
+
this_data = this_data.add_column("videos", videos)
|
| 187 |
+
|
| 188 |
+
if "audios" not in column_names:
|
| 189 |
+
# audios = [[]] * len(this_data)
|
| 190 |
+
audios = [None] * len(this_data)
|
| 191 |
+
this_data = this_data.add_column("audios", audios)
|
| 192 |
+
|
| 193 |
+
if False:
|
| 194 |
+
column_names = list(this_data.features)
|
| 195 |
+
this_data = this_data.map(
|
| 196 |
+
format_function_general,
|
| 197 |
+
batched=True,
|
| 198 |
+
batch_size=2560,
|
| 199 |
+
num_proc=1,
|
| 200 |
+
# batch_size=1,
|
| 201 |
+
# num_proc=1,
|
| 202 |
+
remove_columns=column_names,
|
| 203 |
+
keep_in_memory=False,
|
| 204 |
+
desc="Running format on dataset",
|
| 205 |
+
)
|
| 206 |
+
|
| 207 |
+
this_data = this_data.shuffle(seed=self.seed)
|
| 208 |
+
# this_data = this_data.flatten_indices()
|
| 209 |
+
this_data = this_data.shuffle(seed=self.seed)
|
| 210 |
+
# this_data = this_data.flatten_indices()
|
| 211 |
+
|
| 212 |
+
data_ratio = float(data_ratio)
|
| 213 |
+
total_num = len(this_data)
|
| 214 |
+
used_num = min(int(total_num * data_ratio), data_num)
|
| 215 |
+
logger.info(f"total_num {total_num}")
|
| 216 |
+
logger.info(f"data_ratio {data_ratio}")
|
| 217 |
+
logger.info(f"data_num {data_num}")
|
| 218 |
+
logger.info(f"used_num {used_num}")
|
| 219 |
+
|
| 220 |
+
indices = [x % total_num for x in range(used_num)]
|
| 221 |
+
|
| 222 |
+
this_data = this_data.select(indices)
|
| 223 |
+
|
| 224 |
+
if raw_data is None:
|
| 225 |
+
raw_data = this_data
|
| 226 |
+
else:
|
| 227 |
+
if self.do_dataset_cast:
|
| 228 |
+
this_data = this_data.cast(raw_data.features)
|
| 229 |
+
raw_data = concatenate_datasets([raw_data, this_data])
|
| 230 |
+
|
| 231 |
+
sampled_data[data_path] = {}
|
| 232 |
+
sampled_data[data_path]["data"] = this_data.select(
|
| 233 |
+
range(min(self.xlsx_sample_num, used_num))
|
| 234 |
+
)
|
| 235 |
+
sampled_data[data_path]["total_num"] = total_num
|
| 236 |
+
sampled_data[data_path]["used_num"] = used_num
|
| 237 |
+
|
| 238 |
+
logger.info(f"this_data {this_data}")
|
| 239 |
+
logger.info(f"raw_data {raw_data}")
|
| 240 |
+
# logger.info(f"raw_data {raw_data[0]}")
|
| 241 |
+
# logger.info(f"raw_data {raw_data[-1]}")
|
| 242 |
+
logger.info(f"Successful load {data_path}")
|
| 243 |
+
|
| 244 |
+
raw_data = raw_data.shuffle(seed=self.seed)
|
| 245 |
+
# raw_data = raw_data.flatten_indices()
|
| 246 |
+
raw_data = raw_data.shuffle(seed=self.seed)
|
| 247 |
+
# raw_data = raw_data.flatten_indices()
|
| 248 |
+
|
| 249 |
+
self.raw_data = raw_data
|
| 250 |
+
|
| 251 |
+
if torch.distributed.is_initialized() and torch.distributed.get_rank() == 0:
|
| 252 |
+
output_xlsx = os.path.basename(self.cfg_path).replace("yaml", "xlsx")
|
| 253 |
+
output_xlsx = os.path.join(self.output_dir, output_xlsx)
|
| 254 |
+
logger.info(f"output_xlsx {output_xlsx}")
|
| 255 |
+
draw_data(
|
| 256 |
+
sampled_data,
|
| 257 |
+
output_xlsx,
|
| 258 |
+
tokenizer=self.tokenizer,
|
| 259 |
+
image_processor=self.processor["image"],
|
| 260 |
+
)
|
| 261 |
+
|
| 262 |
+
logger.info(f"raw_data {raw_data}")
|
| 263 |
+
|
| 264 |
+
if torch.distributed.is_initialized() and torch.distributed.get_rank() == 0:
|
| 265 |
+
logger.info(f"raw_data {raw_data[:10]}")
|
| 266 |
+
logger.info(f"raw_data {raw_data[-10:]}")
|
| 267 |
+
|
| 268 |
+
def __len__(self):
|
| 269 |
+
return len(self.raw_data)
|
| 270 |
+
|
| 271 |
+
def update_data_path(self, sample):
|
| 272 |
+
source = sample["source"]
|
| 273 |
+
prefix_path = self.source2prefixpath[source]
|
| 274 |
+
|
| 275 |
+
if has_image(sample):
|
| 276 |
+
sample["images"] = [os.path.join(prefix_path, image) for image in sample["images"]]
|
| 277 |
+
|
| 278 |
+
if has_video(sample):
|
| 279 |
+
sample["videos"] = [os.path.join(prefix_path, video) for video in sample["videos"]]
|
| 280 |
+
|
| 281 |
+
if has_audio(sample):
|
| 282 |
+
sample["audios"] = [os.path.join(prefix_path, audio) for audio in sample["audios"]]
|
| 283 |
+
|
| 284 |
+
return sample
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
def format_function_general(examples):
|
| 289 |
+
messages = [x for x in examples["messages"]]
|
| 290 |
+
|
| 291 |
+
if "images" in examples:
|
| 292 |
+
images = [x for x in examples["images"]]
|
| 293 |
+
else:
|
| 294 |
+
images = [None for _ in messages]
|
| 295 |
+
|
| 296 |
+
if "videos" in examples:
|
| 297 |
+
videos = [x for x in examples["videos"]]
|
| 298 |
+
else:
|
| 299 |
+
videos = [None for _ in messages]
|
| 300 |
+
|
| 301 |
+
if "audios" in examples:
|
| 302 |
+
audios = [x for x in examples["audios"]]
|
| 303 |
+
else:
|
| 304 |
+
audios = [None for _ in messages]
|
| 305 |
+
|
| 306 |
+
return {
|
| 307 |
+
"messages": messages,
|
| 308 |
+
"images": images,
|
| 309 |
+
"videos": videos,
|
| 310 |
+
"audios": audios,
|
| 311 |
+
}
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
def load_json_A(data_file):
|
| 315 |
+
from datasets import Dataset, DatasetDict, concatenate_datasets, load_dataset
|
| 316 |
+
|
| 317 |
+
with open(data_file, "r") as f:
|
| 318 |
+
raw_data = json.load(f)
|
| 319 |
+
this_data = Dataset.from_list(raw_data)
|
| 320 |
+
return this_data
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
def load_json_B(data_file):
|
| 324 |
+
from datasets import Dataset, DatasetDict, concatenate_datasets, load_dataset
|
| 325 |
+
|
| 326 |
+
this_data = load_dataset("json", data_files=data_file, keep_in_memory=False)
|
| 327 |
+
return this_data["train"]
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
def load_json_C(data_file):
|
| 331 |
+
from datasets import Dataset, DatasetDict, concatenate_datasets, load_dataset
|
| 332 |
+
|
| 333 |
+
raw_data = []
|
| 334 |
+
with open(data_file, "r") as f:
|
| 335 |
+
for line in f.readlines():
|
| 336 |
+
d = json.loads(line)
|
| 337 |
+
# raw_data.append({"conversations": d["conversations"], "id": d["id"]})
|
| 338 |
+
if "conversations" in d:
|
| 339 |
+
raw_data.append({"conversations": d["conversations"]})
|
| 340 |
+
if "messages" in d:
|
| 341 |
+
raw_data.append({"messages": d["messages"]})
|
| 342 |
+
this_data = Dataset.from_list(raw_data)
|
| 343 |
+
return this_data
|
| 344 |
+
|
| 345 |
+
|
| 346 |
+
def load_json(data_file, output_dir):
|
| 347 |
+
for func in [load_json_B, load_json_A, load_json_C]:
|
| 348 |
+
try:
|
| 349 |
+
this_data = func(data_file)
|
| 350 |
+
return this_data
|
| 351 |
+
except Exception as error:
|
| 352 |
+
with open(os.path.join(output_dir, "data_error.log"), "a") as f:
|
| 353 |
+
print("-" * 100, file=f)
|
| 354 |
+
print(f"{data_file=}", file=f)
|
| 355 |
+
# print(error, file=f)
|
| 356 |
+
print(traceback.format_exc(), file=f)
|
| 357 |
+
continue
|
| 358 |
+
return None
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
def load_data_one(data_file, output_dir):
|
| 362 |
+
if data_file.endswith("json") or data_file.endswith("jsonl"):
|
| 363 |
+
return load_json(data_file, output_dir)
|
| 364 |
+
from datasets import Dataset, DatasetDict, concatenate_datasets, load_dataset
|
| 365 |
+
|
| 366 |
+
this_data = load_dataset(data_file, keep_in_memory=False)
|
| 367 |
+
return this_data["train"]
|
| 368 |
+
|
| 369 |
+
|
| 370 |
+
@contextlib.contextmanager
|
| 371 |
+
def main_process_first(local=True, desc="work"):
|
| 372 |
+
|
| 373 |
+
if torch.distributed.is_initialized() and torch.distributed.get_world_size() > 1:
|
| 374 |
+
if local:
|
| 375 |
+
rank = int(os.environ["LOCAL_RANK"])
|
| 376 |
+
else:
|
| 377 |
+
rank = torch.distributed.get_rank()
|
| 378 |
+
is_main_process = rank == 0
|
| 379 |
+
|
| 380 |
+
try:
|
| 381 |
+
if not is_main_process:
|
| 382 |
+
torch.distributed.barrier()
|
| 383 |
+
yield
|
| 384 |
+
finally:
|
| 385 |
+
if is_main_process:
|
| 386 |
+
torch.distributed.barrier()
|
| 387 |
+
else:
|
| 388 |
+
yield
|
| 389 |
+
|
| 390 |
+
def has_video(sample):
|
| 391 |
+
# video
|
| 392 |
+
if (
|
| 393 |
+
"videos" in sample
|
| 394 |
+
and isinstance(sample["videos"], list)
|
| 395 |
+
and None not in sample["videos"]
|
| 396 |
+
and len(sample["videos"])
|
| 397 |
+
):
|
| 398 |
+
return True
|
| 399 |
+
return False
|
| 400 |
+
|
| 401 |
+
|
| 402 |
+
def has_image(sample):
|
| 403 |
+
# image
|
| 404 |
+
if (
|
| 405 |
+
"images" in sample
|
| 406 |
+
and isinstance(sample["images"], list)
|
| 407 |
+
and None not in sample["images"]
|
| 408 |
+
and len(sample["images"])
|
| 409 |
+
):
|
| 410 |
+
return True
|
| 411 |
+
return False
|
| 412 |
+
|
| 413 |
+
|
| 414 |
+
def has_audio(sample):
|
| 415 |
+
# audio
|
| 416 |
+
if (
|
| 417 |
+
"audios" in sample
|
| 418 |
+
and isinstance(sample["audios"], list)
|
| 419 |
+
and None not in sample["audios"]
|
| 420 |
+
and len(sample["audios"])
|
| 421 |
+
):
|
| 422 |
+
return True
|
| 423 |
+
return False
|
| 424 |
+
|
omni_diffusion/data/dataset_qwen2.py
ADDED
|
@@ -0,0 +1,981 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import logging
|
| 3 |
+
import math
|
| 4 |
+
import os
|
| 5 |
+
import pdb
|
| 6 |
+
import random
|
| 7 |
+
import re
|
| 8 |
+
import sys
|
| 9 |
+
import time
|
| 10 |
+
import traceback
|
| 11 |
+
from collections import defaultdict
|
| 12 |
+
from typing import Dict, List, Optional, Sequence
|
| 13 |
+
from typing import List, Optional, Tuple, Union
|
| 14 |
+
import copy
|
| 15 |
+
import numpy as np
|
| 16 |
+
import torch
|
| 17 |
+
import transformers
|
| 18 |
+
from transformers.trainer_pt_utils import LabelSmoother
|
| 19 |
+
import soundfile as sf
|
| 20 |
+
|
| 21 |
+
from .dataset_base import BaseDataset
|
| 22 |
+
|
| 23 |
+
logger = logging.getLogger(__name__)
|
| 24 |
+
logger.setLevel(logging.INFO)
|
| 25 |
+
|
| 26 |
+
IGNORE_TOKEN_ID = LabelSmoother.ignore_index
|
| 27 |
+
|
| 28 |
+
def forward_process(
|
| 29 |
+
bsz: int,
|
| 30 |
+
seq_len: int,
|
| 31 |
+
device: torch.device,
|
| 32 |
+
labels: torch.Tensor,
|
| 33 |
+
eps: float = 1e-3,
|
| 34 |
+
special_token_id: int = 151643,
|
| 35 |
+
special_mask_ratio: float = 0.1
|
| 36 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 37 |
+
"""
|
| 38 |
+
Generates a mask for the input sequences, applying different masking probabilities
|
| 39 |
+
for normal tokens and special tokens within the valid (non-padded) regions.
|
| 40 |
+
"""
|
| 41 |
+
|
| 42 |
+
# Initialize the boolean mask matrix and the overall mask ratio for each sample
|
| 43 |
+
b, l = bsz, seq_len
|
| 44 |
+
masked_indices = torch.zeros((b, l), device=device, dtype=torch.bool)
|
| 45 |
+
p_mask = torch.rand(b, device=device)
|
| 46 |
+
p_mask = (1 - eps) * p_mask + eps
|
| 47 |
+
p_mask = p_mask.unsqueeze(1) # [b,1]
|
| 48 |
+
|
| 49 |
+
# Find the first and last valid positions (where label is not -100) for each sequence
|
| 50 |
+
first_idxs = []
|
| 51 |
+
last_idxs = []
|
| 52 |
+
for i in range(b):
|
| 53 |
+
# -100 is the ignore_index
|
| 54 |
+
nonneg = (labels[i] != -100).nonzero(as_tuple=True)[0]
|
| 55 |
+
if nonneg.numel() == 0:
|
| 56 |
+
first_idxs.append(None)
|
| 57 |
+
last_idxs.append(None)
|
| 58 |
+
else:
|
| 59 |
+
first_idxs.append(int(nonneg[0]))
|
| 60 |
+
last_idxs.append(int(nonneg[-1]))
|
| 61 |
+
|
| 62 |
+
# Generate masks specifically for the valid interval of each sequence
|
| 63 |
+
for i in range(b):
|
| 64 |
+
start = first_idxs[i]
|
| 65 |
+
end = last_idxs[i]
|
| 66 |
+
|
| 67 |
+
# Skip if the sequence has no valid tokens or invalid boundaries
|
| 68 |
+
if start is None or end is None or end < start:
|
| 69 |
+
continue
|
| 70 |
+
|
| 71 |
+
# Generate base thresholds for each position in the valid interval
|
| 72 |
+
valid_len = end - start + 1
|
| 73 |
+
t = torch.rand(valid_len, device=device)
|
| 74 |
+
mask_threshold = (1 - eps) * t + eps # [valid_len]
|
| 75 |
+
|
| 76 |
+
# Generate random values to make masking decisions
|
| 77 |
+
rand_vals = torch.rand(valid_len, device=device)
|
| 78 |
+
# Normal token masking decision
|
| 79 |
+
normal_mask = rand_vals <= mask_threshold
|
| 80 |
+
# Special token masking threshold is lower
|
| 81 |
+
special_thresh = mask_threshold * special_mask_ratio
|
| 82 |
+
special_mask = rand_vals <= special_thresh
|
| 83 |
+
|
| 84 |
+
labels_slice = labels[i, start : end + 1]
|
| 85 |
+
# Final mask: special tokens use special_mask, others use normal_mask
|
| 86 |
+
final_mask = torch.where(
|
| 87 |
+
labels_slice == special_token_id,
|
| 88 |
+
special_mask,
|
| 89 |
+
normal_mask
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
masked_indices[i, start : end + 1] = final_mask
|
| 93 |
+
|
| 94 |
+
total_masked = int(masked_indices.sum().item())
|
| 95 |
+
special_masked = int((masked_indices & (labels == special_token_id)).sum().item())
|
| 96 |
+
|
| 97 |
+
return masked_indices, p_mask
|
| 98 |
+
|
| 99 |
+
def update_labels(input_ids, labels, eos_id, max_n=20):
|
| 100 |
+
"""
|
| 101 |
+
Finds the first occurrence of the EOS token in each sequence and updates
|
| 102 |
+
up to `max_n` subsequent labels to the EOS token ID.
|
| 103 |
+
"""
|
| 104 |
+
batch_size, seq_len = input_ids.shape
|
| 105 |
+
first_occurrence_indices = []
|
| 106 |
+
|
| 107 |
+
# Record the first occurrence position of eos_id in each batch sample
|
| 108 |
+
for idx in range(batch_size):
|
| 109 |
+
eos_positions = (input_ids[idx] == eos_id).nonzero(as_tuple=True)[0]
|
| 110 |
+
if len(eos_positions) > 0:
|
| 111 |
+
first_occurrence_indices.append(eos_positions[0].item())
|
| 112 |
+
else:
|
| 113 |
+
first_occurrence_indices.append(-1)
|
| 114 |
+
|
| 115 |
+
# Select a random number of sequential positions (up to max_n) starting from first_idx to update
|
| 116 |
+
for i in range(batch_size):
|
| 117 |
+
first_idx = first_occurrence_indices[i]
|
| 118 |
+
if first_idx == -1:
|
| 119 |
+
continue
|
| 120 |
+
max_possible = seq_len - first_idx
|
| 121 |
+
|
| 122 |
+
if max_possible <= 0:
|
| 123 |
+
continue
|
| 124 |
+
num_to_select = random.randint(1, min(max_n, max_possible))
|
| 125 |
+
|
| 126 |
+
selected_indices = torch.arange(first_idx, first_idx + num_to_select)
|
| 127 |
+
|
| 128 |
+
labels[i, selected_indices] = eos_id
|
| 129 |
+
|
| 130 |
+
return labels
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
import torch
|
| 134 |
+
import random
|
| 135 |
+
|
| 136 |
+
def update_labels_and_inputs(input_ids, labels, eos_id, max_n=20):
|
| 137 |
+
input_ids = torch.tensor(input_ids).unsqueeze(0)
|
| 138 |
+
labels = torch.tensor(labels).unsqueeze(0)
|
| 139 |
+
batch_size, seq_len = input_ids.shape
|
| 140 |
+
input_ids = input_ids.clone()
|
| 141 |
+
labels = labels.clone()
|
| 142 |
+
new_input_ids = []
|
| 143 |
+
new_labels = []
|
| 144 |
+
|
| 145 |
+
for idx in range(batch_size):
|
| 146 |
+
eos_positions = (input_ids[idx] == eos_id).nonzero(as_tuple=True)[0]
|
| 147 |
+
if len(eos_positions) > 0:
|
| 148 |
+
first_idx = eos_positions[0].item()
|
| 149 |
+
cur_input_ids = input_ids[idx]
|
| 150 |
+
cur_labels = labels[idx]
|
| 151 |
+
else:
|
| 152 |
+
random_max_n = random.randint(1, max_n)
|
| 153 |
+
eos_ids = torch.full((random_max_n,), eos_id, device=input_ids.device, dtype=input_ids.dtype)
|
| 154 |
+
cur_input_ids = torch.cat([input_ids[idx], eos_ids])
|
| 155 |
+
pad_labels = torch.full((random_max_n,), eos_id, device=labels.device, dtype=labels.dtype)
|
| 156 |
+
cur_labels = torch.cat([labels[idx], pad_labels])
|
| 157 |
+
# first_idx = len(cur_input_ids) - random_max_n
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
new_input_ids.append(cur_input_ids)
|
| 161 |
+
new_labels.append(cur_labels)
|
| 162 |
+
|
| 163 |
+
max_len = max(len(x) for x in new_input_ids)
|
| 164 |
+
padded_input_ids = torch.stack([
|
| 165 |
+
torch.cat([x, torch.full((max_len - len(x),), eos_id, device=x.device, dtype=x.dtype)])
|
| 166 |
+
for x in new_input_ids
|
| 167 |
+
])
|
| 168 |
+
padded_labels = torch.stack([
|
| 169 |
+
torch.cat([x, torch.full((max_len - len(x),), eos_id, device=x.device, dtype=x.dtype)])
|
| 170 |
+
for x in new_labels
|
| 171 |
+
])
|
| 172 |
+
|
| 173 |
+
return padded_input_ids, padded_labels
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def pad_or_truncate_to_512(
|
| 178 |
+
input_ids,
|
| 179 |
+
labels,
|
| 180 |
+
eos_id,
|
| 181 |
+
target_len: int = 512,
|
| 182 |
+
):
|
| 183 |
+
|
| 184 |
+
input_ids = torch.as_tensor(input_ids).unsqueeze(0) # (B, L)
|
| 185 |
+
labels = torch.as_tensor(labels ).unsqueeze(0)
|
| 186 |
+
|
| 187 |
+
batch_size = input_ids.size(0)
|
| 188 |
+
new_input_ids, new_labels = [], []
|
| 189 |
+
|
| 190 |
+
for i in range(batch_size):
|
| 191 |
+
cur_input = input_ids[i]
|
| 192 |
+
cur_label = labels[i]
|
| 193 |
+
|
| 194 |
+
cur_input = cur_input[:target_len]
|
| 195 |
+
cur_label = cur_label[:target_len]
|
| 196 |
+
|
| 197 |
+
pad_len = target_len - cur_input.size(0)
|
| 198 |
+
if pad_len > 0:
|
| 199 |
+
eos_pad = torch.full((pad_len,), eos_id, device=cur_input.device, dtype=cur_input.dtype)
|
| 200 |
+
label_pad = torch.full((pad_len,), eos_id, device=cur_label.device, dtype=cur_label.dtype)
|
| 201 |
+
cur_input = torch.cat([cur_input, eos_pad])
|
| 202 |
+
cur_label = torch.cat([cur_label, label_pad])
|
| 203 |
+
|
| 204 |
+
new_input_ids.append(cur_input)
|
| 205 |
+
new_labels.append(cur_label)
|
| 206 |
+
|
| 207 |
+
return torch.stack(new_input_ids), torch.stack(new_labels)
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
class Qwen2Dataset(BaseDataset):
|
| 212 |
+
def __init__(
|
| 213 |
+
self,
|
| 214 |
+
*args,
|
| 215 |
+
**kwargs,
|
| 216 |
+
):
|
| 217 |
+
super().__init__(
|
| 218 |
+
*args,
|
| 219 |
+
**kwargs,
|
| 220 |
+
)
|
| 221 |
+
|
| 222 |
+
self.default_system_message = "You are a helpful AI assistant."
|
| 223 |
+
self.default_system_message = None
|
| 224 |
+
|
| 225 |
+
self.ret = defaultdict(dict)
|
| 226 |
+
self.is_cat = True
|
| 227 |
+
|
| 228 |
+
if self.cross_dataset_joint:
|
| 229 |
+
for i in range(2):
|
| 230 |
+
self.maybe_init_ret(f"default_{i}")
|
| 231 |
+
|
| 232 |
+
def maybe_init_ret(self, source, force=False):
|
| 233 |
+
"""Initializes the return dictionary buffer for a specific source if it doesn't exist."""
|
| 234 |
+
if source not in self.ret or force:
|
| 235 |
+
self.ret[source] = {}
|
| 236 |
+
|
| 237 |
+
self.ret[source]["tokens"] = []
|
| 238 |
+
self.ret[source]["labels"] = []
|
| 239 |
+
self.ret[source]["cu_seq_lens"] = [0]
|
| 240 |
+
self.ret[source]["max_seq_len"] = self.max_padding_length
|
| 241 |
+
|
| 242 |
+
if self.create_position_ids:
|
| 243 |
+
self.ret[source]["position_ids"] = []
|
| 244 |
+
|
| 245 |
+
if self.create_attention_mask:
|
| 246 |
+
self.ret[source]["attention_mask"] = []
|
| 247 |
+
|
| 248 |
+
if self.create_attention_mask_2d:
|
| 249 |
+
self.ret[source]["attention_mask_2d"] = torch.tril(
|
| 250 |
+
torch.ones(
|
| 251 |
+
(1, self.max_padding_length, self.max_padding_length), dtype=torch.bool
|
| 252 |
+
)
|
| 253 |
+
)
|
| 254 |
+
return len(self.ret[source]["tokens"]) == 0
|
| 255 |
+
|
| 256 |
+
def get_max_min_ret_length(self):
|
| 257 |
+
"""Finds the buffer with the maximum and minimum current lengths."""
|
| 258 |
+
max_ret_lengh = 0
|
| 259 |
+
min_ret_lengh = self.max_padding_length + 1
|
| 260 |
+
|
| 261 |
+
max_ret_key = None
|
| 262 |
+
min_ret_key = None
|
| 263 |
+
|
| 264 |
+
for k, v in self.ret.items():
|
| 265 |
+
cur_length = len(v["tokens"])
|
| 266 |
+
|
| 267 |
+
if cur_length > max_ret_lengh:
|
| 268 |
+
max_ret_lengh = cur_length
|
| 269 |
+
max_ret_key = k
|
| 270 |
+
|
| 271 |
+
if cur_length < min_ret_lengh:
|
| 272 |
+
min_ret_lengh = cur_length
|
| 273 |
+
min_ret_key = k
|
| 274 |
+
|
| 275 |
+
return max_ret_lengh, max_ret_key, min_ret_lengh, min_ret_key
|
| 276 |
+
|
| 277 |
+
def add_ret(self, ret, source):
|
| 278 |
+
"""
|
| 279 |
+
Appends a processed sample (tokens, images, audios) to the buffer (`self.ret`).
|
| 280 |
+
Updates indices for images/audios to account for the offset in the concatenated sequence.
|
| 281 |
+
"""
|
| 282 |
+
cur_length = len(ret["input_ids"])
|
| 283 |
+
cur_image_length = len(ret["images"])
|
| 284 |
+
cur_audio_length = len(ret["audios"])
|
| 285 |
+
|
| 286 |
+
all_length = len(self.ret[source]["tokens"])
|
| 287 |
+
|
| 288 |
+
if "images" in self.ret[source]:
|
| 289 |
+
all_image_length = len(self.ret[source]["images"])
|
| 290 |
+
else:
|
| 291 |
+
all_image_length = 0
|
| 292 |
+
|
| 293 |
+
if cur_image_length > 0:
|
| 294 |
+
if all_image_length > 0:
|
| 295 |
+
self.ret[source]["images"] = torch.cat(
|
| 296 |
+
[self.ret[source]["images"], ret["images"]], dim=0
|
| 297 |
+
)
|
| 298 |
+
ret["image_indices"][1, :, :] += all_length
|
| 299 |
+
self.ret[source]["image_indices"] = torch.cat(
|
| 300 |
+
[self.ret[source]["image_indices"], ret["image_indices"]], dim=1
|
| 301 |
+
)
|
| 302 |
+
else:
|
| 303 |
+
self.ret[source]["images"] = ret["images"]
|
| 304 |
+
self.ret[source]["image_indices"] = ret["image_indices"]
|
| 305 |
+
|
| 306 |
+
if "audios" in self.ret[source]:
|
| 307 |
+
all_audio_length = len(self.ret[source]["audios"])
|
| 308 |
+
else:
|
| 309 |
+
all_audio_length = 0
|
| 310 |
+
|
| 311 |
+
if cur_audio_length > 0:
|
| 312 |
+
if all_audio_length > 0:
|
| 313 |
+
self.ret[source]["audios"].extend(ret["audios"])
|
| 314 |
+
for audio_indice in ret["audio_indices"]:
|
| 315 |
+
audio_indice[1, :, :] += all_length
|
| 316 |
+
self.ret[source]["audio_indices"].extend(ret["audio_indices"])
|
| 317 |
+
else:
|
| 318 |
+
self.ret[source]["audios"] = ret["audios"]
|
| 319 |
+
self.ret[source]["audio_indices"] = ret["audio_indices"]
|
| 320 |
+
|
| 321 |
+
# print(self.ret[source]["audios"])
|
| 322 |
+
|
| 323 |
+
if self.create_attention_mask:
|
| 324 |
+
self.ret[source]["attention_mask"] += ret["attention_mask"]
|
| 325 |
+
|
| 326 |
+
if self.create_attention_mask_2d:
|
| 327 |
+
self.ret[source]["attention_mask_2d"][:, all_length:, :all_length] = 0
|
| 328 |
+
|
| 329 |
+
if self.create_position_ids:
|
| 330 |
+
self.ret[source]["position_ids"] += list(range(cur_length))
|
| 331 |
+
|
| 332 |
+
self.ret[source]["tokens"] += ret["input_ids"]
|
| 333 |
+
self.ret[source]["labels"] += ret["labels"]
|
| 334 |
+
self.ret[source]["cu_seq_lens"] += [all_length + cur_length]
|
| 335 |
+
|
| 336 |
+
def process_ret(self, to_ret):
|
| 337 |
+
"""
|
| 338 |
+
Finalizes the buffer for return. Handles padding, truncation, tensor conversion,
|
| 339 |
+
and attention mask generation.
|
| 340 |
+
"""
|
| 341 |
+
if "tokens" in to_ret and len(to_ret["tokens"]) > 0:
|
| 342 |
+
pass
|
| 343 |
+
else:
|
| 344 |
+
return to_ret
|
| 345 |
+
|
| 346 |
+
if self.create_position_ids:
|
| 347 |
+
if self.reset_position_ids:
|
| 348 |
+
pass
|
| 349 |
+
else:
|
| 350 |
+
to_ret["position_ids"] = list(range(len(to_ret["tokens"])))
|
| 351 |
+
|
| 352 |
+
if self.create_attention_mask_2d:
|
| 353 |
+
if self.reset_attention_mask:
|
| 354 |
+
pass
|
| 355 |
+
else:
|
| 356 |
+
to_ret["attention_mask_2d"] = torch.tril(
|
| 357 |
+
torch.ones(
|
| 358 |
+
(1, self.max_padding_length, self.max_padding_length), dtype=torch.bool
|
| 359 |
+
)
|
| 360 |
+
)
|
| 361 |
+
|
| 362 |
+
if self.shift_token:
|
| 363 |
+
to_ret["tokens"] = to_ret["tokens"][:-1]
|
| 364 |
+
to_ret["labels"] = to_ret["labels"][1:]
|
| 365 |
+
to_ret["cu_seq_lens"][-1] -= 1
|
| 366 |
+
if self.create_position_ids:
|
| 367 |
+
to_ret["position_ids"] = to_ret["position_ids"][:-1]
|
| 368 |
+
if self.create_attention_mask:
|
| 369 |
+
to_ret["attention_mask"] = to_ret["attention_mask"][:-1]
|
| 370 |
+
|
| 371 |
+
if self.create_attention_mask_2d:
|
| 372 |
+
to_ret["attention_mask_2d"][:, :, -1] = 0
|
| 373 |
+
to_ret["attention_mask_2d"][:, -1, :] = 0
|
| 374 |
+
|
| 375 |
+
assert len(to_ret["tokens"]) == len(
|
| 376 |
+
to_ret["labels"]
|
| 377 |
+
), f"{len(to_ret['tokens'])} {len(to_ret['labels'])}"
|
| 378 |
+
|
| 379 |
+
if not self.variable_length and self.max_padding_length > len(to_ret["tokens"]):
|
| 380 |
+
to_ret["tokens"] += [self.tokenizer.pad_token_id] * (
|
| 381 |
+
self.max_padding_length - len(to_ret["tokens"])
|
| 382 |
+
)
|
| 383 |
+
to_ret["labels"] += [IGNORE_TOKEN_ID] * (
|
| 384 |
+
self.max_padding_length - len(to_ret["labels"])
|
| 385 |
+
)
|
| 386 |
+
to_ret["cu_seq_lens"][-1] = self.max_padding_length
|
| 387 |
+
if self.create_position_ids:
|
| 388 |
+
# to_ret["position_ids"] += to_ret["position_ids"][-1:] * (
|
| 389 |
+
# self.max_padding_length - len(to_ret["position_ids"])
|
| 390 |
+
# )
|
| 391 |
+
to_ret["position_ids"] += list(
|
| 392 |
+
range(to_ret["position_ids"][-1] + 1, self.max_padding_length)
|
| 393 |
+
)
|
| 394 |
+
if self.create_attention_mask:
|
| 395 |
+
to_ret["attention_mask"] += [0] * (
|
| 396 |
+
self.max_padding_length - len(to_ret["attention_mask"])
|
| 397 |
+
)
|
| 398 |
+
|
| 399 |
+
to_ret["tokens"] = to_ret["tokens"][: self.max_padding_length]
|
| 400 |
+
to_ret["labels"] = to_ret["labels"][: self.max_padding_length]
|
| 401 |
+
to_ret["cu_seq_lens"][-1] = self.max_padding_length
|
| 402 |
+
|
| 403 |
+
if self.create_position_ids:
|
| 404 |
+
to_ret["position_ids"] = to_ret["position_ids"][: self.max_padding_length]
|
| 405 |
+
if self.create_attention_mask:
|
| 406 |
+
to_ret["attention_mask"] = to_ret["attention_mask"][: self.max_padding_length]
|
| 407 |
+
|
| 408 |
+
to_ret["tokens"] = torch.tensor(to_ret["tokens"], dtype=torch.int64)
|
| 409 |
+
to_ret["labels"] = torch.tensor(to_ret["labels"], dtype=torch.int64)
|
| 410 |
+
to_ret["cu_seq_lens"] = torch.tensor(to_ret["cu_seq_lens"], dtype=torch.int64)
|
| 411 |
+
if self.create_position_ids:
|
| 412 |
+
to_ret["position_ids"] = torch.tensor(to_ret["position_ids"], dtype=torch.int64)
|
| 413 |
+
if self.create_attention_mask:
|
| 414 |
+
to_ret["attention_mask"] = torch.tensor(to_ret["attention_mask"], dtype=torch.int64)
|
| 415 |
+
|
| 416 |
+
if self.create_attention_mask_2d:
|
| 417 |
+
attention_mask_2d = to_ret.pop("attention_mask_2d")
|
| 418 |
+
attention_mask_2d = attention_mask_2d.masked_fill(
|
| 419 |
+
(to_ret["attention_mask"] < 0.5).view(1, 1, self.max_padding_length), value=0
|
| 420 |
+
)
|
| 421 |
+
attention_mask_2d = attention_mask_2d < 0.5
|
| 422 |
+
|
| 423 |
+
to_ret["attention_mask"] = attention_mask_2d
|
| 424 |
+
|
| 425 |
+
if self.create_loss_mask:
|
| 426 |
+
loss_mask = torch.where(to_ret["labels"] == IGNORE_TOKEN_ID, 0, 1)
|
| 427 |
+
to_ret["loss_mask"] = loss_mask.to(torch.float32)
|
| 428 |
+
|
| 429 |
+
if not self.reset_position_ids and not self.reset_attention_mask:
|
| 430 |
+
to_ret.pop("cu_seq_lens")
|
| 431 |
+
else:
|
| 432 |
+
max_seq_len = max(to_ret["cu_seq_lens"][1:] - to_ret["cu_seq_lens"][:-1])
|
| 433 |
+
to_ret["max_seq_len"] = max_seq_len
|
| 434 |
+
to_ret["input_ids"] = to_ret["tokens"]
|
| 435 |
+
|
| 436 |
+
return to_ret
|
| 437 |
+
|
| 438 |
+
def is_skip(self):
|
| 439 |
+
|
| 440 |
+
processed_samples = sum(self.processed_samples.values())
|
| 441 |
+
if processed_samples < self.skip_samples:
|
| 442 |
+
if processed_samples % 1e3 == 0:
|
| 443 |
+
print(
|
| 444 |
+
f"processed_samples {processed_samples} skip_samples {self.skip_samples}"
|
| 445 |
+
)
|
| 446 |
+
return True
|
| 447 |
+
|
| 448 |
+
def show_statistic(self):
|
| 449 |
+
log_interval = 2000
|
| 450 |
+
if self.max_padding_length >= 2**17:
|
| 451 |
+
log_interval = 1000
|
| 452 |
+
if self.max_padding_length >= 2**20:
|
| 453 |
+
log_interval = 200
|
| 454 |
+
|
| 455 |
+
processed_samples = sum(self.processed_samples.values())
|
| 456 |
+
unjoint_samples = sum(self.unjoint_samples.values())
|
| 457 |
+
joint_samples = sum(self.joint_samples.values())
|
| 458 |
+
if unjoint_samples % log_interval == 1:
|
| 459 |
+
pass
|
| 460 |
+
else:
|
| 461 |
+
return
|
| 462 |
+
|
| 463 |
+
with open(os.path.join(self.output_dir, "data_statistics.log"), "a") as f:
|
| 464 |
+
print("-" * 100, file=f)
|
| 465 |
+
print(
|
| 466 |
+
f"processed_samples {processed_samples}" +
|
| 467 |
+
f" unjoint_samples {unjoint_samples}" +
|
| 468 |
+
f" joint_samples {joint_samples}" +
|
| 469 |
+
f" {[len(v['tokens']) for _, v in self.ret.items()]}", file=f,
|
| 470 |
+
)
|
| 471 |
+
|
| 472 |
+
print("source processed_samples unjoint_samples joint_samples data_path", file=f)
|
| 473 |
+
for source, data_path in self.source2jsonpath.items():
|
| 474 |
+
print(f"{source: >6} {self.processed_samples[source]: >16} \
|
| 475 |
+
{self.unjoint_samples[source]: >16} {self.joint_samples[source]: >16} {data_path}", file=f)
|
| 476 |
+
|
| 477 |
+
def __getitem__(self, index):
|
| 478 |
+
"""
|
| 479 |
+
Main data loading function.
|
| 480 |
+
1. Fetches raw data.
|
| 481 |
+
2. Preprocesses it (tokenization, multimodal handling).
|
| 482 |
+
3. Packs multiple samples into a single buffer (`self.ret`) until `max_padding_length` is reached.
|
| 483 |
+
4. Returns the packed batch when full.
|
| 484 |
+
"""
|
| 485 |
+
self.processor["audio"].load_model()
|
| 486 |
+
self.processor["image"].load_model()
|
| 487 |
+
|
| 488 |
+
while True:
|
| 489 |
+
try:
|
| 490 |
+
sample = self.raw_data[index]
|
| 491 |
+
sample = copy.deepcopy(sample)
|
| 492 |
+
sample = self.update_data_path(sample)
|
| 493 |
+
source = sample["source"]
|
| 494 |
+
|
| 495 |
+
self.processed_samples[source] += 1
|
| 496 |
+
if self.is_skip():
|
| 497 |
+
return {}
|
| 498 |
+
|
| 499 |
+
if self.cross_dataset_joint:
|
| 500 |
+
is_empty = False
|
| 501 |
+
(
|
| 502 |
+
max_ret_lengh,
|
| 503 |
+
max_ret_key,
|
| 504 |
+
min_ret_lengh,
|
| 505 |
+
min_ret_key,
|
| 506 |
+
) = self.get_max_min_ret_length()
|
| 507 |
+
else:
|
| 508 |
+
is_empty = self.maybe_init_ret(source)
|
| 509 |
+
|
| 510 |
+
max_ret_lengh = min_ret_lengh = len(self.ret[source]["tokens"])
|
| 511 |
+
max_ret_key = min_ret_key = source
|
| 512 |
+
|
| 513 |
+
is_begin = is_empty or self.reset_position_ids or self.reset_attention_mask
|
| 514 |
+
|
| 515 |
+
#logger.info("preprocess begin " + str(self.processor["audio"].audio_tokenizer.device))
|
| 516 |
+
|
| 517 |
+
ret = preprocess(
|
| 518 |
+
sample,
|
| 519 |
+
self.tokenizer,
|
| 520 |
+
self.image_token_length,
|
| 521 |
+
default_system_message=self.default_system_message,
|
| 522 |
+
processor=self.processor,
|
| 523 |
+
is_begin=is_begin,
|
| 524 |
+
max_num_frame=self.max_num_frame,
|
| 525 |
+
max_fps=self.max_fps,
|
| 526 |
+
)
|
| 527 |
+
|
| 528 |
+
#logger.info("preprocess end " + str(self.processor["audio"].audio_tokenizer.device))
|
| 529 |
+
|
| 530 |
+
if ret is None:
|
| 531 |
+
#logger.info("ret is None " + str(self.processor["audio"].audio_tokenizer.device))
|
| 532 |
+
return {}
|
| 533 |
+
|
| 534 |
+
cur_length = len(ret["input_ids"])
|
| 535 |
+
|
| 536 |
+
if cur_length > self.max_padding_length:
|
| 537 |
+
return {}
|
| 538 |
+
|
| 539 |
+
self.unjoint_samples[source] += 1
|
| 540 |
+
|
| 541 |
+
if not self.dataset_joint:
|
| 542 |
+
import pdb; pdb.set_trace()
|
| 543 |
+
to_ret = self.ret.pop(max_ret_key)
|
| 544 |
+
|
| 545 |
+
self.maybe_init_ret(max_ret_key, force=True)
|
| 546 |
+
self.add_ret(ret, max_ret_key)
|
| 547 |
+
|
| 548 |
+
elif min_ret_lengh + cur_length > self.max_padding_length:
|
| 549 |
+
#logger.info("data too long " + str(self.processor["audio"].audio_tokenizer.device))
|
| 550 |
+
to_ret = self.ret.pop(max_ret_key)
|
| 551 |
+
self.joint_samples[source] += 1
|
| 552 |
+
|
| 553 |
+
self.maybe_init_ret(max_ret_key, force=True)
|
| 554 |
+
self.add_ret(ret, max_ret_key)
|
| 555 |
+
|
| 556 |
+
else:
|
| 557 |
+
to_ret = {}
|
| 558 |
+
self.add_ret(ret, min_ret_key)
|
| 559 |
+
|
| 560 |
+
to_ret = self.process_ret(to_ret)
|
| 561 |
+
|
| 562 |
+
self.show_statistic()
|
| 563 |
+
#logger.info("output ret " + str(len(ret)) + " " + str(self.processor["audio"].audio_tokenizer.device))
|
| 564 |
+
return to_ret
|
| 565 |
+
|
| 566 |
+
except Exception as error:
|
| 567 |
+
try:
|
| 568 |
+
with open(os.path.join(self.output_dir, "data_error.log"), "a") as f:
|
| 569 |
+
#print("-" * 100)
|
| 570 |
+
#print(traceback.format_exc())
|
| 571 |
+
#print(self.raw_data[index])
|
| 572 |
+
print("-" * 100, file=f)
|
| 573 |
+
print(traceback.format_exc(), file=f)
|
| 574 |
+
print(self.raw_data[index], file=f)
|
| 575 |
+
except Exception as error:
|
| 576 |
+
print(error)
|
| 577 |
+
return {}
|
| 578 |
+
|
| 579 |
+
|
| 580 |
+
def preprocess(
|
| 581 |
+
sample,
|
| 582 |
+
tokenizer: transformers.PreTrainedTokenizer,
|
| 583 |
+
image_token_length: int,
|
| 584 |
+
default_system_message: str = "You are a helpful assistant.",
|
| 585 |
+
processor=None,
|
| 586 |
+
is_begin: bool = True,
|
| 587 |
+
max_num_frame: int = 8,
|
| 588 |
+
max_fps: int = 1,
|
| 589 |
+
) -> Dict:
|
| 590 |
+
|
| 591 |
+
from ..constants import (
|
| 592 |
+
IMG_START_TOKEN,
|
| 593 |
+
IMG_END_TOKEN,
|
| 594 |
+
IMG_CONTEXT_TOKEN,
|
| 595 |
+
VID_START_TOKEN,
|
| 596 |
+
VID_END_TOKEN,
|
| 597 |
+
VID_CONTEXT_TOKEN,
|
| 598 |
+
PATCH_START_TOKEN,
|
| 599 |
+
PATCH_END_TOKEN,
|
| 600 |
+
PATCH_CONTEXT_TOKEN,
|
| 601 |
+
AUD_START_TOKEN,
|
| 602 |
+
AUD_END_TOKEN,
|
| 603 |
+
IMG_TAG_TOKEN,
|
| 604 |
+
VID_TAG_TOKEN,
|
| 605 |
+
AUD_TAG_TOKEN,
|
| 606 |
+
AUD_CONTEXT_TOKEN,
|
| 607 |
+
)
|
| 608 |
+
|
| 609 |
+
human_roles = ["user", "human"]
|
| 610 |
+
gpt_roles = ["assistant", "gpt"]
|
| 611 |
+
system_roles = ["system"]
|
| 612 |
+
|
| 613 |
+
# Ensure special tokens map to exactly one ID
|
| 614 |
+
IMG_CONTEXT_ID = tokenizer(IMG_CONTEXT_TOKEN, add_special_tokens=False).input_ids
|
| 615 |
+
IMG_START_ID = tokenizer(IMG_START_TOKEN, add_special_tokens=False).input_ids
|
| 616 |
+
IMG_END_ID = tokenizer(IMG_END_TOKEN, add_special_tokens=False).input_ids
|
| 617 |
+
|
| 618 |
+
VID_CONTEXT_ID = tokenizer(VID_CONTEXT_TOKEN, add_special_tokens=False).input_ids
|
| 619 |
+
VID_START_ID = tokenizer(VID_START_TOKEN, add_special_tokens=False).input_ids
|
| 620 |
+
VID_END_ID = tokenizer(VID_END_TOKEN, add_special_tokens=False).input_ids
|
| 621 |
+
|
| 622 |
+
PATCH_CONTEXT_ID = tokenizer(PATCH_CONTEXT_TOKEN, add_special_tokens=False).input_ids
|
| 623 |
+
PATCH_START_ID = tokenizer(PATCH_START_TOKEN, add_special_tokens=False).input_ids
|
| 624 |
+
PATCH_END_ID = tokenizer(PATCH_END_TOKEN, add_special_tokens=False).input_ids
|
| 625 |
+
|
| 626 |
+
AUD_CONTEXT_ID = tokenizer(AUD_CONTEXT_TOKEN, add_special_tokens=False).input_ids
|
| 627 |
+
AUD_START_ID = tokenizer(AUD_START_TOKEN, add_special_tokens=False).input_ids
|
| 628 |
+
AUD_END_ID = tokenizer(AUD_END_TOKEN, add_special_tokens=False).input_ids
|
| 629 |
+
|
| 630 |
+
IMG_TAG_ID = tokenizer(IMG_TAG_TOKEN, add_special_tokens=False).input_ids
|
| 631 |
+
VID_TAG_ID = tokenizer(VID_TAG_TOKEN, add_special_tokens=False).input_ids
|
| 632 |
+
AUD_TAG_ID = tokenizer(AUD_TAG_TOKEN, add_special_tokens=False).input_ids
|
| 633 |
+
|
| 634 |
+
assert len(IMG_CONTEXT_ID) == 1
|
| 635 |
+
assert len(IMG_START_ID) == 1
|
| 636 |
+
assert len(IMG_END_ID) == 1
|
| 637 |
+
|
| 638 |
+
assert len(VID_CONTEXT_ID) == 1
|
| 639 |
+
assert len(VID_START_ID) == 1
|
| 640 |
+
assert len(VID_END_ID) == 1
|
| 641 |
+
|
| 642 |
+
assert len(PATCH_CONTEXT_ID) == 1
|
| 643 |
+
assert len(PATCH_START_ID) == 1
|
| 644 |
+
assert len(PATCH_END_ID) == 1
|
| 645 |
+
|
| 646 |
+
IMG_CONTEXT_ID = IMG_CONTEXT_ID[0]
|
| 647 |
+
IMG_START_ID = IMG_START_ID[0]
|
| 648 |
+
IMG_END_ID = IMG_END_ID[0]
|
| 649 |
+
|
| 650 |
+
VID_CONTEXT_ID = VID_CONTEXT_ID[0]
|
| 651 |
+
VID_START_ID = VID_START_ID[0]
|
| 652 |
+
VID_END_ID = VID_END_ID[0]
|
| 653 |
+
|
| 654 |
+
PATCH_CONTEXT_ID = PATCH_CONTEXT_ID[0]
|
| 655 |
+
PATCH_START_ID = PATCH_START_ID[0]
|
| 656 |
+
PATCH_END_ID = PATCH_END_ID[0]
|
| 657 |
+
|
| 658 |
+
AUD_CONTEXT_ID = AUD_CONTEXT_ID[0]
|
| 659 |
+
AUD_START_ID = AUD_START_ID[0]
|
| 660 |
+
AUD_END_ID = AUD_END_ID[0]
|
| 661 |
+
|
| 662 |
+
IMG_TAG_ID = IMG_TAG_ID[0]
|
| 663 |
+
VID_TAG_ID = VID_TAG_ID[0]
|
| 664 |
+
AUD_TAG_ID = AUD_TAG_ID[0]
|
| 665 |
+
|
| 666 |
+
BOS_ID = tokenizer.bos_token_id
|
| 667 |
+
EOS_ID = tokenizer.eos_token_id
|
| 668 |
+
|
| 669 |
+
# ChatML format special tokens
|
| 670 |
+
IM_START = "<|im_start|>"
|
| 671 |
+
IM_END = "<|im_end|>"
|
| 672 |
+
USER = "user"
|
| 673 |
+
ASSISTANT = "assistant"
|
| 674 |
+
SYSTEM = "system"
|
| 675 |
+
|
| 676 |
+
nl_tokens = tokenizer("\n", add_special_tokens=False).input_ids
|
| 677 |
+
IM_START_IDS = tokenizer(IM_START, add_special_tokens=False).input_ids
|
| 678 |
+
IM_END_IDS = tokenizer(IM_END, add_special_tokens=False).input_ids
|
| 679 |
+
USER_IDS = tokenizer(USER, add_special_tokens=False).input_ids
|
| 680 |
+
ASSISTANT_IDS = tokenizer(ASSISTANT, add_special_tokens=False).input_ids
|
| 681 |
+
SYSTEM_IDS = tokenizer(SYSTEM, add_special_tokens=False).input_ids
|
| 682 |
+
|
| 683 |
+
input_ids, targets = [], []
|
| 684 |
+
images = []
|
| 685 |
+
image_indices = []
|
| 686 |
+
audios = []
|
| 687 |
+
audio_indices = []
|
| 688 |
+
|
| 689 |
+
messages = []
|
| 690 |
+
if "conversations" in sample:
|
| 691 |
+
messages = sample["conversations"]
|
| 692 |
+
if len(messages) == 0 and "messages" in sample:
|
| 693 |
+
messages = sample["messages"]
|
| 694 |
+
|
| 695 |
+
# ----------------------------------------------------------------
|
| 696 |
+
# 1. System Prompt Processing
|
| 697 |
+
# ----------------------------------------------------------------
|
| 698 |
+
has_system = False
|
| 699 |
+
if is_begin:
|
| 700 |
+
if messages[0]["role"] == "system":
|
| 701 |
+
has_system = True
|
| 702 |
+
else:
|
| 703 |
+
has_system = False
|
| 704 |
+
|
| 705 |
+
if (
|
| 706 |
+
not has_system
|
| 707 |
+
and default_system_message is not None
|
| 708 |
+
and len(default_system_message) > 0
|
| 709 |
+
):
|
| 710 |
+
messages = [{"role": "system", "content": default_system_message}] + messages
|
| 711 |
+
has_system = True
|
| 712 |
+
|
| 713 |
+
# ----------------------------------------------------------------
|
| 714 |
+
# 2. Audio Processing
|
| 715 |
+
# ----------------------------------------------------------------
|
| 716 |
+
if has_audio(sample) and processor["audio"].is_discrete:
|
| 717 |
+
unused_audio_idxs = list(range(len(sample["audios"])))
|
| 718 |
+
|
| 719 |
+
audio_tokens_list = [
|
| 720 |
+
processor["audio"].process_audios(x, is_discrete=True) for x in sample["audios"]
|
| 721 |
+
]
|
| 722 |
+
audio_tokens_list = ["".join(f"<|audio_{i}|>" for i in x) for x in audio_tokens_list]
|
| 723 |
+
|
| 724 |
+
audio_idx = 0
|
| 725 |
+
for j, sentence in enumerate(messages):
|
| 726 |
+
content = sentence["content"]
|
| 727 |
+
role = sentence["role"]
|
| 728 |
+
# whether apply discrete tokenize to this role
|
| 729 |
+
if processor["audio"].apply_to_role(role, is_discrete=True):
|
| 730 |
+
while AUD_TAG_TOKEN in content:
|
| 731 |
+
content = content.replace(
|
| 732 |
+
AUD_TAG_TOKEN,
|
| 733 |
+
f"{AUD_START_TOKEN}{audio_tokens_list[audio_idx]}{AUD_END_TOKEN}",
|
| 734 |
+
1,
|
| 735 |
+
)
|
| 736 |
+
# <|begin_of_audio|> <|audio_0|> <|audio_1|> ... <|audio_n|> <|end_of_audio|>
|
| 737 |
+
unused_audio_idxs.remove(audio_idx)
|
| 738 |
+
audio_idx += 1
|
| 739 |
+
else:
|
| 740 |
+
audio_idx += content.count(AUD_TAG_TOKEN)
|
| 741 |
+
|
| 742 |
+
sentence["content"] = content
|
| 743 |
+
|
| 744 |
+
# ----------------------------------------------------------------
|
| 745 |
+
# 3. Image Processing
|
| 746 |
+
# ----------------------------------------------------------------
|
| 747 |
+
if has_image(sample):
|
| 748 |
+
# for visual question answering & captioning
|
| 749 |
+
image_tokens_512_list = [
|
| 750 |
+
processor["image"].process_images_with_subpatch(x, 512) for x in sample["images"]
|
| 751 |
+
]
|
| 752 |
+
image_tokens_512_list = [
|
| 753 |
+
processor["image"].get_image_token(x) for x in image_tokens_512_list
|
| 754 |
+
]
|
| 755 |
+
image_tokens_512_list = [x[0].tolist() for x in image_tokens_512_list]
|
| 756 |
+
image_tokens_512_list = ["".join(f"<|image_{i}|>" for i in x) for x in image_tokens_512_list]
|
| 757 |
+
|
| 758 |
+
# for image generation
|
| 759 |
+
image_tokens_256_list = [
|
| 760 |
+
processor["image"].process_images_with_subpatch(x, 256) for x in sample["images"]
|
| 761 |
+
]
|
| 762 |
+
image_tokens_256_list = [
|
| 763 |
+
processor["image"].get_image_token(x) for x in image_tokens_256_list
|
| 764 |
+
]
|
| 765 |
+
image_tokens_256_list = [x[0].tolist() for x in image_tokens_256_list]
|
| 766 |
+
image_tokens_256_list = ["".join(f"<|image_{i}|>" for i in x) for x in image_tokens_256_list]
|
| 767 |
+
|
| 768 |
+
image_idx = 0
|
| 769 |
+
for j, sentence in enumerate(messages):
|
| 770 |
+
content = sentence["content"]
|
| 771 |
+
role = sentence["role"]
|
| 772 |
+
# for image, always apply discrete tokenize to this role
|
| 773 |
+
# if processor["image"].apply_to_role(role) or True:
|
| 774 |
+
if role == "user":
|
| 775 |
+
image_resolution = 512
|
| 776 |
+
else:
|
| 777 |
+
image_resolution = 256
|
| 778 |
+
|
| 779 |
+
while IMG_TAG_TOKEN in content:
|
| 780 |
+
if image_resolution == 256:
|
| 781 |
+
content = content.replace(
|
| 782 |
+
IMG_TAG_TOKEN,
|
| 783 |
+
f"{IMG_START_TOKEN}{image_tokens_256_list[image_idx]}{IMG_END_TOKEN}",
|
| 784 |
+
1,
|
| 785 |
+
)
|
| 786 |
+
# <|begin_of_image|> <|image_0|> <|image_1|> ... <|image_n|> <|end_of_image|>
|
| 787 |
+
else:
|
| 788 |
+
content = content.replace(
|
| 789 |
+
IMG_TAG_TOKEN,
|
| 790 |
+
f"{IMG_START_TOKEN}{image_tokens_512_list[image_idx]}{IMG_END_TOKEN}",
|
| 791 |
+
1,
|
| 792 |
+
)
|
| 793 |
+
# <|begin_of_image|> <|image_0|> <|image_1|> ... <|image_n|> <|end_of_image|>
|
| 794 |
+
image_idx += 1
|
| 795 |
+
else:
|
| 796 |
+
image_idx += content.count(IMG_TAG_TOKEN)
|
| 797 |
+
|
| 798 |
+
sentence["content"] = content
|
| 799 |
+
|
| 800 |
+
# ----------------------------------------------------------------
|
| 801 |
+
# 4. Text Processing
|
| 802 |
+
# ----------------------------------------------------------------
|
| 803 |
+
for j, sentence in enumerate(messages):
|
| 804 |
+
role = sentence["role"]
|
| 805 |
+
content = sentence["content"]
|
| 806 |
+
|
| 807 |
+
if role in human_roles:
|
| 808 |
+
_input_id = (
|
| 809 |
+
IM_START_IDS
|
| 810 |
+
+ USER_IDS
|
| 811 |
+
+ nl_tokens
|
| 812 |
+
+ tokenizer(content, add_special_tokens=False).input_ids
|
| 813 |
+
+ IM_END_IDS
|
| 814 |
+
+ nl_tokens
|
| 815 |
+
)
|
| 816 |
+
_target = [IGNORE_TOKEN_ID] * len(_input_id)
|
| 817 |
+
|
| 818 |
+
elif role in gpt_roles:
|
| 819 |
+
content_input_id = tokenizer(content, add_special_tokens=False).input_ids
|
| 820 |
+
|
| 821 |
+
_input_id = (
|
| 822 |
+
IM_START_IDS + ASSISTANT_IDS + nl_tokens + content_input_id + IM_END_IDS + nl_tokens
|
| 823 |
+
)
|
| 824 |
+
_target = (
|
| 825 |
+
[IGNORE_TOKEN_ID] * len(IM_START_IDS)
|
| 826 |
+
+ [IGNORE_TOKEN_ID] * len(ASSISTANT_IDS)
|
| 827 |
+
+ [IGNORE_TOKEN_ID] * len(nl_tokens)
|
| 828 |
+
+ content_input_id
|
| 829 |
+
+ IM_END_IDS
|
| 830 |
+
+ nl_tokens
|
| 831 |
+
)
|
| 832 |
+
|
| 833 |
+
dream_pad_token = tokenizer.encode("<|endoftext|>")
|
| 834 |
+
_input_id_dream = (
|
| 835 |
+
IM_START_IDS + ASSISTANT_IDS + nl_tokens + content_input_id + IM_END_IDS + dream_pad_token
|
| 836 |
+
)
|
| 837 |
+
_target_dream = (
|
| 838 |
+
[IGNORE_TOKEN_ID] * len(IM_START_IDS)
|
| 839 |
+
+ [IGNORE_TOKEN_ID] * len(ASSISTANT_IDS)
|
| 840 |
+
+ [IGNORE_TOKEN_ID] * len(nl_tokens)
|
| 841 |
+
+ content_input_id
|
| 842 |
+
+ IM_END_IDS
|
| 843 |
+
+ dream_pad_token
|
| 844 |
+
)
|
| 845 |
+
|
| 846 |
+
elif role in system_roles:
|
| 847 |
+
_input_id = (
|
| 848 |
+
IM_START_IDS
|
| 849 |
+
+ SYSTEM_IDS
|
| 850 |
+
+ nl_tokens
|
| 851 |
+
+ tokenizer(content, add_special_tokens=False).input_ids
|
| 852 |
+
+ IM_END_IDS
|
| 853 |
+
+ nl_tokens
|
| 854 |
+
)
|
| 855 |
+
_target = [IGNORE_TOKEN_ID] * len(_input_id)
|
| 856 |
+
|
| 857 |
+
else:
|
| 858 |
+
raise NotImplementedError
|
| 859 |
+
|
| 860 |
+
input_ids += _input_id
|
| 861 |
+
targets += _target
|
| 862 |
+
|
| 863 |
+
|
| 864 |
+
# ----------------------------------------------------------------
|
| 865 |
+
# 5. Contiguous Audio Processing
|
| 866 |
+
# ----------------------------------------------------------------
|
| 867 |
+
if has_audio(sample) and processor["audio"].is_contiguous:
|
| 868 |
+
aud_positions = [i for i, x in enumerate(input_ids) if x == AUD_TAG_ID]
|
| 869 |
+
assert len(aud_positions) == len(unused_audio_idxs), sample
|
| 870 |
+
|
| 871 |
+
new_input_ids = []
|
| 872 |
+
new_targets = []
|
| 873 |
+
st = 0
|
| 874 |
+
|
| 875 |
+
for aud_idx, aud_pos in enumerate(aud_positions):
|
| 876 |
+
aud_idx = unused_audio_idxs[aud_idx]
|
| 877 |
+
audio = processor["audio"].process_audios(sample["audios"][aud_idx], is_contiguous=True)
|
| 878 |
+
audios.append(audio)
|
| 879 |
+
audio_token_length = audio.size(0) + 4
|
| 880 |
+
# audio_token_length = audio.size(0)
|
| 881 |
+
|
| 882 |
+
new_input_ids += input_ids[st:aud_pos]
|
| 883 |
+
new_targets += targets[st:aud_pos]
|
| 884 |
+
|
| 885 |
+
new_input_ids += [AUD_START_ID]
|
| 886 |
+
new_targets += [IGNORE_TOKEN_ID]
|
| 887 |
+
|
| 888 |
+
audio_indice_b = torch.zeros(
|
| 889 |
+
1, audio_token_length, dtype=torch.int64
|
| 890 |
+
) # This will change in collate_fn
|
| 891 |
+
audio_indice_s = (
|
| 892 |
+
torch.arange(len(new_input_ids), len(new_input_ids) + audio_token_length)
|
| 893 |
+
.unsqueeze(0)
|
| 894 |
+
.repeat(1, 1)
|
| 895 |
+
)
|
| 896 |
+
audio_indice_b_s = torch.stack(
|
| 897 |
+
[audio_indice_b, audio_indice_s], dim=0
|
| 898 |
+
) # 2, num_image, image_length
|
| 899 |
+
audio_indices.append(audio_indice_b_s)
|
| 900 |
+
|
| 901 |
+
new_input_ids += [AUD_CONTEXT_ID] * audio_token_length
|
| 902 |
+
new_targets += [IGNORE_TOKEN_ID] * audio_token_length
|
| 903 |
+
|
| 904 |
+
new_input_ids += [AUD_END_ID]
|
| 905 |
+
new_targets += [IGNORE_TOKEN_ID]
|
| 906 |
+
|
| 907 |
+
st = aud_pos + 1
|
| 908 |
+
|
| 909 |
+
new_input_ids += input_ids[st:]
|
| 910 |
+
new_targets += targets[st:]
|
| 911 |
+
|
| 912 |
+
input_ids = new_input_ids
|
| 913 |
+
targets = new_targets
|
| 914 |
+
|
| 915 |
+
if len(images) > 0:
|
| 916 |
+
images = torch.cat(images, dim=0)
|
| 917 |
+
|
| 918 |
+
if len(image_indices) > 0:
|
| 919 |
+
image_indices = torch.cat(image_indices, dim=1)
|
| 920 |
+
|
| 921 |
+
# ----------------------------------------------------------------
|
| 922 |
+
# 6. Final Masking and Label Updating
|
| 923 |
+
# ----------------------------------------------------------------
|
| 924 |
+
origin_input = input_ids
|
| 925 |
+
labels = targets
|
| 926 |
+
eos_id = 151643
|
| 927 |
+
mask_id = 151666
|
| 928 |
+
input_ids, labels = update_labels_and_inputs(input_ids,labels,eos_id,16)
|
| 929 |
+
|
| 930 |
+
labels_mask = ~(labels == -100)
|
| 931 |
+
bsz, seq_len = labels_mask.shape
|
| 932 |
+
masked_indices, p_mask = forward_process(
|
| 933 |
+
bsz, seq_len, input_ids.device, labels,special_mask_ratio=0.6, special_token_id=eos_id
|
| 934 |
+
)
|
| 935 |
+
final_masked_indices = masked_indices & labels_mask
|
| 936 |
+
final_masked_indices_inv = (~masked_indices) & labels_mask
|
| 937 |
+
|
| 938 |
+
mask_id_tensor = torch.full_like(input_ids, mask_id)
|
| 939 |
+
input_ids = torch.where(final_masked_indices, mask_id_tensor, input_ids)
|
| 940 |
+
|
| 941 |
+
new_labels = labels.clone()
|
| 942 |
+
new_labels[final_masked_indices_inv] = -100
|
| 943 |
+
|
| 944 |
+
input_ids = input_ids.squeeze(0).cpu().tolist()
|
| 945 |
+
new_labels = new_labels.squeeze(0).cpu().tolist()
|
| 946 |
+
attention_mask = [1] * len(input_ids)
|
| 947 |
+
assert len(new_labels) == len(input_ids)
|
| 948 |
+
|
| 949 |
+
return dict(
|
| 950 |
+
input_ids=input_ids,
|
| 951 |
+
labels=new_labels,
|
| 952 |
+
attention_mask=attention_mask,
|
| 953 |
+
images=images,
|
| 954 |
+
image_indices=image_indices,
|
| 955 |
+
audios=audios,
|
| 956 |
+
audio_indices=audio_indices,
|
| 957 |
+
)
|
| 958 |
+
|
| 959 |
+
|
| 960 |
+
def has_image(sample):
|
| 961 |
+
# image
|
| 962 |
+
if (
|
| 963 |
+
"images" in sample
|
| 964 |
+
and isinstance(sample["images"], list)
|
| 965 |
+
and None not in sample["images"]
|
| 966 |
+
and len(sample["images"])
|
| 967 |
+
):
|
| 968 |
+
return True
|
| 969 |
+
return False
|
| 970 |
+
|
| 971 |
+
|
| 972 |
+
def has_audio(sample):
|
| 973 |
+
# audio
|
| 974 |
+
if (
|
| 975 |
+
"audios" in sample
|
| 976 |
+
and isinstance(sample["audios"], list)
|
| 977 |
+
and None not in sample["audios"]
|
| 978 |
+
and len(sample["audios"])
|
| 979 |
+
):
|
| 980 |
+
return True
|
| 981 |
+
return False
|
omni_diffusion/data/processor/__init__.py
ADDED
|
File without changes
|
omni_diffusion/data/processor/audio_processor.py
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import math
|
| 3 |
+
import os
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
import torch
|
| 7 |
+
|
| 8 |
+
import natsort
|
| 9 |
+
from omni_diffusion.tokenizer import get_audio_tokenizer
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class AudioProcessor:
|
| 13 |
+
"""
|
| 14 |
+
Handles the initialization of audio tokenizers and processing of audio files.
|
| 15 |
+
"""
|
| 16 |
+
def __init__(
|
| 17 |
+
self,
|
| 18 |
+
audio_tokenizer_path=None,
|
| 19 |
+
audio_tokenizer_type=None,
|
| 20 |
+
text_audio_interval_ratio=None,
|
| 21 |
+
):
|
| 22 |
+
|
| 23 |
+
self.audio_tokenizer = get_audio_tokenizer(
|
| 24 |
+
audio_tokenizer_path,
|
| 25 |
+
audio_tokenizer_type,
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
self.audio_tokenizer_type = audio_tokenizer_type
|
| 29 |
+
|
| 30 |
+
self.text_audio_interval_ratio = text_audio_interval_ratio
|
| 31 |
+
|
| 32 |
+
# self.load_model()
|
| 33 |
+
|
| 34 |
+
def load_model(self):
|
| 35 |
+
"""Loads the underlying tokenizer model weights."""
|
| 36 |
+
if self.audio_tokenizer is not None:
|
| 37 |
+
self.audio_tokenizer.load_model()
|
| 38 |
+
|
| 39 |
+
def process_audios(self, audio_path, is_discrete=False, is_contiguous=False, **kwargs):
|
| 40 |
+
|
| 41 |
+
assert not (is_discrete and is_contiguous)
|
| 42 |
+
assert is_discrete or is_contiguous
|
| 43 |
+
|
| 44 |
+
if is_discrete:
|
| 45 |
+
audio_tokenizer_type = self.audio_tokenizer_type.split("_")[-1]
|
| 46 |
+
cache_path = os.path.splitext(audio_path)[0] + f"_{audio_tokenizer_type}.json"
|
| 47 |
+
try:
|
| 48 |
+
if os.path.isfile(cache_path):
|
| 49 |
+
with open(cache_path, "r") as f:
|
| 50 |
+
audio_data = json.load(f)
|
| 51 |
+
return audio_data
|
| 52 |
+
except Exception as e:
|
| 53 |
+
pass
|
| 54 |
+
|
| 55 |
+
audio_data = self.audio_tokenizer.encode(
|
| 56 |
+
audio_path, is_discrete=is_discrete, is_contiguous=is_contiguous, **kwargs
|
| 57 |
+
)
|
| 58 |
+
# print(f"{len(audio_data)=}")
|
| 59 |
+
|
| 60 |
+
if is_discrete:
|
| 61 |
+
try:
|
| 62 |
+
if isinstance(audio_data, list):
|
| 63 |
+
with open(cache_path, "w") as f:
|
| 64 |
+
json.dump(audio_data, f)
|
| 65 |
+
except Exception as e:
|
| 66 |
+
pass
|
| 67 |
+
|
| 68 |
+
return audio_data
|
| 69 |
+
|
| 70 |
+
@property
|
| 71 |
+
def is_discrete(self):
|
| 72 |
+
return self.audio_tokenizer.is_discrete
|
| 73 |
+
|
| 74 |
+
@property
|
| 75 |
+
def is_contiguous(self):
|
| 76 |
+
return self.audio_tokenizer.is_contiguous
|
| 77 |
+
|
| 78 |
+
def apply_to_role(self, role, **kwargs):
|
| 79 |
+
return self.audio_tokenizer.apply_to_role(role, **kwargs)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def add_audio_input_contiguous(input_ids, audio_paths, tokenizer, audio_tokenizer):
|
| 83 |
+
"""
|
| 84 |
+
Injects contiguous audio embeddings into a text sequence.
|
| 85 |
+
Replaces specific audio tag tokens in the input_ids with a sequence of
|
| 86 |
+
[AUD_START] + [AUD_CONTEXT]... + [AUD_END].
|
| 87 |
+
"""
|
| 88 |
+
from ...constants import (
|
| 89 |
+
AUD_START_TOKEN,
|
| 90 |
+
AUD_END_TOKEN,
|
| 91 |
+
AUD_TAG_TOKEN,
|
| 92 |
+
AUD_CONTEXT_TOKEN,
|
| 93 |
+
)
|
| 94 |
+
|
| 95 |
+
AUD_CONTEXT_ID = tokenizer(AUD_CONTEXT_TOKEN, add_special_tokens=False).input_ids
|
| 96 |
+
AUD_TAG_ID = tokenizer(AUD_TAG_TOKEN, add_special_tokens=False).input_ids
|
| 97 |
+
AUD_START_ID = tokenizer(AUD_START_TOKEN, add_special_tokens=False).input_ids
|
| 98 |
+
AUD_END_ID = tokenizer(AUD_END_TOKEN, add_special_tokens=False).input_ids
|
| 99 |
+
|
| 100 |
+
AUD_CONTEXT_ID = AUD_CONTEXT_ID[0]
|
| 101 |
+
AUD_TAG_ID = AUD_TAG_ID[0]
|
| 102 |
+
AUD_START_ID = AUD_START_ID[0]
|
| 103 |
+
AUD_END_ID = AUD_END_ID[0]
|
| 104 |
+
|
| 105 |
+
aud_positions = [i for i, x in enumerate(input_ids) if x == AUD_TAG_ID]
|
| 106 |
+
|
| 107 |
+
audios = []
|
| 108 |
+
audio_indices = []
|
| 109 |
+
new_input_ids = []
|
| 110 |
+
st = 0
|
| 111 |
+
for aud_idx, aud_pos in enumerate(aud_positions):
|
| 112 |
+
audio = audio_tokenizer.encode(audio_paths[aud_idx], is_contiguous=True)
|
| 113 |
+
audios.append(audio)
|
| 114 |
+
audio_token_length = audio.size(0) + 4
|
| 115 |
+
|
| 116 |
+
new_input_ids += input_ids[st:aud_pos]
|
| 117 |
+
|
| 118 |
+
new_input_ids += [AUD_START_ID]
|
| 119 |
+
|
| 120 |
+
audio_indice_b = torch.zeros(
|
| 121 |
+
1, audio_token_length, dtype=torch.int64
|
| 122 |
+
) # This will change in collate_fn
|
| 123 |
+
audio_indice_s = (
|
| 124 |
+
torch.arange(len(new_input_ids), len(new_input_ids) + audio_token_length)
|
| 125 |
+
.unsqueeze(0)
|
| 126 |
+
.repeat(1, 1)
|
| 127 |
+
)
|
| 128 |
+
audio_indice_b_s = torch.stack(
|
| 129 |
+
[audio_indice_b, audio_indice_s], dim=0
|
| 130 |
+
) # 2, num_image, image_length
|
| 131 |
+
audio_indices.append(audio_indice_b_s)
|
| 132 |
+
|
| 133 |
+
new_input_ids += [AUD_CONTEXT_ID] * audio_token_length
|
| 134 |
+
|
| 135 |
+
new_input_ids += [AUD_END_ID]
|
| 136 |
+
|
| 137 |
+
st = aud_pos + 1
|
| 138 |
+
|
| 139 |
+
new_input_ids += input_ids[st:]
|
| 140 |
+
inputs_ids = new_input_ids
|
| 141 |
+
|
| 142 |
+
return inputs_ids, audios, audio_indices
|
omni_diffusion/data/processor/image_processor.py
ADDED
|
@@ -0,0 +1,358 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
import os
|
| 3 |
+
|
| 4 |
+
import numpy as np
|
| 5 |
+
import torch
|
| 6 |
+
from PIL import Image
|
| 7 |
+
|
| 8 |
+
import decord
|
| 9 |
+
import natsort
|
| 10 |
+
from omni_diffusion.constants import (
|
| 11 |
+
IMAGENET_DEFAULT_MEAN,
|
| 12 |
+
IMAGENET_DEFAULT_STD,
|
| 13 |
+
IMAGENET_STANDARD_MEAN,
|
| 14 |
+
IMAGENET_STANDARD_STD,
|
| 15 |
+
OPENAI_CLIP_MEAN,
|
| 16 |
+
OPENAI_CLIP_STD,
|
| 17 |
+
)
|
| 18 |
+
from torchvision import transforms
|
| 19 |
+
from omni_diffusion.tokenizer_magvitv2 import MagVITV2Tokenizer
|
| 20 |
+
|
| 21 |
+
import logging
|
| 22 |
+
logger = logging.getLogger(__name__)
|
| 23 |
+
logger.setLevel(logging.INFO)
|
| 24 |
+
|
| 25 |
+
def image_transform(image, resolution=256):
|
| 26 |
+
image = transforms.Resize(resolution, interpolation=transforms.InterpolationMode.BICUBIC)(image)
|
| 27 |
+
image = transforms.CenterCrop((resolution, resolution))(image)
|
| 28 |
+
image = transforms.ToTensor()(image)
|
| 29 |
+
image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image)
|
| 30 |
+
return image
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class ImageProcessor:
|
| 34 |
+
def __init__(
|
| 35 |
+
self,
|
| 36 |
+
model_path,
|
| 37 |
+
process_type,
|
| 38 |
+
image_size=256,
|
| 39 |
+
normalize_type="imagenet",
|
| 40 |
+
min_patch_grid=1,
|
| 41 |
+
max_patch_grid=6,
|
| 42 |
+
):
|
| 43 |
+
self.process_type = process_type
|
| 44 |
+
self.image_size = image_size
|
| 45 |
+
|
| 46 |
+
if normalize_type == "imagenet":
|
| 47 |
+
MEAN, STD = IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD
|
| 48 |
+
elif normalize_type == "clip":
|
| 49 |
+
MEAN, STD = OPENAI_CLIP_MEAN, OPENAI_CLIP_STD
|
| 50 |
+
elif normalize_type == "siglip":
|
| 51 |
+
MEAN, STD = IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD
|
| 52 |
+
else:
|
| 53 |
+
raise NotImplementedError(normalize_type)
|
| 54 |
+
self.mean = MEAN
|
| 55 |
+
self.std = STD
|
| 56 |
+
|
| 57 |
+
self.patch_size = image_size
|
| 58 |
+
self.min_patch_grid = min_patch_grid
|
| 59 |
+
self.max_patch_grid = max_patch_grid
|
| 60 |
+
|
| 61 |
+
if self.process_type == "anyres":
|
| 62 |
+
self.grid_pinpoints = [
|
| 63 |
+
(i, j)
|
| 64 |
+
for i in range(min_patch_grid, max_patch_grid + 1)
|
| 65 |
+
for j in range(min_patch_grid, max_patch_grid + 1)
|
| 66 |
+
]
|
| 67 |
+
self.possible_resolutions = [
|
| 68 |
+
[dim * self.patch_size for dim in pair] for pair in self.grid_pinpoints
|
| 69 |
+
]
|
| 70 |
+
print(f"grid_pinpoints {self.grid_pinpoints}")
|
| 71 |
+
print(f"possible_resolutions {self.possible_resolutions}")
|
| 72 |
+
|
| 73 |
+
if self.process_type == "dynamic":
|
| 74 |
+
max_num = self.max_patch_grid
|
| 75 |
+
min_num = self.min_patch_grid
|
| 76 |
+
# calculate the existing image aspect ratio
|
| 77 |
+
target_ratios = set(
|
| 78 |
+
(i, j)
|
| 79 |
+
for n in range(min_num, max_num + 1)
|
| 80 |
+
for i in range(1, n + 1)
|
| 81 |
+
for j in range(1, n + 1)
|
| 82 |
+
if i * j <= max_num and i * j >= min_num
|
| 83 |
+
)
|
| 84 |
+
self.target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
|
| 85 |
+
self.possible_resolutions = [
|
| 86 |
+
[dim * self.patch_size for dim in pair] for pair in self.target_ratios
|
| 87 |
+
]
|
| 88 |
+
print(f"target_ratios {self.target_ratios}")
|
| 89 |
+
print(f"possible_resolutions {self.possible_resolutions}")
|
| 90 |
+
|
| 91 |
+
# self.image_tokenizer = MAGVITv2()
|
| 92 |
+
self.image_tokenizer = MagVITV2Tokenizer(model_path=model_path)
|
| 93 |
+
|
| 94 |
+
def load_model(self):
|
| 95 |
+
if self.image_tokenizer is not None:
|
| 96 |
+
self.image_tokenizer.load_model()
|
| 97 |
+
|
| 98 |
+
def process_images(self, img_or_path_list, image_resolution):
|
| 99 |
+
|
| 100 |
+
if isinstance(img_or_path_list[0], str):
|
| 101 |
+
images = [Image.open(x).convert("RGB") for x in img_or_path_list]
|
| 102 |
+
elif isinstance(img_or_path_list[0], Image.Image):
|
| 103 |
+
images = [x.convert("RGB") for x in img_or_path_list]
|
| 104 |
+
else:
|
| 105 |
+
images = img_or_path_list
|
| 106 |
+
|
| 107 |
+
# image_tensor = torch.ones([len(images), 3, self.image_size, self.image_size])
|
| 108 |
+
image_tensor = torch.ones([len(images), 3, image_resolution, image_resolution])
|
| 109 |
+
|
| 110 |
+
for i, image in enumerate(images):
|
| 111 |
+
image = image_transform(image, resolution=image_resolution)
|
| 112 |
+
|
| 113 |
+
image_tensor[i] = image
|
| 114 |
+
|
| 115 |
+
return image_tensor
|
| 116 |
+
|
| 117 |
+
def process_images_with_subpatch(self, img_or_path, image_resolution):
|
| 118 |
+
return self.process_images([img_or_path], image_resolution)
|
| 119 |
+
|
| 120 |
+
def get_image_token(self, image):
|
| 121 |
+
return self.image_tokenizer.encode(image)
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def select_best_resolution(original_size, possible_resolutions):
|
| 125 |
+
"""
|
| 126 |
+
Selects the best resolution from a list of possible resolutions based on the original size.
|
| 127 |
+
|
| 128 |
+
Args:
|
| 129 |
+
original_size (tuple): The original size of the image in the format (width, height).
|
| 130 |
+
possible_resolutions (list): A list of possible resolutions in the format [(width1, height1),
|
| 131 |
+
(width2, height2), ...].
|
| 132 |
+
|
| 133 |
+
Returns:
|
| 134 |
+
tuple: The best fit resolution in the format (width, height).
|
| 135 |
+
"""
|
| 136 |
+
original_width, original_height = original_size
|
| 137 |
+
best_fit = None
|
| 138 |
+
max_effective_resolution = 0
|
| 139 |
+
min_wasted_resolution = float("inf")
|
| 140 |
+
|
| 141 |
+
for width, height in possible_resolutions:
|
| 142 |
+
# Calculate the downscaled size to keep the aspect ratio
|
| 143 |
+
scale = min(width / original_width, height / original_height)
|
| 144 |
+
downscaled_width, downscaled_height = int(original_width * scale), int(
|
| 145 |
+
original_height * scale
|
| 146 |
+
)
|
| 147 |
+
|
| 148 |
+
# Calculate effective and wasted resolutions
|
| 149 |
+
effective_resolution = min(
|
| 150 |
+
downscaled_width * downscaled_height, original_width * original_height
|
| 151 |
+
)
|
| 152 |
+
wasted_resolution = (width * height) - effective_resolution
|
| 153 |
+
|
| 154 |
+
if effective_resolution > max_effective_resolution or (
|
| 155 |
+
effective_resolution == max_effective_resolution
|
| 156 |
+
and wasted_resolution < min_wasted_resolution
|
| 157 |
+
):
|
| 158 |
+
max_effective_resolution = effective_resolution
|
| 159 |
+
min_wasted_resolution = wasted_resolution
|
| 160 |
+
best_fit = (width, height)
|
| 161 |
+
|
| 162 |
+
return best_fit
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def resize_and_pad_image(image, target_resolution):
|
| 166 |
+
"""
|
| 167 |
+
Resize and pad an image to a target resolution while maintaining aspect ratio.
|
| 168 |
+
|
| 169 |
+
Args:
|
| 170 |
+
image (PIL.Image.Image): The input image.
|
| 171 |
+
target_resolution (tuple): The target resolution (width, height) of the image.
|
| 172 |
+
|
| 173 |
+
Returns:
|
| 174 |
+
PIL.Image.Image: The resized and padded image.
|
| 175 |
+
"""
|
| 176 |
+
original_width, original_height = image.size
|
| 177 |
+
target_width, target_height = target_resolution
|
| 178 |
+
|
| 179 |
+
# Determine which dimension (width or height) to fill
|
| 180 |
+
scale_w = target_width / original_width
|
| 181 |
+
scale_h = target_height / original_height
|
| 182 |
+
|
| 183 |
+
if scale_w < scale_h:
|
| 184 |
+
# Width will be filled completely
|
| 185 |
+
new_width = target_width
|
| 186 |
+
new_height = min(math.ceil(original_height * scale_w), target_height)
|
| 187 |
+
else:
|
| 188 |
+
# Height will be filled completely
|
| 189 |
+
new_height = target_height
|
| 190 |
+
new_width = min(math.ceil(original_width * scale_h), target_width)
|
| 191 |
+
|
| 192 |
+
# Resize the image
|
| 193 |
+
resized_image = image.resize((new_width, new_height))
|
| 194 |
+
|
| 195 |
+
# Create a new image with the target size and paste the resized image onto it
|
| 196 |
+
new_image = Image.new("RGB", (target_width, target_height), (0, 0, 0))
|
| 197 |
+
paste_x = (target_width - new_width) // 2
|
| 198 |
+
paste_y = (target_height - new_height) // 2
|
| 199 |
+
new_image.paste(resized_image, (paste_x, paste_y))
|
| 200 |
+
|
| 201 |
+
return new_image
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def add_image_input_contiguous(input_ids, image_paths, tokenizer):
|
| 205 |
+
|
| 206 |
+
image_processor = ImageProcessor(
|
| 207 |
+
process_type="dynamic",
|
| 208 |
+
image_size=448,
|
| 209 |
+
normalize_type="imagenet",
|
| 210 |
+
min_patch_grid=1,
|
| 211 |
+
max_patch_grid=12,
|
| 212 |
+
)
|
| 213 |
+
|
| 214 |
+
image_token_length = 256
|
| 215 |
+
max_num_frame = 4096
|
| 216 |
+
max_fps = 1
|
| 217 |
+
|
| 218 |
+
from ...constants import (
|
| 219 |
+
IMG_START_TOKEN,
|
| 220 |
+
IMG_END_TOKEN,
|
| 221 |
+
IMG_TAG_TOKEN,
|
| 222 |
+
IMG_CONTEXT_TOKEN,
|
| 223 |
+
VID_START_TOKEN,
|
| 224 |
+
VID_END_TOKEN,
|
| 225 |
+
VID_TAG_TOKEN,
|
| 226 |
+
VID_CONTEXT_TOKEN,
|
| 227 |
+
PATCH_START_TOKEN,
|
| 228 |
+
PATCH_END_TOKEN,
|
| 229 |
+
PATCH_CONTEXT_TOKEN,
|
| 230 |
+
)
|
| 231 |
+
|
| 232 |
+
IMG_CONTEXT_ID = tokenizer(IMG_CONTEXT_TOKEN, add_special_tokens=False).input_ids
|
| 233 |
+
IMG_START_ID = tokenizer(IMG_START_TOKEN, add_special_tokens=False).input_ids
|
| 234 |
+
IMG_END_ID = tokenizer(IMG_END_TOKEN, add_special_tokens=False).input_ids
|
| 235 |
+
|
| 236 |
+
VID_CONTEXT_ID = tokenizer(VID_CONTEXT_TOKEN, add_special_tokens=False).input_ids
|
| 237 |
+
VID_START_ID = tokenizer(VID_START_TOKEN, add_special_tokens=False).input_ids
|
| 238 |
+
VID_END_ID = tokenizer(VID_END_TOKEN, add_special_tokens=False).input_ids
|
| 239 |
+
|
| 240 |
+
PATCH_CONTEXT_ID = tokenizer(PATCH_CONTEXT_TOKEN, add_special_tokens=False).input_ids
|
| 241 |
+
PATCH_START_ID = tokenizer(PATCH_START_TOKEN, add_special_tokens=False).input_ids
|
| 242 |
+
PATCH_END_ID = tokenizer(PATCH_END_TOKEN, add_special_tokens=False).input_ids
|
| 243 |
+
|
| 244 |
+
IMG_TAG_ID = tokenizer(IMG_TAG_TOKEN, add_special_tokens=False).input_ids
|
| 245 |
+
VID_TAG_ID = tokenizer(VID_TAG_TOKEN, add_special_tokens=False).input_ids
|
| 246 |
+
|
| 247 |
+
assert len(IMG_CONTEXT_ID) == 1
|
| 248 |
+
assert len(IMG_START_ID) == 1
|
| 249 |
+
assert len(IMG_END_ID) == 1
|
| 250 |
+
|
| 251 |
+
assert len(VID_CONTEXT_ID) == 1
|
| 252 |
+
assert len(VID_START_ID) == 1
|
| 253 |
+
assert len(VID_END_ID) == 1
|
| 254 |
+
|
| 255 |
+
assert len(PATCH_CONTEXT_ID) == 1
|
| 256 |
+
assert len(PATCH_START_ID) == 1
|
| 257 |
+
assert len(PATCH_END_ID) == 1
|
| 258 |
+
|
| 259 |
+
IMG_CONTEXT_ID = IMG_CONTEXT_ID[0]
|
| 260 |
+
IMG_START_ID = IMG_START_ID[0]
|
| 261 |
+
IMG_END_ID = IMG_END_ID[0]
|
| 262 |
+
|
| 263 |
+
VID_CONTEXT_ID = VID_CONTEXT_ID[0]
|
| 264 |
+
VID_START_ID = VID_START_ID[0]
|
| 265 |
+
VID_END_ID = VID_END_ID[0]
|
| 266 |
+
|
| 267 |
+
PATCH_CONTEXT_ID = PATCH_CONTEXT_ID[0]
|
| 268 |
+
PATCH_START_ID = PATCH_START_ID[0]
|
| 269 |
+
PATCH_END_ID = PATCH_END_ID[0]
|
| 270 |
+
|
| 271 |
+
IMG_TAG_ID = IMG_TAG_ID[0]
|
| 272 |
+
VID_TAG_ID = VID_TAG_ID[0]
|
| 273 |
+
|
| 274 |
+
nl_tokens = tokenizer("\n", add_special_tokens=False).input_ids
|
| 275 |
+
|
| 276 |
+
img_positions = [i for i, x in enumerate(input_ids) if x == IMG_TAG_ID]
|
| 277 |
+
|
| 278 |
+
images = []
|
| 279 |
+
image_indices = []
|
| 280 |
+
new_input_ids = []
|
| 281 |
+
st = 0
|
| 282 |
+
for img_idx, img_pos in enumerate(img_positions):
|
| 283 |
+
image_patches, (
|
| 284 |
+
best_width,
|
| 285 |
+
best_height,
|
| 286 |
+
) = image_processor.process_images_with_subpatch(image_paths[img_idx])
|
| 287 |
+
images.append(image_patches)
|
| 288 |
+
print(f"add_image_input_contiguous best_width {best_width} best_height {best_height}")
|
| 289 |
+
|
| 290 |
+
new_input_ids += input_ids[st:img_pos]
|
| 291 |
+
|
| 292 |
+
new_input_ids += [IMG_START_ID]
|
| 293 |
+
|
| 294 |
+
image_indice_b = torch.zeros(
|
| 295 |
+
1, image_token_length, dtype=torch.int64
|
| 296 |
+
) # This will change in collate_fn
|
| 297 |
+
image_indice_s = (
|
| 298 |
+
torch.arange(len(new_input_ids), len(new_input_ids) + image_token_length)
|
| 299 |
+
.unsqueeze(0)
|
| 300 |
+
.repeat(1, 1)
|
| 301 |
+
)
|
| 302 |
+
image_indice_b_s = torch.stack(
|
| 303 |
+
[image_indice_b, image_indice_s], dim=0
|
| 304 |
+
) # 2, num_image, image_length
|
| 305 |
+
image_indices.append(image_indice_b_s)
|
| 306 |
+
|
| 307 |
+
new_input_ids += [IMG_CONTEXT_ID] * image_token_length
|
| 308 |
+
|
| 309 |
+
new_input_ids += [IMG_END_ID]
|
| 310 |
+
|
| 311 |
+
if len(image_patches) > 1:
|
| 312 |
+
for i in range(0, best_height, image_processor.patch_size):
|
| 313 |
+
new_input_ids += nl_tokens
|
| 314 |
+
|
| 315 |
+
for j in range(0, best_width, image_processor.patch_size):
|
| 316 |
+
new_input_ids += [PATCH_START_ID]
|
| 317 |
+
|
| 318 |
+
image_indice_b = torch.zeros(
|
| 319 |
+
1, image_token_length, dtype=torch.int64
|
| 320 |
+
) # This will change in collate_fn
|
| 321 |
+
image_indice_s = (
|
| 322 |
+
torch.arange(
|
| 323 |
+
len(new_input_ids), len(new_input_ids) + image_token_length
|
| 324 |
+
)
|
| 325 |
+
.unsqueeze(0)
|
| 326 |
+
.repeat(1, 1)
|
| 327 |
+
)
|
| 328 |
+
image_indice_b_s = torch.stack(
|
| 329 |
+
[image_indice_b, image_indice_s], dim=0
|
| 330 |
+
) # 2, num_image, image_length
|
| 331 |
+
image_indices.append(image_indice_b_s)
|
| 332 |
+
|
| 333 |
+
new_input_ids += [PATCH_CONTEXT_ID] * image_token_length
|
| 334 |
+
|
| 335 |
+
new_input_ids += [PATCH_END_ID]
|
| 336 |
+
# print(f"get_external_dict i {i} j {j} new_input_ids {len(new_input_ids)}")
|
| 337 |
+
|
| 338 |
+
st = img_pos + 1
|
| 339 |
+
|
| 340 |
+
new_input_ids += input_ids[st:]
|
| 341 |
+
|
| 342 |
+
inputs_ids = new_input_ids
|
| 343 |
+
|
| 344 |
+
images = torch.cat(images, dim=0)
|
| 345 |
+
image_indices = torch.cat(image_indices, dim=1)
|
| 346 |
+
|
| 347 |
+
image_indices = image_indices.contiguous().to(torch.cuda.current_device())
|
| 348 |
+
if True:
|
| 349 |
+
images = (
|
| 350 |
+
torch.tensor(images, dtype=torch.bfloat16).contiguous().to(torch.cuda.current_device())
|
| 351 |
+
)
|
| 352 |
+
|
| 353 |
+
else:
|
| 354 |
+
images = (
|
| 355 |
+
torch.tensor(images, dtype=torch.float16).contiguous().to(torch.cuda.current_device())
|
| 356 |
+
)
|
| 357 |
+
|
| 358 |
+
return inputs_ids, images, image_indices
|
omni_diffusion/data/utils.py
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
import copy
|
| 3 |
+
import hashlib
|
| 4 |
+
import io
|
| 5 |
+
import itertools
|
| 6 |
+
import json
|
| 7 |
+
import logging
|
| 8 |
+
import os
|
| 9 |
+
import re
|
| 10 |
+
from typing import Dict, Sequence, Tuple
|
| 11 |
+
|
| 12 |
+
import numpy as np
|
| 13 |
+
import xlsxwriter
|
| 14 |
+
from datasets import concatenate_datasets, load_dataset
|
| 15 |
+
from PIL import Image
|
| 16 |
+
from tabulate import tabulate
|
| 17 |
+
from termcolor import colored
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def buffer_image(image: Image, format: str = "JPEG"):
|
| 21 |
+
# Store image in buffer, so we don't have to write it to disk.
|
| 22 |
+
buffer = io.BytesIO()
|
| 23 |
+
image.save(buffer, format=format)
|
| 24 |
+
return buffer, image
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def resize(img_or_path: str, size: Tuple[int, int], format="JPEG"):
|
| 28 |
+
"""
|
| 29 |
+
Opens an image (from a file path or existing object), creates a thumbnail
|
| 30 |
+
preserving aspect ratio using LANCZOS filter, and returns the image buffer.
|
| 31 |
+
"""
|
| 32 |
+
if isinstance(img_or_path, str):
|
| 33 |
+
image = Image.open(img_or_path)
|
| 34 |
+
else:
|
| 35 |
+
image = img_or_path
|
| 36 |
+
# image = image.resize(size)
|
| 37 |
+
image.thumbnail(size, Image.LANCZOS)
|
| 38 |
+
image = image.convert("RGB")
|
| 39 |
+
|
| 40 |
+
return buffer_image(image, format)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def calculate_scale(file_path, bound_size):
|
| 44 |
+
# check the image size without loading it into memory
|
| 45 |
+
im = Image.open(file_path)
|
| 46 |
+
original_width, original_height = im.size
|
| 47 |
+
|
| 48 |
+
# calculate the resize factor, keeping original aspect and staying within boundary
|
| 49 |
+
bound_width, bound_height = bound_size
|
| 50 |
+
ratios = (float(bound_width) / original_width, float(bound_height) / original_height)
|
| 51 |
+
return min(ratios)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def draw_data(all_datasets, output_path, tokenizer=None, image_processor=None):
|
| 55 |
+
"""
|
| 56 |
+
Visualizes dataset contents by exporting them to an Excel file (.xlsx).
|
| 57 |
+
It creates a summary sheet and individual sheets for each dataset,
|
| 58 |
+
embedding text, images, and video frames into the cells.
|
| 59 |
+
"""
|
| 60 |
+
if hasattr(tokenizer, "image_start_tag"):
|
| 61 |
+
image_start_tag = tokenizer.image_start_tag
|
| 62 |
+
image_end_tag = tokenizer.image_end_tag
|
| 63 |
+
else:
|
| 64 |
+
from ..constants import IMG_START_TOKEN, IMG_END_TOKEN
|
| 65 |
+
|
| 66 |
+
image_start_tag = IMG_START_TOKEN
|
| 67 |
+
image_end_tag = IMG_END_TOKEN
|
| 68 |
+
|
| 69 |
+
# Initialize Excel Workbook
|
| 70 |
+
workbook = xlsxwriter.Workbook(output_path)
|
| 71 |
+
cell_format = workbook.add_format({"text_wrap": True, "font_size": 12})
|
| 72 |
+
|
| 73 |
+
worksheet = workbook.add_worksheet("ALL")
|
| 74 |
+
worksheet.set_column(1, 2, 20, cell_format)
|
| 75 |
+
worksheet.set_column(3, 3, 240, cell_format)
|
| 76 |
+
worksheet.write(0, 1, "total_num")
|
| 77 |
+
worksheet.write(0, 2, "used_num")
|
| 78 |
+
worksheet.write(0, 3, "name")
|
| 79 |
+
row = 1
|
| 80 |
+
|
| 81 |
+
# Write statistics for each dataset
|
| 82 |
+
for this_name, this_dataset in all_datasets.items():
|
| 83 |
+
total_num = this_dataset["total_num"]
|
| 84 |
+
used_num = this_dataset["used_num"]
|
| 85 |
+
worksheet.write(row, 1, total_num)
|
| 86 |
+
worksheet.write(row, 2, used_num)
|
| 87 |
+
worksheet.write(row, 3, this_name)
|
| 88 |
+
row += 1
|
| 89 |
+
worksheet.write(
|
| 90 |
+
row, 2, sum([this_dataset["used_num"] for this_dataset in all_datasets.values()])
|
| 91 |
+
)
|
| 92 |
+
|
| 93 |
+
all_base_name = ["all"]
|
| 94 |
+
for this_name, this_dataset in all_datasets.items():
|
| 95 |
+
base_name = os.path.basename(this_name)
|
| 96 |
+
base_name = os.path.splitext(base_name)[0]
|
| 97 |
+
|
| 98 |
+
base_name = base_name[:24]
|
| 99 |
+
all_base_name.append(base_name)
|
| 100 |
+
|
| 101 |
+
if all_base_name.count(base_name) > 1:
|
| 102 |
+
base_name = base_name + "_" + str(all_base_name.count(base_name))
|
| 103 |
+
|
| 104 |
+
worksheet = workbook.add_worksheet(base_name)
|
| 105 |
+
worksheet.set_column(1, 2, 120, cell_format)
|
| 106 |
+
worksheet.write(0, 1, "user")
|
| 107 |
+
worksheet.write(0, 2, "assistant")
|
| 108 |
+
row = 1
|
| 109 |
+
|
| 110 |
+
data = this_dataset["data"]
|
| 111 |
+
for this_data in data:
|
| 112 |
+
# print(this_data)
|
| 113 |
+
if isinstance(this_data, Dict):
|
| 114 |
+
# print(this_data.keys())
|
| 115 |
+
messages = this_data["messages"]
|
| 116 |
+
if "images" in this_data:
|
| 117 |
+
images = this_data["images"]
|
| 118 |
+
if "videos" in this_data:
|
| 119 |
+
videos = this_data["videos"]
|
| 120 |
+
else:
|
| 121 |
+
messages = this_data
|
| 122 |
+
|
| 123 |
+
image_count = 0
|
| 124 |
+
video_count = 0
|
| 125 |
+
for message in messages:
|
| 126 |
+
content = message["content"]
|
| 127 |
+
role = message["role"]
|
| 128 |
+
if role == "user" or role == "human":
|
| 129 |
+
col = 1
|
| 130 |
+
else:
|
| 131 |
+
col = 2
|
| 132 |
+
worksheet.write(row, col, content)
|
| 133 |
+
row += 1
|
| 134 |
+
|
| 135 |
+
bos_pos = [m.start() for m in re.finditer(image_start_tag, content)]
|
| 136 |
+
eos_pos = [m.start() for m in re.finditer(image_end_tag, content)]
|
| 137 |
+
# print(bos_pos, eos_pos)
|
| 138 |
+
for a, b in zip(bos_pos, eos_pos):
|
| 139 |
+
# print(content[a+len(image_start_tag:b])
|
| 140 |
+
img_path = content[a + len(image_start_tag) : b]
|
| 141 |
+
# print(img_path)
|
| 142 |
+
worksheet.set_row(row, 200)
|
| 143 |
+
|
| 144 |
+
try:
|
| 145 |
+
image_buffer, image = resize(img_path, (512, 512), format="JPEG")
|
| 146 |
+
except:
|
| 147 |
+
continue
|
| 148 |
+
|
| 149 |
+
scale = min(256 / image.width, 256 / image.height)
|
| 150 |
+
data = {"x_scale": scale, "y_scale": scale, "object_position": 1}
|
| 151 |
+
|
| 152 |
+
worksheet.insert_image(row, col, img_path, {"image_data": image_buffer, **data})
|
| 153 |
+
|
| 154 |
+
row += 1
|
| 155 |
+
|
| 156 |
+
for _ in range(content.count("<image>") + content.count("<|image|>")):
|
| 157 |
+
if images is None:
|
| 158 |
+
continue
|
| 159 |
+
img_path = images[image_count]
|
| 160 |
+
# print(img_path)
|
| 161 |
+
worksheet.set_row(row, 200)
|
| 162 |
+
|
| 163 |
+
try:
|
| 164 |
+
image_buffer, image = resize(img_path, (512, 512), format="JPEG")
|
| 165 |
+
except:
|
| 166 |
+
continue
|
| 167 |
+
|
| 168 |
+
scale = min(256 / image.width, 256 / image.height)
|
| 169 |
+
data = {"x_scale": scale, "y_scale": scale, "object_position": 1}
|
| 170 |
+
|
| 171 |
+
worksheet.insert_image(row, col, img_path, {"image_data": image_buffer, **data})
|
| 172 |
+
|
| 173 |
+
row += 1
|
| 174 |
+
image_count += 1
|
| 175 |
+
|
| 176 |
+
for _ in range(content.count("<video>") + content.count("<|video|>")):
|
| 177 |
+
if videos is None:
|
| 178 |
+
continue
|
| 179 |
+
vid_path = videos[video_count]
|
| 180 |
+
try:
|
| 181 |
+
_, video_frames = image_processor.process_video(vid_path, max_num_frame=4)
|
| 182 |
+
# print(vid_path)
|
| 183 |
+
except:
|
| 184 |
+
continue
|
| 185 |
+
|
| 186 |
+
for video_frame in video_frames:
|
| 187 |
+
worksheet.set_row(row, 200)
|
| 188 |
+
try:
|
| 189 |
+
image_buffer, image = resize(video_frame, (512, 512), format="JPEG")
|
| 190 |
+
except:
|
| 191 |
+
continue
|
| 192 |
+
|
| 193 |
+
scale = min(256 / image.width, 256 / image.height)
|
| 194 |
+
data = {"x_scale": scale, "y_scale": scale, "object_position": 1}
|
| 195 |
+
|
| 196 |
+
if isinstance(video_frame, str):
|
| 197 |
+
video_path = video_frame
|
| 198 |
+
else:
|
| 199 |
+
video_file = hashlib.md5(video_frame.tobytes()).hexdigest() + ".png"
|
| 200 |
+
video_path = os.path.join("/tmp/", video_file)
|
| 201 |
+
video_frame.save(video_path)
|
| 202 |
+
|
| 203 |
+
worksheet.insert_image(
|
| 204 |
+
row, col, video_path, {"image_data": image_buffer, **data}
|
| 205 |
+
)
|
| 206 |
+
|
| 207 |
+
row += 1
|
| 208 |
+
video_count += 1
|
| 209 |
+
|
| 210 |
+
row += 8
|
| 211 |
+
|
| 212 |
+
workbook.close()
|
omni_diffusion/models/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# models.__init__
|
| 2 |
+
|
| 3 |
+
from .dream import DreamConfig, DreamModel, DreamTokenizer
|
omni_diffusion/models/dream/__init__.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# models.dream.__init__
|
| 2 |
+
|
| 3 |
+
from .modeling_dream import DreamModel
|
| 4 |
+
from .configuration_dream import DreamConfig
|
| 5 |
+
from .tokenization_dream import DreamTokenizer
|
| 6 |
+
from transformers import AutoConfig, AutoTokenizer, AutoModel, AutoModelForCausalLM
|
| 7 |
+
|
| 8 |
+
AutoConfig.register("Dream", DreamConfig)
|
| 9 |
+
AutoModelForCausalLM.register(DreamConfig, DreamModel)
|
| 10 |
+
|
| 11 |
+
DreamConfig.register_for_auto_class()
|
| 12 |
+
DreamModel.register_for_auto_class("AutoModelForCausalLM")
|
omni_diffusion/models/dream/config.yaml
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
encoder: SenseVoiceEncoderSmall
|
| 2 |
+
encoder_conf:
|
| 3 |
+
output_size: 512
|
| 4 |
+
attention_heads: 4
|
| 5 |
+
linear_units: 2048
|
| 6 |
+
num_blocks: 50
|
| 7 |
+
tp_blocks: 20
|
| 8 |
+
dropout_rate: 0.1
|
| 9 |
+
positional_dropout_rate: 0.1
|
| 10 |
+
attention_dropout_rate: 0.1
|
| 11 |
+
input_layer: pe
|
| 12 |
+
pos_enc_class: SinusoidalPositionEncoder
|
| 13 |
+
normalize_before: true
|
| 14 |
+
kernel_size: 11
|
| 15 |
+
sanm_shfit: 0
|
| 16 |
+
selfattention_layer_type: sanm
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
model: SenseVoiceSmall
|
| 20 |
+
model_conf:
|
| 21 |
+
length_normalized_loss: true
|
| 22 |
+
sos: 1
|
| 23 |
+
eos: 2
|
| 24 |
+
ignore_id: -1
|
| 25 |
+
|
| 26 |
+
tokenizer: SentencepiecesTokenizer
|
| 27 |
+
tokenizer_conf:
|
| 28 |
+
bpemodel: null
|
| 29 |
+
unk_symbol: <unk>
|
| 30 |
+
split_with_space: true
|
| 31 |
+
|
| 32 |
+
frontend: WavFrontend
|
| 33 |
+
frontend_conf:
|
| 34 |
+
fs: 16000
|
| 35 |
+
window: hamming
|
| 36 |
+
n_mels: 80
|
| 37 |
+
frame_length: 25
|
| 38 |
+
frame_shift: 10
|
| 39 |
+
lfr_m: 7
|
| 40 |
+
lfr_n: 6
|
| 41 |
+
cmvn_file: null
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
dataset: SenseVoiceCTCDataset
|
| 45 |
+
dataset_conf:
|
| 46 |
+
index_ds: IndexDSJsonl
|
| 47 |
+
batch_sampler: EspnetStyleBatchSampler
|
| 48 |
+
data_split_num: 32
|
| 49 |
+
batch_type: token
|
| 50 |
+
batch_size: 14000
|
| 51 |
+
max_token_length: 2000
|
| 52 |
+
min_token_length: 60
|
| 53 |
+
max_source_length: 2000
|
| 54 |
+
min_source_length: 60
|
| 55 |
+
max_target_length: 200
|
| 56 |
+
min_target_length: 0
|
| 57 |
+
shuffle: true
|
| 58 |
+
num_workers: 4
|
| 59 |
+
sos: ${model_conf.sos}
|
| 60 |
+
eos: ${model_conf.eos}
|
| 61 |
+
IndexDSJsonl: IndexDSJsonl
|
| 62 |
+
retry: 20
|
| 63 |
+
|
| 64 |
+
train_conf:
|
| 65 |
+
accum_grad: 1
|
| 66 |
+
grad_clip: 5
|
| 67 |
+
max_epoch: 20
|
| 68 |
+
keep_nbest_models: 10
|
| 69 |
+
avg_nbest_model: 10
|
| 70 |
+
log_interval: 100
|
| 71 |
+
resume: true
|
| 72 |
+
validate_interval: 10000
|
| 73 |
+
save_checkpoint_interval: 10000
|
| 74 |
+
|
| 75 |
+
optim: adamw
|
| 76 |
+
optim_conf:
|
| 77 |
+
lr: 0.00002
|
| 78 |
+
scheduler: warmuplr
|
| 79 |
+
scheduler_conf:
|
| 80 |
+
warmup_steps: 25000
|
| 81 |
+
|
| 82 |
+
specaug: SpecAugLFR
|
| 83 |
+
specaug_conf:
|
| 84 |
+
apply_time_warp: false
|
| 85 |
+
time_warp_window: 5
|
| 86 |
+
time_warp_mode: bicubic
|
| 87 |
+
apply_freq_mask: true
|
| 88 |
+
freq_mask_width_range:
|
| 89 |
+
- 0
|
| 90 |
+
- 30
|
| 91 |
+
lfr_rate: 6
|
| 92 |
+
num_freq_mask: 1
|
| 93 |
+
apply_time_mask: true
|
| 94 |
+
time_mask_width_range:
|
| 95 |
+
- 0
|
| 96 |
+
- 12
|
| 97 |
+
num_time_mask: 1
|
| 98 |
+
|
omni_diffusion/models/dream/config_dream_resume.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_name_or_path": "Instruct-7B",
|
| 3 |
+
"architectures": [
|
| 4 |
+
"DreamModel"
|
| 5 |
+
],
|
| 6 |
+
"attention_dropout": 0.0,
|
| 7 |
+
"auto_map": {
|
| 8 |
+
"AutoConfig": "configuration_dream.DreamConfig",
|
| 9 |
+
"AutoModel": "modeling_dream.DreamModel"
|
| 10 |
+
},
|
| 11 |
+
"bos_token_id": 151643,
|
| 12 |
+
"eos_token_id": 151643,
|
| 13 |
+
"hidden_act": "silu",
|
| 14 |
+
"hidden_size": 3584,
|
| 15 |
+
"initializer_range": 0.02,
|
| 16 |
+
"intermediate_size": 18944,
|
| 17 |
+
"mask_token_id": 151666,
|
| 18 |
+
"max_position_embeddings": 131072,
|
| 19 |
+
"max_window_layers": 28,
|
| 20 |
+
"model_type": "Dream",
|
| 21 |
+
"num_attention_heads": 28,
|
| 22 |
+
"num_hidden_layers": 28,
|
| 23 |
+
"num_key_value_heads": 4,
|
| 24 |
+
"pad_token_id": 151643,
|
| 25 |
+
"rms_norm_eps": 1e-06,
|
| 26 |
+
"rope_scaling": null,
|
| 27 |
+
"rope_theta": 1000000.0,
|
| 28 |
+
"sliding_window": null,
|
| 29 |
+
"tie_word_embeddings": false,
|
| 30 |
+
"torch_dtype": "bfloat16",
|
| 31 |
+
"transformers_version": "4.46.2",
|
| 32 |
+
"use_cache": true,
|
| 33 |
+
"use_mrope": false,
|
| 34 |
+
"use_sliding_window": false,
|
| 35 |
+
"vocab_size": 176264,
|
| 36 |
+
"chunk_size": -1
|
| 37 |
+
}
|
omni_diffusion/models/dream/configuration.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"framework": "pytorch",
|
| 3 |
+
"task" : "auto-speech-recognition",
|
| 4 |
+
"model": {"type" : "funasr"},
|
| 5 |
+
"pipeline": {"type":"funasr-pipeline"},
|
| 6 |
+
"model_name_in_hub": {
|
| 7 |
+
"ms":"",
|
| 8 |
+
"hf":""},
|
| 9 |
+
"file_path_metas": {
|
| 10 |
+
"config":"config.yaml",
|
| 11 |
+
"tokenizer_conf": {"bpemodel": "chn_jpn_yue_eng_ko_spectok.bpe.model"},
|
| 12 |
+
"frontend_conf":{"cmvn_file": "am.mvn"}}
|
| 13 |
+
}
|
| 14 |
+
|
omni_diffusion/models/dream/configuration_dream.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# coding=utf-8
|
| 2 |
+
# Copyright 2024 The Dream team, HKUNLP Group and the HuggingFace Inc. team. All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 5 |
+
# you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at
|
| 7 |
+
#
|
| 8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 9 |
+
#
|
| 10 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 13 |
+
# See the License for the specific language governing permissions and
|
| 14 |
+
# limitations under the License.
|
| 15 |
+
"""Dream model configuration"""
|
| 16 |
+
|
| 17 |
+
from transformers.configuration_utils import PretrainedConfig
|
| 18 |
+
from transformers.modeling_rope_utils import rope_config_validation
|
| 19 |
+
from transformers.utils import logging
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
logger = logging.get_logger(__name__)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class DreamConfig(PretrainedConfig):
|
| 26 |
+
model_type = "Dream"
|
| 27 |
+
keys_to_ignore_at_inference = ["past_key_values"]
|
| 28 |
+
|
| 29 |
+
def __init__(
|
| 30 |
+
self,
|
| 31 |
+
vocab_size=151936,
|
| 32 |
+
hidden_size=4096,
|
| 33 |
+
intermediate_size=22016,
|
| 34 |
+
num_hidden_layers=32,
|
| 35 |
+
num_attention_heads=32,
|
| 36 |
+
num_key_value_heads=32,
|
| 37 |
+
hidden_act="silu",
|
| 38 |
+
max_position_embeddings=32768,
|
| 39 |
+
initializer_range=0.02,
|
| 40 |
+
rms_norm_eps=1e-6,
|
| 41 |
+
use_cache=False, # cache not used in diffusion
|
| 42 |
+
tie_word_embeddings=False,
|
| 43 |
+
rope_theta=10000.0,
|
| 44 |
+
rope_scaling=None,
|
| 45 |
+
use_sliding_window=False,
|
| 46 |
+
sliding_window=4096,
|
| 47 |
+
max_window_layers=28,
|
| 48 |
+
attention_dropout=0.0,
|
| 49 |
+
mask_token_id=151666,
|
| 50 |
+
pad_token_id=151643,
|
| 51 |
+
**kwargs,
|
| 52 |
+
):
|
| 53 |
+
self.vocab_size = vocab_size
|
| 54 |
+
self.max_position_embeddings = max_position_embeddings
|
| 55 |
+
self.hidden_size = hidden_size
|
| 56 |
+
self.intermediate_size = intermediate_size
|
| 57 |
+
self.num_hidden_layers = num_hidden_layers
|
| 58 |
+
self.num_attention_heads = num_attention_heads
|
| 59 |
+
self.use_sliding_window = use_sliding_window
|
| 60 |
+
self.sliding_window = sliding_window if use_sliding_window else None
|
| 61 |
+
self.max_window_layers = max_window_layers
|
| 62 |
+
|
| 63 |
+
# for backward compatibility
|
| 64 |
+
if num_key_value_heads is None:
|
| 65 |
+
num_key_value_heads = num_attention_heads
|
| 66 |
+
|
| 67 |
+
self.num_key_value_heads = num_key_value_heads
|
| 68 |
+
self.hidden_act = hidden_act
|
| 69 |
+
self.initializer_range = initializer_range
|
| 70 |
+
self.rms_norm_eps = rms_norm_eps
|
| 71 |
+
self.use_cache = use_cache
|
| 72 |
+
self.rope_theta = rope_theta
|
| 73 |
+
self.rope_scaling = rope_scaling
|
| 74 |
+
self.attention_dropout = attention_dropout
|
| 75 |
+
# Validate the correctness of rotary position embeddings parameters
|
| 76 |
+
# BC: if there is a 'type' field, move it to 'rope_type'.
|
| 77 |
+
if self.rope_scaling is not None and "type" in self.rope_scaling:
|
| 78 |
+
self.rope_scaling["rope_type"] = self.rope_scaling["type"]
|
| 79 |
+
rope_config_validation(self)
|
| 80 |
+
|
| 81 |
+
super().__init__(
|
| 82 |
+
tie_word_embeddings=tie_word_embeddings,
|
| 83 |
+
**kwargs,
|
| 84 |
+
)
|
| 85 |
+
self.mask_token_id = mask_token_id
|
| 86 |
+
self.pad_token_id = pad_token_id
|
omni_diffusion/models/dream/generation_utils.py
ADDED
|
@@ -0,0 +1,623 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# coding=utf-8
|
| 2 |
+
# Copyright 2024 The Dream team, HKUNLP Group and the HuggingFace Inc. team. All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 5 |
+
# you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at
|
| 7 |
+
#
|
| 8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 9 |
+
#
|
| 10 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 13 |
+
# See the License for the specific language governing permissions and
|
| 14 |
+
# limitations under the License.
|
| 15 |
+
|
| 16 |
+
import warnings
|
| 17 |
+
import copy
|
| 18 |
+
from dataclasses import dataclass
|
| 19 |
+
from typing import Any, Dict, Optional, Tuple, Union
|
| 20 |
+
|
| 21 |
+
import torch
|
| 22 |
+
import torch.distributions as dists
|
| 23 |
+
from torch.nn import functional as F
|
| 24 |
+
from transformers import __version__
|
| 25 |
+
from transformers.generation.configuration_utils import (
|
| 26 |
+
GenerationConfig
|
| 27 |
+
)
|
| 28 |
+
from transformers.utils import (
|
| 29 |
+
ModelOutput,
|
| 30 |
+
is_torchdynamo_compiling,
|
| 31 |
+
logging,
|
| 32 |
+
)
|
| 33 |
+
logger = logging.get_logger(__name__)
|
| 34 |
+
from tqdm import tqdm
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def top_p_logits(logits, top_p=None):
|
| 38 |
+
sorted_logits, sorted_indices = torch.sort(logits, descending=True)
|
| 39 |
+
cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
|
| 40 |
+
sorted_indices_to_remove = cumulative_probs > top_p
|
| 41 |
+
# Shift the indices to the right to keep the first token above the threshold
|
| 42 |
+
sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
|
| 43 |
+
sorted_indices_to_remove[..., 0] = 0
|
| 44 |
+
|
| 45 |
+
mask = torch.zeros_like(logits, dtype=torch.bool, device=logits.device)
|
| 46 |
+
mask = mask.scatter_(-1, sorted_indices, sorted_indices_to_remove)
|
| 47 |
+
logits = logits.masked_fill(mask, torch.finfo(logits.dtype).min)
|
| 48 |
+
return logits
|
| 49 |
+
|
| 50 |
+
def top_k_logits(logits, top_k=None):
|
| 51 |
+
top_k = min(top_k, logits.size(-1)) # Safety check
|
| 52 |
+
# Remove all tokens with a probability less than the last token of the top-k
|
| 53 |
+
indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]
|
| 54 |
+
logits = logits.masked_fill(indices_to_remove, torch.finfo(logits.dtype).min)
|
| 55 |
+
return logits
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def sample_tokens(logits, temperature=0.0, top_p=None, top_k=None, margin_confidence=False, neg_entropy=False,
|
| 59 |
+
repeat_penalty=1.0, max_position_penalty=1.0, past_x=None, mask_id=None,):
|
| 60 |
+
if temperature > 0:
|
| 61 |
+
logits = logits / temperature
|
| 62 |
+
|
| 63 |
+
if top_p is not None and top_p < 1:
|
| 64 |
+
logits = top_p_logits(logits, top_p)
|
| 65 |
+
if top_k is not None:
|
| 66 |
+
logits = top_k_logits(logits, top_k)
|
| 67 |
+
|
| 68 |
+
if repeat_penalty != 1.0:
|
| 69 |
+
select_mask = torch.logical_and((past_x != 0), (past_x != mask_id))
|
| 70 |
+
generated_tokens = set(past_x[select_mask].tolist())
|
| 71 |
+
for token in set(generated_tokens):
|
| 72 |
+
logits[:, token][logits[:, token] < 0] *= repeat_penalty
|
| 73 |
+
logits[:, token][logits[:, token] >= 0] /= repeat_penalty
|
| 74 |
+
|
| 75 |
+
if max_position_penalty != 1.0:
|
| 76 |
+
token_length = logits.shape[-2]
|
| 77 |
+
if token_length > 100:
|
| 78 |
+
penalty_map = [i / (token_length - 100) * (max_position_penalty - 1.0) + 1.0
|
| 79 |
+
for i in range(token_length - 100)]
|
| 80 |
+
penalty_map = torch.tensor(penalty_map).unsqueeze(-1).to(logits.device).to(logits.dtype)
|
| 81 |
+
penalty_map = torch.cat([torch.ones_like(logits[:100, :1]), penalty_map], dim=0)
|
| 82 |
+
penalty_map = penalty_map.repeat(1, logits.shape[-1])
|
| 83 |
+
|
| 84 |
+
logits[logits < 0] *= penalty_map[logits < 0]
|
| 85 |
+
logits[logits >= 0] /= penalty_map[logits >= 0]
|
| 86 |
+
|
| 87 |
+
probs = torch.softmax(logits, dim=-1)
|
| 88 |
+
if temperature > 0:
|
| 89 |
+
try:
|
| 90 |
+
x0 = dists.Categorical(probs=probs).sample()
|
| 91 |
+
confidence = torch.gather(probs, -1, x0.unsqueeze(-1)).squeeze(-1)
|
| 92 |
+
except:
|
| 93 |
+
confidence, x0 = probs.max(dim=-1)
|
| 94 |
+
else:
|
| 95 |
+
confidence, x0 = probs.max(dim=-1)
|
| 96 |
+
|
| 97 |
+
if margin_confidence:
|
| 98 |
+
sorted_probs, _ = torch.sort(probs, dim=-1, descending=True)
|
| 99 |
+
top1_probs = sorted_probs[:, 0]
|
| 100 |
+
top2_probs = sorted_probs[:, 1]
|
| 101 |
+
confidence = top1_probs - top2_probs
|
| 102 |
+
|
| 103 |
+
if neg_entropy:
|
| 104 |
+
epsilon = 1e-10
|
| 105 |
+
log_probs = torch.log(probs + epsilon)
|
| 106 |
+
confidence = torch.sum(probs * log_probs, dim=-1)
|
| 107 |
+
|
| 108 |
+
return confidence, x0
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
@dataclass
|
| 112 |
+
class DreamModelOutput(ModelOutput):
|
| 113 |
+
sequences: torch.LongTensor = None
|
| 114 |
+
history: Optional[Tuple[torch.FloatTensor]] = None
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
class DreamGenerationConfig(GenerationConfig):
|
| 118 |
+
def __init__(self, **kwargs):
|
| 119 |
+
self.temperature: float = kwargs.pop("temperature", 0.0)
|
| 120 |
+
self.top_p: Optional[float] = kwargs.pop("top_p", None)
|
| 121 |
+
self.top_k: Optional[int] = kwargs.pop("top_k", None)
|
| 122 |
+
self.max_length = kwargs.pop("max_length", 20)
|
| 123 |
+
self.max_new_tokens = kwargs.pop("max_new_tokens", None)
|
| 124 |
+
# diffusion specific params
|
| 125 |
+
self.eps: float = kwargs.pop("eps", 1e-3)
|
| 126 |
+
self.steps: int = kwargs.pop("steps", 512)
|
| 127 |
+
self.alg: str = kwargs.pop("alg", 'origin')
|
| 128 |
+
self.alg_temp: Optional[float] = kwargs.pop("alg_temp", None)
|
| 129 |
+
|
| 130 |
+
# Parameters that define the output variables of `generate`
|
| 131 |
+
self.num_return_sequences: int = kwargs.pop("num_return_sequences", 1)
|
| 132 |
+
self.return_dict_in_generate: bool = kwargs.pop("return_dict_in_generate", False)
|
| 133 |
+
self.output_history: bool = kwargs.pop("output_history", False)
|
| 134 |
+
|
| 135 |
+
# Special tokens that can be used at generation time
|
| 136 |
+
self.mask_token_id = kwargs.pop("mask_token_id", None)
|
| 137 |
+
self.pad_token_id = kwargs.pop("pad_token_id", None)
|
| 138 |
+
self.bos_token_id = kwargs.pop("bos_token_id", None)
|
| 139 |
+
self.eos_token_id = kwargs.pop("eos_token_id", None)
|
| 140 |
+
|
| 141 |
+
# Wild card
|
| 142 |
+
self.generation_kwargs = kwargs.pop("generation_kwargs", {})
|
| 143 |
+
|
| 144 |
+
# The remaining attributes do not parametrize `.generate()`, but are informative and/or used by the hub
|
| 145 |
+
# interface.
|
| 146 |
+
self._from_model_config = kwargs.pop("_from_model_config", False)
|
| 147 |
+
self._commit_hash = kwargs.pop("_commit_hash", None)
|
| 148 |
+
self.transformers_version = kwargs.pop("transformers_version", __version__)
|
| 149 |
+
|
| 150 |
+
# Additional attributes without default values
|
| 151 |
+
if not self._from_model_config:
|
| 152 |
+
# we don't want to copy values from the model config if we're initializing a `GenerationConfig` from a
|
| 153 |
+
# model's default configuration file
|
| 154 |
+
for key, value in kwargs.items():
|
| 155 |
+
try:
|
| 156 |
+
setattr(self, key, value)
|
| 157 |
+
except AttributeError as err:
|
| 158 |
+
logger.error(f"Can't set {key} with value {value} for {self}")
|
| 159 |
+
raise err
|
| 160 |
+
|
| 161 |
+
# Validate the values of the attributes
|
| 162 |
+
self.validate(is_init=True)
|
| 163 |
+
|
| 164 |
+
def validate(self, is_init=False):
|
| 165 |
+
pass
|
| 166 |
+
|
| 167 |
+
class DreamGenerationMixin:
|
| 168 |
+
@staticmethod
|
| 169 |
+
def _expand_inputs_for_generation(
|
| 170 |
+
expand_size: int = 1,
|
| 171 |
+
input_ids: Optional[torch.LongTensor] = None,
|
| 172 |
+
attention_mask: Optional[torch.LongTensor] = None
|
| 173 |
+
) -> Tuple[torch.LongTensor, Dict[str, Any]]:
|
| 174 |
+
"""Expands tensors from [batch_size, ...] to [batch_size * expand_size, ...]"""
|
| 175 |
+
# Do not call torch.repeat_interleave if expand_size is 1 because it clones
|
| 176 |
+
# the input tensor and thus requires more memory although no change is applied
|
| 177 |
+
if expand_size == 1:
|
| 178 |
+
return input_ids, attention_mask
|
| 179 |
+
if input_ids is not None:
|
| 180 |
+
input_ids = input_ids.repeat_interleave(expand_size, dim=0)
|
| 181 |
+
if attention_mask is not None:
|
| 182 |
+
attention_mask = attention_mask.repeat_interleave(expand_size, dim=0)
|
| 183 |
+
return input_ids, attention_mask
|
| 184 |
+
|
| 185 |
+
def _validate_generated_length(self, generation_config, input_ids_length, has_default_max_length):
|
| 186 |
+
"""Performs validation related to the resulting generated length"""
|
| 187 |
+
|
| 188 |
+
# Can't throw warnings/exceptions during compilation
|
| 189 |
+
if is_torchdynamo_compiling():
|
| 190 |
+
return
|
| 191 |
+
|
| 192 |
+
# 1. Max length warnings related to poor parameterization
|
| 193 |
+
if has_default_max_length and generation_config.max_new_tokens is None and generation_config.max_length == 20:
|
| 194 |
+
# 20 is the default max_length of the generation config
|
| 195 |
+
warnings.warn(
|
| 196 |
+
f"Using the model-agnostic default `max_length` (={generation_config.max_length}) to control the "
|
| 197 |
+
"generation length. We recommend setting `max_new_tokens` to control the maximum length of the "
|
| 198 |
+
"generation.",
|
| 199 |
+
UserWarning,
|
| 200 |
+
)
|
| 201 |
+
if input_ids_length >= generation_config.max_length:
|
| 202 |
+
input_ids_string = "input_ids"
|
| 203 |
+
raise ValueError(
|
| 204 |
+
f"Input length of {input_ids_string} is {input_ids_length}, but `max_length` is set to"
|
| 205 |
+
f" {generation_config.max_length}. This can lead to unexpected behavior. You should consider"
|
| 206 |
+
" increasing `max_length` or, better yet, setting `max_new_tokens`."
|
| 207 |
+
)
|
| 208 |
+
|
| 209 |
+
def _prepare_generated_length(
|
| 210 |
+
self,
|
| 211 |
+
generation_config,
|
| 212 |
+
has_default_max_length,
|
| 213 |
+
input_ids_length,
|
| 214 |
+
):
|
| 215 |
+
"""Prepared max and min length in generation configs to avoid clashes between similar attributes"""
|
| 216 |
+
|
| 217 |
+
if generation_config.max_new_tokens is not None:
|
| 218 |
+
if not has_default_max_length and generation_config.max_length is not None:
|
| 219 |
+
logger.warning(
|
| 220 |
+
f"Both `max_new_tokens` (={generation_config.max_new_tokens}) and `max_length`(="
|
| 221 |
+
f"{generation_config.max_length}) seem to have been set. `max_new_tokens` will take precedence. "
|
| 222 |
+
"Please refer to the documentation for more information. "
|
| 223 |
+
"(https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)"
|
| 224 |
+
)
|
| 225 |
+
generation_config.max_length = generation_config.max_new_tokens + input_ids_length
|
| 226 |
+
|
| 227 |
+
elif has_default_max_length:
|
| 228 |
+
if generation_config.max_length == DreamGenerationConfig().max_length:
|
| 229 |
+
generation_config.max_length = generation_config.max_length + input_ids_length
|
| 230 |
+
max_position_embeddings = getattr(self.config, "max_position_embeddings", None)
|
| 231 |
+
if max_position_embeddings is not None:
|
| 232 |
+
generation_config.max_length = min(generation_config.max_length, max_position_embeddings)
|
| 233 |
+
|
| 234 |
+
return generation_config
|
| 235 |
+
|
| 236 |
+
def _prepare_generation_config(
|
| 237 |
+
self, generation_config: Optional[DreamGenerationConfig], **kwargs: Dict
|
| 238 |
+
) -> DreamGenerationConfig:
|
| 239 |
+
"""
|
| 240 |
+
Prepares the base generation config, then applies any generation configuration options from kwargs. This
|
| 241 |
+
function handles retrocompatibility with respect to configuration files.
|
| 242 |
+
"""
|
| 243 |
+
# priority: `generation_config` argument > `model.generation_config` (the default generation config)
|
| 244 |
+
using_model_generation_config = False
|
| 245 |
+
if generation_config is None:
|
| 246 |
+
generation_config = DreamGenerationConfig.from_model_config(self.config)
|
| 247 |
+
using_model_generation_config = True
|
| 248 |
+
|
| 249 |
+
# `torch.compile` can't compile `copy.deepcopy`, arguments in `kwargs` that are part of `generation_config`
|
| 250 |
+
# will mutate the object with `.update`. As such, passing these arguments through `kwargs` is disabled -- an
|
| 251 |
+
# exception will be raised in `_validate_model_kwargs`
|
| 252 |
+
if not is_torchdynamo_compiling():
|
| 253 |
+
generation_config = copy.deepcopy(generation_config)
|
| 254 |
+
_kwargs = generation_config.update(**kwargs)
|
| 255 |
+
# If `generation_config` is provided, let's fallback ALL special tokens to the default values for the model
|
| 256 |
+
if not using_model_generation_config:
|
| 257 |
+
if generation_config.bos_token_id is None:
|
| 258 |
+
generation_config.bos_token_id = self.generation_config.bos_token_id
|
| 259 |
+
if generation_config.eos_token_id is None:
|
| 260 |
+
generation_config.eos_token_id = self.generation_config.eos_token_id
|
| 261 |
+
if generation_config.pad_token_id is None:
|
| 262 |
+
generation_config.pad_token_id = self.generation_config.pad_token_id
|
| 263 |
+
if generation_config.mask_token_id is None:
|
| 264 |
+
generation_config.mask_token_id = self.generation_config.mask_token_id
|
| 265 |
+
|
| 266 |
+
return generation_config
|
| 267 |
+
|
| 268 |
+
def _prepare_special_tokens(
|
| 269 |
+
self,
|
| 270 |
+
generation_config: DreamGenerationConfig,
|
| 271 |
+
device: Optional[Union[torch.device, str]] = None,
|
| 272 |
+
):
|
| 273 |
+
"""
|
| 274 |
+
Prepares the special tokens for generation, overwriting the generation config with their processed versions
|
| 275 |
+
converted to tensor.
|
| 276 |
+
|
| 277 |
+
Note that `generation_config` is changed in place and stops being serializable after this method is called.
|
| 278 |
+
That is no problem if called within `generate` (`generation_config` is a local copy that doesn't leave the
|
| 279 |
+
function). However, if called outside `generate`, consider creating a copy of `generation_config` first.
|
| 280 |
+
"""
|
| 281 |
+
|
| 282 |
+
# Convert special tokens to tensors
|
| 283 |
+
def _tensor_or_none(token, device=None):
|
| 284 |
+
if token is None:
|
| 285 |
+
return token
|
| 286 |
+
|
| 287 |
+
device = device if device is not None else self.device
|
| 288 |
+
if isinstance(token, torch.Tensor):
|
| 289 |
+
return token.to(device)
|
| 290 |
+
return torch.tensor(token, device=device, dtype=torch.long)
|
| 291 |
+
|
| 292 |
+
bos_token_tensor = _tensor_or_none(generation_config.bos_token_id, device=device)
|
| 293 |
+
eos_token_tensor = _tensor_or_none(generation_config.eos_token_id, device=device)
|
| 294 |
+
pad_token_tensor = _tensor_or_none(generation_config.pad_token_id, device=device)
|
| 295 |
+
mask_token_tensor = _tensor_or_none(generation_config.mask_token_id, device=device)
|
| 296 |
+
|
| 297 |
+
# We can have more than one eos token. Always treat it as a 1D tensor (when it exists).
|
| 298 |
+
if eos_token_tensor is not None and eos_token_tensor.ndim == 0:
|
| 299 |
+
eos_token_tensor = eos_token_tensor.unsqueeze(0)
|
| 300 |
+
|
| 301 |
+
# Set pad token if unset (and there are conditions to do so)
|
| 302 |
+
if pad_token_tensor is None and eos_token_tensor is not None:
|
| 303 |
+
pad_token_tensor = eos_token_tensor[0]
|
| 304 |
+
logger.warning(f"Setting `pad_token_id` to `eos_token_id`:{pad_token_tensor} for open-end generation.")
|
| 305 |
+
|
| 306 |
+
# Update generation config with the updated special tokens tensors
|
| 307 |
+
# NOTE: this must be written into a different attribute name than the one holding the original special tokens
|
| 308 |
+
# (in their non-tensor form), in order to enable end-to-end compilation. See
|
| 309 |
+
# https://pytorch.org/docs/stable/torch.compiler_cudagraph_trees.html#limitations
|
| 310 |
+
generation_config._bos_token_tensor = bos_token_tensor
|
| 311 |
+
generation_config._eos_token_tensor = eos_token_tensor
|
| 312 |
+
generation_config._pad_token_tensor = pad_token_tensor
|
| 313 |
+
generation_config._mask_token_tensor = mask_token_tensor
|
| 314 |
+
|
| 315 |
+
@torch.no_grad()
|
| 316 |
+
def diffusion_generate(
|
| 317 |
+
self,
|
| 318 |
+
inputs: Optional[torch.Tensor] = None,
|
| 319 |
+
generation_config: Optional[DreamGenerationConfig] = None,
|
| 320 |
+
inputs_embeds=None,
|
| 321 |
+
prefix_lm=False,
|
| 322 |
+
alg=None,
|
| 323 |
+
block_size=-1,
|
| 324 |
+
cfg=0.0,
|
| 325 |
+
add_boa_token=False,
|
| 326 |
+
**kwargs,
|
| 327 |
+
) -> Union[DreamModelOutput, torch.LongTensor]:
|
| 328 |
+
# 1. Handle `generation_config` and kwargs that might update it, and validate the `.generate()` call
|
| 329 |
+
generation_config = self._prepare_generation_config(generation_config, **kwargs)
|
| 330 |
+
generation_tokens_hook_func = kwargs.pop("generation_tokens_hook_func", lambda step, x, logits: x)
|
| 331 |
+
generation_logits_hook_func = kwargs.pop("generation_logits_hook_func", lambda step, x, logits: logits)
|
| 332 |
+
# breakpoint()
|
| 333 |
+
# 2. Define model inputs
|
| 334 |
+
if inputs is not None:
|
| 335 |
+
input_ids = inputs
|
| 336 |
+
device = input_ids.device
|
| 337 |
+
input_ids_length = input_ids.shape[-1]
|
| 338 |
+
else:
|
| 339 |
+
input_ids = None
|
| 340 |
+
device = inputs_embeds.device
|
| 341 |
+
input_ids_length = inputs_embeds.shape[1]
|
| 342 |
+
attention_mask = kwargs.pop("attention_mask", None)
|
| 343 |
+
self._prepare_special_tokens(generation_config, device=device)
|
| 344 |
+
|
| 345 |
+
# 3. Prepare `max_length`.
|
| 346 |
+
|
| 347 |
+
has_default_max_length = kwargs.get("max_length") is None and generation_config.max_length is not None
|
| 348 |
+
generation_config = self._prepare_generated_length(
|
| 349 |
+
generation_config=generation_config,
|
| 350 |
+
has_default_max_length=has_default_max_length,
|
| 351 |
+
input_ids_length=input_ids_length,
|
| 352 |
+
)
|
| 353 |
+
|
| 354 |
+
self._validate_generated_length(generation_config, input_ids_length, has_default_max_length)
|
| 355 |
+
# import pdb;pdb.set_trace()
|
| 356 |
+
# 4. Check input_ids
|
| 357 |
+
#if not is_torchdynamo_compiling() and self.device.type != input_ids.device.type:
|
| 358 |
+
if not is_torchdynamo_compiling() and self.device.type != device.type:
|
| 359 |
+
warnings.warn(
|
| 360 |
+
"You are calling .generate() with the `input_ids` being on a device type different"
|
| 361 |
+
f" than your model's device. `input_ids` is on {input_ids.device.type}, whereas the model"
|
| 362 |
+
f" is on {self.device.type}. You may experience unexpected behaviors or slower generation."
|
| 363 |
+
" Please make sure that you have put `input_ids` to the"
|
| 364 |
+
f" correct device by calling for example input_ids = input_ids.to('{self.device.type}') before"
|
| 365 |
+
" running `.generate()`.",
|
| 366 |
+
UserWarning,
|
| 367 |
+
)
|
| 368 |
+
# breakpoint()
|
| 369 |
+
if (
|
| 370 |
+
hasattr(generation_config, "pad_token_id") and
|
| 371 |
+
input_ids is not None and
|
| 372 |
+
torch.any(input_ids == generation_config.pad_token_id) and
|
| 373 |
+
attention_mask is None
|
| 374 |
+
):
|
| 375 |
+
warnings.warn(
|
| 376 |
+
"Padding was detected but no attention mask is passed here. For correct "
|
| 377 |
+
"generation results, please set `attention_mask` when batch-padding inputs.",
|
| 378 |
+
UserWarning,
|
| 379 |
+
)
|
| 380 |
+
assert generation_config.num_return_sequences == 1, \
|
| 381 |
+
"Currently, we only support num_return_sequences = 1 for diffusion generation."
|
| 382 |
+
input_ids, attention_mask = self._expand_inputs_for_generation(
|
| 383 |
+
expand_size=generation_config.num_return_sequences,
|
| 384 |
+
input_ids=input_ids,
|
| 385 |
+
attention_mask=attention_mask
|
| 386 |
+
)
|
| 387 |
+
result = self._sample(
|
| 388 |
+
input_ids,
|
| 389 |
+
attention_mask=attention_mask,
|
| 390 |
+
generation_config=generation_config,
|
| 391 |
+
generation_tokens_hook_func=generation_tokens_hook_func,
|
| 392 |
+
generation_logits_hook_func=generation_logits_hook_func,
|
| 393 |
+
inputs_embeds=inputs_embeds,
|
| 394 |
+
device=device,
|
| 395 |
+
prefix_lm=prefix_lm,
|
| 396 |
+
alg=alg,
|
| 397 |
+
block_size=block_size,
|
| 398 |
+
cfg=cfg,
|
| 399 |
+
add_boa_token=add_boa_token,
|
| 400 |
+
**kwargs,
|
| 401 |
+
)
|
| 402 |
+
return result
|
| 403 |
+
|
| 404 |
+
def _sample(
|
| 405 |
+
self,
|
| 406 |
+
input_ids: torch.LongTensor,
|
| 407 |
+
attention_mask: Optional[torch.LongTensor],
|
| 408 |
+
generation_config: DreamGenerationConfig,
|
| 409 |
+
generation_tokens_hook_func,
|
| 410 |
+
generation_logits_hook_func,
|
| 411 |
+
inputs_embeds=None,
|
| 412 |
+
prefix_lm=False,
|
| 413 |
+
device=None,
|
| 414 |
+
step_ratio=None,
|
| 415 |
+
penalty=1.2,
|
| 416 |
+
alg=None,
|
| 417 |
+
block_size=None,
|
| 418 |
+
add_boa_token=False,
|
| 419 |
+
max_position_penalty=1.0,
|
| 420 |
+
repeat_penalty=1.0,
|
| 421 |
+
cfg=0.0,
|
| 422 |
+
**kwargs,
|
| 423 |
+
) -> Union[DreamModelOutput, torch.LongTensor]:
|
| 424 |
+
output_history = True
|
| 425 |
+
return_dict_in_generate = generation_config.return_dict_in_generate
|
| 426 |
+
max_length = generation_config.max_length
|
| 427 |
+
mask_token_id = generation_config.mask_token_id
|
| 428 |
+
max_new_tokens = generation_config.max_new_tokens
|
| 429 |
+
steps = min(generation_config.steps, max_new_tokens)
|
| 430 |
+
eps = generation_config.eps
|
| 431 |
+
alg = generation_config.alg if alg is None else alg
|
| 432 |
+
print("denoise algorithm: " + alg)
|
| 433 |
+
alg_temp = generation_config.alg_temp
|
| 434 |
+
temperature = generation_config.temperature
|
| 435 |
+
top_p = generation_config.top_p
|
| 436 |
+
top_k = generation_config.top_k
|
| 437 |
+
|
| 438 |
+
histories = [] if (return_dict_in_generate and output_history) else None
|
| 439 |
+
all_logit = []
|
| 440 |
+
generated_tokens = []
|
| 441 |
+
block_size = max_new_tokens if block_size < 0 else block_size
|
| 442 |
+
|
| 443 |
+
if input_ids is None:
|
| 444 |
+
assert device is not None
|
| 445 |
+
assert inputs_embeds is not None
|
| 446 |
+
bsz, seq_len = inputs_embeds.shape[:2]
|
| 447 |
+
max_length = seq_len + max_new_tokens
|
| 448 |
+
input_ids = torch.full((bsz, seq_len), 0, dtype=torch.long).to(device)
|
| 449 |
+
|
| 450 |
+
tok_idx = None
|
| 451 |
+
past_key_values = None
|
| 452 |
+
|
| 453 |
+
x = F.pad(input_ids, (0, max_length - input_ids.shape[1]), value=mask_token_id)
|
| 454 |
+
|
| 455 |
+
timesteps = torch.linspace(1, eps, steps + 1, device=x.device)
|
| 456 |
+
x = generation_tokens_hook_func(None, x, None)
|
| 457 |
+
|
| 458 |
+
if step_ratio is not None:
|
| 459 |
+
steps = int(max_new_tokens * step_ratio)
|
| 460 |
+
|
| 461 |
+
if add_boa_token:
|
| 462 |
+
bos_index = int((x.shape[1] - (x == mask_token_id).sum()) + (x == mask_token_id).sum() * 0.2)
|
| 463 |
+
x[:, bos_index] = 151684 # <|begin_of_audio|>
|
| 464 |
+
|
| 465 |
+
input_x = x.clone()
|
| 466 |
+
total_steps = steps
|
| 467 |
+
block_num = (x == mask_token_id).sum() // block_size
|
| 468 |
+
if block_num * block_size < (x == mask_token_id).sum(): block_num += 1
|
| 469 |
+
input_length = input_ids.shape[-1]
|
| 470 |
+
|
| 471 |
+
task = None
|
| 472 |
+
if "task" in kwargs: task = kwargs['task']
|
| 473 |
+
if cfg > 0:
|
| 474 |
+
import random
|
| 475 |
+
empty_prompt = ""
|
| 476 |
+
if task == "S2I":
|
| 477 |
+
empty_prompt = "<|im_start|>system\nPlease generate an image based on the input audio.<|im_end|>\n"
|
| 478 |
+
empty_prompt += "<|im_start|>user\n<|im_end|>\n<|im_start|>assistant\n"
|
| 479 |
+
un_x = kwargs['tokenizer'].encode(empty_prompt)
|
| 480 |
+
elif task == "T2I":
|
| 481 |
+
empty_prompt = "<|im_start|>user\nGenerate an image based on the provided text description.\n"
|
| 482 |
+
empty_prompt += "<|im_end|>\n<|im_start|>assistant\n"
|
| 483 |
+
first_audio_token = kwargs['tokenizer'].encode("<|begin_of_audio|>")[0]
|
| 484 |
+
un_x_text = random.sample([_ for _ in range(first_audio_token)],
|
| 485 |
+
input_ids.shape[1] - len(kwargs['tokenizer'].encode(empty_prompt)))
|
| 486 |
+
un_x = kwargs['tokenizer'].encode("<|im_start|>user\nGenerate an image based on the provided \
|
| 487 |
+
text description.\n")
|
| 488 |
+
un_x = un_x + un_x_text + kwargs['tokenizer'].encode("<|im_end|>\n<|im_start|>assistant\n")
|
| 489 |
+
|
| 490 |
+
for block_idx in range(block_num):
|
| 491 |
+
block_mask = torch.zeros([x.shape[-1]]).to(torch.bool).to(x.device)
|
| 492 |
+
block_mask[input_length + block_idx * block_size: input_length + (block_idx + 1) * block_size] = True
|
| 493 |
+
steps = int(block_mask.sum() / (x.shape[-1] - input_length) * total_steps)
|
| 494 |
+
timesteps = torch.linspace(1, eps, steps + 1, device=x.device)
|
| 495 |
+
for i in tqdm(range(steps)):
|
| 496 |
+
mask_index = (x == mask_token_id)
|
| 497 |
+
if mask_index.sum() == 0: break
|
| 498 |
+
inputs_embeds_curr = self.model.embed_tokens(x)
|
| 499 |
+
|
| 500 |
+
if inputs_embeds is not None:
|
| 501 |
+
inputs_embeds_curr[:, :inputs_embeds.shape[1]] = inputs_embeds
|
| 502 |
+
|
| 503 |
+
if cfg > 0:
|
| 504 |
+
input_un_x = torch.tensor(un_x).unsqueeze(0).to(x.dtype).to(x.device)
|
| 505 |
+
input_un_x = torch.cat([input_un_x, x[:, input_ids.shape[1]:]], dim=1)
|
| 506 |
+
un_inpus_embeds = self.model.embed_tokens(input_un_x)
|
| 507 |
+
|
| 508 |
+
attention_mask_cond = torch.ones([1, inputs_embeds_curr.shape[1], inputs_embeds_curr.shape[1]])
|
| 509 |
+
attention_mask_cond = attention_mask_cond.to(torch.bool).to(inputs_embeds_curr.device)
|
| 510 |
+
attention_mask_uncond = torch.zeros([1, inputs_embeds_curr.shape[1], inputs_embeds_curr.shape[1]])
|
| 511 |
+
attention_mask_uncond[:, :un_inpus_embeds.shape[1], :un_inpus_embeds.shape[1]] = 1
|
| 512 |
+
attention_mask_uncond = attention_mask_uncond.to(torch.bool).to(inputs_embeds.device)
|
| 513 |
+
attention_mask = torch.cat([attention_mask_cond, attention_mask_uncond])
|
| 514 |
+
attention_mask = attention_mask.unsqueeze(1)
|
| 515 |
+
|
| 516 |
+
if inputs_embeds_curr.shape[1] != un_inpus_embeds.shape[1]:
|
| 517 |
+
un_inpus_embeds = torch.cat([un_inpus_embeds,
|
| 518 |
+
torch.zeros_like(inputs_embeds_curr[:, :inputs_embeds_curr.shape[1] -
|
| 519 |
+
un_inpus_embeds.shape[1], :])], dim=1)
|
| 520 |
+
input_inputs_embeds_curr = torch.cat([inputs_embeds_curr, un_inpus_embeds])
|
| 521 |
+
|
| 522 |
+
model_logits = self.forward_dream(None, attention_mask, tok_idx,
|
| 523 |
+
inputs_embeds=input_inputs_embeds_curr).logits
|
| 524 |
+
logits = model_logits[:1]; un_logits = model_logits[1:]
|
| 525 |
+
logits = un_logits + (cfg + 1) * (logits - un_logits)
|
| 526 |
+
logits = torch.cat([logits[:, :1], logits[:, :-1]], dim=1)
|
| 527 |
+
|
| 528 |
+
else:
|
| 529 |
+
logits = self.forward_dream(None, attention_mask, tok_idx,
|
| 530 |
+
inputs_embeds=inputs_embeds_curr).logits
|
| 531 |
+
logits = torch.cat([logits[:, :1], logits[:, :-1]], dim=1)
|
| 532 |
+
|
| 533 |
+
logits = generation_logits_hook_func(i, x, logits)
|
| 534 |
+
|
| 535 |
+
mask_logits = logits[mask_index]
|
| 536 |
+
if i == 0:
|
| 537 |
+
input_index = torch.where(mask_index[0]==True)[0][0]
|
| 538 |
+
|
| 539 |
+
t = timesteps[i]
|
| 540 |
+
s = timesteps[i + 1]
|
| 541 |
+
|
| 542 |
+
if alg == 'origin':
|
| 543 |
+
p_transfer = 1 - s / t if i < steps - 1 else 1
|
| 544 |
+
x0 = torch.zeros_like(x[mask_index], device=self.device, dtype=torch.long) + mask_token_id
|
| 545 |
+
transfer_index_t_s = torch.rand(*x0.shape, device=self.device) < p_transfer
|
| 546 |
+
_, x0[transfer_index_t_s] = sample_tokens(
|
| 547 |
+
mask_logits[transfer_index_t_s],
|
| 548 |
+
temperature=temperature,
|
| 549 |
+
top_p=top_p,
|
| 550 |
+
top_k=top_k,
|
| 551 |
+
max_position_penalty=max_position_penalty,
|
| 552 |
+
)
|
| 553 |
+
x[mask_index] = x0.clone()
|
| 554 |
+
|
| 555 |
+
else:
|
| 556 |
+
if alg == 'maskgit_plus':
|
| 557 |
+
confidence, x0 = sample_tokens(mask_logits, temperature=temperature, top_p=top_p, top_k=top_k,
|
| 558 |
+
max_position_penalty=max_position_penalty)
|
| 559 |
+
elif alg == 'topk_margin':
|
| 560 |
+
confidence, x0 = sample_tokens(
|
| 561 |
+
mask_logits,
|
| 562 |
+
temperature=temperature,
|
| 563 |
+
top_p=top_p,
|
| 564 |
+
top_k=top_k,
|
| 565 |
+
margin_confidence=True,
|
| 566 |
+
max_position_penalty=max_position_penalty,
|
| 567 |
+
)
|
| 568 |
+
elif alg == 'entropy':
|
| 569 |
+
confidence, x0 = sample_tokens(
|
| 570 |
+
mask_logits,
|
| 571 |
+
temperature,
|
| 572 |
+
top_p=top_p,
|
| 573 |
+
top_k=top_k,
|
| 574 |
+
neg_entropy=True,
|
| 575 |
+
max_position_penalty=max_position_penalty,
|
| 576 |
+
)
|
| 577 |
+
elif alg == "entropy-penalty":
|
| 578 |
+
confidence, x0 = sample_tokens(
|
| 579 |
+
mask_logits,
|
| 580 |
+
temperature,
|
| 581 |
+
top_p=top_p,
|
| 582 |
+
top_k=top_k,
|
| 583 |
+
neg_entropy=True,
|
| 584 |
+
repeat_penalty=repeat_penalty if len(histories) != 0 else 1.0,
|
| 585 |
+
past_x=histories[-1] if len(histories) != 0 else [],
|
| 586 |
+
mask_id=mask_token_id,
|
| 587 |
+
max_position_penalty=max_position_penalty,
|
| 588 |
+
)
|
| 589 |
+
else:
|
| 590 |
+
raise RuntimeError(f"Unknown alg: {alg}")
|
| 591 |
+
|
| 592 |
+
block_mask_1 = block_mask[mask_index[0]]
|
| 593 |
+
confidence = confidence + torch.where(block_mask_1, 0, -torch.inf).to(confidence.device)
|
| 594 |
+
|
| 595 |
+
num_mask_token = mask_index.sum()
|
| 596 |
+
num_mask_token = (x[:, block_mask] == mask_token_id).sum()
|
| 597 |
+
number_transfer_tokens = int(num_mask_token * (1 - s / t)) if i < steps - 1 else num_mask_token
|
| 598 |
+
number_transfer_tokens = max(number_transfer_tokens, 1)
|
| 599 |
+
|
| 600 |
+
if number_transfer_tokens > 0:
|
| 601 |
+
if alg_temp is None or alg_temp == 0:
|
| 602 |
+
_, transfer_index = torch.topk(confidence, number_transfer_tokens)
|
| 603 |
+
else:
|
| 604 |
+
confidence = confidence / alg_temp
|
| 605 |
+
confidence = F.softmax(confidence, dim=-1)
|
| 606 |
+
transfer_index = torch.multinomial(confidence, num_samples=number_transfer_tokens)
|
| 607 |
+
|
| 608 |
+
x0_ = torch.zeros_like(x0, device=self.device, dtype=torch.long) + mask_token_id
|
| 609 |
+
x0_[transfer_index] = x0[transfer_index].clone()
|
| 610 |
+
x[mask_index] = x0_
|
| 611 |
+
|
| 612 |
+
logit,indic = torch.max(torch.softmax(logits.clone(),dim=-1),-1)
|
| 613 |
+
logit = logit[0][x[0]!=0]
|
| 614 |
+
indic = indic[0][x[0]!=0]
|
| 615 |
+
temp_X = x[0][x[0]!=0]
|
| 616 |
+
|
| 617 |
+
x = generation_tokens_hook_func(i, x, logits)
|
| 618 |
+
|
| 619 |
+
if histories is not None:
|
| 620 |
+
histories.append(x.clone())
|
| 621 |
+
all_logit.append(torch.max(logits.clone(),-1)[-1])
|
| 622 |
+
|
| 623 |
+
return (x, histories)
|
omni_diffusion/models/dream/modeling_dream.py
ADDED
|
@@ -0,0 +1,1136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# coding=utf-8
|
| 2 |
+
# Copyright 2024 The Dream team, HKUNLP Group and the HuggingFace Inc. team. All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
|
| 5 |
+
# and OPT and Qwen implementations in this library. It has been modified from its
|
| 6 |
+
# original forms to accommodate minor architectural differences compared
|
| 7 |
+
# to GPT-NeoX and OPT and Qwen used by the Meta AI and Qwen team that trained the model.
|
| 8 |
+
#
|
| 9 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 10 |
+
# you may not use this file except in compliance with the License.
|
| 11 |
+
# You may obtain a copy of the License at
|
| 12 |
+
#
|
| 13 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 14 |
+
#
|
| 15 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 16 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 17 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 18 |
+
# See the License for the specific language governing permissions and
|
| 19 |
+
# limitations under the License.
|
| 20 |
+
"""PyTorch Dream model."""
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
from .modeling_sensevoice import AudioEncoder
|
| 25 |
+
from .resampler_projector import ResamplerProjector
|
| 26 |
+
import random
|
| 27 |
+
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
|
| 28 |
+
|
| 29 |
+
import math
|
| 30 |
+
from typing import List, Optional, Tuple, Union
|
| 31 |
+
import os
|
| 32 |
+
import torch
|
| 33 |
+
import torch.utils.checkpoint
|
| 34 |
+
from torch import nn
|
| 35 |
+
|
| 36 |
+
from transformers.activations import ACT2FN
|
| 37 |
+
from transformers.cache_utils import Cache, DynamicCache
|
| 38 |
+
from transformers.modeling_outputs import (
|
| 39 |
+
# BaseModelOutput,
|
| 40 |
+
MaskedLMOutput,
|
| 41 |
+
ModelOutput
|
| 42 |
+
)
|
| 43 |
+
from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
|
| 44 |
+
from transformers.modeling_utils import PreTrainedModel
|
| 45 |
+
from transformers.utils import (
|
| 46 |
+
add_start_docstrings,
|
| 47 |
+
add_start_docstrings_to_model_forward,
|
| 48 |
+
is_flash_attn_2_available,
|
| 49 |
+
is_flash_attn_greater_or_equal_2_10,
|
| 50 |
+
logging,
|
| 51 |
+
)
|
| 52 |
+
from transformers import PretrainedConfig
|
| 53 |
+
from .configuration_dream import DreamConfig
|
| 54 |
+
from .generation_utils import DreamGenerationMixin, DreamGenerationConfig
|
| 55 |
+
from dataclasses import dataclass
|
| 56 |
+
from typing import Any
|
| 57 |
+
if is_flash_attn_2_available():
|
| 58 |
+
from transformers.modeling_flash_attention_utils import _flash_attention_forward
|
| 59 |
+
from transformers.modeling_outputs import CausalLMOutputWithPast
|
| 60 |
+
|
| 61 |
+
from .modeling_sensevoice import AudioEncoder
|
| 62 |
+
from .resampler_projector import ResamplerProjector
|
| 63 |
+
|
| 64 |
+
import torch
|
| 65 |
+
import random
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
logger = logging.get_logger(__name__)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
@dataclass
|
| 72 |
+
class MaskedLMOutput(ModelOutput):
|
| 73 |
+
"""
|
| 74 |
+
Base class for masked language models outputs.
|
| 75 |
+
|
| 76 |
+
Args:
|
| 77 |
+
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
|
| 78 |
+
Masked language modeling (MLM) loss.
|
| 79 |
+
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
|
| 80 |
+
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
|
| 81 |
+
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed
|
| 82 |
+
or when `config.output_hidden_states=True`):
|
| 83 |
+
Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has
|
| 84 |
+
an embedding layer, + one for the output of each layer) of shape `(batch_size,
|
| 85 |
+
sequence_length, hidden_size)`.
|
| 86 |
+
|
| 87 |
+
Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
|
| 88 |
+
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed
|
| 89 |
+
or when `config.output_attentions=True`):
|
| 90 |
+
Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
|
| 91 |
+
sequence_length)`.
|
| 92 |
+
|
| 93 |
+
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
|
| 94 |
+
heads.
|
| 95 |
+
"""
|
| 96 |
+
|
| 97 |
+
loss: Optional[torch.FloatTensor] = None
|
| 98 |
+
logits: torch.FloatTensor = None
|
| 99 |
+
hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
|
| 100 |
+
attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
|
| 101 |
+
past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
|
| 102 |
+
|
| 103 |
+
_CHECKPOINT_FOR_DOC = "Dream-7B"
|
| 104 |
+
_CONFIG_FOR_DOC = "DreamConfig"
|
| 105 |
+
import os
|
| 106 |
+
ENFORCE_NUM_ITEMIN_BATCH = os.environ.get("ENFORCE_NUM_ITEMIN_BATCH", False)
|
| 107 |
+
|
| 108 |
+
@dataclass
|
| 109 |
+
class BaseModelOutput(ModelOutput):
|
| 110 |
+
"""
|
| 111 |
+
Base class for model's outputs, with potential hidden states and attentions.
|
| 112 |
+
|
| 113 |
+
Args:
|
| 114 |
+
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
|
| 115 |
+
Sequence of hidden-states at the output of the last layer of the model.
|
| 116 |
+
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed
|
| 117 |
+
or when `config.output_hidden_states=True`):
|
| 118 |
+
Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
|
| 119 |
+
one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
|
| 120 |
+
|
| 121 |
+
Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
|
| 122 |
+
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed
|
| 123 |
+
or when `config.output_attentions=True`):
|
| 124 |
+
Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
|
| 125 |
+
sequence_length)`.
|
| 126 |
+
|
| 127 |
+
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
|
| 128 |
+
heads.
|
| 129 |
+
"""
|
| 130 |
+
|
| 131 |
+
last_hidden_state: torch.FloatTensor = None
|
| 132 |
+
hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
|
| 133 |
+
attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
|
| 134 |
+
past_key_values: Optional[Cache] = None
|
| 135 |
+
# Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->Dream
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
class DreamRMSNorm(nn.Module):
|
| 139 |
+
def __init__(self, hidden_size, eps=1e-6):
|
| 140 |
+
"""
|
| 141 |
+
DreamRMSNorm is equivalent to T5LayerNorm
|
| 142 |
+
"""
|
| 143 |
+
super().__init__()
|
| 144 |
+
self.weight = nn.Parameter(torch.ones(hidden_size))
|
| 145 |
+
self.variance_epsilon = eps
|
| 146 |
+
|
| 147 |
+
def forward(self, hidden_states):
|
| 148 |
+
input_dtype = hidden_states.dtype
|
| 149 |
+
hidden_states = hidden_states.to(torch.float32)
|
| 150 |
+
variance = hidden_states.pow(2).mean(-1, keepdim=True)
|
| 151 |
+
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
|
| 152 |
+
return self.weight * hidden_states.to(input_dtype)
|
| 153 |
+
|
| 154 |
+
def extra_repr(self):
|
| 155 |
+
return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
# Copied from transformers.models.llama.modeling_llama.LlamaRotaryEmbedding with Llama->Dream
|
| 159 |
+
class DreamRotaryEmbedding(nn.Module):
|
| 160 |
+
def __init__(
|
| 161 |
+
self,
|
| 162 |
+
dim=None,
|
| 163 |
+
max_position_embeddings=2048,
|
| 164 |
+
base=10000,
|
| 165 |
+
device=None,
|
| 166 |
+
scaling_factor=1.0,
|
| 167 |
+
rope_type="default",
|
| 168 |
+
config: Optional[DreamConfig] = None,
|
| 169 |
+
):
|
| 170 |
+
super().__init__()
|
| 171 |
+
# TODO (joao): remove the `if` below, only used for BC
|
| 172 |
+
self.rope_kwargs = {}
|
| 173 |
+
if config is None:
|
| 174 |
+
logger.warning_once(
|
| 175 |
+
"`DreamRotaryEmbedding` can now be fully parameterized by passing the model config through the "
|
| 176 |
+
"`config` argument. All other arguments will be removed in v4.46"
|
| 177 |
+
)
|
| 178 |
+
self.rope_kwargs = {
|
| 179 |
+
"rope_type": rope_type,
|
| 180 |
+
"factor": scaling_factor,
|
| 181 |
+
"dim": dim,
|
| 182 |
+
"base": base,
|
| 183 |
+
"max_position_embeddings": max_position_embeddings,
|
| 184 |
+
}
|
| 185 |
+
self.rope_type = rope_type
|
| 186 |
+
self.max_seq_len_cached = max_position_embeddings
|
| 187 |
+
self.original_max_seq_len = max_position_embeddings
|
| 188 |
+
else:
|
| 189 |
+
# BC: "rope_type" was originally "type"
|
| 190 |
+
if config.rope_scaling is not None:
|
| 191 |
+
self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type"))
|
| 192 |
+
else:
|
| 193 |
+
self.rope_type = "default"
|
| 194 |
+
self.max_seq_len_cached = config.max_position_embeddings
|
| 195 |
+
self.original_max_seq_len = config.max_position_embeddings
|
| 196 |
+
|
| 197 |
+
self.config = config
|
| 198 |
+
self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
|
| 199 |
+
|
| 200 |
+
inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device, **self.rope_kwargs)
|
| 201 |
+
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
| 202 |
+
self.original_inv_freq = self.inv_freq
|
| 203 |
+
|
| 204 |
+
def reset_parameters(self):
|
| 205 |
+
inv_freq, self.attention_scaling = self.rope_init_fn(self.config, self.inv_freq.device, **self.rope_kwargs)
|
| 206 |
+
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
| 207 |
+
self.original_inv_freq = self.inv_freq
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def _dynamic_frequency_update(self, position_ids, device):
|
| 211 |
+
"""
|
| 212 |
+
dynamic RoPE layers should recompute `inv_freq` in the following situations:
|
| 213 |
+
1 - growing beyond the cached sequence length (allow scaling)
|
| 214 |
+
2 - the current sequence length is in the original scale (avoid losing precision with small sequences)
|
| 215 |
+
"""
|
| 216 |
+
seq_len = torch.max(position_ids) + 1
|
| 217 |
+
if seq_len > self.max_seq_len_cached: # growth
|
| 218 |
+
inv_freq, self.attention_scaling = self.rope_init_fn(
|
| 219 |
+
self.config, device, seq_len=seq_len, **self.rope_kwargs
|
| 220 |
+
)
|
| 221 |
+
self.register_buffer("inv_freq", inv_freq, persistent=False) # TODO joao: may break with compilation
|
| 222 |
+
self.max_seq_len_cached = seq_len
|
| 223 |
+
|
| 224 |
+
if seq_len < self.original_max_seq_len and self.max_seq_len_cached > self.original_max_seq_len: # reset
|
| 225 |
+
self.register_buffer("inv_freq", self.original_inv_freq, persistent=False)
|
| 226 |
+
self.max_seq_len_cached = self.original_max_seq_len
|
| 227 |
+
|
| 228 |
+
@torch.no_grad()
|
| 229 |
+
def forward(self, x, position_ids):
|
| 230 |
+
if "dynamic" in self.rope_type:
|
| 231 |
+
self._dynamic_frequency_update(position_ids, device=x.device)
|
| 232 |
+
|
| 233 |
+
# Core RoPE block
|
| 234 |
+
inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
|
| 235 |
+
position_ids_expanded = position_ids[:, None, :].float()
|
| 236 |
+
# Force float32 (see https://github.com/huggingface/transformers/pull/29285)
|
| 237 |
+
device_type = x.device.type
|
| 238 |
+
device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"
|
| 239 |
+
with torch.autocast(device_type=device_type, enabled=False):
|
| 240 |
+
freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
|
| 241 |
+
emb = torch.cat((freqs, freqs), dim=-1)
|
| 242 |
+
cos = emb.cos()
|
| 243 |
+
sin = emb.sin()
|
| 244 |
+
|
| 245 |
+
# Advanced RoPE types (e.g. yarn) apply a post-processing scaling factor, equivalent to scaling attention
|
| 246 |
+
cos = cos * self.attention_scaling
|
| 247 |
+
sin = sin * self.attention_scaling
|
| 248 |
+
|
| 249 |
+
return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
# Copied from transformers.models.llama.modeling_llama.rotate_half
|
| 253 |
+
def rotate_half(x):
|
| 254 |
+
"""Rotates half the hidden dims of the input."""
|
| 255 |
+
x1 = x[..., : x.shape[-1] // 2]
|
| 256 |
+
x2 = x[..., x.shape[-1] // 2 :]
|
| 257 |
+
return torch.cat((-x2, x1), dim=-1)
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
# Copied from transformers.models.llama.modeling_llama.apply_rotary_pos_emb
|
| 261 |
+
def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
|
| 262 |
+
"""Applies Rotary Position Embedding to the query and key tensors.
|
| 263 |
+
|
| 264 |
+
Args:
|
| 265 |
+
q (`torch.Tensor`): The query tensor.
|
| 266 |
+
k (`torch.Tensor`): The key tensor.
|
| 267 |
+
cos (`torch.Tensor`): The cosine part of the rotary embedding.
|
| 268 |
+
sin (`torch.Tensor`): The sine part of the rotary embedding.
|
| 269 |
+
position_ids (`torch.Tensor`, *optional*):
|
| 270 |
+
Deprecated and unused.
|
| 271 |
+
unsqueeze_dim (`int`, *optional*, defaults to 1):
|
| 272 |
+
The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
|
| 273 |
+
sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
|
| 274 |
+
that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
|
| 275 |
+
k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
|
| 276 |
+
cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
|
| 277 |
+
the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
|
| 278 |
+
Returns:
|
| 279 |
+
`tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
|
| 280 |
+
"""
|
| 281 |
+
cos = cos.unsqueeze(unsqueeze_dim)
|
| 282 |
+
sin = sin.unsqueeze(unsqueeze_dim)
|
| 283 |
+
q_embed = (q * cos) + (rotate_half(q) * sin)
|
| 284 |
+
k_embed = (k * cos) + (rotate_half(k) * sin)
|
| 285 |
+
return q_embed, k_embed
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
# Copied from transformers.models.mistral.modeling_mistral.MistralMLP with Mistral->Dream
|
| 289 |
+
class DreamMLP(nn.Module):
|
| 290 |
+
def __init__(self, config):
|
| 291 |
+
super().__init__()
|
| 292 |
+
self.hidden_size = config.hidden_size
|
| 293 |
+
self.intermediate_size = config.intermediate_size
|
| 294 |
+
self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
|
| 295 |
+
self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
|
| 296 |
+
self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
|
| 297 |
+
self.act_fn = ACT2FN[config.hidden_act]
|
| 298 |
+
|
| 299 |
+
def forward(self, hidden_state):
|
| 300 |
+
return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state))
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
# Copied from transformers.models.llama.modeling_llama.repeat_kv
|
| 304 |
+
def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
|
| 305 |
+
"""
|
| 306 |
+
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
|
| 307 |
+
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
|
| 308 |
+
"""
|
| 309 |
+
batch, num_key_value_heads, slen, head_dim = hidden_states.shape
|
| 310 |
+
if n_rep == 1:
|
| 311 |
+
return hidden_states
|
| 312 |
+
hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
|
| 313 |
+
return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
class DreamAttention(nn.Module):
|
| 317 |
+
"""
|
| 318 |
+
Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer
|
| 319 |
+
and "Generating Long Sequences with Sparse Transformers".
|
| 320 |
+
"""
|
| 321 |
+
|
| 322 |
+
def __init__(self, config: DreamConfig, layer_idx: Optional[int] = None):
|
| 323 |
+
super().__init__()
|
| 324 |
+
self.config = config
|
| 325 |
+
self.layer_idx = layer_idx
|
| 326 |
+
if layer_idx is None:
|
| 327 |
+
logger.warning_once(
|
| 328 |
+
f"Instantiating {self.__class__.__name__} without passing `layer_idx` is not recommended and will "
|
| 329 |
+
"to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` "
|
| 330 |
+
"when creating this class."
|
| 331 |
+
)
|
| 332 |
+
|
| 333 |
+
self.hidden_size = config.hidden_size
|
| 334 |
+
self.num_heads = config.num_attention_heads
|
| 335 |
+
self.head_dim = self.hidden_size // self.num_heads
|
| 336 |
+
self.num_key_value_heads = config.num_key_value_heads
|
| 337 |
+
self.num_key_value_groups = self.num_heads // self.num_key_value_heads
|
| 338 |
+
self.max_position_embeddings = config.max_position_embeddings
|
| 339 |
+
self.rope_theta = config.rope_theta
|
| 340 |
+
self.is_causal = False
|
| 341 |
+
self.attention_dropout = config.attention_dropout
|
| 342 |
+
|
| 343 |
+
if (self.head_dim * self.num_heads) != self.hidden_size:
|
| 344 |
+
raise ValueError(
|
| 345 |
+
f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
|
| 346 |
+
f" and `num_heads`: {self.num_heads})."
|
| 347 |
+
)
|
| 348 |
+
self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=True)
|
| 349 |
+
self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True)
|
| 350 |
+
self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True)
|
| 351 |
+
self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
|
| 352 |
+
|
| 353 |
+
self.rotary_emb = DreamRotaryEmbedding(config=self.config)
|
| 354 |
+
|
| 355 |
+
def forward(
|
| 356 |
+
self,
|
| 357 |
+
hidden_states: torch.Tensor,
|
| 358 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 359 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 360 |
+
past_key_value: Optional[Cache] = None,
|
| 361 |
+
output_attentions: bool = False,
|
| 362 |
+
use_cache: bool = False,
|
| 363 |
+
cache_position: Optional[torch.LongTensor] = None,
|
| 364 |
+
position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46
|
| 365 |
+
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
|
| 366 |
+
bsz, q_len, _ = hidden_states.size()
|
| 367 |
+
|
| 368 |
+
query_states = self.q_proj(hidden_states)
|
| 369 |
+
key_states = self.k_proj(hidden_states)
|
| 370 |
+
value_states = self.v_proj(hidden_states)
|
| 371 |
+
|
| 372 |
+
query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
| 373 |
+
key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
|
| 374 |
+
value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
|
| 375 |
+
|
| 376 |
+
if position_embeddings is None:
|
| 377 |
+
logger.warning_once(
|
| 378 |
+
"The attention layers in this model are transitioning from computing the RoPE embeddings internally "
|
| 379 |
+
"through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed "
|
| 380 |
+
"`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be "
|
| 381 |
+
"removed and `position_embeddings` will be mandatory."
|
| 382 |
+
)
|
| 383 |
+
cos, sin = self.rotary_emb(value_states, position_ids)
|
| 384 |
+
else:
|
| 385 |
+
cos, sin = position_embeddings
|
| 386 |
+
query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
|
| 387 |
+
|
| 388 |
+
if past_key_value is not None:
|
| 389 |
+
cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models
|
| 390 |
+
key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
|
| 391 |
+
|
| 392 |
+
# repeat k/v heads if n_kv_heads < n_heads
|
| 393 |
+
key_states = repeat_kv(key_states, self.num_key_value_groups)
|
| 394 |
+
value_states = repeat_kv(value_states, self.num_key_value_groups)
|
| 395 |
+
|
| 396 |
+
attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
|
| 397 |
+
if attention_mask is not None: # no matter the length, we just slice it
|
| 398 |
+
causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
|
| 399 |
+
attn_weights = attn_weights + causal_mask
|
| 400 |
+
|
| 401 |
+
# upcast attention to fp32
|
| 402 |
+
attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
|
| 403 |
+
attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training)
|
| 404 |
+
attn_output = torch.matmul(attn_weights, value_states)
|
| 405 |
+
|
| 406 |
+
if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
|
| 407 |
+
raise ValueError(
|
| 408 |
+
f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
|
| 409 |
+
f" {attn_output.size()}"
|
| 410 |
+
)
|
| 411 |
+
|
| 412 |
+
attn_output = attn_output.transpose(1, 2).contiguous()
|
| 413 |
+
attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
|
| 414 |
+
|
| 415 |
+
attn_output = self.o_proj(attn_output)
|
| 416 |
+
|
| 417 |
+
if not output_attentions:
|
| 418 |
+
attn_weights = None
|
| 419 |
+
|
| 420 |
+
return attn_output, attn_weights, past_key_value
|
| 421 |
+
|
| 422 |
+
|
| 423 |
+
class DreamSdpaAttention(DreamAttention):
|
| 424 |
+
"""
|
| 425 |
+
Dream attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from
|
| 426 |
+
`DreamAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to
|
| 427 |
+
SDPA API.
|
| 428 |
+
"""
|
| 429 |
+
|
| 430 |
+
# Adapted from DreamAttention.forward
|
| 431 |
+
def forward(
|
| 432 |
+
self,
|
| 433 |
+
hidden_states: torch.Tensor,
|
| 434 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 435 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 436 |
+
past_key_value: Optional[Cache] = None,
|
| 437 |
+
use_cache: bool = False,
|
| 438 |
+
output_attentions: bool = False,
|
| 439 |
+
cache_position: Optional[torch.LongTensor] = None,
|
| 440 |
+
position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46
|
| 441 |
+
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
|
| 442 |
+
if output_attentions:
|
| 443 |
+
# TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"`
|
| 444 |
+
# once this is implemented.
|
| 445 |
+
logger.warning_once(
|
| 446 |
+
"DreamModel is using DreamSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` " \
|
| 447 |
+
" does not support" \
|
| 448 |
+
" `output_attentions=True`. Falling back to the manual attention implementation, "
|
| 449 |
+
'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. ' \
|
| 450 |
+
'This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'
|
| 451 |
+
)
|
| 452 |
+
return super().forward(
|
| 453 |
+
hidden_states=hidden_states,
|
| 454 |
+
attention_mask=attention_mask,
|
| 455 |
+
position_ids=position_ids,
|
| 456 |
+
past_key_value=past_key_value,
|
| 457 |
+
output_attentions=output_attentions,
|
| 458 |
+
use_cache=use_cache,
|
| 459 |
+
)
|
| 460 |
+
bsz, q_len, _ = hidden_states.size()
|
| 461 |
+
|
| 462 |
+
query_states = self.q_proj(hidden_states)
|
| 463 |
+
key_states = self.k_proj(hidden_states)
|
| 464 |
+
value_states = self.v_proj(hidden_states)
|
| 465 |
+
|
| 466 |
+
query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
| 467 |
+
key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
|
| 468 |
+
value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
|
| 469 |
+
|
| 470 |
+
if position_embeddings is None:
|
| 471 |
+
logger.warning_once(
|
| 472 |
+
"The attention layers in this model are transitioning from computing the RoPE embeddings internally "
|
| 473 |
+
"through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed "
|
| 474 |
+
"`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be "
|
| 475 |
+
"removed and `position_embeddings` will be mandatory."
|
| 476 |
+
)
|
| 477 |
+
cos, sin = self.rotary_emb(value_states, position_ids)
|
| 478 |
+
else:
|
| 479 |
+
cos, sin = position_embeddings
|
| 480 |
+
query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
|
| 481 |
+
|
| 482 |
+
if past_key_value is not None:
|
| 483 |
+
cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models
|
| 484 |
+
key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
|
| 485 |
+
|
| 486 |
+
key_states = repeat_kv(key_states, self.num_key_value_groups)
|
| 487 |
+
value_states = repeat_kv(value_states, self.num_key_value_groups)
|
| 488 |
+
|
| 489 |
+
# SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs
|
| 490 |
+
# with custom attn_mask,
|
| 491 |
+
# Reference: https://github.com/pytorch/pytorch/issues/112577.
|
| 492 |
+
if query_states.device.type == "cuda" and attention_mask is not None:
|
| 493 |
+
query_states = query_states.contiguous()
|
| 494 |
+
key_states = key_states.contiguous()
|
| 495 |
+
value_states = value_states.contiguous()
|
| 496 |
+
|
| 497 |
+
# We dispatch to SDPA's Flash Attention or Efficient kernels via this `is_causal` if statement
|
| 498 |
+
# instead of an inline conditional assignment in SDPA to support both torch.compile's dynamic shapes
|
| 499 |
+
# and full graph options. An inline conditional prevents dynamic shapes from compiling.
|
| 500 |
+
# The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create
|
| 501 |
+
# a causal mask in case q_len == 1.
|
| 502 |
+
|
| 503 |
+
if attention_mask == None:
|
| 504 |
+
attention_mask = torch.ones([1, 1,
|
| 505 |
+
query_states.shape[2], query_states.shape[2]]).to(torch.bool).to(query_states.device)
|
| 506 |
+
bool_mask = attention_mask.to(torch.bool)
|
| 507 |
+
|
| 508 |
+
attn_output = torch.nn.functional.scaled_dot_product_attention(
|
| 509 |
+
query_states,
|
| 510 |
+
key_states,
|
| 511 |
+
value_states,
|
| 512 |
+
attn_mask=bool_mask ,
|
| 513 |
+
dropout_p=self.attention_dropout if self.training else 0.0,
|
| 514 |
+
is_causal=False, # hard coded
|
| 515 |
+
)
|
| 516 |
+
|
| 517 |
+
attn_output = attn_output.transpose(1, 2).contiguous()
|
| 518 |
+
attn_output = attn_output.view(bsz, q_len, self.hidden_size)
|
| 519 |
+
|
| 520 |
+
attn_output = self.o_proj(attn_output)
|
| 521 |
+
|
| 522 |
+
return attn_output, None, past_key_value
|
| 523 |
+
|
| 524 |
+
class DreamDecoderLayer(nn.Module):
|
| 525 |
+
def __init__(self, config: DreamConfig, layer_idx: int):
|
| 526 |
+
super().__init__()
|
| 527 |
+
self.hidden_size = config.hidden_size
|
| 528 |
+
|
| 529 |
+
if config.sliding_window and config._attn_implementation != "flash_attention_2":
|
| 530 |
+
logger.warning_once(
|
| 531 |
+
f"Sliding Window Attention is enabled but not implemented for "
|
| 532 |
+
f"`{config._attn_implementation}`; "
|
| 533 |
+
"unexpected results may be encountered."
|
| 534 |
+
)
|
| 535 |
+
|
| 536 |
+
# self.self_attn = Dream_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx)
|
| 537 |
+
self.self_attn = DreamSdpaAttention(config, layer_idx)
|
| 538 |
+
|
| 539 |
+
self.mlp = DreamMLP(config)
|
| 540 |
+
self.input_layernorm = DreamRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 541 |
+
self.post_attention_layernorm = DreamRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 542 |
+
|
| 543 |
+
# @torch.compile
|
| 544 |
+
def forward(
|
| 545 |
+
self,
|
| 546 |
+
hidden_states: torch.Tensor,
|
| 547 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 548 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 549 |
+
past_key_value: Optional[Tuple[torch.Tensor]] = None,
|
| 550 |
+
output_attentions: Optional[bool] = False,
|
| 551 |
+
use_cache: Optional[bool] = False,
|
| 552 |
+
cache_position: Optional[torch.LongTensor] = None,
|
| 553 |
+
position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
| 554 |
+
**kwargs,
|
| 555 |
+
) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
|
| 556 |
+
"""
|
| 557 |
+
Args:
|
| 558 |
+
hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
|
| 559 |
+
attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
|
| 560 |
+
`(batch, sequence_length)` where padding elements are indicated by 0.
|
| 561 |
+
output_attentions (`bool`, *optional*):
|
| 562 |
+
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
|
| 563 |
+
returned tensors for more detail.
|
| 564 |
+
use_cache (`bool`, *optional*):
|
| 565 |
+
If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
|
| 566 |
+
(see `past_key_values`).
|
| 567 |
+
past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
|
| 568 |
+
cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
|
| 569 |
+
Indices depicting the position of the input sequence tokens in the sequence.
|
| 570 |
+
position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
|
| 571 |
+
Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
|
| 572 |
+
with `head_dim` being the embedding dimension of each attention head.
|
| 573 |
+
kwargs (`dict`, *optional*):
|
| 574 |
+
Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
|
| 575 |
+
into the model
|
| 576 |
+
"""
|
| 577 |
+
|
| 578 |
+
residual = hidden_states
|
| 579 |
+
|
| 580 |
+
hidden_states = self.input_layernorm(hidden_states)
|
| 581 |
+
|
| 582 |
+
hidden_states, self_attn_weights, present_key_value = self.self_attn(
|
| 583 |
+
hidden_states=hidden_states,
|
| 584 |
+
attention_mask=attention_mask,
|
| 585 |
+
position_ids=position_ids,
|
| 586 |
+
past_key_value=past_key_value,
|
| 587 |
+
output_attentions=output_attentions,
|
| 588 |
+
use_cache=use_cache,
|
| 589 |
+
cache_position=cache_position,
|
| 590 |
+
position_embeddings=position_embeddings,
|
| 591 |
+
)
|
| 592 |
+
hidden_states = residual + hidden_states
|
| 593 |
+
|
| 594 |
+
# Fully Connected
|
| 595 |
+
residual = hidden_states
|
| 596 |
+
hidden_states = self.post_attention_layernorm(hidden_states)
|
| 597 |
+
hidden_states = self.mlp(hidden_states)
|
| 598 |
+
hidden_states = residual + hidden_states
|
| 599 |
+
|
| 600 |
+
outputs = (hidden_states,)
|
| 601 |
+
|
| 602 |
+
if output_attentions:
|
| 603 |
+
outputs += (self_attn_weights,)
|
| 604 |
+
|
| 605 |
+
if use_cache:
|
| 606 |
+
outputs += (present_key_value,)
|
| 607 |
+
|
| 608 |
+
return outputs
|
| 609 |
+
|
| 610 |
+
class DreamPreTrainedModel(PreTrainedModel):
|
| 611 |
+
config_class = DreamConfig
|
| 612 |
+
base_model_prefix = "model"
|
| 613 |
+
supports_gradient_checkpointing = True
|
| 614 |
+
_no_split_modules = ["DreamDecoderLayer"]
|
| 615 |
+
_skip_keys_device_placement = "past_key_values"
|
| 616 |
+
_supports_flash_attn_2 = True
|
| 617 |
+
_supports_sdpa = True
|
| 618 |
+
_supports_cache_class = True
|
| 619 |
+
_supports_quantized_cache = True
|
| 620 |
+
_supports_static_cache = True
|
| 621 |
+
|
| 622 |
+
def _init_weights(self, module):
|
| 623 |
+
std = self.config.initializer_range
|
| 624 |
+
if isinstance(module, nn.Linear):
|
| 625 |
+
module.weight.data.normal_(mean=0.0, std=std)
|
| 626 |
+
if module.bias is not None:
|
| 627 |
+
module.bias.data.zero_()
|
| 628 |
+
elif isinstance(module, nn.Embedding):
|
| 629 |
+
module.weight.data.normal_(mean=0.0, std=std)
|
| 630 |
+
if module.padding_idx is not None:
|
| 631 |
+
module.weight.data[module.padding_idx].zero_()
|
| 632 |
+
|
| 633 |
+
@classmethod
|
| 634 |
+
def from_pretrained(
|
| 635 |
+
cls,
|
| 636 |
+
pretrained_model_name_or_path: Optional[Union[str, os.PathLike]],
|
| 637 |
+
*model_args,
|
| 638 |
+
config: Optional[Union[PretrainedConfig, str, os.PathLike]] = None,
|
| 639 |
+
cache_dir: Optional[Union[str, os.PathLike]] = None,
|
| 640 |
+
ignore_mismatched_sizes: bool = False,
|
| 641 |
+
force_download: bool = False,
|
| 642 |
+
local_files_only: bool = False,
|
| 643 |
+
token: Optional[Union[str, bool]] = None,
|
| 644 |
+
revision: str = "main",
|
| 645 |
+
use_safetensors: Optional[bool] = None,
|
| 646 |
+
weights_only: bool = True,
|
| 647 |
+
**kwargs,
|
| 648 |
+
):
|
| 649 |
+
_ = None
|
| 650 |
+
try:
|
| 651 |
+
_model,_ = super().from_pretrained(
|
| 652 |
+
pretrained_model_name_or_path,
|
| 653 |
+
*model_args,
|
| 654 |
+
config=config,
|
| 655 |
+
cache_dir=cache_dir,
|
| 656 |
+
ignore_mismatched_sizes=ignore_mismatched_sizes,
|
| 657 |
+
force_download=force_download,
|
| 658 |
+
local_files_only=local_files_only,
|
| 659 |
+
token=token,
|
| 660 |
+
revision=revision,
|
| 661 |
+
use_safetensors=use_safetensors,
|
| 662 |
+
weights_only=weights_only,
|
| 663 |
+
**kwargs,
|
| 664 |
+
)
|
| 665 |
+
except Exception as e:
|
| 666 |
+
_model = super().from_pretrained(
|
| 667 |
+
pretrained_model_name_or_path,
|
| 668 |
+
*model_args,
|
| 669 |
+
config=config,
|
| 670 |
+
cache_dir=cache_dir,
|
| 671 |
+
ignore_mismatched_sizes=ignore_mismatched_sizes,
|
| 672 |
+
force_download=force_download,
|
| 673 |
+
local_files_only=local_files_only,
|
| 674 |
+
token=token,
|
| 675 |
+
revision=revision,
|
| 676 |
+
use_safetensors=use_safetensors,
|
| 677 |
+
weights_only=weights_only,
|
| 678 |
+
**kwargs,
|
| 679 |
+
)
|
| 680 |
+
|
| 681 |
+
resume_download = kwargs.get("resume_download", None)
|
| 682 |
+
proxies = kwargs.get("proxies", None)
|
| 683 |
+
subfolder = kwargs.get("subfolder", "")
|
| 684 |
+
from_auto_class = kwargs.get("_from_auto", False)
|
| 685 |
+
from_pipeline = kwargs.get("_from_pipeline", None)
|
| 686 |
+
_model.generation_config= DreamGenerationConfig.from_pretrained(
|
| 687 |
+
pretrained_model_name_or_path,
|
| 688 |
+
cache_dir=cache_dir,
|
| 689 |
+
force_download=force_download,
|
| 690 |
+
resume_download=resume_download,
|
| 691 |
+
proxies=proxies,
|
| 692 |
+
local_files_only=local_files_only,
|
| 693 |
+
token=token,
|
| 694 |
+
revision=revision,
|
| 695 |
+
subfolder=subfolder,
|
| 696 |
+
_from_auto=from_auto_class,
|
| 697 |
+
_from_pipeline=from_pipeline,
|
| 698 |
+
)
|
| 699 |
+
if _ is not None:
|
| 700 |
+
return _model,_
|
| 701 |
+
return _model
|
| 702 |
+
|
| 703 |
+
class DreamPrefixLMCache(Cache):
|
| 704 |
+
|
| 705 |
+
def __init__(self):
|
| 706 |
+
super().__init__()
|
| 707 |
+
self.past_key_values = {}
|
| 708 |
+
# this will not be updated beyond the prefilling phase
|
| 709 |
+
|
| 710 |
+
def update(
|
| 711 |
+
self,
|
| 712 |
+
key_states: torch.Tensor,
|
| 713 |
+
value_states: torch.Tensor,
|
| 714 |
+
layer_idx: int,
|
| 715 |
+
cache_kwargs = None,
|
| 716 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 717 |
+
if layer_idx in self.past_key_values:
|
| 718 |
+
past_key, past_value = self.past_key_values[layer_idx]
|
| 719 |
+
key_states = torch.cat((past_key, key_states), dim=-2)
|
| 720 |
+
value_states = torch.cat((past_value, value_states), dim=-2)
|
| 721 |
+
return key_states,value_states
|
| 722 |
+
else:
|
| 723 |
+
self.past_key_values[layer_idx] = (key_states, value_states)
|
| 724 |
+
return key_states, value_states
|
| 725 |
+
|
| 726 |
+
def get_seq_length(self, layer_idx: Optional[int] = 0) -> int:
|
| 727 |
+
"""Returns the sequence length of the cached states. A layer index can be optionally passed."""
|
| 728 |
+
# TODO: deprecate this function in favor of `cache_position`
|
| 729 |
+
if len(self.past_key_values) == 0:
|
| 730 |
+
return 0
|
| 731 |
+
else:
|
| 732 |
+
return self.past_key_values[0][0].shape[-2]
|
| 733 |
+
|
| 734 |
+
def get_max_cache_shape(self) -> Optional[int]:
|
| 735 |
+
return None
|
| 736 |
+
|
| 737 |
+
|
| 738 |
+
import deepspeed
|
| 739 |
+
class DreamBaseModel(DreamPreTrainedModel):#
|
| 740 |
+
"""
|
| 741 |
+
Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`DreamDecoderLayer`]
|
| 742 |
+
|
| 743 |
+
Args:
|
| 744 |
+
config: DreamConfig
|
| 745 |
+
"""
|
| 746 |
+
|
| 747 |
+
def __init__(self, config: DreamConfig):
|
| 748 |
+
super().__init__(config)
|
| 749 |
+
self.padding_idx = config.pad_token_id
|
| 750 |
+
self.vocab_size = config.vocab_size
|
| 751 |
+
|
| 752 |
+
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
|
| 753 |
+
self.layers = nn.ModuleList(
|
| 754 |
+
[DreamDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
|
| 755 |
+
)
|
| 756 |
+
self._attn_implementation = config._attn_implementation
|
| 757 |
+
self.norm = DreamRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 758 |
+
self.rotary_emb = DreamRotaryEmbedding(config=config)
|
| 759 |
+
|
| 760 |
+
self.gradient_checkpointing = False
|
| 761 |
+
# Initialize weights and apply final processing
|
| 762 |
+
|
| 763 |
+
self.audio_model = AudioEncoder()
|
| 764 |
+
self.audio_projection = ResamplerProjector(512, config.hidden_size)
|
| 765 |
+
|
| 766 |
+
self.post_init()
|
| 767 |
+
|
| 768 |
+
def get_input_embeddings(self):
|
| 769 |
+
return self.embed_tokens
|
| 770 |
+
|
| 771 |
+
def set_input_embeddings(self, value):
|
| 772 |
+
self.embed_tokens = value
|
| 773 |
+
|
| 774 |
+
def forward(
|
| 775 |
+
self,
|
| 776 |
+
input_ids: torch.LongTensor = None,
|
| 777 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 778 |
+
audios: Optional[torch.FloatTensor] = None,
|
| 779 |
+
audio_indices: Optional[torch.LongTensor] = None,
|
| 780 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 781 |
+
past_key_values: Optional[List[torch.FloatTensor]] = None,
|
| 782 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
| 783 |
+
use_cache: Optional[bool] = None,
|
| 784 |
+
output_attentions: Optional[bool] = None,
|
| 785 |
+
output_hidden_states: Optional[bool] = None,
|
| 786 |
+
return_dict: Optional[bool] = None,
|
| 787 |
+
cache_position: Optional[torch.LongTensor] = None,
|
| 788 |
+
) -> Union[Tuple, BaseModelOutput]:
|
| 789 |
+
|
| 790 |
+
if (past_key_values is None or len(past_key_values) == 0) and audios is not None:
|
| 791 |
+
audio_embeds, audio_lengths = self.audio_model(audios)
|
| 792 |
+
assert audio_embeds.shape[0] == len(audios)
|
| 793 |
+
fake_audios = None
|
| 794 |
+
|
| 795 |
+
audio_embeds = self.audio_projection(audio_embeds)
|
| 796 |
+
|
| 797 |
+
elif self.training:
|
| 798 |
+
device = self.get_input_embeddings().weight.data.device
|
| 799 |
+
dtype = self.get_input_embeddings().weight.data.dtype
|
| 800 |
+
fake_audios = torch.ones((1, 1, 560), dtype=dtype, device=device)
|
| 801 |
+
audio_embeds, audio_lengths = self.audio_model(fake_audios)
|
| 802 |
+
audio_embeds = self.audio_projection(audio_embeds)
|
| 803 |
+
|
| 804 |
+
else:
|
| 805 |
+
fake_audios = None
|
| 806 |
+
audio_embeds = None
|
| 807 |
+
|
| 808 |
+
|
| 809 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 810 |
+
output_hidden_states = (
|
| 811 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 812 |
+
)
|
| 813 |
+
use_cache = use_cache if use_cache is not None else self.config.use_cache
|
| 814 |
+
|
| 815 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 816 |
+
|
| 817 |
+
if (input_ids is None) ^ (inputs_embeds is not None):
|
| 818 |
+
raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
|
| 819 |
+
|
| 820 |
+
if self.gradient_checkpointing and self.training:
|
| 821 |
+
if use_cache:
|
| 822 |
+
logger.warning_once(
|
| 823 |
+
"`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
|
| 824 |
+
)
|
| 825 |
+
use_cache = False
|
| 826 |
+
|
| 827 |
+
if inputs_embeds is None:
|
| 828 |
+
inputs_embeds = self.embed_tokens(input_ids)
|
| 829 |
+
|
| 830 |
+
|
| 831 |
+
|
| 832 |
+
if fake_audios is not None:
|
| 833 |
+
inputs_embeds = inputs_embeds + audio_embeds.mean() * 0.0
|
| 834 |
+
elif audio_embeds is not None:
|
| 835 |
+
inputs_embeds = inputs_embeds.clone()
|
| 836 |
+
for audio_embeds_, audio_lengths_, audio_indices_ in zip(audio_embeds, audio_lengths, audio_indices,):
|
| 837 |
+
# print(f"{audio_embeds_.size()=} {audio_lengths_=} {audio_indices_.size()=}")
|
| 838 |
+
audio_embeds_ = audio_embeds_[:audio_lengths_, ...]
|
| 839 |
+
audio_embeds_ = audio_embeds_.to(inputs_embeds.device)
|
| 840 |
+
indices_b, indices_s = audio_indices_.to(inputs_embeds.device).unbind(dim=0)
|
| 841 |
+
inputs_embeds[indices_b.view(-1), indices_s.view(-1)] = audio_embeds_.view(-1, audio_embeds_.shape[-1])
|
| 842 |
+
|
| 843 |
+
|
| 844 |
+
if use_cache and past_key_values is None:
|
| 845 |
+
past_key_values = DreamPrefixLMCache()
|
| 846 |
+
|
| 847 |
+
if cache_position is None:
|
| 848 |
+
past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
|
| 849 |
+
cache_position = torch.arange(
|
| 850 |
+
past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
|
| 851 |
+
)
|
| 852 |
+
|
| 853 |
+
if position_ids is None:
|
| 854 |
+
position_ids = cache_position.unsqueeze(0)
|
| 855 |
+
|
| 856 |
+
hidden_states = inputs_embeds
|
| 857 |
+
|
| 858 |
+
# create position embeddings to be shared across the decoder layers
|
| 859 |
+
position_embeddings = self.rotary_emb(hidden_states, position_ids)
|
| 860 |
+
|
| 861 |
+
# decoder layers
|
| 862 |
+
all_hidden_states = () if output_hidden_states else None
|
| 863 |
+
all_self_attns = () if output_attentions else None
|
| 864 |
+
|
| 865 |
+
for decoder_layer in self.layers:
|
| 866 |
+
if output_hidden_states:
|
| 867 |
+
all_hidden_states += (hidden_states,)
|
| 868 |
+
|
| 869 |
+
if self.gradient_checkpointing and self.training:
|
| 870 |
+
layer_outputs = deepspeed.checkpointing.checkpoint(
|
| 871 |
+
decoder_layer,
|
| 872 |
+
hidden_states,
|
| 873 |
+
attention_mask,
|
| 874 |
+
position_ids,
|
| 875 |
+
past_key_values,
|
| 876 |
+
output_attentions,
|
| 877 |
+
use_cache,
|
| 878 |
+
cache_position,
|
| 879 |
+
position_embeddings,
|
| 880 |
+
)
|
| 881 |
+
else:
|
| 882 |
+
layer_outputs = decoder_layer(
|
| 883 |
+
hidden_states,
|
| 884 |
+
attention_mask=attention_mask,
|
| 885 |
+
position_ids=position_ids,
|
| 886 |
+
past_key_value=past_key_values,
|
| 887 |
+
output_attentions=output_attentions,
|
| 888 |
+
use_cache=use_cache,
|
| 889 |
+
cache_position=cache_position,
|
| 890 |
+
position_embeddings=position_embeddings,
|
| 891 |
+
)
|
| 892 |
+
|
| 893 |
+
# breakpoint()
|
| 894 |
+
if isinstance(layer_outputs,torch.Tensor):
|
| 895 |
+
layer_outputs = (layer_outputs,None)
|
| 896 |
+
hidden_states = layer_outputs[0]
|
| 897 |
+
|
| 898 |
+
if output_attentions:
|
| 899 |
+
all_self_attns += (layer_outputs[1],)
|
| 900 |
+
|
| 901 |
+
hidden_states = self.norm(hidden_states)
|
| 902 |
+
|
| 903 |
+
# add hidden states from the last decoder layer
|
| 904 |
+
if output_hidden_states:
|
| 905 |
+
all_hidden_states += (hidden_states,)
|
| 906 |
+
|
| 907 |
+
if not return_dict:
|
| 908 |
+
return tuple(v for v in [hidden_states, all_hidden_states, all_self_attns] if v is not None)
|
| 909 |
+
return BaseModelOutput(
|
| 910 |
+
last_hidden_state=hidden_states,
|
| 911 |
+
hidden_states=all_hidden_states,
|
| 912 |
+
attentions=all_self_attns,
|
| 913 |
+
past_key_values=past_key_values,
|
| 914 |
+
)
|
| 915 |
+
|
| 916 |
+
|
| 917 |
+
class DreamModel(DreamGenerationMixin, DreamPreTrainedModel):
|
| 918 |
+
_tied_weights_keys = ["lm_head.weight"]
|
| 919 |
+
|
| 920 |
+
def __init__(self, config):
|
| 921 |
+
super().__init__(config)
|
| 922 |
+
self.model = DreamBaseModel(config)
|
| 923 |
+
self.vocab_size = config.vocab_size
|
| 924 |
+
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
| 925 |
+
|
| 926 |
+
# Initialize weights and apply final processing
|
| 927 |
+
self.tokenizer = None
|
| 928 |
+
self.post_init()
|
| 929 |
+
|
| 930 |
+
def reset_rope_parameters(self):
|
| 931 |
+
self.model.rotary_emb.reset_parameters()
|
| 932 |
+
for layer in self.model.layers:
|
| 933 |
+
layer.self_attn.rotary_emb.reset_parameters()
|
| 934 |
+
|
| 935 |
+
def get_input_embeddings(self):
|
| 936 |
+
return self.model.embed_tokens
|
| 937 |
+
|
| 938 |
+
def set_input_embeddings(self, value):
|
| 939 |
+
self.model.embed_tokens = value
|
| 940 |
+
|
| 941 |
+
def get_output_embeddings(self):
|
| 942 |
+
return self.lm_head
|
| 943 |
+
|
| 944 |
+
def set_output_embeddings(self, new_embeddings):
|
| 945 |
+
self.lm_head = new_embeddings
|
| 946 |
+
|
| 947 |
+
def set_decoder(self, decoder):
|
| 948 |
+
self.model = decoder
|
| 949 |
+
|
| 950 |
+
def get_decoder(self):
|
| 951 |
+
return self.model
|
| 952 |
+
|
| 953 |
+
def forward(
|
| 954 |
+
self,
|
| 955 |
+
input_ids: torch.LongTensor = None,
|
| 956 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 957 |
+
audios: Optional[torch.FloatTensor] = None,
|
| 958 |
+
audio_indices: Optional[torch.LongTensor] = None,
|
| 959 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 960 |
+
past_key_values: Optional[List[torch.FloatTensor]] = None,
|
| 961 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
| 962 |
+
labels: Optional[torch.LongTensor] = None,
|
| 963 |
+
use_cache: Optional[bool] = None,
|
| 964 |
+
output_attentions: Optional[bool] = None,
|
| 965 |
+
output_hidden_states: Optional[bool] = None,
|
| 966 |
+
return_dict: Optional[bool] = None,
|
| 967 |
+
cache_position: Optional[torch.LongTensor] = None,
|
| 968 |
+
num_logits_to_keep: int = 0,
|
| 969 |
+
num_items_in_batch: int = None,
|
| 970 |
+
**loss_kwargs,
|
| 971 |
+
) -> Union[Tuple, MaskedLMOutput]:
|
| 972 |
+
|
| 973 |
+
num_items_in_batch = None
|
| 974 |
+
if ENFORCE_NUM_ITEMIN_BATCH:
|
| 975 |
+
num_items_in_batch = labels.ne(-100).sum()
|
| 976 |
+
num_items_in_batch = torch.distributed.reduce(num_items_in_batch)
|
| 977 |
+
|
| 978 |
+
is_new = position_ids == 0
|
| 979 |
+
# is_new[0] = True
|
| 980 |
+
segment_id = torch.cumsum(is_new.long(), dim=1) - 1
|
| 981 |
+
new_attention_mask = (segment_id.unsqueeze(1) == segment_id.unsqueeze(2)).long()
|
| 982 |
+
mask = attention_mask.unsqueeze(-1) # [bs, len, 1]
|
| 983 |
+
new_attention_mask = new_attention_mask * mask
|
| 984 |
+
|
| 985 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 986 |
+
output_hidden_states = (
|
| 987 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 988 |
+
)
|
| 989 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 990 |
+
|
| 991 |
+
outputs = self.model(
|
| 992 |
+
input_ids=input_ids,
|
| 993 |
+
attention_mask=new_attention_mask,
|
| 994 |
+
audios=audios,
|
| 995 |
+
audio_indices=audio_indices,
|
| 996 |
+
position_ids=position_ids,
|
| 997 |
+
past_key_values=past_key_values,
|
| 998 |
+
inputs_embeds=inputs_embeds,
|
| 999 |
+
use_cache=use_cache,
|
| 1000 |
+
output_attentions=output_attentions,
|
| 1001 |
+
output_hidden_states=output_hidden_states,
|
| 1002 |
+
return_dict=return_dict,
|
| 1003 |
+
cache_position=cache_position,
|
| 1004 |
+
)
|
| 1005 |
+
hidden_states = outputs[0]
|
| 1006 |
+
# Only compute necessary logits, and do not upcast them to float if we are not computing the loss
|
| 1007 |
+
logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :])
|
| 1008 |
+
|
| 1009 |
+
loss = None
|
| 1010 |
+
if labels is not None:
|
| 1011 |
+
if ENFORCE_NUM_ITEMIN_BATCH:
|
| 1012 |
+
assert num_items_in_batch is not None, \
|
| 1013 |
+
"num_items_in_batch must be provided if ENFORCE_NUM_ITEMIN_BATCH is True"
|
| 1014 |
+
loss = self.loss_function(logits, labels,
|
| 1015 |
+
self.vocab_size,num_items_in_batch=num_items_in_batch, **loss_kwargs)
|
| 1016 |
+
|
| 1017 |
+
if not return_dict:
|
| 1018 |
+
output = (logits,) + outputs[1:]
|
| 1019 |
+
return (loss,) + output if loss is not None else output
|
| 1020 |
+
|
| 1021 |
+
return MaskedLMOutput(
|
| 1022 |
+
loss=loss,
|
| 1023 |
+
logits=logits,
|
| 1024 |
+
hidden_states=outputs.hidden_states,
|
| 1025 |
+
attentions=outputs.attentions,
|
| 1026 |
+
past_key_values=outputs.past_key_values
|
| 1027 |
+
)
|
| 1028 |
+
|
| 1029 |
+
def forward_dream(
|
| 1030 |
+
self,
|
| 1031 |
+
input_ids: torch.LongTensor = None,
|
| 1032 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 1033 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 1034 |
+
past_key_values: Optional[List[torch.FloatTensor]] = None,
|
| 1035 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
| 1036 |
+
labels: Optional[torch.LongTensor] = None,
|
| 1037 |
+
use_cache: Optional[bool] = None,
|
| 1038 |
+
output_attentions: Optional[bool] = None,
|
| 1039 |
+
output_hidden_states: Optional[bool] = None,
|
| 1040 |
+
return_dict: Optional[bool] = None,
|
| 1041 |
+
cache_position: Optional[torch.LongTensor] = None,
|
| 1042 |
+
num_logits_to_keep: int = 0,
|
| 1043 |
+
**loss_kwargs,
|
| 1044 |
+
) -> Union[Tuple, MaskedLMOutput]:
|
| 1045 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 1046 |
+
output_hidden_states = (
|
| 1047 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 1048 |
+
)
|
| 1049 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1050 |
+
attention_mask = None
|
| 1051 |
+
# decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
|
| 1052 |
+
outputs = self.model(
|
| 1053 |
+
input_ids=input_ids,
|
| 1054 |
+
attention_mask=attention_mask,
|
| 1055 |
+
position_ids=position_ids,
|
| 1056 |
+
past_key_values=past_key_values,
|
| 1057 |
+
inputs_embeds=inputs_embeds,
|
| 1058 |
+
use_cache=use_cache,
|
| 1059 |
+
output_attentions=output_attentions,
|
| 1060 |
+
output_hidden_states=output_hidden_states,
|
| 1061 |
+
return_dict=return_dict,
|
| 1062 |
+
cache_position=cache_position,
|
| 1063 |
+
)
|
| 1064 |
+
|
| 1065 |
+
hidden_states = outputs[0]
|
| 1066 |
+
# Only compute necessary logits, and do not upcast them to float if we are not computing the loss
|
| 1067 |
+
logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :])
|
| 1068 |
+
|
| 1069 |
+
loss = None
|
| 1070 |
+
if labels is not None:
|
| 1071 |
+
loss = self.loss_function(logits, labels, self.vocab_size, **loss_kwargs)
|
| 1072 |
+
|
| 1073 |
+
if not return_dict:
|
| 1074 |
+
output = (logits,) + outputs[1:]
|
| 1075 |
+
return (loss,) + output if loss is not None else output
|
| 1076 |
+
|
| 1077 |
+
return MaskedLMOutput(
|
| 1078 |
+
loss=loss,
|
| 1079 |
+
logits=logits,
|
| 1080 |
+
hidden_states=outputs.hidden_states,
|
| 1081 |
+
attentions=outputs.attentions,
|
| 1082 |
+
past_key_values=outputs.past_key_values,
|
| 1083 |
+
)
|
| 1084 |
+
|
| 1085 |
+
@torch.no_grad()
|
| 1086 |
+
def generate(
|
| 1087 |
+
self,
|
| 1088 |
+
input_ids: Optional[torch.Tensor] = None,
|
| 1089 |
+
audios: Optional[torch.FloatTensor] = None,
|
| 1090 |
+
audio_indices: Optional[torch.LongTensor] = None,
|
| 1091 |
+
max_new_tokens=512,
|
| 1092 |
+
steps=512,
|
| 1093 |
+
temperature=0.2,
|
| 1094 |
+
top_p=0.95,
|
| 1095 |
+
alg_temp=0.,
|
| 1096 |
+
alg="entropy",
|
| 1097 |
+
output_history=False,
|
| 1098 |
+
**kwargs,
|
| 1099 |
+
):
|
| 1100 |
+
# modalities = kwargs.pop("modalities", None) if "modalities" in kwargs and modalities is None else modalities
|
| 1101 |
+
position_ids = kwargs.pop("position_ids", None)
|
| 1102 |
+
attention_mask = kwargs.pop("attention_mask", None)
|
| 1103 |
+
if "inputs_embeds" in kwargs:
|
| 1104 |
+
raise NotImplementedError("`inputs_embeds` is not supported")
|
| 1105 |
+
|
| 1106 |
+
if audios is not None:
|
| 1107 |
+
audio_embeds, audio_lengths = self.model.audio_model(audios)
|
| 1108 |
+
assert audio_embeds.shape[0] == len(audios)
|
| 1109 |
+
audio_embeds = self.model.audio_projection(audio_embeds)
|
| 1110 |
+
else:
|
| 1111 |
+
audio_embeds = None
|
| 1112 |
+
|
| 1113 |
+
inputs_embeds = self.model.embed_tokens(input_ids)
|
| 1114 |
+
|
| 1115 |
+
if audio_embeds is not None:
|
| 1116 |
+
inputs_embeds = inputs_embeds.clone()
|
| 1117 |
+
for audio_embeds_, audio_lengths_, audio_indices_ in zip(audio_embeds, audio_lengths, audio_indices,):
|
| 1118 |
+
# print(f"{audio_embeds_.size()=} {audio_lengths_=} {audio_indices_.size()=}")
|
| 1119 |
+
audio_embeds_ = audio_embeds_[:audio_lengths_, ...]
|
| 1120 |
+
audio_embeds_ = audio_embeds_.to(inputs_embeds.device)
|
| 1121 |
+
indices_b, indices_s = audio_indices_.to(inputs_embeds.device).unbind(dim=0)
|
| 1122 |
+
inputs_embeds[indices_b.view(-1), indices_s.view(-1)] = audio_embeds_.view(-1, audio_embeds_.shape[-1])
|
| 1123 |
+
|
| 1124 |
+
return self.diffusion_generate(
|
| 1125 |
+
None,
|
| 1126 |
+
inputs_embeds=inputs_embeds,
|
| 1127 |
+
max_new_tokens=max_new_tokens,
|
| 1128 |
+
output_history=output_history,
|
| 1129 |
+
return_dict_in_generate=True,
|
| 1130 |
+
steps=steps,
|
| 1131 |
+
temperature=temperature,
|
| 1132 |
+
top_p=top_p,
|
| 1133 |
+
alg=alg,
|
| 1134 |
+
alg_temp=alg_temp,
|
| 1135 |
+
**kwargs
|
| 1136 |
+
)
|
omni_diffusion/models/dream/modeling_sensevoice.py
ADDED
|
@@ -0,0 +1,1258 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
import time
|
| 3 |
+
import torch
|
| 4 |
+
from torch import nn
|
| 5 |
+
import torch.nn.functional as F
|
| 6 |
+
from typing import Iterable, Optional
|
| 7 |
+
|
| 8 |
+
from funasr.register import tables
|
| 9 |
+
from funasr.models.ctc.ctc import CTC
|
| 10 |
+
from funasr.utils.datadir_writer import DatadirWriter
|
| 11 |
+
from funasr.models.paraformer.search import Hypothesis
|
| 12 |
+
from funasr.train_utils.device_funcs import force_gatherable
|
| 13 |
+
from funasr.losses.label_smoothing_loss import LabelSmoothingLoss
|
| 14 |
+
from funasr.metrics.compute_acc import compute_accuracy, th_accuracy
|
| 15 |
+
from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank
|
| 16 |
+
# from utils.ctc_alignment import ctc_forced_align
|
| 17 |
+
|
| 18 |
+
def ctc_forced_align(
|
| 19 |
+
log_probs: torch.Tensor,
|
| 20 |
+
targets: torch.Tensor,
|
| 21 |
+
input_lengths: torch.Tensor,
|
| 22 |
+
target_lengths: torch.Tensor,
|
| 23 |
+
blank: int = 0,
|
| 24 |
+
ignore_id: int = -1,
|
| 25 |
+
) -> torch.Tensor:
|
| 26 |
+
"""Align a CTC label sequence to an emission.
|
| 27 |
+
|
| 28 |
+
Args:
|
| 29 |
+
log_probs (Tensor): log probability of CTC emission output.
|
| 30 |
+
Tensor of shape `(B, T, C)`. where `B` is the batch size, `T` is the input length,
|
| 31 |
+
`C` is the number of characters in alphabet including blank.
|
| 32 |
+
targets (Tensor): Target sequence. Tensor of shape `(B, L)`,
|
| 33 |
+
where `L` is the target length.
|
| 34 |
+
input_lengths (Tensor):
|
| 35 |
+
Lengths of the inputs (max value must each be <= `T`). 1-D Tensor of shape `(B,)`.
|
| 36 |
+
target_lengths (Tensor):
|
| 37 |
+
Lengths of the targets. 1-D Tensor of shape `(B,)`.
|
| 38 |
+
blank_id (int, optional): The index of blank symbol in CTC emission. (Default: 0)
|
| 39 |
+
ignore_id (int, optional): The index of ignore symbol in CTC emission. (Default: -1)
|
| 40 |
+
"""
|
| 41 |
+
targets[targets == ignore_id] = blank
|
| 42 |
+
|
| 43 |
+
batch_size, input_time_size, _ = log_probs.size()
|
| 44 |
+
bsz_indices = torch.arange(batch_size, device=input_lengths.device)
|
| 45 |
+
|
| 46 |
+
_t_a_r_g_e_t_s_ = torch.cat(
|
| 47 |
+
(
|
| 48 |
+
torch.stack((torch.full_like(targets, blank), targets), dim=-1).flatten(start_dim=1),
|
| 49 |
+
torch.full_like(targets[:, :1], blank),
|
| 50 |
+
),
|
| 51 |
+
dim=-1,
|
| 52 |
+
)
|
| 53 |
+
diff_labels = torch.cat(
|
| 54 |
+
(
|
| 55 |
+
torch.as_tensor([[False, False]], device=targets.device).expand(batch_size, -1),
|
| 56 |
+
_t_a_r_g_e_t_s_[:, 2:] != _t_a_r_g_e_t_s_[:, :-2],
|
| 57 |
+
),
|
| 58 |
+
dim=1,
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
neg_inf = torch.tensor(float("-inf"), device=log_probs.device, dtype=log_probs.dtype)
|
| 62 |
+
padding_num = 2
|
| 63 |
+
padded_t = padding_num + _t_a_r_g_e_t_s_.size(-1)
|
| 64 |
+
best_score = torch.full((batch_size, padded_t), neg_inf, device=log_probs.device, dtype=log_probs.dtype)
|
| 65 |
+
best_score[:, padding_num + 0] = log_probs[:, 0, blank]
|
| 66 |
+
best_score[:, padding_num + 1] = log_probs[bsz_indices, 0, _t_a_r_g_e_t_s_[:, 1]]
|
| 67 |
+
|
| 68 |
+
backpointers = torch.zeros((batch_size, input_time_size, padded_t), device=log_probs.device, dtype=targets.dtype)
|
| 69 |
+
|
| 70 |
+
for t in range(1, input_time_size):
|
| 71 |
+
prev = torch.stack(
|
| 72 |
+
(best_score[:, 2:], best_score[:, 1:-1], torch.where(diff_labels, best_score[:, :-2], neg_inf))
|
| 73 |
+
)
|
| 74 |
+
prev_max_value, prev_max_idx = prev.max(dim=0)
|
| 75 |
+
best_score[:, padding_num:] = log_probs[:, t].gather(-1, _t_a_r_g_e_t_s_) + prev_max_value
|
| 76 |
+
backpointers[:, t, padding_num:] = prev_max_idx
|
| 77 |
+
|
| 78 |
+
l1l2 = best_score.gather(
|
| 79 |
+
-1, torch.stack((padding_num + target_lengths * 2 - 1, padding_num + target_lengths * 2), dim=-1)
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
path = torch.zeros((batch_size, input_time_size), device=best_score.device, dtype=torch.long)
|
| 83 |
+
path[bsz_indices, input_lengths - 1] = padding_num + target_lengths * 2 - 1 + l1l2.argmax(dim=-1)
|
| 84 |
+
|
| 85 |
+
for t in range(input_time_size - 1, 0, -1):
|
| 86 |
+
target_indices = path[:, t]
|
| 87 |
+
prev_max_idx = backpointers[bsz_indices, t, target_indices]
|
| 88 |
+
path[:, t - 1] += target_indices - prev_max_idx
|
| 89 |
+
|
| 90 |
+
alignments = _t_a_r_g_e_t_s_.gather(dim=-1, index=(path - padding_num).clamp(min=0))
|
| 91 |
+
return alignments
|
| 92 |
+
|
| 93 |
+
class SinusoidalPositionEncoder(torch.nn.Module):
|
| 94 |
+
""" """
|
| 95 |
+
|
| 96 |
+
def __int__(self, d_model=80, dropout_rate=0.1):
|
| 97 |
+
pass
|
| 98 |
+
|
| 99 |
+
def encode(
|
| 100 |
+
self, positions: torch.Tensor = None, depth: int = None, dtype: torch.dtype = torch.float32
|
| 101 |
+
):
|
| 102 |
+
batch_size = positions.size(0)
|
| 103 |
+
positions = positions.type(dtype)
|
| 104 |
+
device = positions.device
|
| 105 |
+
log_timescale_increment = torch.log(torch.tensor([10000], dtype=dtype, device=device)) / (
|
| 106 |
+
depth / 2 - 1
|
| 107 |
+
)
|
| 108 |
+
inv_timescales = torch.exp(
|
| 109 |
+
torch.arange(depth / 2, device=device).type(dtype) * (-log_timescale_increment)
|
| 110 |
+
)
|
| 111 |
+
inv_timescales = torch.reshape(inv_timescales, [batch_size, -1])
|
| 112 |
+
scaled_time = torch.reshape(positions, [1, -1, 1]) * torch.reshape(
|
| 113 |
+
inv_timescales, [1, 1, -1]
|
| 114 |
+
)
|
| 115 |
+
encoding = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=2)
|
| 116 |
+
return encoding.type(dtype)
|
| 117 |
+
|
| 118 |
+
def forward(self, x):
|
| 119 |
+
batch_size, timesteps, input_dim = x.size()
|
| 120 |
+
positions = torch.arange(1, timesteps + 1, device=x.device)[None, :]
|
| 121 |
+
position_encoding = self.encode(positions, input_dim, x.dtype).to(x.device)
|
| 122 |
+
|
| 123 |
+
return x + position_encoding
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
class PositionwiseFeedForward(torch.nn.Module):
|
| 127 |
+
"""Positionwise feed forward layer.
|
| 128 |
+
|
| 129 |
+
Args:
|
| 130 |
+
idim (int): Input dimenstion.
|
| 131 |
+
hidden_units (int): The number of hidden units.
|
| 132 |
+
dropout_rate (float): Dropout rate.
|
| 133 |
+
|
| 134 |
+
"""
|
| 135 |
+
|
| 136 |
+
def __init__(self, idim, hidden_units, dropout_rate, activation=torch.nn.ReLU()):
|
| 137 |
+
"""Construct an PositionwiseFeedForward object."""
|
| 138 |
+
super(PositionwiseFeedForward, self).__init__()
|
| 139 |
+
self.w_1 = torch.nn.Linear(idim, hidden_units)
|
| 140 |
+
self.w_2 = torch.nn.Linear(hidden_units, idim)
|
| 141 |
+
self.dropout = torch.nn.Dropout(dropout_rate)
|
| 142 |
+
self.activation = activation
|
| 143 |
+
|
| 144 |
+
def forward(self, x):
|
| 145 |
+
"""Forward function."""
|
| 146 |
+
return self.w_2(self.dropout(self.activation(self.w_1(x))))
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
class MultiHeadedAttentionSANM(nn.Module):
|
| 150 |
+
"""Multi-Head Attention layer.
|
| 151 |
+
|
| 152 |
+
Args:
|
| 153 |
+
n_head (int): The number of heads.
|
| 154 |
+
n_feat (int): The number of features.
|
| 155 |
+
dropout_rate (float): Dropout rate.
|
| 156 |
+
|
| 157 |
+
"""
|
| 158 |
+
|
| 159 |
+
def __init__(
|
| 160 |
+
self,
|
| 161 |
+
n_head,
|
| 162 |
+
in_feat,
|
| 163 |
+
n_feat,
|
| 164 |
+
dropout_rate,
|
| 165 |
+
kernel_size,
|
| 166 |
+
sanm_shfit=0,
|
| 167 |
+
lora_list=None,
|
| 168 |
+
lora_rank=8,
|
| 169 |
+
lora_alpha=16,
|
| 170 |
+
lora_dropout=0.1,
|
| 171 |
+
):
|
| 172 |
+
"""Construct an MultiHeadedAttention object."""
|
| 173 |
+
super().__init__()
|
| 174 |
+
assert n_feat % n_head == 0
|
| 175 |
+
# We assume d_v always equals d_k
|
| 176 |
+
self.d_k = n_feat // n_head
|
| 177 |
+
self.h = n_head
|
| 178 |
+
# self.linear_q = nn.Linear(n_feat, n_feat)
|
| 179 |
+
# self.linear_k = nn.Linear(n_feat, n_feat)
|
| 180 |
+
# self.linear_v = nn.Linear(n_feat, n_feat)
|
| 181 |
+
|
| 182 |
+
self.linear_out = nn.Linear(n_feat, n_feat)
|
| 183 |
+
self.linear_q_k_v = nn.Linear(in_feat, n_feat * 3)
|
| 184 |
+
self.attn = None
|
| 185 |
+
self.dropout = nn.Dropout(p=dropout_rate)
|
| 186 |
+
|
| 187 |
+
self.fsmn_block = nn.Conv1d(
|
| 188 |
+
n_feat, n_feat, kernel_size, stride=1, padding=0, groups=n_feat, bias=False
|
| 189 |
+
)
|
| 190 |
+
# padding
|
| 191 |
+
left_padding = (kernel_size - 1) // 2
|
| 192 |
+
if sanm_shfit > 0:
|
| 193 |
+
left_padding = left_padding + sanm_shfit
|
| 194 |
+
right_padding = kernel_size - 1 - left_padding
|
| 195 |
+
self.pad_fn = nn.ConstantPad1d((left_padding, right_padding), 0.0)
|
| 196 |
+
|
| 197 |
+
def forward_fsmn(self, inputs, mask, mask_shfit_chunk=None):
|
| 198 |
+
b, t, d = inputs.size()
|
| 199 |
+
if mask is not None:
|
| 200 |
+
mask = torch.reshape(mask, (b, -1, 1))
|
| 201 |
+
if mask_shfit_chunk is not None:
|
| 202 |
+
mask = mask * mask_shfit_chunk
|
| 203 |
+
inputs = inputs * mask
|
| 204 |
+
|
| 205 |
+
x = inputs.transpose(1, 2)
|
| 206 |
+
x = self.pad_fn(x)
|
| 207 |
+
x = self.fsmn_block(x)
|
| 208 |
+
x = x.transpose(1, 2)
|
| 209 |
+
x += inputs
|
| 210 |
+
x = self.dropout(x)
|
| 211 |
+
if mask is not None:
|
| 212 |
+
x = x * mask
|
| 213 |
+
return x
|
| 214 |
+
|
| 215 |
+
def forward_qkv(self, x):
|
| 216 |
+
"""Transform query, key and value.
|
| 217 |
+
|
| 218 |
+
Args:
|
| 219 |
+
query (torch.Tensor): Query tensor (#batch, time1, size).
|
| 220 |
+
key (torch.Tensor): Key tensor (#batch, time2, size).
|
| 221 |
+
value (torch.Tensor): Value tensor (#batch, time2, size).
|
| 222 |
+
|
| 223 |
+
Returns:
|
| 224 |
+
torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k).
|
| 225 |
+
torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k).
|
| 226 |
+
torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k).
|
| 227 |
+
|
| 228 |
+
"""
|
| 229 |
+
b, t, d = x.size()
|
| 230 |
+
q_k_v = self.linear_q_k_v(x)
|
| 231 |
+
q, k, v = torch.split(q_k_v, int(self.h * self.d_k), dim=-1)
|
| 232 |
+
q_h = torch.reshape(q, (b, t, self.h, self.d_k)).transpose(
|
| 233 |
+
1, 2
|
| 234 |
+
) # (batch, head, time1, d_k)
|
| 235 |
+
k_h = torch.reshape(k, (b, t, self.h, self.d_k)).transpose(
|
| 236 |
+
1, 2
|
| 237 |
+
) # (batch, head, time2, d_k)
|
| 238 |
+
v_h = torch.reshape(v, (b, t, self.h, self.d_k)).transpose(
|
| 239 |
+
1, 2
|
| 240 |
+
) # (batch, head, time2, d_k)
|
| 241 |
+
|
| 242 |
+
return q_h, k_h, v_h, v
|
| 243 |
+
|
| 244 |
+
def forward_attention(self, value, scores, mask, mask_att_chunk_encoder=None):
|
| 245 |
+
"""Compute attention context vector.
|
| 246 |
+
|
| 247 |
+
Args:
|
| 248 |
+
value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k).
|
| 249 |
+
scores (torch.Tensor): Attention score (#batch, n_head, time1, time2).
|
| 250 |
+
mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2).
|
| 251 |
+
|
| 252 |
+
Returns:
|
| 253 |
+
torch.Tensor: Transformed value (#batch, time1, d_model)
|
| 254 |
+
weighted by the attention score (#batch, time1, time2).
|
| 255 |
+
|
| 256 |
+
"""
|
| 257 |
+
n_batch = value.size(0)
|
| 258 |
+
if mask is not None:
|
| 259 |
+
if mask_att_chunk_encoder is not None:
|
| 260 |
+
mask = mask * mask_att_chunk_encoder
|
| 261 |
+
|
| 262 |
+
mask = mask.unsqueeze(1).eq(0) # (batch, 1, *, time2)
|
| 263 |
+
|
| 264 |
+
min_value = -float(
|
| 265 |
+
"inf"
|
| 266 |
+
) # float(numpy.finfo(torch.tensor(0, dtype=scores.dtype).numpy().dtype).min)
|
| 267 |
+
scores = scores.masked_fill(mask, min_value)
|
| 268 |
+
attn = torch.softmax(scores, dim=-1).masked_fill(
|
| 269 |
+
mask, 0.0
|
| 270 |
+
) # (batch, head, time1, time2)
|
| 271 |
+
else:
|
| 272 |
+
attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2)
|
| 273 |
+
|
| 274 |
+
p_attn = self.dropout(attn)
|
| 275 |
+
x = torch.matmul(p_attn, value) # (batch, head, time1, d_k)
|
| 276 |
+
x = (
|
| 277 |
+
x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k)
|
| 278 |
+
) # (batch, time1, d_model)
|
| 279 |
+
|
| 280 |
+
return self.linear_out(x) # (batch, time1, d_model)
|
| 281 |
+
|
| 282 |
+
def forward(self, x, mask, mask_shfit_chunk=None, mask_att_chunk_encoder=None):
|
| 283 |
+
"""Compute scaled dot product attention.
|
| 284 |
+
|
| 285 |
+
Args:
|
| 286 |
+
query (torch.Tensor): Query tensor (#batch, time1, size).
|
| 287 |
+
key (torch.Tensor): Key tensor (#batch, time2, size).
|
| 288 |
+
value (torch.Tensor): Value tensor (#batch, time2, size).
|
| 289 |
+
mask (torch.Tensor): Mask tensor (#batch, 1, time2) or
|
| 290 |
+
(#batch, time1, time2).
|
| 291 |
+
|
| 292 |
+
Returns:
|
| 293 |
+
torch.Tensor: Output tensor (#batch, time1, d_model).
|
| 294 |
+
|
| 295 |
+
"""
|
| 296 |
+
q_h, k_h, v_h, v = self.forward_qkv(x)
|
| 297 |
+
fsmn_memory = self.forward_fsmn(v, mask, mask_shfit_chunk)
|
| 298 |
+
q_h = q_h * self.d_k ** (-0.5)
|
| 299 |
+
scores = torch.matmul(q_h, k_h.transpose(-2, -1))
|
| 300 |
+
att_outs = self.forward_attention(v_h, scores, mask, mask_att_chunk_encoder)
|
| 301 |
+
return att_outs + fsmn_memory
|
| 302 |
+
|
| 303 |
+
def forward_chunk(self, x, cache=None, chunk_size=None, look_back=0):
|
| 304 |
+
"""Compute scaled dot product attention.
|
| 305 |
+
|
| 306 |
+
Args:
|
| 307 |
+
query (torch.Tensor): Query tensor (#batch, time1, size).
|
| 308 |
+
key (torch.Tensor): Key tensor (#batch, time2, size).
|
| 309 |
+
value (torch.Tensor): Value tensor (#batch, time2, size).
|
| 310 |
+
mask (torch.Tensor): Mask tensor (#batch, 1, time2) or
|
| 311 |
+
(#batch, time1, time2).
|
| 312 |
+
|
| 313 |
+
Returns:
|
| 314 |
+
torch.Tensor: Output tensor (#batch, time1, d_model).
|
| 315 |
+
|
| 316 |
+
"""
|
| 317 |
+
q_h, k_h, v_h, v = self.forward_qkv(x)
|
| 318 |
+
if chunk_size is not None and look_back > 0 or look_back == -1:
|
| 319 |
+
if cache is not None:
|
| 320 |
+
k_h_stride = k_h[:, :, : -(chunk_size[2]), :]
|
| 321 |
+
v_h_stride = v_h[:, :, : -(chunk_size[2]), :]
|
| 322 |
+
k_h = torch.cat((cache["k"], k_h), dim=2)
|
| 323 |
+
v_h = torch.cat((cache["v"], v_h), dim=2)
|
| 324 |
+
|
| 325 |
+
cache["k"] = torch.cat((cache["k"], k_h_stride), dim=2)
|
| 326 |
+
cache["v"] = torch.cat((cache["v"], v_h_stride), dim=2)
|
| 327 |
+
if look_back != -1:
|
| 328 |
+
cache["k"] = cache["k"][:, :, -(look_back * chunk_size[1]) :, :]
|
| 329 |
+
cache["v"] = cache["v"][:, :, -(look_back * chunk_size[1]) :, :]
|
| 330 |
+
else:
|
| 331 |
+
cache_tmp = {
|
| 332 |
+
"k": k_h[:, :, : -(chunk_size[2]), :],
|
| 333 |
+
"v": v_h[:, :, : -(chunk_size[2]), :],
|
| 334 |
+
}
|
| 335 |
+
cache = cache_tmp
|
| 336 |
+
fsmn_memory = self.forward_fsmn(v, None)
|
| 337 |
+
q_h = q_h * self.d_k ** (-0.5)
|
| 338 |
+
scores = torch.matmul(q_h, k_h.transpose(-2, -1))
|
| 339 |
+
att_outs = self.forward_attention(v_h, scores, None)
|
| 340 |
+
return att_outs + fsmn_memory, cache
|
| 341 |
+
|
| 342 |
+
|
| 343 |
+
class LayerNorm(nn.LayerNorm):
|
| 344 |
+
def __init__(self, *args, **kwargs):
|
| 345 |
+
super().__init__(*args, **kwargs)
|
| 346 |
+
|
| 347 |
+
def forward(self, input):
|
| 348 |
+
output = F.layer_norm(
|
| 349 |
+
input.float(),
|
| 350 |
+
self.normalized_shape,
|
| 351 |
+
self.weight.float() if self.weight is not None else None,
|
| 352 |
+
self.bias.float() if self.bias is not None else None,
|
| 353 |
+
self.eps,
|
| 354 |
+
)
|
| 355 |
+
return output.type_as(input)
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
def sequence_mask(lengths, maxlen=None, dtype=torch.float32, device=None):
|
| 359 |
+
if maxlen is None:
|
| 360 |
+
maxlen = lengths.max()
|
| 361 |
+
row_vector = torch.arange(0, maxlen, 1).to(lengths.device)
|
| 362 |
+
matrix = torch.unsqueeze(lengths, dim=-1)
|
| 363 |
+
mask = row_vector < matrix
|
| 364 |
+
mask = mask.detach()
|
| 365 |
+
|
| 366 |
+
return mask.to(dtype).to(device) if device is not None else mask.to(dtype)
|
| 367 |
+
# return mask.type(dtype).to(device) if device is not None else mask.type(dtype)
|
| 368 |
+
|
| 369 |
+
|
| 370 |
+
class EncoderLayerSANM(nn.Module):
|
| 371 |
+
def __init__(
|
| 372 |
+
self,
|
| 373 |
+
in_size,
|
| 374 |
+
size,
|
| 375 |
+
self_attn,
|
| 376 |
+
feed_forward,
|
| 377 |
+
dropout_rate,
|
| 378 |
+
normalize_before=True,
|
| 379 |
+
concat_after=False,
|
| 380 |
+
stochastic_depth_rate=0.0,
|
| 381 |
+
):
|
| 382 |
+
"""Construct an EncoderLayer object."""
|
| 383 |
+
super(EncoderLayerSANM, self).__init__()
|
| 384 |
+
self.self_attn = self_attn
|
| 385 |
+
self.feed_forward = feed_forward
|
| 386 |
+
self.norm1 = LayerNorm(in_size)
|
| 387 |
+
self.norm2 = LayerNorm(size)
|
| 388 |
+
self.dropout = nn.Dropout(dropout_rate)
|
| 389 |
+
self.in_size = in_size
|
| 390 |
+
self.size = size
|
| 391 |
+
self.normalize_before = normalize_before
|
| 392 |
+
self.concat_after = concat_after
|
| 393 |
+
if self.concat_after:
|
| 394 |
+
self.concat_linear = nn.Linear(size + size, size)
|
| 395 |
+
self.stochastic_depth_rate = stochastic_depth_rate
|
| 396 |
+
self.dropout_rate = dropout_rate
|
| 397 |
+
|
| 398 |
+
def forward(self, x, mask, cache=None, mask_shfit_chunk=None, mask_att_chunk_encoder=None):
|
| 399 |
+
"""Compute encoded features.
|
| 400 |
+
|
| 401 |
+
Args:
|
| 402 |
+
x_input (torch.Tensor): Input tensor (#batch, time, size).
|
| 403 |
+
mask (torch.Tensor): Mask tensor for the input (#batch, time).
|
| 404 |
+
cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size).
|
| 405 |
+
|
| 406 |
+
Returns:
|
| 407 |
+
torch.Tensor: Output tensor (#batch, time, size).
|
| 408 |
+
torch.Tensor: Mask tensor (#batch, time).
|
| 409 |
+
|
| 410 |
+
"""
|
| 411 |
+
skip_layer = False
|
| 412 |
+
# with stochastic depth, residual connection `x + f(x)` becomes
|
| 413 |
+
# `x <- x + 1 / (1 - p) * f(x)` at training time.
|
| 414 |
+
stoch_layer_coeff = 1.0
|
| 415 |
+
if self.training and self.stochastic_depth_rate > 0:
|
| 416 |
+
skip_layer = torch.rand(1).item() < self.stochastic_depth_rate
|
| 417 |
+
stoch_layer_coeff = 1.0 / (1 - self.stochastic_depth_rate)
|
| 418 |
+
|
| 419 |
+
if skip_layer:
|
| 420 |
+
if cache is not None:
|
| 421 |
+
x = torch.cat([cache, x], dim=1)
|
| 422 |
+
return x, mask
|
| 423 |
+
|
| 424 |
+
residual = x
|
| 425 |
+
if self.normalize_before:
|
| 426 |
+
x = self.norm1(x)
|
| 427 |
+
|
| 428 |
+
if self.concat_after:
|
| 429 |
+
x_concat = torch.cat(
|
| 430 |
+
(
|
| 431 |
+
x,
|
| 432 |
+
self.self_attn(
|
| 433 |
+
x,
|
| 434 |
+
mask,
|
| 435 |
+
mask_shfit_chunk=mask_shfit_chunk,
|
| 436 |
+
mask_att_chunk_encoder=mask_att_chunk_encoder,
|
| 437 |
+
),
|
| 438 |
+
),
|
| 439 |
+
dim=-1,
|
| 440 |
+
)
|
| 441 |
+
if self.in_size == self.size:
|
| 442 |
+
x = residual + stoch_layer_coeff * self.concat_linear(x_concat)
|
| 443 |
+
else:
|
| 444 |
+
x = stoch_layer_coeff * self.concat_linear(x_concat)
|
| 445 |
+
else:
|
| 446 |
+
if self.in_size == self.size:
|
| 447 |
+
x = residual + stoch_layer_coeff * self.dropout(
|
| 448 |
+
self.self_attn(
|
| 449 |
+
x,
|
| 450 |
+
mask,
|
| 451 |
+
mask_shfit_chunk=mask_shfit_chunk,
|
| 452 |
+
mask_att_chunk_encoder=mask_att_chunk_encoder,
|
| 453 |
+
)
|
| 454 |
+
)
|
| 455 |
+
else:
|
| 456 |
+
x = stoch_layer_coeff * self.dropout(
|
| 457 |
+
self.self_attn(
|
| 458 |
+
x,
|
| 459 |
+
mask,
|
| 460 |
+
mask_shfit_chunk=mask_shfit_chunk,
|
| 461 |
+
mask_att_chunk_encoder=mask_att_chunk_encoder,
|
| 462 |
+
)
|
| 463 |
+
)
|
| 464 |
+
if not self.normalize_before:
|
| 465 |
+
x = self.norm1(x)
|
| 466 |
+
|
| 467 |
+
residual = x
|
| 468 |
+
if self.normalize_before:
|
| 469 |
+
x = self.norm2(x)
|
| 470 |
+
x = residual + stoch_layer_coeff * self.dropout(self.feed_forward(x))
|
| 471 |
+
if not self.normalize_before:
|
| 472 |
+
x = self.norm2(x)
|
| 473 |
+
|
| 474 |
+
return x, mask, cache, mask_shfit_chunk, mask_att_chunk_encoder
|
| 475 |
+
|
| 476 |
+
def forward_chunk(self, x, cache=None, chunk_size=None, look_back=0):
|
| 477 |
+
"""Compute encoded features.
|
| 478 |
+
|
| 479 |
+
Args:
|
| 480 |
+
x_input (torch.Tensor): Input tensor (#batch, time, size).
|
| 481 |
+
mask (torch.Tensor): Mask tensor for the input (#batch, time).
|
| 482 |
+
cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size).
|
| 483 |
+
|
| 484 |
+
Returns:
|
| 485 |
+
torch.Tensor: Output tensor (#batch, time, size).
|
| 486 |
+
torch.Tensor: Mask tensor (#batch, time).
|
| 487 |
+
|
| 488 |
+
"""
|
| 489 |
+
|
| 490 |
+
residual = x
|
| 491 |
+
if self.normalize_before:
|
| 492 |
+
x = self.norm1(x)
|
| 493 |
+
|
| 494 |
+
if self.in_size == self.size:
|
| 495 |
+
attn, cache = self.self_attn.forward_chunk(x, cache, chunk_size, look_back)
|
| 496 |
+
x = residual + attn
|
| 497 |
+
else:
|
| 498 |
+
x, cache = self.self_attn.forward_chunk(x, cache, chunk_size, look_back)
|
| 499 |
+
|
| 500 |
+
if not self.normalize_before:
|
| 501 |
+
x = self.norm1(x)
|
| 502 |
+
|
| 503 |
+
residual = x
|
| 504 |
+
if self.normalize_before:
|
| 505 |
+
x = self.norm2(x)
|
| 506 |
+
x = residual + self.feed_forward(x)
|
| 507 |
+
if not self.normalize_before:
|
| 508 |
+
x = self.norm2(x)
|
| 509 |
+
|
| 510 |
+
return x, cache
|
| 511 |
+
|
| 512 |
+
|
| 513 |
+
@tables.register("encoder_classes", "SenseVoiceEncoderSmall")
|
| 514 |
+
class SenseVoiceEncoderSmall(nn.Module):
|
| 515 |
+
"""
|
| 516 |
+
Author: Speech Lab of DAMO Academy, Alibaba Group
|
| 517 |
+
SCAMA: Streaming chunk-aware multihead attention for online end-to-end speech recognition
|
| 518 |
+
https://arxiv.org/abs/2006.01713
|
| 519 |
+
"""
|
| 520 |
+
|
| 521 |
+
def __init__(
|
| 522 |
+
self,
|
| 523 |
+
input_size: int,
|
| 524 |
+
output_size: int = 256,
|
| 525 |
+
attention_heads: int = 4,
|
| 526 |
+
linear_units: int = 2048,
|
| 527 |
+
num_blocks: int = 6,
|
| 528 |
+
tp_blocks: int = 0,
|
| 529 |
+
dropout_rate: float = 0.1,
|
| 530 |
+
positional_dropout_rate: float = 0.1,
|
| 531 |
+
attention_dropout_rate: float = 0.0,
|
| 532 |
+
stochastic_depth_rate: float = 0.0,
|
| 533 |
+
input_layer: Optional[str] = "conv2d",
|
| 534 |
+
pos_enc_class=SinusoidalPositionEncoder,
|
| 535 |
+
normalize_before: bool = True,
|
| 536 |
+
concat_after: bool = False,
|
| 537 |
+
positionwise_layer_type: str = "linear",
|
| 538 |
+
positionwise_conv_kernel_size: int = 1,
|
| 539 |
+
padding_idx: int = -1,
|
| 540 |
+
kernel_size: int = 11,
|
| 541 |
+
sanm_shfit: int = 0,
|
| 542 |
+
selfattention_layer_type: str = "sanm",
|
| 543 |
+
**kwargs,
|
| 544 |
+
):
|
| 545 |
+
super().__init__()
|
| 546 |
+
self._output_size = output_size
|
| 547 |
+
|
| 548 |
+
self.embed = SinusoidalPositionEncoder()
|
| 549 |
+
|
| 550 |
+
self.normalize_before = normalize_before
|
| 551 |
+
|
| 552 |
+
positionwise_layer = PositionwiseFeedForward
|
| 553 |
+
positionwise_layer_args = (
|
| 554 |
+
output_size,
|
| 555 |
+
linear_units,
|
| 556 |
+
dropout_rate,
|
| 557 |
+
)
|
| 558 |
+
|
| 559 |
+
encoder_selfattn_layer = MultiHeadedAttentionSANM
|
| 560 |
+
encoder_selfattn_layer_args0 = (
|
| 561 |
+
attention_heads,
|
| 562 |
+
input_size,
|
| 563 |
+
output_size,
|
| 564 |
+
attention_dropout_rate,
|
| 565 |
+
kernel_size,
|
| 566 |
+
sanm_shfit,
|
| 567 |
+
)
|
| 568 |
+
encoder_selfattn_layer_args = (
|
| 569 |
+
attention_heads,
|
| 570 |
+
output_size,
|
| 571 |
+
output_size,
|
| 572 |
+
attention_dropout_rate,
|
| 573 |
+
kernel_size,
|
| 574 |
+
sanm_shfit,
|
| 575 |
+
)
|
| 576 |
+
|
| 577 |
+
self.encoders0 = nn.ModuleList(
|
| 578 |
+
[
|
| 579 |
+
EncoderLayerSANM(
|
| 580 |
+
input_size,
|
| 581 |
+
output_size,
|
| 582 |
+
encoder_selfattn_layer(*encoder_selfattn_layer_args0),
|
| 583 |
+
positionwise_layer(*positionwise_layer_args),
|
| 584 |
+
dropout_rate,
|
| 585 |
+
)
|
| 586 |
+
for i in range(1)
|
| 587 |
+
]
|
| 588 |
+
)
|
| 589 |
+
self.encoders = nn.ModuleList(
|
| 590 |
+
[
|
| 591 |
+
EncoderLayerSANM(
|
| 592 |
+
output_size,
|
| 593 |
+
output_size,
|
| 594 |
+
encoder_selfattn_layer(*encoder_selfattn_layer_args),
|
| 595 |
+
positionwise_layer(*positionwise_layer_args),
|
| 596 |
+
dropout_rate,
|
| 597 |
+
)
|
| 598 |
+
for i in range(num_blocks - 1)
|
| 599 |
+
]
|
| 600 |
+
)
|
| 601 |
+
|
| 602 |
+
self.tp_encoders = nn.ModuleList(
|
| 603 |
+
[
|
| 604 |
+
EncoderLayerSANM(
|
| 605 |
+
output_size,
|
| 606 |
+
output_size,
|
| 607 |
+
encoder_selfattn_layer(*encoder_selfattn_layer_args),
|
| 608 |
+
positionwise_layer(*positionwise_layer_args),
|
| 609 |
+
dropout_rate,
|
| 610 |
+
)
|
| 611 |
+
for i in range(tp_blocks)
|
| 612 |
+
]
|
| 613 |
+
)
|
| 614 |
+
|
| 615 |
+
self.after_norm = LayerNorm(output_size)
|
| 616 |
+
|
| 617 |
+
self.tp_norm = LayerNorm(output_size)
|
| 618 |
+
|
| 619 |
+
def output_size(self) -> int:
|
| 620 |
+
return self._output_size
|
| 621 |
+
|
| 622 |
+
def forward(
|
| 623 |
+
self,
|
| 624 |
+
xs_pad: torch.Tensor,
|
| 625 |
+
ilens: torch.Tensor,
|
| 626 |
+
):
|
| 627 |
+
"""Embed positions in tensor."""
|
| 628 |
+
masks = sequence_mask(ilens, dtype=torch.bfloat16, device=ilens.device)[:, None, :]
|
| 629 |
+
# print(f"{masks=}")
|
| 630 |
+
# print(f"{ilens=}")
|
| 631 |
+
# print(f"{(masks>0.5).squeeze(1).sum(1).int()=}")
|
| 632 |
+
|
| 633 |
+
xs_pad *= self.output_size() ** 0.5
|
| 634 |
+
|
| 635 |
+
xs_pad = self.embed(xs_pad)
|
| 636 |
+
|
| 637 |
+
# forward encoder1
|
| 638 |
+
for layer_idx, encoder_layer in enumerate(self.encoders0):
|
| 639 |
+
encoder_outs = encoder_layer(xs_pad, masks)
|
| 640 |
+
xs_pad, masks = encoder_outs[0], encoder_outs[1]
|
| 641 |
+
|
| 642 |
+
for layer_idx, encoder_layer in enumerate(self.encoders):
|
| 643 |
+
encoder_outs = encoder_layer(xs_pad, masks)
|
| 644 |
+
xs_pad, masks = encoder_outs[0], encoder_outs[1]
|
| 645 |
+
|
| 646 |
+
xs_pad = self.after_norm(xs_pad)
|
| 647 |
+
|
| 648 |
+
# forward encoder2
|
| 649 |
+
# olens = masks.squeeze(1).sum(1).int()
|
| 650 |
+
olens = (masks > 0.5).squeeze(1).sum(1).int()
|
| 651 |
+
|
| 652 |
+
for layer_idx, encoder_layer in enumerate(self.tp_encoders):
|
| 653 |
+
encoder_outs = encoder_layer(xs_pad, masks)
|
| 654 |
+
xs_pad, masks = encoder_outs[0], encoder_outs[1]
|
| 655 |
+
|
| 656 |
+
xs_pad = self.tp_norm(xs_pad)
|
| 657 |
+
return xs_pad, olens
|
| 658 |
+
|
| 659 |
+
|
| 660 |
+
@tables.register("model_classes", "SenseVoiceSmall")
|
| 661 |
+
class SenseVoiceSmall(nn.Module):
|
| 662 |
+
"""CTC-attention hybrid Encoder-Decoder model"""
|
| 663 |
+
|
| 664 |
+
def __init__(
|
| 665 |
+
self,
|
| 666 |
+
specaug: str = None,
|
| 667 |
+
specaug_conf: dict = None,
|
| 668 |
+
normalize: str = None,
|
| 669 |
+
normalize_conf: dict = None,
|
| 670 |
+
encoder: str = None,
|
| 671 |
+
encoder_conf: dict = None,
|
| 672 |
+
ctc_conf: dict = None,
|
| 673 |
+
input_size: int = 80,
|
| 674 |
+
vocab_size: int = -1,
|
| 675 |
+
ignore_id: int = -1,
|
| 676 |
+
blank_id: int = 0,
|
| 677 |
+
sos: int = 1,
|
| 678 |
+
eos: int = 2,
|
| 679 |
+
length_normalized_loss: bool = False,
|
| 680 |
+
**kwargs,
|
| 681 |
+
):
|
| 682 |
+
|
| 683 |
+
super().__init__()
|
| 684 |
+
|
| 685 |
+
if specaug is not None:
|
| 686 |
+
specaug_class = tables.specaug_classes.get(specaug)
|
| 687 |
+
specaug = specaug_class(**specaug_conf)
|
| 688 |
+
if normalize is not None:
|
| 689 |
+
normalize_class = tables.normalize_classes.get(normalize)
|
| 690 |
+
normalize = normalize_class(**normalize_conf)
|
| 691 |
+
encoder_class = tables.encoder_classes.get(encoder)
|
| 692 |
+
encoder = encoder_class(input_size=input_size, **encoder_conf)
|
| 693 |
+
encoder_output_size = encoder.output_size()
|
| 694 |
+
|
| 695 |
+
if ctc_conf is None:
|
| 696 |
+
ctc_conf = {}
|
| 697 |
+
ctc = CTC(odim=vocab_size, encoder_output_size=encoder_output_size, **ctc_conf)
|
| 698 |
+
|
| 699 |
+
self.blank_id = blank_id
|
| 700 |
+
self.sos = sos if sos is not None else vocab_size - 1
|
| 701 |
+
self.eos = eos if eos is not None else vocab_size - 1
|
| 702 |
+
self.vocab_size = vocab_size
|
| 703 |
+
self.ignore_id = ignore_id
|
| 704 |
+
self.specaug = specaug
|
| 705 |
+
self.normalize = normalize
|
| 706 |
+
self.encoder = encoder
|
| 707 |
+
self.error_calculator = None
|
| 708 |
+
|
| 709 |
+
self.ctc = ctc
|
| 710 |
+
|
| 711 |
+
self.length_normalized_loss = length_normalized_loss
|
| 712 |
+
self.encoder_output_size = encoder_output_size
|
| 713 |
+
|
| 714 |
+
self.lid_dict = {"auto": 0, "zh": 3, "en": 4, "yue": 7, "ja": 11, "ko": 12, "nospeech": 13}
|
| 715 |
+
self.lid_int_dict = {24884: 3, 24885: 4, 24888: 7, 24892: 11, 24896: 12, 24992: 13}
|
| 716 |
+
self.textnorm_dict = {"withitn": 14, "woitn": 15}
|
| 717 |
+
self.textnorm_int_dict = {25016: 14, 25017: 15}
|
| 718 |
+
self.embed = torch.nn.Embedding(7 + len(self.lid_dict) + len(self.textnorm_dict), input_size)
|
| 719 |
+
self.emo_dict = {"unk": 25009, "happy": 25001, "sad": 25002, "angry": 25003, "neutral": 25004}
|
| 720 |
+
|
| 721 |
+
self.criterion_att = LabelSmoothingLoss(
|
| 722 |
+
size=self.vocab_size,
|
| 723 |
+
padding_idx=self.ignore_id,
|
| 724 |
+
smoothing=kwargs.get("lsm_weight", 0.0),
|
| 725 |
+
normalize_length=self.length_normalized_loss,
|
| 726 |
+
)
|
| 727 |
+
|
| 728 |
+
@staticmethod
|
| 729 |
+
def from_pretrained(model:str=None, **kwargs):
|
| 730 |
+
from funasr import AutoModel
|
| 731 |
+
model, kwargs = AutoModel.build_model(model=model, trust_remote_code=True, **kwargs)
|
| 732 |
+
|
| 733 |
+
return model, kwargs
|
| 734 |
+
|
| 735 |
+
def forward(
|
| 736 |
+
self,
|
| 737 |
+
speech: torch.Tensor,
|
| 738 |
+
speech_lengths: torch.Tensor,
|
| 739 |
+
text: torch.Tensor,
|
| 740 |
+
text_lengths: torch.Tensor,
|
| 741 |
+
**kwargs,
|
| 742 |
+
):
|
| 743 |
+
"""Encoder + Decoder + Calc loss
|
| 744 |
+
Args:
|
| 745 |
+
speech: (Batch, Length, ...)
|
| 746 |
+
speech_lengths: (Batch, )
|
| 747 |
+
text: (Batch, Length)
|
| 748 |
+
text_lengths: (Batch,)
|
| 749 |
+
"""
|
| 750 |
+
# import pdb;
|
| 751 |
+
# pdb.set_trace()
|
| 752 |
+
if len(text_lengths.size()) > 1:
|
| 753 |
+
text_lengths = text_lengths[:, 0]
|
| 754 |
+
if len(speech_lengths.size()) > 1:
|
| 755 |
+
speech_lengths = speech_lengths[:, 0]
|
| 756 |
+
|
| 757 |
+
batch_size = speech.shape[0]
|
| 758 |
+
|
| 759 |
+
# 1. Encoder
|
| 760 |
+
encoder_out, encoder_out_lens = self.encode(speech, speech_lengths, text)
|
| 761 |
+
|
| 762 |
+
loss_ctc, cer_ctc = None, None
|
| 763 |
+
loss_rich, acc_rich = None, None
|
| 764 |
+
stats = dict()
|
| 765 |
+
|
| 766 |
+
loss_ctc, cer_ctc = self._calc_ctc_loss(
|
| 767 |
+
encoder_out[:, 4:, :], encoder_out_lens - 4, text[:, 4:], text_lengths - 4
|
| 768 |
+
)
|
| 769 |
+
|
| 770 |
+
loss_rich, acc_rich = self._calc_rich_ce_loss(
|
| 771 |
+
encoder_out[:, :4, :], text[:, :4]
|
| 772 |
+
)
|
| 773 |
+
|
| 774 |
+
loss = loss_ctc + loss_rich
|
| 775 |
+
# Collect total loss stats
|
| 776 |
+
stats["loss_ctc"] = torch.clone(loss_ctc.detach()) if loss_ctc is not None else None
|
| 777 |
+
stats["loss_rich"] = torch.clone(loss_rich.detach()) if loss_rich is not None else None
|
| 778 |
+
stats["loss"] = torch.clone(loss.detach()) if loss is not None else None
|
| 779 |
+
stats["acc_rich"] = acc_rich
|
| 780 |
+
|
| 781 |
+
# force_gatherable: to-device and to-tensor if scalar for DataParallel
|
| 782 |
+
if self.length_normalized_loss:
|
| 783 |
+
batch_size = int((text_lengths + 1).sum())
|
| 784 |
+
loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device)
|
| 785 |
+
return loss, stats, weight
|
| 786 |
+
|
| 787 |
+
def encode(
|
| 788 |
+
self,
|
| 789 |
+
speech: torch.Tensor,
|
| 790 |
+
speech_lengths: torch.Tensor,
|
| 791 |
+
text: torch.Tensor,
|
| 792 |
+
**kwargs,
|
| 793 |
+
):
|
| 794 |
+
"""Frontend + Encoder. Note that this method is used by asr_inference.py
|
| 795 |
+
Args:
|
| 796 |
+
speech: (Batch, Length, ...)
|
| 797 |
+
speech_lengths: (Batch, )
|
| 798 |
+
ind: int
|
| 799 |
+
"""
|
| 800 |
+
|
| 801 |
+
# Data augmentation
|
| 802 |
+
if self.specaug is not None and self.training:
|
| 803 |
+
speech, speech_lengths = self.specaug(speech, speech_lengths)
|
| 804 |
+
|
| 805 |
+
# Normalization for feature: e.g. Global-CMVN, Utterance-CMVN
|
| 806 |
+
if self.normalize is not None:
|
| 807 |
+
speech, speech_lengths = self.normalize(speech, speech_lengths)
|
| 808 |
+
|
| 809 |
+
|
| 810 |
+
lids = torch.LongTensor([[self.lid_int_dict[int(lid)]
|
| 811 |
+
if torch.rand(1) > 0.2 and int(lid) in self.lid_int_dict else 0 ]
|
| 812 |
+
for lid in text[:, 0]]).to(speech.device)
|
| 813 |
+
language_query = self.embed(lids)
|
| 814 |
+
|
| 815 |
+
styles = torch.LongTensor([[self.textnorm_int_dict[int(style)]] for style in text[:, 3]]).to(speech.device)
|
| 816 |
+
style_query = self.embed(styles)
|
| 817 |
+
speech = torch.cat((style_query, speech), dim=1)
|
| 818 |
+
speech_lengths += 1
|
| 819 |
+
|
| 820 |
+
event_emo_query = self.embed(torch.LongTensor([[1, 2]]).to(speech.device)).repeat(speech.size(0), 1, 1)
|
| 821 |
+
input_query = torch.cat((language_query, event_emo_query), dim=1)
|
| 822 |
+
speech = torch.cat((input_query, speech), dim=1)
|
| 823 |
+
speech_lengths += 3
|
| 824 |
+
|
| 825 |
+
encoder_out, encoder_out_lens = self.encoder(speech, speech_lengths)
|
| 826 |
+
|
| 827 |
+
return encoder_out, encoder_out_lens
|
| 828 |
+
|
| 829 |
+
def _calc_ctc_loss(
|
| 830 |
+
self,
|
| 831 |
+
encoder_out: torch.Tensor,
|
| 832 |
+
encoder_out_lens: torch.Tensor,
|
| 833 |
+
ys_pad: torch.Tensor,
|
| 834 |
+
ys_pad_lens: torch.Tensor,
|
| 835 |
+
):
|
| 836 |
+
# Calc CTC loss
|
| 837 |
+
loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
|
| 838 |
+
|
| 839 |
+
# Calc CER using CTC
|
| 840 |
+
cer_ctc = None
|
| 841 |
+
if not self.training and self.error_calculator is not None:
|
| 842 |
+
ys_hat = self.ctc.argmax(encoder_out).data
|
| 843 |
+
cer_ctc = self.error_calculator(ys_hat.cpu(), ys_pad.cpu(), is_ctc=True)
|
| 844 |
+
return loss_ctc, cer_ctc
|
| 845 |
+
|
| 846 |
+
def _calc_rich_ce_loss(
|
| 847 |
+
self,
|
| 848 |
+
encoder_out: torch.Tensor,
|
| 849 |
+
ys_pad: torch.Tensor,
|
| 850 |
+
):
|
| 851 |
+
decoder_out = self.ctc.ctc_lo(encoder_out)
|
| 852 |
+
# 2. Compute attention loss
|
| 853 |
+
loss_rich = self.criterion_att(decoder_out, ys_pad.contiguous())
|
| 854 |
+
acc_rich = th_accuracy(
|
| 855 |
+
decoder_out.view(-1, self.vocab_size),
|
| 856 |
+
ys_pad.contiguous(),
|
| 857 |
+
ignore_label=self.ignore_id,
|
| 858 |
+
)
|
| 859 |
+
|
| 860 |
+
return loss_rich, acc_rich
|
| 861 |
+
|
| 862 |
+
|
| 863 |
+
def inference(
|
| 864 |
+
self,
|
| 865 |
+
data_in,
|
| 866 |
+
data_lengths=None,
|
| 867 |
+
key: list = ["wav_file_tmp_name"],
|
| 868 |
+
tokenizer=None,
|
| 869 |
+
frontend=None,
|
| 870 |
+
**kwargs,
|
| 871 |
+
):
|
| 872 |
+
|
| 873 |
+
|
| 874 |
+
meta_data = {}
|
| 875 |
+
if (
|
| 876 |
+
isinstance(data_in, torch.Tensor) and kwargs.get("data_type", "sound") == "fbank"
|
| 877 |
+
): # fbank
|
| 878 |
+
speech, speech_lengths = data_in, data_lengths
|
| 879 |
+
if len(speech.shape) < 3:
|
| 880 |
+
speech = speech[None, :, :]
|
| 881 |
+
if speech_lengths is None:
|
| 882 |
+
speech_lengths = speech.shape[1]
|
| 883 |
+
else:
|
| 884 |
+
# extract fbank feats
|
| 885 |
+
time1 = time.perf_counter()
|
| 886 |
+
audio_sample_list = load_audio_text_image_video(
|
| 887 |
+
data_in,
|
| 888 |
+
fs=frontend.fs,
|
| 889 |
+
audio_fs=kwargs.get("fs", 16000),
|
| 890 |
+
data_type=kwargs.get("data_type", "sound"),
|
| 891 |
+
tokenizer=tokenizer,
|
| 892 |
+
)
|
| 893 |
+
time2 = time.perf_counter()
|
| 894 |
+
meta_data["load_data"] = f"{time2 - time1:0.3f}"
|
| 895 |
+
speech, speech_lengths = extract_fbank(
|
| 896 |
+
audio_sample_list, data_type=kwargs.get("data_type", "sound"), frontend=frontend
|
| 897 |
+
)
|
| 898 |
+
time3 = time.perf_counter()
|
| 899 |
+
meta_data["extract_feat"] = f"{time3 - time2:0.3f}"
|
| 900 |
+
meta_data["batch_data_time"] = (
|
| 901 |
+
speech_lengths.sum().item() * frontend.frame_shift * frontend.lfr_n / 1000
|
| 902 |
+
)
|
| 903 |
+
|
| 904 |
+
speech = speech.to(device=kwargs["device"])
|
| 905 |
+
speech_lengths = speech_lengths.to(device=kwargs["device"])
|
| 906 |
+
|
| 907 |
+
language = kwargs.get("language", "auto")
|
| 908 |
+
language_query = self.embed(
|
| 909 |
+
torch.LongTensor(
|
| 910 |
+
[[self.lid_dict[language] if language in self.lid_dict else 0]]
|
| 911 |
+
).to(speech.device)
|
| 912 |
+
).repeat(speech.size(0), 1, 1)
|
| 913 |
+
|
| 914 |
+
use_itn = kwargs.get("use_itn", False)
|
| 915 |
+
output_timestamp = kwargs.get("output_timestamp", False)
|
| 916 |
+
|
| 917 |
+
textnorm = kwargs.get("text_norm", None)
|
| 918 |
+
if textnorm is None:
|
| 919 |
+
textnorm = "withitn" if use_itn else "woitn"
|
| 920 |
+
textnorm_query = self.embed(
|
| 921 |
+
torch.LongTensor([[self.textnorm_dict[textnorm]]]).to(speech.device)
|
| 922 |
+
).repeat(speech.size(0), 1, 1)
|
| 923 |
+
speech = torch.cat((textnorm_query, speech), dim=1)
|
| 924 |
+
speech_lengths += 1
|
| 925 |
+
|
| 926 |
+
event_emo_query = self.embed(torch.LongTensor([[1, 2]]).to(speech.device)).repeat(
|
| 927 |
+
speech.size(0), 1, 1
|
| 928 |
+
)
|
| 929 |
+
input_query = torch.cat((language_query, event_emo_query), dim=1)
|
| 930 |
+
speech = torch.cat((input_query, speech), dim=1)
|
| 931 |
+
speech_lengths += 3
|
| 932 |
+
|
| 933 |
+
# Encoder
|
| 934 |
+
encoder_out, encoder_out_lens = self.encoder(speech, speech_lengths)
|
| 935 |
+
if isinstance(encoder_out, tuple):
|
| 936 |
+
encoder_out = encoder_out[0]
|
| 937 |
+
|
| 938 |
+
# c. Passed the encoder result and the beam search
|
| 939 |
+
ctc_logits = self.ctc.log_softmax(encoder_out)
|
| 940 |
+
if kwargs.get("ban_emo_unk", False):
|
| 941 |
+
ctc_logits[:, :, self.emo_dict["unk"]] = -float("inf")
|
| 942 |
+
|
| 943 |
+
results = []
|
| 944 |
+
b, n, d = encoder_out.size()
|
| 945 |
+
if isinstance(key[0], (list, tuple)):
|
| 946 |
+
key = key[0]
|
| 947 |
+
if len(key) < b:
|
| 948 |
+
key = key * b
|
| 949 |
+
for i in range(b):
|
| 950 |
+
x = ctc_logits[i, : encoder_out_lens[i].item(), :]
|
| 951 |
+
yseq = x.argmax(dim=-1)
|
| 952 |
+
yseq = torch.unique_consecutive(yseq, dim=-1)
|
| 953 |
+
|
| 954 |
+
ibest_writer = None
|
| 955 |
+
if kwargs.get("output_dir") is not None:
|
| 956 |
+
if not hasattr(self, "writer"):
|
| 957 |
+
self.writer = DatadirWriter(kwargs.get("output_dir"))
|
| 958 |
+
ibest_writer = self.writer[f"1best_recog"]
|
| 959 |
+
|
| 960 |
+
mask = yseq != self.blank_id
|
| 961 |
+
token_int = yseq[mask].tolist()
|
| 962 |
+
|
| 963 |
+
# Change integer-ids to tokens
|
| 964 |
+
text = tokenizer.decode(token_int)
|
| 965 |
+
if ibest_writer is not None:
|
| 966 |
+
ibest_writer["text"][key[i]] = text
|
| 967 |
+
|
| 968 |
+
if output_timestamp:
|
| 969 |
+
from itertools import groupby
|
| 970 |
+
timestamp = []
|
| 971 |
+
tokens = tokenizer.text2tokens(text)[4:]
|
| 972 |
+
|
| 973 |
+
logits_speech = self.ctc.softmax(encoder_out)[i, 4:encoder_out_lens[i].item(), :]
|
| 974 |
+
|
| 975 |
+
pred = logits_speech.argmax(-1).cpu()
|
| 976 |
+
logits_speech[pred==self.blank_id, self.blank_id] = 0
|
| 977 |
+
|
| 978 |
+
align = ctc_forced_align(
|
| 979 |
+
logits_speech.unsqueeze(0).float(),
|
| 980 |
+
torch.Tensor(token_int[4:]).unsqueeze(0).long().to(logits_speech.device),
|
| 981 |
+
(encoder_out_lens-4).long(),
|
| 982 |
+
torch.tensor(len(token_int)-4).unsqueeze(0).long().to(logits_speech.device),
|
| 983 |
+
ignore_id=self.ignore_id,
|
| 984 |
+
)
|
| 985 |
+
|
| 986 |
+
pred = groupby(align[0, :encoder_out_lens[0]])
|
| 987 |
+
_start = 0
|
| 988 |
+
token_id = 0
|
| 989 |
+
ts_max = encoder_out_lens[i] - 4
|
| 990 |
+
for pred_token, pred_frame in pred:
|
| 991 |
+
_end = _start + len(list(pred_frame))
|
| 992 |
+
if pred_token != 0:
|
| 993 |
+
ts_left = max((_start*60-30)/1000, 0)
|
| 994 |
+
ts_right = min((_end*60-30)/1000, (ts_max*60-30)/1000)
|
| 995 |
+
timestamp.append([tokens[token_id], ts_left, ts_right])
|
| 996 |
+
token_id += 1
|
| 997 |
+
_start = _end
|
| 998 |
+
|
| 999 |
+
result_i = {"key": key[i], "text": text, "timestamp": timestamp}
|
| 1000 |
+
results.append(result_i)
|
| 1001 |
+
else:
|
| 1002 |
+
result_i = {"key": key[i], "text": text}
|
| 1003 |
+
results.append(result_i)
|
| 1004 |
+
return results, meta_data
|
| 1005 |
+
|
| 1006 |
+
|
| 1007 |
+
def inference_encode(
|
| 1008 |
+
self,
|
| 1009 |
+
data_in,
|
| 1010 |
+
data_lengths=None,
|
| 1011 |
+
key: list = ["wav_file_tmp_name"],
|
| 1012 |
+
tokenizer=None,
|
| 1013 |
+
frontend=None,
|
| 1014 |
+
**kwargs,
|
| 1015 |
+
):
|
| 1016 |
+
|
| 1017 |
+
# fbank
|
| 1018 |
+
speech, speech_lengths = data_in, data_lengths
|
| 1019 |
+
if len(speech.shape) < 3:
|
| 1020 |
+
speech = speech[None, :, :]
|
| 1021 |
+
if speech_lengths is None:
|
| 1022 |
+
speech_lengths = speech.shape[1]
|
| 1023 |
+
|
| 1024 |
+
speech = speech.to(device=kwargs["device"])
|
| 1025 |
+
speech_lengths = speech_lengths.to(device=kwargs["device"])
|
| 1026 |
+
|
| 1027 |
+
language = kwargs.get("language", "auto")
|
| 1028 |
+
language_query = self.embed(
|
| 1029 |
+
torch.LongTensor(
|
| 1030 |
+
[[self.lid_dict[language] if language in self.lid_dict else 0]]
|
| 1031 |
+
).to(speech.device)
|
| 1032 |
+
).repeat(speech.size(0), 1, 1)
|
| 1033 |
+
|
| 1034 |
+
use_itn = kwargs.get("use_itn", False)
|
| 1035 |
+
output_timestamp = kwargs.get("output_timestamp", False)
|
| 1036 |
+
|
| 1037 |
+
textnorm = kwargs.get("text_norm", None)
|
| 1038 |
+
if textnorm is None:
|
| 1039 |
+
textnorm = "withitn" if use_itn else "woitn"
|
| 1040 |
+
textnorm_query = self.embed(
|
| 1041 |
+
torch.LongTensor([[self.textnorm_dict[textnorm]]]).to(speech.device)
|
| 1042 |
+
).repeat(speech.size(0), 1, 1)
|
| 1043 |
+
speech = torch.cat((textnorm_query, speech), dim=1)
|
| 1044 |
+
speech_lengths += 1
|
| 1045 |
+
|
| 1046 |
+
event_emo_query = self.embed(torch.LongTensor([[1, 2]]).to(speech.device)).repeat(
|
| 1047 |
+
speech.size(0), 1, 1
|
| 1048 |
+
)
|
| 1049 |
+
input_query = torch.cat((language_query, event_emo_query), dim=1)
|
| 1050 |
+
speech = torch.cat((input_query, speech), dim=1)
|
| 1051 |
+
speech_lengths += 3
|
| 1052 |
+
|
| 1053 |
+
# Encoder
|
| 1054 |
+
encoder_out, encoder_out_lens = self.encoder(speech, speech_lengths)
|
| 1055 |
+
if isinstance(encoder_out, tuple):
|
| 1056 |
+
encoder_out = encoder_out[0]
|
| 1057 |
+
|
| 1058 |
+
return encoder_out, encoder_out_lens
|
| 1059 |
+
|
| 1060 |
+
def export(self, **kwargs):
|
| 1061 |
+
from export_meta import export_rebuild_model
|
| 1062 |
+
|
| 1063 |
+
if "max_seq_len" not in kwargs:
|
| 1064 |
+
kwargs["max_seq_len"] = 512
|
| 1065 |
+
models = export_rebuild_model(model=self, **kwargs)
|
| 1066 |
+
return models
|
| 1067 |
+
|
| 1068 |
+
|
| 1069 |
+
|
| 1070 |
+
class AudioEncoder(nn.Module):
|
| 1071 |
+
|
| 1072 |
+
def __init__(
|
| 1073 |
+
self,
|
| 1074 |
+
model_dir=None
|
| 1075 |
+
):
|
| 1076 |
+
|
| 1077 |
+
super().__init__()
|
| 1078 |
+
|
| 1079 |
+
# model_dir = "SenceVoiceSmall"
|
| 1080 |
+
from huggingface_hub import snapshot_download
|
| 1081 |
+
model_dir = snapshot_download(repo_id="FunAudioLLM/SenseVoiceSmall")
|
| 1082 |
+
self.model, self.kwargs = self.build_model(model=model_dir, trust_remote_code=True,)
|
| 1083 |
+
|
| 1084 |
+
|
| 1085 |
+
def forward(
|
| 1086 |
+
self,
|
| 1087 |
+
audios,
|
| 1088 |
+
):
|
| 1089 |
+
|
| 1090 |
+
# print(f"{[x for x in audios]=}")
|
| 1091 |
+
# print(f"{[x.sum() for x in audios]=}")
|
| 1092 |
+
# print(f"{[x.size() for x in audios]=}")
|
| 1093 |
+
|
| 1094 |
+
from torch.nn.utils.rnn import pad_sequence
|
| 1095 |
+
feats_pad = pad_sequence(audios, batch_first=True, padding_value=0.0)
|
| 1096 |
+
# feats_lens = torch.as_tensor([len(x) + 4 for x in audios])
|
| 1097 |
+
feats_lens = torch.as_tensor([len(x) for x in audios])
|
| 1098 |
+
|
| 1099 |
+
feats_pad = feats_pad.to(torch.bfloat16)
|
| 1100 |
+
|
| 1101 |
+
# print(f"{feats_pad.sum()=}")
|
| 1102 |
+
# print(f"{feats_pad.size()=}")
|
| 1103 |
+
# print(f"{feats_lens=}")
|
| 1104 |
+
|
| 1105 |
+
encoder_out, encoder_out_lens = self.model.inference_encode(
|
| 1106 |
+
feats_pad,
|
| 1107 |
+
data_lengths=feats_lens,
|
| 1108 |
+
language="auto", # "zh", "en", "yue", "ja", "ko", "nospeech"
|
| 1109 |
+
use_itn=False,
|
| 1110 |
+
ban_emo_unk=False,
|
| 1111 |
+
**self.kwargs,
|
| 1112 |
+
)
|
| 1113 |
+
|
| 1114 |
+
# print(f"{encoder_out.sum()=}")
|
| 1115 |
+
# print(f"{encoder_out.size()=}")
|
| 1116 |
+
# print(f"{encoder_out_lens=}")
|
| 1117 |
+
return encoder_out, encoder_out_lens
|
| 1118 |
+
|
| 1119 |
+
audio_embeds = []
|
| 1120 |
+
for x, y in zip(encoder_out, encoder_out_lens):
|
| 1121 |
+
audio_embeds.append(x[:y, ...])
|
| 1122 |
+
|
| 1123 |
+
audio_embeds = torch.stack(audio_embeds, dim=0)
|
| 1124 |
+
|
| 1125 |
+
return audio_embeds
|
| 1126 |
+
|
| 1127 |
+
# https://github.com/modelscope/FunASR/blob/main/funasr/auto/auto_model.py
|
| 1128 |
+
@staticmethod
|
| 1129 |
+
def build_model(**kwargs):
|
| 1130 |
+
from omegaconf import DictConfig, ListConfig
|
| 1131 |
+
import os
|
| 1132 |
+
|
| 1133 |
+
from funasr.download.download_model_from_hub import download_model
|
| 1134 |
+
from funasr.train_utils.set_all_random_seed import set_all_random_seed
|
| 1135 |
+
from funasr.register import tables
|
| 1136 |
+
from funasr.train_utils.load_pretrained_model import load_pretrained_model
|
| 1137 |
+
from funasr.utils.misc import deep_update
|
| 1138 |
+
|
| 1139 |
+
import logging
|
| 1140 |
+
|
| 1141 |
+
assert "model" in kwargs
|
| 1142 |
+
if "model_conf" not in kwargs:
|
| 1143 |
+
logging.info("download models from model hub: {}".format(kwargs.get("hub", "ms")))
|
| 1144 |
+
kwargs = download_model(**kwargs)
|
| 1145 |
+
|
| 1146 |
+
set_all_random_seed(kwargs.get("seed", 0))
|
| 1147 |
+
|
| 1148 |
+
device = kwargs.get("device", "cuda")
|
| 1149 |
+
if not torch.cuda.is_available() or kwargs.get("ngpu", 1) == 0:
|
| 1150 |
+
device = "cpu"
|
| 1151 |
+
kwargs["batch_size"] = 1
|
| 1152 |
+
kwargs["device"] = device
|
| 1153 |
+
|
| 1154 |
+
torch.set_num_threads(kwargs.get("ncpu", 4))
|
| 1155 |
+
|
| 1156 |
+
# build tokenizer
|
| 1157 |
+
tokenizer = kwargs.get("tokenizer", None)
|
| 1158 |
+
kwargs["tokenizer"] = tokenizer
|
| 1159 |
+
kwargs["vocab_size"] = -1
|
| 1160 |
+
|
| 1161 |
+
if tokenizer is not None:
|
| 1162 |
+
tokenizers = (
|
| 1163 |
+
tokenizer.split(",") if isinstance(tokenizer, str) else tokenizer
|
| 1164 |
+
) # type of tokenizers is list!!!
|
| 1165 |
+
tokenizers_conf = kwargs.get("tokenizer_conf", {})
|
| 1166 |
+
tokenizers_build = []
|
| 1167 |
+
vocab_sizes = []
|
| 1168 |
+
token_lists = []
|
| 1169 |
+
|
| 1170 |
+
### === only for kws ===
|
| 1171 |
+
token_list_files = kwargs.get("token_lists", [])
|
| 1172 |
+
seg_dicts = kwargs.get("seg_dicts", [])
|
| 1173 |
+
### === only for kws ===
|
| 1174 |
+
|
| 1175 |
+
if not isinstance(tokenizers_conf, (list, tuple, ListConfig)):
|
| 1176 |
+
tokenizers_conf = [tokenizers_conf] * len(tokenizers)
|
| 1177 |
+
|
| 1178 |
+
for i, tokenizer in enumerate(tokenizers):
|
| 1179 |
+
tokenizer_class = tables.tokenizer_classes.get(tokenizer)
|
| 1180 |
+
tokenizer_conf = tokenizers_conf[i]
|
| 1181 |
+
|
| 1182 |
+
### === only for kws ===
|
| 1183 |
+
if len(token_list_files) > 1:
|
| 1184 |
+
tokenizer_conf["token_list"] = token_list_files[i]
|
| 1185 |
+
if len(seg_dicts) > 1:
|
| 1186 |
+
tokenizer_conf["seg_dict"] = seg_dicts[i]
|
| 1187 |
+
### === only for kws ===
|
| 1188 |
+
|
| 1189 |
+
tokenizer = tokenizer_class(**tokenizer_conf)
|
| 1190 |
+
tokenizers_build.append(tokenizer)
|
| 1191 |
+
token_list = tokenizer.token_list if hasattr(tokenizer, "token_list") else None
|
| 1192 |
+
token_list = (
|
| 1193 |
+
tokenizer.get_vocab() if hasattr(tokenizer, "get_vocab") else token_list
|
| 1194 |
+
)
|
| 1195 |
+
vocab_size = -1
|
| 1196 |
+
if token_list is not None:
|
| 1197 |
+
vocab_size = len(token_list)
|
| 1198 |
+
|
| 1199 |
+
if vocab_size == -1 and hasattr(tokenizer, "get_vocab_size"):
|
| 1200 |
+
vocab_size = tokenizer.get_vocab_size()
|
| 1201 |
+
token_lists.append(token_list)
|
| 1202 |
+
vocab_sizes.append(vocab_size)
|
| 1203 |
+
|
| 1204 |
+
if len(tokenizers_build) <= 1:
|
| 1205 |
+
tokenizers_build = tokenizers_build[0]
|
| 1206 |
+
token_lists = token_lists[0]
|
| 1207 |
+
vocab_sizes = vocab_sizes[0]
|
| 1208 |
+
|
| 1209 |
+
kwargs["tokenizer"] = tokenizers_build
|
| 1210 |
+
kwargs["vocab_size"] = vocab_sizes
|
| 1211 |
+
kwargs["token_list"] = token_lists
|
| 1212 |
+
|
| 1213 |
+
# build frontend
|
| 1214 |
+
frontend = kwargs.get("frontend", None)
|
| 1215 |
+
kwargs["input_size"] = None
|
| 1216 |
+
if frontend is not None:
|
| 1217 |
+
frontend_class = tables.frontend_classes.get(frontend)
|
| 1218 |
+
frontend = frontend_class(**kwargs.get("frontend_conf", {}))
|
| 1219 |
+
kwargs["input_size"] = (
|
| 1220 |
+
frontend.output_size() if hasattr(frontend, "output_size") else None
|
| 1221 |
+
)
|
| 1222 |
+
kwargs["frontend"] = frontend
|
| 1223 |
+
# build model
|
| 1224 |
+
model_class = tables.model_classes.get(kwargs["model"])
|
| 1225 |
+
assert model_class is not None, f'{kwargs["model"]} is not registered'
|
| 1226 |
+
model_conf = {}
|
| 1227 |
+
deep_update(model_conf, kwargs.get("model_conf", {}))
|
| 1228 |
+
deep_update(model_conf, kwargs)
|
| 1229 |
+
model = model_class(**model_conf)
|
| 1230 |
+
|
| 1231 |
+
# init_param
|
| 1232 |
+
init_param = kwargs.get("init_param", None)
|
| 1233 |
+
if init_param is not None:
|
| 1234 |
+
if os.path.exists(init_param):
|
| 1235 |
+
logging.info(f"Loading pretrained params from {init_param}")
|
| 1236 |
+
load_pretrained_model(
|
| 1237 |
+
model=model,
|
| 1238 |
+
path=init_param,
|
| 1239 |
+
ignore_init_mismatch=kwargs.get("ignore_init_mismatch", True),
|
| 1240 |
+
oss_bucket=kwargs.get("oss_bucket", None),
|
| 1241 |
+
scope_map=kwargs.get("scope_map", []),
|
| 1242 |
+
excludes=kwargs.get("excludes", None),
|
| 1243 |
+
)
|
| 1244 |
+
else:
|
| 1245 |
+
print(f"error, init_param does not exist!: {init_param}")
|
| 1246 |
+
|
| 1247 |
+
# fp16
|
| 1248 |
+
if kwargs.get("fp16", False):
|
| 1249 |
+
model.to(torch.float16)
|
| 1250 |
+
elif kwargs.get("bf16", False):
|
| 1251 |
+
model.to(torch.bfloat16)
|
| 1252 |
+
# model.to(device)
|
| 1253 |
+
|
| 1254 |
+
if not kwargs.get("disable_log", True):
|
| 1255 |
+
tables.print()
|
| 1256 |
+
|
| 1257 |
+
return model, kwargs
|
| 1258 |
+
|
omni_diffusion/models/dream/resampler_projector.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
|
| 4 |
+
import math
|
| 5 |
+
|
| 6 |
+
class ResamplerProjector(nn.Module):
|
| 7 |
+
"""
|
| 8 |
+
A projection module designed to map input features (e.g., from a vision encoder)
|
| 9 |
+
to a target hidden size. It flattens spatial dimensions into a sequence
|
| 10 |
+
and applies an MLP.
|
| 11 |
+
"""
|
| 12 |
+
def __init__(self, proj_input_size, hidden_size):
|
| 13 |
+
super().__init__()
|
| 14 |
+
|
| 15 |
+
self.pre_proj_layernorm = torch.nn.LayerNorm(proj_input_size)
|
| 16 |
+
|
| 17 |
+
self.mlp = nn.Sequential(
|
| 18 |
+
nn.Linear(proj_input_size, hidden_size, bias=False),
|
| 19 |
+
nn.GELU(),
|
| 20 |
+
nn.Linear(hidden_size, hidden_size, bias=False),
|
| 21 |
+
)
|
| 22 |
+
self.mlp.apply(init_weights)
|
| 23 |
+
self.pre_proj_layernorm.apply(init_weights)
|
| 24 |
+
|
| 25 |
+
def forward(self, x, *args, **kwargs):
|
| 26 |
+
x = x.reshape(x.shape[0], -1, x.shape[-1])
|
| 27 |
+
x = self.pre_proj_layernorm(x)
|
| 28 |
+
x = self.mlp(x)
|
| 29 |
+
return x
|
| 30 |
+
|
| 31 |
+
def init_weights(m):
|
| 32 |
+
if isinstance(m, nn.Linear):
|
| 33 |
+
torch.nn.init.normal_(m.weight, mean=0.0, std=0.02)
|
| 34 |
+
if m.bias is not None:
|
| 35 |
+
torch.nn.init.zeros_(m.bias)
|
| 36 |
+
|
| 37 |
+
if isinstance(m, nn.LayerNorm):
|
| 38 |
+
torch.nn.init.ones_(m.weight)
|
| 39 |
+
torch.nn.init.zeros_(m.bias)
|
| 40 |
+
|
omni_diffusion/models/dream/tokenization_dream.py
ADDED
|
@@ -0,0 +1,351 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# coding=utf-8
|
| 2 |
+
# Copyright 2024 The Dream team, HKUNLP Group and The HuggingFace Inc. team. All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This code is based on Qwen's implementations in this library.
|
| 5 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 6 |
+
# you may not use this file except in compliance with the License.
|
| 7 |
+
# You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 12 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 13 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 14 |
+
# See the License for the specific language governing permissions and
|
| 15 |
+
# limitations under the License.
|
| 16 |
+
"""Tokenization classes for Dream."""
|
| 17 |
+
|
| 18 |
+
import json
|
| 19 |
+
import os
|
| 20 |
+
import unicodedata
|
| 21 |
+
from functools import lru_cache
|
| 22 |
+
from typing import Optional, Tuple
|
| 23 |
+
|
| 24 |
+
import regex as re
|
| 25 |
+
|
| 26 |
+
from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
|
| 27 |
+
from transformers.utils import logging
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
logger = logging.get_logger(__name__)
|
| 31 |
+
|
| 32 |
+
VOCAB_FILES_NAMES = {
|
| 33 |
+
"vocab_file": "vocab.json",
|
| 34 |
+
"merges_file": "merges.txt",
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
MAX_MODEL_INPUT_SIZES = {"dream/dream-tokenizer": 32768}
|
| 39 |
+
|
| 40 |
+
regex_parts = [
|
| 41 |
+
r"(?i:'s|'t|'re|'ve|'m|'ll|'d)", # Contractions (case-insensitive)
|
| 42 |
+
r"[^\r\n\p{L}\p{N}]?\p{L}+", # Words (with optional leading symbol)
|
| 43 |
+
r"\p{N}", # Numbers
|
| 44 |
+
r" ?[^\s\p{L}\p{N}]+[\r\n]*", # Punctuation and symbols
|
| 45 |
+
r"\s*[\r\n]+", # Newlines (handling indentation)
|
| 46 |
+
r"\s+(?!\S)", # Trailing whitespace
|
| 47 |
+
r"\s+", # General whitespace
|
| 48 |
+
]
|
| 49 |
+
PRETOKENIZE_REGEX = "|".join(regex_parts)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
@lru_cache()
|
| 53 |
+
# Copied from transformers.models.gpt2.tokenization_gpt2.bytes_to_unicode
|
| 54 |
+
def bytes_to_unicode():
|
| 55 |
+
"""
|
| 56 |
+
Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
|
| 57 |
+
characters the bpe code barfs on.
|
| 58 |
+
|
| 59 |
+
The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
|
| 60 |
+
if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
|
| 61 |
+
decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
|
| 62 |
+
tables between utf-8 bytes and unicode strings.
|
| 63 |
+
"""
|
| 64 |
+
bs = (
|
| 65 |
+
list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
|
| 66 |
+
)
|
| 67 |
+
cs = bs[:]
|
| 68 |
+
n = 0
|
| 69 |
+
for b in range(2**8):
|
| 70 |
+
if b not in bs:
|
| 71 |
+
bs.append(b)
|
| 72 |
+
cs.append(2**8 + n)
|
| 73 |
+
n += 1
|
| 74 |
+
cs = [chr(n) for n in cs]
|
| 75 |
+
return dict(zip(bs, cs))
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
# Copied from transformers.models.gpt2.tokenization_gpt2.get_pairs
|
| 79 |
+
def get_pairs(word):
|
| 80 |
+
"""
|
| 81 |
+
Return set of symbol pairs in a word.
|
| 82 |
+
|
| 83 |
+
Word is represented as tuple of symbols (symbols being variable-length strings).
|
| 84 |
+
"""
|
| 85 |
+
pairs = set()
|
| 86 |
+
prev_char = word[0]
|
| 87 |
+
for char in word[1:]:
|
| 88 |
+
pairs.add((prev_char, char))
|
| 89 |
+
prev_char = char
|
| 90 |
+
return pairs
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
class DreamTokenizer(PreTrainedTokenizer):
|
| 94 |
+
"""
|
| 95 |
+
Construct a Dream tokenizer. Based on byte-level Byte-Pair-Encoding.
|
| 96 |
+
|
| 97 |
+
Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
|
| 98 |
+
be encoded differently whether it is at the beginning of the sentence (without space) or not:
|
| 99 |
+
|
| 100 |
+
```python
|
| 101 |
+
>>> from transformers import AutoTokenizer
|
| 102 |
+
|
| 103 |
+
>>> tokenizer = AutoTokenizer.from_pretrained("Dream-org/Dream-v0-Base-7B", trust_remote_code=True)
|
| 104 |
+
>>> tokenizer("Hello world")["input_ids"]
|
| 105 |
+
[9707, 1879]
|
| 106 |
+
|
| 107 |
+
>>> tokenizer(" Hello world")["input_ids"]
|
| 108 |
+
[21927, 1879]
|
| 109 |
+
```
|
| 110 |
+
This is expected.
|
| 111 |
+
|
| 112 |
+
You should not use GPT2Tokenizer instead, because of the different pretokenization rules.
|
| 113 |
+
|
| 114 |
+
This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
|
| 115 |
+
this superclass for more information regarding those methods.
|
| 116 |
+
|
| 117 |
+
Args:
|
| 118 |
+
vocab_file (`str`):
|
| 119 |
+
Path to the vocabulary file.
|
| 120 |
+
merges_file (`str`):
|
| 121 |
+
Path to the merges file.
|
| 122 |
+
errors (`str`, *optional*, defaults to `"replace"`):
|
| 123 |
+
Paradigm to follow when decoding bytes to UTF-8. See
|
| 124 |
+
[bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
|
| 125 |
+
unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
|
| 126 |
+
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
|
| 127 |
+
token instead.
|
| 128 |
+
bos_token (`str`, *optional*):
|
| 129 |
+
The beginning of sequence token. Not applicable for this tokenizer.
|
| 130 |
+
eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
|
| 131 |
+
The end of sequence token.
|
| 132 |
+
pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
|
| 133 |
+
The token used for padding, for example when batching sequences of different lengths.
|
| 134 |
+
clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
|
| 135 |
+
Whether or not the model should cleanup the spaces that were added when splitting the input text during the
|
| 136 |
+
tokenization process. Not applicable to this tokenizer, since tokenization does not add spaces.
|
| 137 |
+
split_special_tokens (`bool`, *optional*, defaults to `False`):
|
| 138 |
+
Whether or not the special tokens should be split during the tokenization process. The default behavior is
|
| 139 |
+
to not split special tokens. This means that if `<|endoftext|>` is the `eos_token`, then
|
| 140 |
+
`tokenizer.tokenize("<|endoftext|>") = ['<|endoftext|>`]. Otherwise, if `split_special_tokens=True`,
|
| 141 |
+
then `tokenizer.tokenize("<|endoftext|>")` will be give `['<', '|', 'endo', 'ft', 'ext', '|', '>']`.
|
| 142 |
+
This argument is only supported for `slow` tokenizers for the moment.
|
| 143 |
+
"""
|
| 144 |
+
|
| 145 |
+
vocab_files_names = VOCAB_FILES_NAMES
|
| 146 |
+
model_input_names = ["input_ids", "attention_mask"]
|
| 147 |
+
|
| 148 |
+
def __init__(
|
| 149 |
+
self,
|
| 150 |
+
vocab_file,
|
| 151 |
+
merges_file,
|
| 152 |
+
errors="replace",
|
| 153 |
+
unk_token="<|endoftext|>",
|
| 154 |
+
bos_token=None,
|
| 155 |
+
eos_token="<|endoftext|>",
|
| 156 |
+
pad_token="<|endoftext|>",
|
| 157 |
+
clean_up_tokenization_spaces=False,
|
| 158 |
+
split_special_tokens=False,
|
| 159 |
+
**kwargs,
|
| 160 |
+
):
|
| 161 |
+
# Dream vocab does not contain control tokens; added tokens need to be special
|
| 162 |
+
bos_token = (
|
| 163 |
+
AddedToken(bos_token, lstrip=False, rstrip=False, special=True, normalized=False)
|
| 164 |
+
if isinstance(bos_token, str)
|
| 165 |
+
else bos_token
|
| 166 |
+
)
|
| 167 |
+
eos_token = (
|
| 168 |
+
AddedToken(eos_token, lstrip=False, rstrip=False, special=True, normalized=False)
|
| 169 |
+
if isinstance(eos_token, str)
|
| 170 |
+
else eos_token
|
| 171 |
+
)
|
| 172 |
+
unk_token = (
|
| 173 |
+
AddedToken(unk_token, lstrip=False, rstrip=False, special=True, normalized=False)
|
| 174 |
+
if isinstance(unk_token, str)
|
| 175 |
+
else unk_token
|
| 176 |
+
)
|
| 177 |
+
pad_token = (
|
| 178 |
+
AddedToken(pad_token, lstrip=False, rstrip=False, special=True, normalized=False)
|
| 179 |
+
if isinstance(pad_token, str)
|
| 180 |
+
else pad_token
|
| 181 |
+
)
|
| 182 |
+
|
| 183 |
+
with open(vocab_file, encoding="utf-8") as vocab_handle:
|
| 184 |
+
self.encoder = json.load(vocab_handle)
|
| 185 |
+
self.decoder = {v: k for k, v in self.encoder.items()}
|
| 186 |
+
self.errors = errors # how to handle errors in decoding
|
| 187 |
+
self.byte_encoder = bytes_to_unicode()
|
| 188 |
+
self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
|
| 189 |
+
bpe_merges = []
|
| 190 |
+
with open(merges_file, encoding="utf-8") as merges_handle:
|
| 191 |
+
for i, line in enumerate(merges_handle):
|
| 192 |
+
line = line.strip()
|
| 193 |
+
if (i == 0 and line.startswith("#version:")) or not line:
|
| 194 |
+
continue
|
| 195 |
+
bpe_merges.append(tuple(line.split()))
|
| 196 |
+
self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
|
| 197 |
+
# NOTE: the cache can grow without bound and will get really large for long running processes
|
| 198 |
+
# (esp. for texts of language that do not use space between word, e.g. Chinese); technically
|
| 199 |
+
# not a memory leak but appears as one.
|
| 200 |
+
# GPT2Tokenizer has the same problem, so let's be consistent.
|
| 201 |
+
self.cache = {}
|
| 202 |
+
|
| 203 |
+
self.pat = re.compile(PRETOKENIZE_REGEX)
|
| 204 |
+
|
| 205 |
+
if kwargs.get("add_prefix_space", False):
|
| 206 |
+
logger.warning_once(
|
| 207 |
+
f"{self.__class__.__name} does not support `add_prefix_space`, setting it to True has no effect."
|
| 208 |
+
)
|
| 209 |
+
|
| 210 |
+
super().__init__(
|
| 211 |
+
errors=errors,
|
| 212 |
+
bos_token=bos_token,
|
| 213 |
+
eos_token=eos_token,
|
| 214 |
+
pad_token=pad_token,
|
| 215 |
+
unk_token=unk_token,
|
| 216 |
+
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
| 217 |
+
split_special_tokens=split_special_tokens,
|
| 218 |
+
**kwargs,
|
| 219 |
+
)
|
| 220 |
+
|
| 221 |
+
@property
|
| 222 |
+
def vocab_size(self) -> int:
|
| 223 |
+
return len(self.encoder)
|
| 224 |
+
|
| 225 |
+
# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.get_vocab
|
| 226 |
+
def get_vocab(self):
|
| 227 |
+
return dict(self.encoder, **self.added_tokens_encoder)
|
| 228 |
+
|
| 229 |
+
# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.bpe
|
| 230 |
+
def bpe(self, token):
|
| 231 |
+
if token in self.cache:
|
| 232 |
+
return self.cache[token]
|
| 233 |
+
word = tuple(token)
|
| 234 |
+
pairs = get_pairs(word)
|
| 235 |
+
|
| 236 |
+
if not pairs:
|
| 237 |
+
return token
|
| 238 |
+
|
| 239 |
+
while True:
|
| 240 |
+
bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
|
| 241 |
+
if bigram not in self.bpe_ranks:
|
| 242 |
+
break
|
| 243 |
+
first, second = bigram
|
| 244 |
+
new_word = []
|
| 245 |
+
i = 0
|
| 246 |
+
while i < len(word):
|
| 247 |
+
try:
|
| 248 |
+
j = word.index(first, i)
|
| 249 |
+
except ValueError:
|
| 250 |
+
new_word.extend(word[i:])
|
| 251 |
+
break
|
| 252 |
+
else:
|
| 253 |
+
new_word.extend(word[i:j])
|
| 254 |
+
i = j
|
| 255 |
+
|
| 256 |
+
if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
|
| 257 |
+
new_word.append(first + second)
|
| 258 |
+
i += 2
|
| 259 |
+
else:
|
| 260 |
+
new_word.append(word[i])
|
| 261 |
+
i += 1
|
| 262 |
+
new_word = tuple(new_word)
|
| 263 |
+
word = new_word
|
| 264 |
+
if len(word) == 1:
|
| 265 |
+
break
|
| 266 |
+
else:
|
| 267 |
+
pairs = get_pairs(word)
|
| 268 |
+
word = " ".join(word)
|
| 269 |
+
self.cache[token] = word
|
| 270 |
+
return word
|
| 271 |
+
|
| 272 |
+
# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._tokenize
|
| 273 |
+
def _tokenize(self, text):
|
| 274 |
+
"""Tokenize a string."""
|
| 275 |
+
bpe_tokens = []
|
| 276 |
+
for token in re.findall(self.pat, text):
|
| 277 |
+
token = "".join(
|
| 278 |
+
self.byte_encoder[b] for b in token.encode("utf-8")
|
| 279 |
+
) # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
|
| 280 |
+
bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" "))
|
| 281 |
+
return bpe_tokens
|
| 282 |
+
|
| 283 |
+
# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._convert_token_to_id
|
| 284 |
+
def _convert_token_to_id(self, token):
|
| 285 |
+
"""Converts a token (str) in an id using the vocab."""
|
| 286 |
+
return self.encoder.get(token, self.encoder.get(self.unk_token))
|
| 287 |
+
|
| 288 |
+
# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._convert_id_to_token
|
| 289 |
+
def _convert_id_to_token(self, index):
|
| 290 |
+
"""Converts an index (integer) in a token (str) using the vocab."""
|
| 291 |
+
return self.decoder.get(index)
|
| 292 |
+
|
| 293 |
+
# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.convert_tokens_to_string
|
| 294 |
+
def convert_tokens_to_string(self, tokens):
|
| 295 |
+
"""Converts a sequence of tokens (string) in a single string."""
|
| 296 |
+
text = "".join(tokens)
|
| 297 |
+
text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors)
|
| 298 |
+
return text
|
| 299 |
+
|
| 300 |
+
def decode(
|
| 301 |
+
self,
|
| 302 |
+
token_ids,
|
| 303 |
+
skip_special_tokens: bool = False,
|
| 304 |
+
clean_up_tokenization_spaces: Optional[bool] = False,
|
| 305 |
+
spaces_between_special_tokens: bool = False,
|
| 306 |
+
**kwargs,
|
| 307 |
+
) -> str:
|
| 308 |
+
# `spaces_between_special_tokens` defaults to True for _decode in slow tokenizers
|
| 309 |
+
# and cannot be configured elsewhere, but it should default to False for DreamTokenizer
|
| 310 |
+
return super().decode(
|
| 311 |
+
token_ids,
|
| 312 |
+
skip_special_tokens=skip_special_tokens,
|
| 313 |
+
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
| 314 |
+
spaces_between_special_tokens=spaces_between_special_tokens,
|
| 315 |
+
**kwargs,
|
| 316 |
+
)
|
| 317 |
+
|
| 318 |
+
# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.save_vocabulary
|
| 319 |
+
def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
|
| 320 |
+
if not os.path.isdir(save_directory):
|
| 321 |
+
logger.error(f"Vocabulary path ({save_directory}) should be a directory")
|
| 322 |
+
return
|
| 323 |
+
vocab_file = os.path.join(
|
| 324 |
+
save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
|
| 325 |
+
)
|
| 326 |
+
merge_file = os.path.join(
|
| 327 |
+
save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
|
| 328 |
+
)
|
| 329 |
+
|
| 330 |
+
with open(vocab_file, "w", encoding="utf-8") as f:
|
| 331 |
+
f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")
|
| 332 |
+
|
| 333 |
+
index = 0
|
| 334 |
+
with open(merge_file, "w", encoding="utf-8") as writer:
|
| 335 |
+
writer.write("#version: 0.2\n")
|
| 336 |
+
for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
|
| 337 |
+
if index != token_index:
|
| 338 |
+
logger.warning(
|
| 339 |
+
f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
|
| 340 |
+
" Please check that the tokenizer is not corrupted!"
|
| 341 |
+
)
|
| 342 |
+
index = token_index
|
| 343 |
+
writer.write(" ".join(bpe_tokens) + "\n")
|
| 344 |
+
index += 1
|
| 345 |
+
|
| 346 |
+
return vocab_file, merge_file
|
| 347 |
+
|
| 348 |
+
def prepare_for_tokenization(self, text, **kwargs):
|
| 349 |
+
text = unicodedata.normalize("NFC", text)
|
| 350 |
+
return (text, kwargs)
|
| 351 |
+
|
omni_diffusion/models/dream/tokenizer_config.json
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": false,
|
| 3 |
+
"add_prefix_space": false,
|
| 4 |
+
"added_tokens_decoder": {
|
| 5 |
+
"151643": {
|
| 6 |
+
"content": "<|endoftext|>",
|
| 7 |
+
"lstrip": false,
|
| 8 |
+
"normalized": false,
|
| 9 |
+
"rstrip": false,
|
| 10 |
+
"single_word": false,
|
| 11 |
+
"special": true
|
| 12 |
+
},
|
| 13 |
+
"151644": {
|
| 14 |
+
"content": "<|im_start|>",
|
| 15 |
+
"lstrip": false,
|
| 16 |
+
"normalized": false,
|
| 17 |
+
"rstrip": false,
|
| 18 |
+
"single_word": false,
|
| 19 |
+
"special": true
|
| 20 |
+
},
|
| 21 |
+
"151645": {
|
| 22 |
+
"content": "<|im_end|>",
|
| 23 |
+
"lstrip": false,
|
| 24 |
+
"normalized": false,
|
| 25 |
+
"rstrip": false,
|
| 26 |
+
"single_word": false,
|
| 27 |
+
"special": true
|
| 28 |
+
},
|
| 29 |
+
"151646": {
|
| 30 |
+
"content": "<|object_ref_start|>",
|
| 31 |
+
"lstrip": false,
|
| 32 |
+
"normalized": false,
|
| 33 |
+
"rstrip": false,
|
| 34 |
+
"single_word": false,
|
| 35 |
+
"special": true
|
| 36 |
+
},
|
| 37 |
+
"151647": {
|
| 38 |
+
"content": "<|object_ref_end|>",
|
| 39 |
+
"lstrip": false,
|
| 40 |
+
"normalized": false,
|
| 41 |
+
"rstrip": false,
|
| 42 |
+
"single_word": false,
|
| 43 |
+
"special": true
|
| 44 |
+
},
|
| 45 |
+
"151648": {
|
| 46 |
+
"content": "<|box_start|>",
|
| 47 |
+
"lstrip": false,
|
| 48 |
+
"normalized": false,
|
| 49 |
+
"rstrip": false,
|
| 50 |
+
"single_word": false,
|
| 51 |
+
"special": true
|
| 52 |
+
},
|
| 53 |
+
"151649": {
|
| 54 |
+
"content": "<|box_end|>",
|
| 55 |
+
"lstrip": false,
|
| 56 |
+
"normalized": false,
|
| 57 |
+
"rstrip": false,
|
| 58 |
+
"single_word": false,
|
| 59 |
+
"special": true
|
| 60 |
+
},
|
| 61 |
+
"151650": {
|
| 62 |
+
"content": "<|quad_start|>",
|
| 63 |
+
"lstrip": false,
|
| 64 |
+
"normalized": false,
|
| 65 |
+
"rstrip": false,
|
| 66 |
+
"single_word": false,
|
| 67 |
+
"special": true
|
| 68 |
+
},
|
| 69 |
+
"151651": {
|
| 70 |
+
"content": "<|quad_end|>",
|
| 71 |
+
"lstrip": false,
|
| 72 |
+
"normalized": false,
|
| 73 |
+
"rstrip": false,
|
| 74 |
+
"single_word": false,
|
| 75 |
+
"special": true
|
| 76 |
+
},
|
| 77 |
+
"151652": {
|
| 78 |
+
"content": "<|vision_start|>",
|
| 79 |
+
"lstrip": false,
|
| 80 |
+
"normalized": false,
|
| 81 |
+
"rstrip": false,
|
| 82 |
+
"single_word": false,
|
| 83 |
+
"special": true
|
| 84 |
+
},
|
| 85 |
+
"151653": {
|
| 86 |
+
"content": "<|vision_end|>",
|
| 87 |
+
"lstrip": false,
|
| 88 |
+
"normalized": false,
|
| 89 |
+
"rstrip": false,
|
| 90 |
+
"single_word": false,
|
| 91 |
+
"special": true
|
| 92 |
+
},
|
| 93 |
+
"151654": {
|
| 94 |
+
"content": "<|vision_pad|>",
|
| 95 |
+
"lstrip": false,
|
| 96 |
+
"normalized": false,
|
| 97 |
+
"rstrip": false,
|
| 98 |
+
"single_word": false,
|
| 99 |
+
"special": true
|
| 100 |
+
},
|
| 101 |
+
"151655": {
|
| 102 |
+
"content": "<|image_pad|>",
|
| 103 |
+
"lstrip": false,
|
| 104 |
+
"normalized": false,
|
| 105 |
+
"rstrip": false,
|
| 106 |
+
"single_word": false,
|
| 107 |
+
"special": true
|
| 108 |
+
},
|
| 109 |
+
"151656": {
|
| 110 |
+
"content": "<|video_pad|>",
|
| 111 |
+
"lstrip": false,
|
| 112 |
+
"normalized": false,
|
| 113 |
+
"rstrip": false,
|
| 114 |
+
"single_word": false,
|
| 115 |
+
"special": true
|
| 116 |
+
},
|
| 117 |
+
"151657": {
|
| 118 |
+
"content": "<tool_call>",
|
| 119 |
+
"lstrip": false,
|
| 120 |
+
"normalized": false,
|
| 121 |
+
"rstrip": false,
|
| 122 |
+
"single_word": false,
|
| 123 |
+
"special": false
|
| 124 |
+
},
|
| 125 |
+
"151658": {
|
| 126 |
+
"content": "</tool_call>",
|
| 127 |
+
"lstrip": false,
|
| 128 |
+
"normalized": false,
|
| 129 |
+
"rstrip": false,
|
| 130 |
+
"single_word": false,
|
| 131 |
+
"special": false
|
| 132 |
+
},
|
| 133 |
+
"151659": {
|
| 134 |
+
"content": "<|fim_prefix|>",
|
| 135 |
+
"lstrip": false,
|
| 136 |
+
"normalized": false,
|
| 137 |
+
"rstrip": false,
|
| 138 |
+
"single_word": false,
|
| 139 |
+
"special": false
|
| 140 |
+
},
|
| 141 |
+
"151660": {
|
| 142 |
+
"content": "<|fim_middle|>",
|
| 143 |
+
"lstrip": false,
|
| 144 |
+
"normalized": false,
|
| 145 |
+
"rstrip": false,
|
| 146 |
+
"single_word": false,
|
| 147 |
+
"special": false
|
| 148 |
+
},
|
| 149 |
+
"151661": {
|
| 150 |
+
"content": "<|fim_suffix|>",
|
| 151 |
+
"lstrip": false,
|
| 152 |
+
"normalized": false,
|
| 153 |
+
"rstrip": false,
|
| 154 |
+
"single_word": false,
|
| 155 |
+
"special": false
|
| 156 |
+
},
|
| 157 |
+
"151662": {
|
| 158 |
+
"content": "<|fim_pad|>",
|
| 159 |
+
"lstrip": false,
|
| 160 |
+
"normalized": false,
|
| 161 |
+
"rstrip": false,
|
| 162 |
+
"single_word": false,
|
| 163 |
+
"special": false
|
| 164 |
+
},
|
| 165 |
+
"151663": {
|
| 166 |
+
"content": "<|repo_name|>",
|
| 167 |
+
"lstrip": false,
|
| 168 |
+
"normalized": false,
|
| 169 |
+
"rstrip": false,
|
| 170 |
+
"single_word": false,
|
| 171 |
+
"special": false
|
| 172 |
+
},
|
| 173 |
+
"151664": {
|
| 174 |
+
"content": "<|file_sep|>",
|
| 175 |
+
"lstrip": false,
|
| 176 |
+
"normalized": false,
|
| 177 |
+
"rstrip": false,
|
| 178 |
+
"single_word": false,
|
| 179 |
+
"special": false
|
| 180 |
+
},
|
| 181 |
+
"151665": {
|
| 182 |
+
"content": "<|beginoftext|>",
|
| 183 |
+
"lstrip": false,
|
| 184 |
+
"normalized": false,
|
| 185 |
+
"rstrip": false,
|
| 186 |
+
"single_word": false,
|
| 187 |
+
"special": true
|
| 188 |
+
},
|
| 189 |
+
"151666": {
|
| 190 |
+
"content": "<|mask|>",
|
| 191 |
+
"lstrip": false,
|
| 192 |
+
"normalized": false,
|
| 193 |
+
"rstrip": false,
|
| 194 |
+
"single_word": false,
|
| 195 |
+
"special": true
|
| 196 |
+
}
|
| 197 |
+
},
|
| 198 |
+
"additional_special_tokens": [
|
| 199 |
+
"<|beginoftext|>",
|
| 200 |
+
"<|mask|>"
|
| 201 |
+
],
|
| 202 |
+
"auto_map": {
|
| 203 |
+
"AutoTokenizer": [
|
| 204 |
+
"tokenization_dream.DreamTokenizer",
|
| 205 |
+
null
|
| 206 |
+
]
|
| 207 |
+
},
|
| 208 |
+
"bos_token": "<|beginoftext|>",
|
| 209 |
+
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0]['role'] == 'system' %}\n {{- messages[0]['content'] }}\n {%- else %}\n {{- 'You are a helpful assistant.' }}\n {%- endif %}\n {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>system\\nYou are a helpful assistant.<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role }}\n {%- if message.content %}\n {{- '\\n' + message.content }}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- message.content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
|
| 210 |
+
"clean_up_tokenization_spaces": false,
|
| 211 |
+
"eos_token": "<|endoftext|>",
|
| 212 |
+
"errors": "replace",
|
| 213 |
+
"mask_token": "<|mask|>",
|
| 214 |
+
"model_max_length": 131072,
|
| 215 |
+
"pad_token": "<|endoftext|>",
|
| 216 |
+
"split_special_tokens": false,
|
| 217 |
+
"tokenizer_class": "DreamTokenizer",
|
| 218 |
+
"unk_token": null
|
| 219 |
+
}
|
omni_diffusion/models/magvit/common_modules.py
ADDED
|
@@ -0,0 +1,393 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Modified from https://github.com/CompVis/taming-transformers/blob/master/taming/modules/diffusionmodules/model.py#L34
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import math
|
| 6 |
+
from typing import Tuple, Union
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
import torch
|
| 10 |
+
import torch.nn as nn
|
| 11 |
+
import torch.nn.functional as F
|
| 12 |
+
from einops import rearrange, repeat
|
| 13 |
+
from einops.layers.torch import Rearrange
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def nonlinearity(x):
|
| 17 |
+
# swish
|
| 18 |
+
return x * torch.sigmoid(x)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def Normalize(in_channels):
|
| 22 |
+
return torch.nn.GroupNorm(
|
| 23 |
+
num_groups=32, num_channels=in_channels, eps=1e-6, affine=True
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class Upsample(nn.Module):
|
| 28 |
+
"""
|
| 29 |
+
Spatial upsampling layer.
|
| 30 |
+
Upsamples by a factor of 2 using nearest neighbor interpolation,
|
| 31 |
+
optionally followed by a convolution.
|
| 32 |
+
"""
|
| 33 |
+
def __init__(self, in_channels, with_conv):
|
| 34 |
+
super().__init__()
|
| 35 |
+
self.with_conv = with_conv
|
| 36 |
+
if self.with_conv:
|
| 37 |
+
self.conv = torch.nn.Conv2d(
|
| 38 |
+
in_channels, in_channels, kernel_size=3, stride=1, padding=1
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
def forward(self, x):
|
| 42 |
+
x = torch.nn.functional.interpolate(x, scale_factor=2.0, mode="nearest")
|
| 43 |
+
if self.with_conv:
|
| 44 |
+
x = self.conv(x)
|
| 45 |
+
return x
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
class DepthToSpaceUpsample(nn.Module):
|
| 49 |
+
"""
|
| 50 |
+
Upsamples spatially by moving channel depth to spatial dimensions (PixelShuffle).
|
| 51 |
+
Expands channels by 4, then rearranges (c * 2 * 2) -> (h * 2, w * 2).
|
| 52 |
+
"""
|
| 53 |
+
def __init__(
|
| 54 |
+
self,
|
| 55 |
+
in_channels,
|
| 56 |
+
):
|
| 57 |
+
super().__init__()
|
| 58 |
+
conv = nn.Conv2d(in_channels, in_channels * 4, 1)
|
| 59 |
+
|
| 60 |
+
self.net = nn.Sequential(
|
| 61 |
+
conv,
|
| 62 |
+
nn.SiLU(),
|
| 63 |
+
Rearrange("b (c p1 p2) h w -> b c (h p1) (w p2)", p1=2, p2=2),
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
self.init_conv_(conv)
|
| 67 |
+
|
| 68 |
+
def init_conv_(self, conv):
|
| 69 |
+
o, i, h, w = conv.weight.shape
|
| 70 |
+
conv_weight = torch.empty(o // 4, i, h, w)
|
| 71 |
+
nn.init.kaiming_uniform_(conv_weight)
|
| 72 |
+
conv_weight = repeat(conv_weight, "o ... -> (o 4) ...")
|
| 73 |
+
|
| 74 |
+
conv.weight.data.copy_(conv_weight)
|
| 75 |
+
nn.init.zeros_(conv.bias.data)
|
| 76 |
+
|
| 77 |
+
def forward(self, x):
|
| 78 |
+
out = self.net(x)
|
| 79 |
+
return out
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class Downsample(nn.Module):
|
| 83 |
+
"""
|
| 84 |
+
Spatial downsampling layer.
|
| 85 |
+
Uses either a strided convolution (with padding) or average pooling.
|
| 86 |
+
"""
|
| 87 |
+
def __init__(self, in_channels, with_conv):
|
| 88 |
+
super().__init__()
|
| 89 |
+
self.with_conv = with_conv
|
| 90 |
+
if self.with_conv:
|
| 91 |
+
# no asymmetric padding in torch conv, must do it ourselves
|
| 92 |
+
self.conv = torch.nn.Conv2d(
|
| 93 |
+
in_channels, in_channels, kernel_size=3, stride=2, padding=0
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
def forward(self, x):
|
| 97 |
+
if self.with_conv:
|
| 98 |
+
pad = (0, 1, 0, 1)
|
| 99 |
+
x = torch.nn.functional.pad(x, pad, mode="constant", value=0)
|
| 100 |
+
x = self.conv(x)
|
| 101 |
+
else:
|
| 102 |
+
x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2)
|
| 103 |
+
return x
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def unpack_time(t, batch):
|
| 107 |
+
_, c, w, h = t.size()
|
| 108 |
+
out = torch.reshape(t, [batch, -1, c, w, h])
|
| 109 |
+
out = rearrange(out, "b t c h w -> b c t h w")
|
| 110 |
+
return out
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def pack_time(t):
|
| 114 |
+
out = rearrange(t, "b c t h w -> b t c h w")
|
| 115 |
+
_, _, c, w, h = out.size()
|
| 116 |
+
return torch.reshape(out, [-1, c, w, h])
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
class TimeDownsample2x(nn.Module):
|
| 120 |
+
"""
|
| 121 |
+
Temporal downsampling layer.
|
| 122 |
+
Reduces the time dimension (T) by half using 1D convolution with causal padding.
|
| 123 |
+
"""
|
| 124 |
+
def __init__(
|
| 125 |
+
self,
|
| 126 |
+
dim,
|
| 127 |
+
dim_out=None,
|
| 128 |
+
kernel_size=3,
|
| 129 |
+
):
|
| 130 |
+
super().__init__()
|
| 131 |
+
if dim_out is None:
|
| 132 |
+
dim_out = dim
|
| 133 |
+
self.time_causal_padding = (kernel_size - 1, 0)
|
| 134 |
+
self.conv = nn.Conv1d(dim, dim_out, kernel_size, stride=2)
|
| 135 |
+
|
| 136 |
+
def forward(self, x):
|
| 137 |
+
x = rearrange(x, "b c t h w -> b h w c t")
|
| 138 |
+
b, h, w, c, t = x.size()
|
| 139 |
+
x = torch.reshape(x, [-1, c, t])
|
| 140 |
+
|
| 141 |
+
x = F.pad(x, self.time_causal_padding)
|
| 142 |
+
out = self.conv(x)
|
| 143 |
+
|
| 144 |
+
out = torch.reshape(out, [b, h, w, c, t])
|
| 145 |
+
out = rearrange(out, "b h w c t -> b c t h w")
|
| 146 |
+
out = rearrange(out, "b h w c t -> b c t h w")
|
| 147 |
+
return out
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
class TimeUpsample2x(nn.Module):
|
| 151 |
+
"""
|
| 152 |
+
Temporal upsampling layer.
|
| 153 |
+
Doubles the time dimension (T) using a PixelShuffle-like approach on the time axis.
|
| 154 |
+
"""
|
| 155 |
+
def __init__(self, dim, dim_out=None):
|
| 156 |
+
super().__init__()
|
| 157 |
+
if dim_out is None:
|
| 158 |
+
dim_out = dim
|
| 159 |
+
conv = nn.Conv1d(dim, dim_out * 2, 1)
|
| 160 |
+
|
| 161 |
+
self.net = nn.Sequential(
|
| 162 |
+
nn.SiLU(), conv, Rearrange("b (c p) t -> b c (t p)", p=2)
|
| 163 |
+
)
|
| 164 |
+
|
| 165 |
+
self.init_conv_(conv)
|
| 166 |
+
|
| 167 |
+
def init_conv_(self, conv):
|
| 168 |
+
o, i, t = conv.weight.shape
|
| 169 |
+
conv_weight = torch.empty(o // 2, i, t)
|
| 170 |
+
nn.init.kaiming_uniform_(conv_weight)
|
| 171 |
+
conv_weight = repeat(conv_weight, "o ... -> (o 2) ...")
|
| 172 |
+
|
| 173 |
+
conv.weight.data.copy_(conv_weight)
|
| 174 |
+
nn.init.zeros_(conv.bias.data)
|
| 175 |
+
|
| 176 |
+
def forward(self, x):
|
| 177 |
+
x = rearrange(x, "b c t h w -> b h w c t")
|
| 178 |
+
b, h, w, c, t = x.size()
|
| 179 |
+
x = torch.reshape(x, [-1, c, t])
|
| 180 |
+
|
| 181 |
+
out = self.net(x)
|
| 182 |
+
out = out[:, :, 1:].contiguous()
|
| 183 |
+
|
| 184 |
+
out = torch.reshape(out, [b, h, w, c, t])
|
| 185 |
+
out = rearrange(out, "b h w c t -> b c t h w")
|
| 186 |
+
return out
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
class AttnBlock(nn.Module):
|
| 190 |
+
"""
|
| 191 |
+
Standard Self-Attention Block for spatial dimensions.
|
| 192 |
+
"""
|
| 193 |
+
def __init__(self, in_channels):
|
| 194 |
+
super().__init__()
|
| 195 |
+
self.in_channels = in_channels
|
| 196 |
+
|
| 197 |
+
self.norm = Normalize(in_channels)
|
| 198 |
+
self.q = torch.nn.Conv2d(
|
| 199 |
+
in_channels, in_channels, kernel_size=1, stride=1, padding=0
|
| 200 |
+
)
|
| 201 |
+
self.k = torch.nn.Conv2d(
|
| 202 |
+
in_channels, in_channels, kernel_size=1, stride=1, padding=0
|
| 203 |
+
)
|
| 204 |
+
self.v = torch.nn.Conv2d(
|
| 205 |
+
in_channels, in_channels, kernel_size=1, stride=1, padding=0
|
| 206 |
+
)
|
| 207 |
+
self.proj_out = torch.nn.Conv2d(
|
| 208 |
+
in_channels, in_channels, kernel_size=1, stride=1, padding=0
|
| 209 |
+
)
|
| 210 |
+
|
| 211 |
+
def forward(self, x):
|
| 212 |
+
h_ = x
|
| 213 |
+
h_ = self.norm(h_)
|
| 214 |
+
q = self.q(h_)
|
| 215 |
+
k = self.k(h_)
|
| 216 |
+
v = self.v(h_)
|
| 217 |
+
|
| 218 |
+
# compute attention
|
| 219 |
+
b, c, h, w = q.shape
|
| 220 |
+
q = q.reshape(b, c, h * w)
|
| 221 |
+
q = q.permute(0, 2, 1) # b,hw,c
|
| 222 |
+
k = k.reshape(b, c, h * w) # b,c,hw
|
| 223 |
+
w_ = torch.bmm(q, k) # b,hw,hw w[b,i,j]=sum_c q[b,i,c]k[b,c,j]
|
| 224 |
+
w_ = w_ * (int(c) ** (-0.5))
|
| 225 |
+
w_ = torch.nn.functional.softmax(w_, dim=2)
|
| 226 |
+
|
| 227 |
+
# attend to values
|
| 228 |
+
v = v.reshape(b, c, h * w)
|
| 229 |
+
w_ = w_.permute(0, 2, 1) # b,hw,hw (first hw of k, second of q)
|
| 230 |
+
h_ = torch.bmm(v, w_) # b, c,hw (hw of q) h_[b,c,j] = sum_i v[b,c,i] w_[b,i,j]
|
| 231 |
+
h_ = h_.reshape(b, c, h, w)
|
| 232 |
+
|
| 233 |
+
h_ = self.proj_out(h_)
|
| 234 |
+
|
| 235 |
+
return x + h_
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
class TimeAttention(AttnBlock):
|
| 239 |
+
"""
|
| 240 |
+
Applies the AttnBlock over the temporal dimension (T).
|
| 241 |
+
It treats spatial dimensions (H, W) as part of the batch.
|
| 242 |
+
"""
|
| 243 |
+
def forward(self, x, *args, **kwargs):
|
| 244 |
+
x = rearrange(x, "b c t h w -> b h w t c")
|
| 245 |
+
b, h, w, t, c = x.size()
|
| 246 |
+
x = torch.reshape(x, (-1, t, c))
|
| 247 |
+
|
| 248 |
+
x = super().forward(x, *args, **kwargs)
|
| 249 |
+
|
| 250 |
+
x = torch.reshape(x, [b, h, w, t, c])
|
| 251 |
+
return rearrange(x, "b h w t c -> b c t h w")
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
class Residual(nn.Module):
|
| 255 |
+
def __init__(self, fn: nn.Module):
|
| 256 |
+
super().__init__()
|
| 257 |
+
self.fn = fn
|
| 258 |
+
|
| 259 |
+
def forward(self, x, **kwargs):
|
| 260 |
+
return self.fn(x, **kwargs) + x
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
def cast_tuple(t, length=1):
|
| 264 |
+
return t if isinstance(t, tuple) else ((t,) * length)
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
class CausalConv3d(nn.Module):
|
| 268 |
+
"""
|
| 269 |
+
3D Convolution with Causal Padding on the time axis.
|
| 270 |
+
Ensures that the convolution at time T only depends on frames <= T.
|
| 271 |
+
"""
|
| 272 |
+
def __init__(
|
| 273 |
+
self,
|
| 274 |
+
chan_in,
|
| 275 |
+
chan_out,
|
| 276 |
+
kernel_size: Union[int, Tuple[int, int, int]],
|
| 277 |
+
pad_mode="constant",
|
| 278 |
+
**kwargs
|
| 279 |
+
):
|
| 280 |
+
super().__init__()
|
| 281 |
+
kernel_size = cast_tuple(kernel_size, 3)
|
| 282 |
+
|
| 283 |
+
time_kernel_size, height_kernel_size, width_kernel_size = kernel_size
|
| 284 |
+
|
| 285 |
+
dilation = kwargs.pop("dilation", 1)
|
| 286 |
+
stride = kwargs.pop("stride", 1)
|
| 287 |
+
|
| 288 |
+
self.pad_mode = pad_mode
|
| 289 |
+
time_pad = dilation * (time_kernel_size - 1) + (1 - stride)
|
| 290 |
+
height_pad = height_kernel_size // 2
|
| 291 |
+
width_pad = width_kernel_size // 2
|
| 292 |
+
|
| 293 |
+
self.time_pad = time_pad
|
| 294 |
+
self.time_causal_padding = (
|
| 295 |
+
width_pad,
|
| 296 |
+
width_pad,
|
| 297 |
+
height_pad,
|
| 298 |
+
height_pad,
|
| 299 |
+
time_pad,
|
| 300 |
+
0,
|
| 301 |
+
)
|
| 302 |
+
|
| 303 |
+
stride = (stride, 1, 1)
|
| 304 |
+
dilation = (dilation, 1, 1)
|
| 305 |
+
self.conv = nn.Conv3d(
|
| 306 |
+
chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs
|
| 307 |
+
)
|
| 308 |
+
|
| 309 |
+
def forward(self, x):
|
| 310 |
+
pad_mode = self.pad_mode if self.time_pad < x.shape[2] else "constant"
|
| 311 |
+
|
| 312 |
+
x = F.pad(x, self.time_causal_padding, mode=pad_mode)
|
| 313 |
+
return self.conv(x)
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
def ResnetBlockCausal3D(
|
| 317 |
+
dim, kernel_size: Union[int, Tuple[int, int, int]], pad_mode: str = "constant"
|
| 318 |
+
):
|
| 319 |
+
net = nn.Sequential(
|
| 320 |
+
Normalize(dim),
|
| 321 |
+
nn.SiLU(),
|
| 322 |
+
CausalConv3d(dim, dim, kernel_size, pad_mode),
|
| 323 |
+
Normalize(dim),
|
| 324 |
+
nn.SiLU(),
|
| 325 |
+
CausalConv3d(dim, dim, kernel_size, pad_mode),
|
| 326 |
+
)
|
| 327 |
+
return Residual(net)
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
class ResnetBlock(nn.Module):
|
| 331 |
+
"""
|
| 332 |
+
Standard ResNet Block commonly used in Diffusion Models.
|
| 333 |
+
Includes support for Time Embeddings (temb) injection.
|
| 334 |
+
"""
|
| 335 |
+
def __init__(
|
| 336 |
+
self,
|
| 337 |
+
*,
|
| 338 |
+
in_channels,
|
| 339 |
+
out_channels=None,
|
| 340 |
+
conv_shortcut=False,
|
| 341 |
+
dropout,
|
| 342 |
+
temb_channels=512
|
| 343 |
+
):
|
| 344 |
+
super().__init__()
|
| 345 |
+
self.in_channels = in_channels
|
| 346 |
+
out_channels = in_channels if out_channels is None else out_channels
|
| 347 |
+
self.out_channels = out_channels
|
| 348 |
+
self.use_conv_shortcut = conv_shortcut
|
| 349 |
+
|
| 350 |
+
self.norm1 = Normalize(in_channels)
|
| 351 |
+
self.conv1 = torch.nn.Conv2d(
|
| 352 |
+
in_channels, out_channels, kernel_size=3, stride=1, padding=1
|
| 353 |
+
)
|
| 354 |
+
if temb_channels > 0:
|
| 355 |
+
self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
|
| 356 |
+
else:
|
| 357 |
+
self.temb_proj = None
|
| 358 |
+
self.norm2 = Normalize(out_channels)
|
| 359 |
+
self.dropout = torch.nn.Dropout(dropout)
|
| 360 |
+
self.conv2 = torch.nn.Conv2d(
|
| 361 |
+
out_channels, out_channels, kernel_size=3, stride=1, padding=1
|
| 362 |
+
)
|
| 363 |
+
if self.in_channels != self.out_channels:
|
| 364 |
+
if self.use_conv_shortcut:
|
| 365 |
+
self.conv_shortcut = torch.nn.Conv2d(
|
| 366 |
+
in_channels, out_channels, kernel_size=3, stride=1, padding=1
|
| 367 |
+
)
|
| 368 |
+
else:
|
| 369 |
+
self.nin_shortcut = torch.nn.Conv2d(
|
| 370 |
+
in_channels, out_channels, kernel_size=1, stride=1, padding=0
|
| 371 |
+
)
|
| 372 |
+
|
| 373 |
+
def forward(self, x, temb):
|
| 374 |
+
h = x
|
| 375 |
+
h = self.norm1(h)
|
| 376 |
+
h = nonlinearity(h)
|
| 377 |
+
h = self.conv1(h)
|
| 378 |
+
|
| 379 |
+
if temb is not None:
|
| 380 |
+
h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
|
| 381 |
+
|
| 382 |
+
h = self.norm2(h)
|
| 383 |
+
h = nonlinearity(h)
|
| 384 |
+
h = self.dropout(h)
|
| 385 |
+
h = self.conv2(h)
|
| 386 |
+
|
| 387 |
+
if self.in_channels != self.out_channels:
|
| 388 |
+
if self.use_conv_shortcut:
|
| 389 |
+
x = self.conv_shortcut(x)
|
| 390 |
+
else:
|
| 391 |
+
x = self.nin_shortcut(x)
|
| 392 |
+
|
| 393 |
+
return x + h
|
omni_diffusion/models/magvit/misc.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# copy from MMaDA
|
| 2 |
+
|
| 3 |
+
from omegaconf import OmegaConf
|
| 4 |
+
import torch
|
| 5 |
+
from typing import (
|
| 6 |
+
Any,
|
| 7 |
+
Callable,
|
| 8 |
+
Dict,
|
| 9 |
+
Iterable,
|
| 10 |
+
List,
|
| 11 |
+
NamedTuple,
|
| 12 |
+
NewType,
|
| 13 |
+
Optional,
|
| 14 |
+
Sized,
|
| 15 |
+
Tuple,
|
| 16 |
+
Type,
|
| 17 |
+
TypeVar,
|
| 18 |
+
Union,
|
| 19 |
+
)
|
| 20 |
+
try:
|
| 21 |
+
from typing import Literal
|
| 22 |
+
except ImportError:
|
| 23 |
+
from typing_extensions import Literal
|
| 24 |
+
|
| 25 |
+
# Tensor dtype
|
| 26 |
+
# for jaxtyping usage, see https://github.com/google/jaxtyping/blob/main/API.md
|
| 27 |
+
from jaxtyping import Bool, Complex, Float, Inexact, Int, Integer, Num, Shaped, UInt
|
| 28 |
+
|
| 29 |
+
# Config type
|
| 30 |
+
from omegaconf import DictConfig
|
| 31 |
+
|
| 32 |
+
# PyTorch Tensor type
|
| 33 |
+
from torch import Tensor
|
| 34 |
+
|
| 35 |
+
# Runtime type checking decorator
|
| 36 |
+
from typeguard import typechecked as typechecker
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def broadcast(tensor, src=0):
|
| 40 |
+
if not _distributed_available():
|
| 41 |
+
return tensor
|
| 42 |
+
else:
|
| 43 |
+
torch.distributed.broadcast(tensor, src=src)
|
| 44 |
+
return tensor
|
| 45 |
+
|
| 46 |
+
def _distributed_available():
|
| 47 |
+
return torch.distributed.is_available() and torch.distributed.is_initialized()
|
| 48 |
+
|
| 49 |
+
def parse_structured(fields: Any, cfg: Optional[Union[dict, DictConfig]] = None) -> Any:
|
| 50 |
+
# added by Xavier -- delete '--local-rank' in multi-nodes training, don't know why there is such a keyword
|
| 51 |
+
if '--local-rank' in cfg:
|
| 52 |
+
del cfg['--local-rank']
|
| 53 |
+
# added by Xavier -- delete '--local-rank' in multi-nodes training, don't know why there is such a keyword
|
| 54 |
+
scfg = OmegaConf.structured(fields(**cfg))
|
| 55 |
+
return scfg
|
omni_diffusion/models/magvit/modeling_magvitv2.py
ADDED
|
@@ -0,0 +1,445 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Modified from https://github.com/Gen-Verse/MMaDA/blob/main/models/modeling_magvitv2.py
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from dataclasses import dataclass, field
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch
|
| 8 |
+
import torch.nn as nn
|
| 9 |
+
from .common_modules import *
|
| 10 |
+
from .modeling_utils import ConfigMixin, ModelMixin, register_to_config
|
| 11 |
+
from .misc import *
|
| 12 |
+
import math
|
| 13 |
+
|
| 14 |
+
class Updateable:
|
| 15 |
+
def do_update_step(
|
| 16 |
+
self, epoch: int, global_step: int, on_load_weights: bool = False
|
| 17 |
+
):
|
| 18 |
+
for attr in self.__dir__():
|
| 19 |
+
if attr.startswith("_"):
|
| 20 |
+
continue
|
| 21 |
+
try:
|
| 22 |
+
module = getattr(self, attr)
|
| 23 |
+
except:
|
| 24 |
+
continue # ignore attributes like property, which can't be retrived using getattr?
|
| 25 |
+
if isinstance(module, Updateable):
|
| 26 |
+
module.do_update_step(
|
| 27 |
+
epoch, global_step, on_load_weights=on_load_weights
|
| 28 |
+
)
|
| 29 |
+
self.update_step(epoch, global_step, on_load_weights=on_load_weights)
|
| 30 |
+
|
| 31 |
+
def do_update_step_end(self, epoch: int, global_step: int):
|
| 32 |
+
for attr in self.__dir__():
|
| 33 |
+
if attr.startswith("_"):
|
| 34 |
+
continue
|
| 35 |
+
try:
|
| 36 |
+
module = getattr(self, attr)
|
| 37 |
+
except:
|
| 38 |
+
continue # ignore attributes like property, which can't be retrived using getattr?
|
| 39 |
+
if isinstance(module, Updateable):
|
| 40 |
+
module.do_update_step_end(epoch, global_step)
|
| 41 |
+
self.update_step_end(epoch, global_step)
|
| 42 |
+
|
| 43 |
+
def update_step(self, epoch: int, global_step: int, on_load_weights: bool = False):
|
| 44 |
+
# override this method to implement custom update logic
|
| 45 |
+
# if on_load_weights is True, you should be careful doing things related to model evaluations,
|
| 46 |
+
# as the models and tensors are not guarenteed to be on the same device
|
| 47 |
+
pass
|
| 48 |
+
|
| 49 |
+
def update_step_end(self, epoch: int, global_step: int):
|
| 50 |
+
pass
|
| 51 |
+
|
| 52 |
+
class VQGANEncoder(ModelMixin, ConfigMixin):
|
| 53 |
+
@dataclass
|
| 54 |
+
class Config:
|
| 55 |
+
ch: int = 128
|
| 56 |
+
ch_mult: List[int] = field(default_factory=lambda: [1, 2, 2, 4, 4])
|
| 57 |
+
num_res_blocks: List[int] = field(default_factory=lambda: [4, 3, 4, 3, 4])
|
| 58 |
+
attn_resolutions: List[int] = field(default_factory=lambda: [5])
|
| 59 |
+
dropout: float = 0.0
|
| 60 |
+
in_ch: int = 3
|
| 61 |
+
out_ch: int = 3
|
| 62 |
+
resolution: int = 256
|
| 63 |
+
z_channels: int = 13
|
| 64 |
+
double_z: bool = False
|
| 65 |
+
|
| 66 |
+
def __init__(self,
|
| 67 |
+
ch: int = 128,
|
| 68 |
+
ch_mult: List[int] = [1, 2, 2, 4, 4],
|
| 69 |
+
num_res_blocks: List[int] = [4, 3, 4, 3, 4],
|
| 70 |
+
attn_resolutions: List[int] = [5],
|
| 71 |
+
dropout: float = 0.0,
|
| 72 |
+
in_ch: int = 3,
|
| 73 |
+
out_ch: int = 3,
|
| 74 |
+
resolution: int = 256,
|
| 75 |
+
z_channels: int = 13,
|
| 76 |
+
double_z: bool = False):
|
| 77 |
+
super().__init__()
|
| 78 |
+
self.ch = ch
|
| 79 |
+
self.temb_ch = 0
|
| 80 |
+
self.num_resolutions = len(ch_mult)
|
| 81 |
+
self.num_res_blocks = num_res_blocks
|
| 82 |
+
self.resolution = resolution
|
| 83 |
+
self.in_ch = in_ch
|
| 84 |
+
# downsampling
|
| 85 |
+
self.conv_in = torch.nn.Conv2d(
|
| 86 |
+
self.in_ch, self.ch, kernel_size=3, stride=1, padding=1
|
| 87 |
+
)
|
| 88 |
+
|
| 89 |
+
curr_res = self.resolution
|
| 90 |
+
in_ch_mult = (1,) + tuple(ch_mult)
|
| 91 |
+
self.down = nn.ModuleList()
|
| 92 |
+
for i_level in range(self.num_resolutions):
|
| 93 |
+
block = nn.ModuleList()
|
| 94 |
+
attn = nn.ModuleList()
|
| 95 |
+
block_in = self.ch * in_ch_mult[i_level]
|
| 96 |
+
block_out = self.ch * ch_mult[i_level]
|
| 97 |
+
for i_block in range(self.num_res_blocks[i_level]):
|
| 98 |
+
block.append(
|
| 99 |
+
ResnetBlock(
|
| 100 |
+
in_channels=block_in,
|
| 101 |
+
out_channels=block_out,
|
| 102 |
+
temb_channels=self.temb_ch,
|
| 103 |
+
dropout=dropout,
|
| 104 |
+
)
|
| 105 |
+
)
|
| 106 |
+
block_in = block_out
|
| 107 |
+
if curr_res in attn_resolutions:
|
| 108 |
+
attn.append(AttnBlock(block_in))
|
| 109 |
+
down = nn.Module()
|
| 110 |
+
down.block = block
|
| 111 |
+
down.attn = attn
|
| 112 |
+
if i_level != self.num_resolutions - 1:
|
| 113 |
+
down.downsample = Downsample(block_in, True)
|
| 114 |
+
curr_res = curr_res // 2
|
| 115 |
+
self.down.append(down)
|
| 116 |
+
|
| 117 |
+
# middle
|
| 118 |
+
self.mid = nn.Module()
|
| 119 |
+
self.mid.block_1 = ResnetBlock(
|
| 120 |
+
in_channels=block_in,
|
| 121 |
+
out_channels=block_in,
|
| 122 |
+
temb_channels=self.temb_ch,
|
| 123 |
+
dropout=dropout,
|
| 124 |
+
)
|
| 125 |
+
self.mid.attn_1 = AttnBlock(block_in)
|
| 126 |
+
self.mid.block_2 = ResnetBlock(
|
| 127 |
+
in_channels=block_in,
|
| 128 |
+
out_channels=block_in,
|
| 129 |
+
temb_channels=self.temb_ch,
|
| 130 |
+
dropout=dropout,
|
| 131 |
+
)
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
self.norm_out = Normalize(block_in)
|
| 135 |
+
self.conv_out = torch.nn.Conv2d(
|
| 136 |
+
block_in,
|
| 137 |
+
2 * z_channels if double_z else z_channels,
|
| 138 |
+
kernel_size=3,
|
| 139 |
+
stride=1,
|
| 140 |
+
padding=1,
|
| 141 |
+
)
|
| 142 |
+
|
| 143 |
+
self.quant_conv = torch.nn.Conv2d(z_channels, z_channels, 1)
|
| 144 |
+
# for param in self.parameters():
|
| 145 |
+
# broadcast(param, src=0)
|
| 146 |
+
|
| 147 |
+
def forward(self, x):
|
| 148 |
+
# timestep embedding
|
| 149 |
+
temb = None
|
| 150 |
+
|
| 151 |
+
# downsampling
|
| 152 |
+
hs = [self.conv_in(x)]
|
| 153 |
+
for i_level in range(self.num_resolutions):
|
| 154 |
+
for i_block in range(self.num_res_blocks[i_level]):
|
| 155 |
+
h = self.down[i_level].block[i_block](hs[-1], temb)
|
| 156 |
+
if len(self.down[i_level].attn) > 0:
|
| 157 |
+
h = self.down[i_level].attn[i_block](h)
|
| 158 |
+
hs.append(h)
|
| 159 |
+
if i_level != self.num_resolutions - 1:
|
| 160 |
+
hs.append(self.down[i_level].downsample(hs[-1]))
|
| 161 |
+
|
| 162 |
+
# middle
|
| 163 |
+
h = hs[-1]
|
| 164 |
+
h = self.mid.block_1(h, temb)
|
| 165 |
+
h = self.mid.attn_1(h)
|
| 166 |
+
h = self.mid.block_2(h, temb)
|
| 167 |
+
|
| 168 |
+
# end
|
| 169 |
+
h = self.norm_out(h)
|
| 170 |
+
h = nonlinearity(h)
|
| 171 |
+
h = self.conv_out(h)
|
| 172 |
+
h = self.quant_conv(h)
|
| 173 |
+
return h
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
class LFQuantizer(nn.Module):
|
| 177 |
+
def __init__(self, num_codebook_entry: int = -1,
|
| 178 |
+
codebook_dim: int = 13,
|
| 179 |
+
beta: float = 0.25,
|
| 180 |
+
entropy_multiplier: float = 0.1,
|
| 181 |
+
commit_loss_multiplier: float = 0.1, ):
|
| 182 |
+
super().__init__()
|
| 183 |
+
self.codebook_size = 2 ** codebook_dim
|
| 184 |
+
print(
|
| 185 |
+
f"Look-up free quantizer with codebook size: {self.codebook_size}"
|
| 186 |
+
)
|
| 187 |
+
self.e_dim = codebook_dim
|
| 188 |
+
self.beta = beta
|
| 189 |
+
|
| 190 |
+
indices = torch.arange(self.codebook_size)
|
| 191 |
+
|
| 192 |
+
binary = (
|
| 193 |
+
indices.unsqueeze(1)
|
| 194 |
+
>> torch.arange(codebook_dim - 1, -1, -1, dtype=torch.long)
|
| 195 |
+
) & 1
|
| 196 |
+
|
| 197 |
+
embedding = binary.float() * 2 - 1
|
| 198 |
+
self.register_buffer("embedding", embedding)
|
| 199 |
+
self.register_buffer(
|
| 200 |
+
"power_vals", 2 ** torch.arange(codebook_dim - 1, -1, -1)
|
| 201 |
+
)
|
| 202 |
+
self.commit_loss_multiplier = commit_loss_multiplier
|
| 203 |
+
self.entropy_multiplier = entropy_multiplier
|
| 204 |
+
|
| 205 |
+
def get_indices(self, z_q):
|
| 206 |
+
return (
|
| 207 |
+
(self.power_vals.reshape(1, -1, 1, 1) * (z_q > 0).float())
|
| 208 |
+
.sum(1, keepdim=True)
|
| 209 |
+
.long()
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
def get_codebook_entry(self, indices, shape=None):
|
| 213 |
+
if shape is None:
|
| 214 |
+
h, w = int(math.sqrt(indices.shape[-1])), int(math.sqrt(indices.shape[-1]))
|
| 215 |
+
else:
|
| 216 |
+
h, w = shape
|
| 217 |
+
b, _ = indices.shape
|
| 218 |
+
indices = indices.reshape(-1)
|
| 219 |
+
z_q = self.embedding[indices]
|
| 220 |
+
z_q = z_q.view(b, h, w, -1)
|
| 221 |
+
|
| 222 |
+
# reshape back to match original input shape
|
| 223 |
+
z_q = z_q.permute(0, 3, 1, 2).contiguous()
|
| 224 |
+
|
| 225 |
+
return z_q
|
| 226 |
+
|
| 227 |
+
def forward(self, z, get_code=False):
|
| 228 |
+
"""
|
| 229 |
+
Inputs the output of the encoder network z and maps it to a discrete
|
| 230 |
+
one-hot vector that is the index of the closest embedding vector e_j
|
| 231 |
+
z (continuous) -> z_q (discrete)
|
| 232 |
+
z.shape = (batch, channel, height, width)
|
| 233 |
+
quantization pipeline:
|
| 234 |
+
1. get encoder input (B,C,H,W)
|
| 235 |
+
2. flatten input to (B*H*W,C)
|
| 236 |
+
"""
|
| 237 |
+
if get_code:
|
| 238 |
+
return self.get_codebook_entry(z)
|
| 239 |
+
|
| 240 |
+
# reshape z -> (batch, height, width, channel) and flatten
|
| 241 |
+
z = z.permute(0, 2, 3, 1).contiguous()
|
| 242 |
+
z_flattened = z.view(-1, self.e_dim)
|
| 243 |
+
ge_zero = (z_flattened > 0).float()
|
| 244 |
+
ones = torch.ones_like(z_flattened)
|
| 245 |
+
z_q = ones * ge_zero + -ones * (1 - ge_zero)
|
| 246 |
+
|
| 247 |
+
# preserve gradients
|
| 248 |
+
z_q = z_flattened + (z_q - z_flattened).detach()
|
| 249 |
+
|
| 250 |
+
# compute entropy loss
|
| 251 |
+
CatDist = torch.distributions.categorical.Categorical
|
| 252 |
+
logit = torch.stack(
|
| 253 |
+
[
|
| 254 |
+
-(z_flattened - torch.ones_like(z_q)).pow(2),
|
| 255 |
+
-(z_flattened - torch.ones_like(z_q) * -1).pow(2),
|
| 256 |
+
],
|
| 257 |
+
dim=-1,
|
| 258 |
+
)
|
| 259 |
+
cat_dist = CatDist(logits=logit)
|
| 260 |
+
entropy = cat_dist.entropy().mean()
|
| 261 |
+
mean_prob = cat_dist.probs.mean(0)
|
| 262 |
+
mean_entropy = CatDist(probs=mean_prob).entropy().mean()
|
| 263 |
+
|
| 264 |
+
# compute loss for embedding
|
| 265 |
+
commit_loss = torch.mean(
|
| 266 |
+
(z_q.detach() - z_flattened) ** 2
|
| 267 |
+
) + self.beta * torch.mean((z_q - z_flattened.detach()) ** 2)
|
| 268 |
+
|
| 269 |
+
# reshape back to match original input shape
|
| 270 |
+
z_q = z_q.view(z.shape)
|
| 271 |
+
z_q = z_q.permute(0, 3, 1, 2).contiguous()
|
| 272 |
+
|
| 273 |
+
return {
|
| 274 |
+
"z": z_q,
|
| 275 |
+
"quantizer_loss": commit_loss * self.commit_loss_multiplier,
|
| 276 |
+
"entropy_loss": (entropy - mean_entropy) * self.entropy_multiplier,
|
| 277 |
+
"indices": self.get_indices(z_q),
|
| 278 |
+
}
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
class VQGANDecoder(ModelMixin, ConfigMixin):
|
| 282 |
+
def __init__(self, ch: int = 128,
|
| 283 |
+
ch_mult: List[int] = [1, 1, 2, 2, 4],
|
| 284 |
+
num_res_blocks: List[int] = [4, 4, 3, 4, 3],
|
| 285 |
+
attn_resolutions: List[int] = [5],
|
| 286 |
+
dropout: float = 0.0,
|
| 287 |
+
in_ch: int = 3,
|
| 288 |
+
out_ch: int = 3,
|
| 289 |
+
resolution: int = 256,
|
| 290 |
+
z_channels: int = 13,
|
| 291 |
+
double_z: bool = False):
|
| 292 |
+
super().__init__()
|
| 293 |
+
self.ch = ch
|
| 294 |
+
self.temb_ch = 0
|
| 295 |
+
self.num_resolutions = len(ch_mult)
|
| 296 |
+
self.num_res_blocks = num_res_blocks
|
| 297 |
+
self.resolution = resolution
|
| 298 |
+
self.in_ch = in_ch
|
| 299 |
+
self.give_pre_end = False
|
| 300 |
+
|
| 301 |
+
self.z_channels = z_channels
|
| 302 |
+
# compute in_ch_mult, block_in and curr_res at lowest res
|
| 303 |
+
in_ch_mult = (1,) + tuple(ch_mult)
|
| 304 |
+
block_in = ch * ch_mult[self.num_resolutions - 1]
|
| 305 |
+
curr_res = self.resolution // 2 ** (self.num_resolutions - 1)
|
| 306 |
+
self.z_shape = (1, z_channels, curr_res, curr_res)
|
| 307 |
+
print(
|
| 308 |
+
"Working with z of shape {} = {} dimensions.".format(
|
| 309 |
+
self.z_shape, np.prod(self.z_shape)
|
| 310 |
+
)
|
| 311 |
+
)
|
| 312 |
+
|
| 313 |
+
# z to block_in
|
| 314 |
+
self.conv_in = torch.nn.Conv2d(
|
| 315 |
+
z_channels, block_in, kernel_size=3, stride=1, padding=1
|
| 316 |
+
)
|
| 317 |
+
|
| 318 |
+
# middle
|
| 319 |
+
self.mid = nn.Module()
|
| 320 |
+
self.mid.block_1 = ResnetBlock(
|
| 321 |
+
in_channels=block_in,
|
| 322 |
+
out_channels=block_in,
|
| 323 |
+
temb_channels=self.temb_ch,
|
| 324 |
+
dropout=dropout,
|
| 325 |
+
)
|
| 326 |
+
self.mid.attn_1 = AttnBlock(block_in)
|
| 327 |
+
self.mid.block_2 = ResnetBlock(
|
| 328 |
+
in_channels=block_in,
|
| 329 |
+
out_channels=block_in,
|
| 330 |
+
temb_channels=self.temb_ch,
|
| 331 |
+
dropout=dropout,
|
| 332 |
+
)
|
| 333 |
+
|
| 334 |
+
# upsampling
|
| 335 |
+
self.up = nn.ModuleList()
|
| 336 |
+
for i_level in reversed(range(self.num_resolutions)):
|
| 337 |
+
block = nn.ModuleList()
|
| 338 |
+
attn = nn.ModuleList()
|
| 339 |
+
block_out = ch * ch_mult[i_level]
|
| 340 |
+
for i_block in range(self.num_res_blocks[i_level]):
|
| 341 |
+
block.append(
|
| 342 |
+
ResnetBlock(
|
| 343 |
+
in_channels=block_in,
|
| 344 |
+
out_channels=block_out,
|
| 345 |
+
temb_channels=self.temb_ch,
|
| 346 |
+
dropout=dropout,
|
| 347 |
+
)
|
| 348 |
+
)
|
| 349 |
+
block_in = block_out
|
| 350 |
+
if curr_res in attn_resolutions:
|
| 351 |
+
attn.append(AttnBlock(block_in))
|
| 352 |
+
up = nn.Module()
|
| 353 |
+
up.block = block
|
| 354 |
+
up.attn = attn
|
| 355 |
+
if i_level != 0:
|
| 356 |
+
up.upsample = Upsample(block_in, True)
|
| 357 |
+
curr_res = curr_res * 2
|
| 358 |
+
self.up.insert(0, up) # prepend to get consistent order
|
| 359 |
+
|
| 360 |
+
self.norm_out = Normalize(block_in)
|
| 361 |
+
self.conv_out = torch.nn.Conv2d(
|
| 362 |
+
block_in, out_ch, kernel_size=3, stride=1, padding=1
|
| 363 |
+
)
|
| 364 |
+
self.post_quant_conv = torch.nn.Conv2d(
|
| 365 |
+
z_channels, z_channels, 1
|
| 366 |
+
)
|
| 367 |
+
|
| 368 |
+
|
| 369 |
+
def forward(self, z):
|
| 370 |
+
# assert z.shape[1:] == self.z_shape[1:]
|
| 371 |
+
self.last_z_shape = z.shape
|
| 372 |
+
# timestep embedding
|
| 373 |
+
temb = None
|
| 374 |
+
output = dict()
|
| 375 |
+
z = self.post_quant_conv(z)
|
| 376 |
+
|
| 377 |
+
# z to block_in
|
| 378 |
+
h = self.conv_in(z)
|
| 379 |
+
|
| 380 |
+
# middle
|
| 381 |
+
h = self.mid.block_1(h, temb)
|
| 382 |
+
h = self.mid.attn_1(h)
|
| 383 |
+
h = self.mid.block_2(h, temb)
|
| 384 |
+
|
| 385 |
+
# upsampling
|
| 386 |
+
for i_level in reversed(range(self.num_resolutions)):
|
| 387 |
+
for i_block in range(self.num_res_blocks[i_level]):
|
| 388 |
+
h = self.up[i_level].block[i_block](h, temb)
|
| 389 |
+
if len(self.up[i_level].attn) > 0:
|
| 390 |
+
h = self.up[i_level].attn[i_block](h)
|
| 391 |
+
if i_level != 0:
|
| 392 |
+
h = self.up[i_level].upsample(h)
|
| 393 |
+
|
| 394 |
+
# end
|
| 395 |
+
output["output"] = h
|
| 396 |
+
if self.give_pre_end:
|
| 397 |
+
return output
|
| 398 |
+
|
| 399 |
+
h = self.norm_out(h)
|
| 400 |
+
h = nonlinearity(h)
|
| 401 |
+
h = self.conv_out(h)
|
| 402 |
+
output["output"] = h
|
| 403 |
+
return output
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
class MAGVITv2(ModelMixin, ConfigMixin):
|
| 407 |
+
@register_to_config
|
| 408 |
+
def __init__(
|
| 409 |
+
self,
|
| 410 |
+
):
|
| 411 |
+
super().__init__()
|
| 412 |
+
|
| 413 |
+
self.encoder = VQGANEncoder()
|
| 414 |
+
self.decoder = VQGANDecoder()
|
| 415 |
+
self.quantize = LFQuantizer()
|
| 416 |
+
|
| 417 |
+
def forward(self, pixel_values, return_loss=False):
|
| 418 |
+
pass
|
| 419 |
+
|
| 420 |
+
def encode(self, pixel_values, return_loss=False):
|
| 421 |
+
hidden_states = self.encoder(pixel_values)
|
| 422 |
+
quantized_states = self.quantize(hidden_states)['z']
|
| 423 |
+
codebook_indices = self.quantize.get_indices(quantized_states).reshape(pixel_values.shape[0], -1)
|
| 424 |
+
output = (quantized_states, codebook_indices)
|
| 425 |
+
return output
|
| 426 |
+
|
| 427 |
+
def get_code(self, pixel_values):
|
| 428 |
+
hidden_states = self.encoder(pixel_values)
|
| 429 |
+
codebook_indices = self.quantize.get_indices(self.quantize(hidden_states)['z'])
|
| 430 |
+
codebook_indices = codebook_indices.reshape(pixel_values.shape[0], -1)
|
| 431 |
+
|
| 432 |
+
return codebook_indices
|
| 433 |
+
|
| 434 |
+
def decode_code(self, codebook_indices, shape=None):
|
| 435 |
+
z_q = self.quantize.get_codebook_entry(codebook_indices, shape=shape)
|
| 436 |
+
|
| 437 |
+
reconstructed_pixel_values = self.decoder(z_q)["output"]
|
| 438 |
+
return reconstructed_pixel_values
|
| 439 |
+
|
| 440 |
+
|
| 441 |
+
if __name__ == '__main__':
|
| 442 |
+
encoder = VQGANEncoder()
|
| 443 |
+
import ipdb
|
| 444 |
+
ipdb.set_trace()
|
| 445 |
+
print()
|
omni_diffusion/models/magvit/modeling_utils.py
ADDED
|
@@ -0,0 +1,1184 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Modified from https://github.com/Gen-Verse/MMaDA/blob/main/models/modeling_utils.py
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
# coding=utf-8
|
| 6 |
+
# Copyright 2024 The HuggingFace Inc. team.
|
| 7 |
+
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
|
| 8 |
+
#
|
| 9 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 10 |
+
# you may not use this file except in compliance with the License.
|
| 11 |
+
# You may obtain a copy of the License at
|
| 12 |
+
#
|
| 13 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 14 |
+
#
|
| 15 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 16 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 17 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 18 |
+
# See the License for the specific language governing permissions and
|
| 19 |
+
# limitations under the License.
|
| 20 |
+
|
| 21 |
+
import inspect
|
| 22 |
+
import itertools
|
| 23 |
+
import json
|
| 24 |
+
import os
|
| 25 |
+
import re
|
| 26 |
+
from collections import OrderedDict
|
| 27 |
+
from functools import partial
|
| 28 |
+
from pathlib import Path
|
| 29 |
+
from typing import Any, Callable, List, Optional, Tuple, Union
|
| 30 |
+
|
| 31 |
+
import safetensors
|
| 32 |
+
import torch
|
| 33 |
+
from huggingface_hub import create_repo, split_torch_state_dict_into_shards
|
| 34 |
+
from huggingface_hub.utils import validate_hf_hub_args
|
| 35 |
+
from torch import Tensor, nn
|
| 36 |
+
|
| 37 |
+
from diffusers import __version__
|
| 38 |
+
from diffusers.utils import (
|
| 39 |
+
FLAX_WEIGHTS_NAME,
|
| 40 |
+
SAFE_WEIGHTS_INDEX_NAME,
|
| 41 |
+
WEIGHTS_INDEX_NAME,
|
| 42 |
+
_add_variant,
|
| 43 |
+
_get_checkpoint_shard_files,
|
| 44 |
+
_get_model_file,
|
| 45 |
+
deprecate,
|
| 46 |
+
is_accelerate_available,
|
| 47 |
+
is_torch_version,
|
| 48 |
+
logging,
|
| 49 |
+
)
|
| 50 |
+
|
| 51 |
+
CONFIG_NAME = "config.json"
|
| 52 |
+
WEIGHTS_NAME = "pytorch_model.bin"
|
| 53 |
+
SAFETENSORS_WEIGHTS_NAME = "pytorch_model.safetensors"
|
| 54 |
+
HUGGINGFACE_CO_RESOLVE_ENDPOINT = "https://huggingface.co"
|
| 55 |
+
|
| 56 |
+
from diffusers.utils.hub_utils import (
|
| 57 |
+
PushToHubMixin,
|
| 58 |
+
load_or_create_model_card,
|
| 59 |
+
populate_model_card,
|
| 60 |
+
)
|
| 61 |
+
from diffusers.models.model_loading_utils import (
|
| 62 |
+
_determine_device_map,
|
| 63 |
+
_fetch_index_file,
|
| 64 |
+
_load_state_dict_into_model,
|
| 65 |
+
load_model_dict_into_meta,
|
| 66 |
+
)
|
| 67 |
+
|
| 68 |
+
from diffusers.configuration_utils import ConfigMixin, register_to_config
|
| 69 |
+
import diffusers
|
| 70 |
+
|
| 71 |
+
logger = logging.get_logger(__name__)
|
| 72 |
+
|
| 73 |
+
_REGEX_SHARD = re.compile(r"(.*?)-\d{5}-of-\d{5}")
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
if is_torch_version(">=", "1.9.0"):
|
| 77 |
+
_LOW_CPU_MEM_USAGE_DEFAULT = True
|
| 78 |
+
else:
|
| 79 |
+
_LOW_CPU_MEM_USAGE_DEFAULT = False
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
if is_accelerate_available():
|
| 83 |
+
import accelerate
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def get_parameter_device(parameter: torch.nn.Module) -> torch.device:
|
| 87 |
+
try:
|
| 88 |
+
parameters_and_buffers = itertools.chain(parameter.parameters(), parameter.buffers())
|
| 89 |
+
return next(parameters_and_buffers).device
|
| 90 |
+
except StopIteration:
|
| 91 |
+
# For torch.nn.DataParallel compatibility in PyTorch 1.5
|
| 92 |
+
|
| 93 |
+
def find_tensor_attributes(module: torch.nn.Module) -> List[Tuple[str, Tensor]]:
|
| 94 |
+
tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)]
|
| 95 |
+
return tuples
|
| 96 |
+
|
| 97 |
+
gen = parameter._named_members(get_members_fn=find_tensor_attributes)
|
| 98 |
+
first_tuple = next(gen)
|
| 99 |
+
return first_tuple[1].device
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def get_parameter_dtype(parameter: torch.nn.Module) -> torch.dtype:
|
| 103 |
+
try:
|
| 104 |
+
params = tuple(parameter.parameters())
|
| 105 |
+
if len(params) > 0:
|
| 106 |
+
return params[0].dtype
|
| 107 |
+
|
| 108 |
+
buffers = tuple(parameter.buffers())
|
| 109 |
+
if len(buffers) > 0:
|
| 110 |
+
return buffers[0].dtype
|
| 111 |
+
|
| 112 |
+
except StopIteration:
|
| 113 |
+
# For torch.nn.DataParallel compatibility in PyTorch 1.5
|
| 114 |
+
|
| 115 |
+
def find_tensor_attributes(module: torch.nn.Module) -> List[Tuple[str, Tensor]]:
|
| 116 |
+
tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)]
|
| 117 |
+
return tuples
|
| 118 |
+
|
| 119 |
+
gen = parameter._named_members(get_members_fn=find_tensor_attributes)
|
| 120 |
+
first_tuple = next(gen)
|
| 121 |
+
return first_tuple[1].dtype
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
class ModelMixin(torch.nn.Module, PushToHubMixin):
|
| 125 |
+
r"""
|
| 126 |
+
Base class for all models.
|
| 127 |
+
|
| 128 |
+
[`ModelMixin`] takes care of storing the model configuration and provides methods for loading, downloading and
|
| 129 |
+
saving models.
|
| 130 |
+
|
| 131 |
+
- **config_name** ([`str`]) -- Filename to save a model to when calling [`~models.ModelMixin.save_pretrained`].
|
| 132 |
+
"""
|
| 133 |
+
|
| 134 |
+
config_name = CONFIG_NAME
|
| 135 |
+
_automatically_saved_args = ["_diffusers_version", "_class_name", "_name_or_path"]
|
| 136 |
+
_supports_gradient_checkpointing = False
|
| 137 |
+
_keys_to_ignore_on_load_unexpected = None
|
| 138 |
+
_no_split_modules = None
|
| 139 |
+
|
| 140 |
+
def __init__(self):
|
| 141 |
+
super().__init__()
|
| 142 |
+
|
| 143 |
+
def __getattr__(self, name: str) -> Any:
|
| 144 |
+
"""The only reason we overwrite `getattr` here is to gracefully deprecate accessing
|
| 145 |
+
config attributes directly. See https://github.com/huggingface/diffusers/pull/3129 We need to overwrite
|
| 146 |
+
__getattr__ here in addition so that we don't trigger `torch.nn.Module`'s __getattr__':
|
| 147 |
+
https://pytorch.org/docs/stable/_modules/torch/nn/modules/module.html#Module
|
| 148 |
+
"""
|
| 149 |
+
|
| 150 |
+
is_in_config = "_internal_dict" in self.__dict__ and hasattr(self.__dict__["_internal_dict"], name)
|
| 151 |
+
is_attribute = name in self.__dict__
|
| 152 |
+
|
| 153 |
+
if is_in_config and not is_attribute:
|
| 154 |
+
deprecation_message = f"Accessing config attribute `{name}` directly via \
|
| 155 |
+
'{type(self).__name__}' object attribute is deprecated. \
|
| 156 |
+
Please access '{name}' over '{type(self).__name__}'s config object instead, \
|
| 157 |
+
e.g. 'unet.config.{name}'."
|
| 158 |
+
deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False, stacklevel=3)
|
| 159 |
+
return self._internal_dict[name]
|
| 160 |
+
|
| 161 |
+
# call PyTorch's https://pytorch.org/docs/stable/_modules/torch/nn/modules/module.html#Module
|
| 162 |
+
return super().__getattr__(name)
|
| 163 |
+
|
| 164 |
+
@property
|
| 165 |
+
def is_gradient_checkpointing(self) -> bool:
|
| 166 |
+
"""
|
| 167 |
+
Whether gradient checkpointing is activated for this model or not.
|
| 168 |
+
"""
|
| 169 |
+
return any(hasattr(m, "gradient_checkpointing") and m.gradient_checkpointing for m in self.modules())
|
| 170 |
+
|
| 171 |
+
def enable_gradient_checkpointing(self) -> None:
|
| 172 |
+
"""
|
| 173 |
+
Activates gradient checkpointing for the current model (may be referred to as *activation checkpointing* or
|
| 174 |
+
*checkpoint activations* in other frameworks).
|
| 175 |
+
"""
|
| 176 |
+
if not self._supports_gradient_checkpointing:
|
| 177 |
+
raise ValueError(f"{self.__class__.__name__} does not support gradient checkpointing.")
|
| 178 |
+
self.apply(partial(self._set_gradient_checkpointing, value=True))
|
| 179 |
+
|
| 180 |
+
def disable_gradient_checkpointing(self) -> None:
|
| 181 |
+
"""
|
| 182 |
+
Deactivates gradient checkpointing for the current model (may be referred to as *activation checkpointing* or
|
| 183 |
+
*checkpoint activations* in other frameworks).
|
| 184 |
+
"""
|
| 185 |
+
if self._supports_gradient_checkpointing:
|
| 186 |
+
self.apply(partial(self._set_gradient_checkpointing, value=False))
|
| 187 |
+
|
| 188 |
+
# def set_use_npu_flash_attention(self, valid: bool) -> None:
|
| 189 |
+
# r"""
|
| 190 |
+
# Set the switch for the npu flash attention.
|
| 191 |
+
# """
|
| 192 |
+
|
| 193 |
+
# def fn_recursive_set_npu_flash_attention(module: torch.nn.Module):
|
| 194 |
+
# if hasattr(module, "set_use_npu_flash_attention"):
|
| 195 |
+
# module.set_use_npu_flash_attention(valid)
|
| 196 |
+
|
| 197 |
+
# for child in module.children():
|
| 198 |
+
# fn_recursive_set_npu_flash_attention(child)
|
| 199 |
+
|
| 200 |
+
# for module in self.children():
|
| 201 |
+
# if isinstance(module, torch.nn.Module):
|
| 202 |
+
# fn_recursive_set_npu_flash_attention(module)
|
| 203 |
+
|
| 204 |
+
# def enable_npu_flash_attention(self) -> None:
|
| 205 |
+
# r"""
|
| 206 |
+
# Enable npu flash attention from torch_npu
|
| 207 |
+
|
| 208 |
+
# """
|
| 209 |
+
# self.set_use_npu_flash_attention(True)
|
| 210 |
+
|
| 211 |
+
# def disable_npu_flash_attention(self) -> None:
|
| 212 |
+
# r"""
|
| 213 |
+
# disable npu flash attention from torch_npu
|
| 214 |
+
|
| 215 |
+
# """
|
| 216 |
+
# self.set_use_npu_flash_attention(False)
|
| 217 |
+
|
| 218 |
+
# def set_use_memory_efficient_attention_xformers(
|
| 219 |
+
# self, valid: bool, attention_op: Optional[Callable] = None
|
| 220 |
+
# ) -> None:
|
| 221 |
+
# # Recursively walk through all the children.
|
| 222 |
+
# # Any children which exposes the set_use_memory_efficient_attention_xformers method
|
| 223 |
+
# # gets the message
|
| 224 |
+
# def fn_recursive_set_mem_eff(module: torch.nn.Module):
|
| 225 |
+
# if hasattr(module, "set_use_memory_efficient_attention_xformers"):
|
| 226 |
+
# module.set_use_memory_efficient_attention_xformers(valid, attention_op)
|
| 227 |
+
|
| 228 |
+
# for child in module.children():
|
| 229 |
+
# fn_recursive_set_mem_eff(child)
|
| 230 |
+
|
| 231 |
+
# for module in self.children():
|
| 232 |
+
# if isinstance(module, torch.nn.Module):
|
| 233 |
+
# fn_recursive_set_mem_eff(module)
|
| 234 |
+
|
| 235 |
+
# def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None) -> None:
|
| 236 |
+
# r"""
|
| 237 |
+
# Enable memory efficient attention from [xFormers](https://facebookresearch.github.io/xformers/).
|
| 238 |
+
|
| 239 |
+
# When this option is enabled, you should observe lower GPU memory usage and a potential speed up during
|
| 240 |
+
# inference. Speed up during training is not guaranteed.
|
| 241 |
+
|
| 242 |
+
# <Tip warning={true}>
|
| 243 |
+
|
| 244 |
+
# ⚠️ When memory efficient attention and sliced attention are both enabled, memory efficient attention takes
|
| 245 |
+
# precedent.
|
| 246 |
+
|
| 247 |
+
# </Tip>
|
| 248 |
+
|
| 249 |
+
# Parameters:
|
| 250 |
+
# attention_op (`Callable`, *optional*):
|
| 251 |
+
# Override the default `None` operator for use as `op` argument to the
|
| 252 |
+
# [`memory_efficient_attention()`](https://facebookresearch.github.io/xformers/components/ops.html#x
|
| 253 |
+
# formers.ops.memory_efficient_attention)
|
| 254 |
+
# function of xFormers.
|
| 255 |
+
|
| 256 |
+
# Examples:
|
| 257 |
+
|
| 258 |
+
# ```py
|
| 259 |
+
# >>> import torch
|
| 260 |
+
# >>> from diffusers import UNet2DConditionModel
|
| 261 |
+
# >>> from xformers.ops import MemoryEfficientAttentionFlashAttentionOp
|
| 262 |
+
|
| 263 |
+
# >>> model = UNet2DConditionModel.from_pretrained(
|
| 264 |
+
# ... "stabilityai/stable-diffusion-2-1", subfolder="unet", torch_dtype=torch.float16
|
| 265 |
+
# ... )
|
| 266 |
+
# >>> model = model.to("cuda")
|
| 267 |
+
# >>> model.enable_xformers_memory_efficient_attention(attention_op=MemoryEfficientAttentionFlashAttentionOp)
|
| 268 |
+
# ```
|
| 269 |
+
# """
|
| 270 |
+
# self.set_use_memory_efficient_attention_xformers(True, attention_op)
|
| 271 |
+
|
| 272 |
+
# def disable_xformers_memory_efficient_attention(self) -> None:
|
| 273 |
+
# r"""
|
| 274 |
+
# Disable memory efficient attention from [xFormers](https://facebookresearch.github.io/xformers/).
|
| 275 |
+
# """
|
| 276 |
+
# self.set_use_memory_efficient_attention_xformers(False)
|
| 277 |
+
|
| 278 |
+
def save_pretrained(
|
| 279 |
+
self,
|
| 280 |
+
save_directory: Union[str, os.PathLike],
|
| 281 |
+
is_main_process: bool = True,
|
| 282 |
+
save_function: Optional[Callable] = None,
|
| 283 |
+
safe_serialization: bool = True,
|
| 284 |
+
variant: Optional[str] = None,
|
| 285 |
+
max_shard_size: Union[int, str] = "10GB",
|
| 286 |
+
push_to_hub: bool = False,
|
| 287 |
+
**kwargs,
|
| 288 |
+
):
|
| 289 |
+
"""
|
| 290 |
+
Save a model and its configuration file to a directory so that it can be reloaded using the
|
| 291 |
+
[`~models.ModelMixin.from_pretrained`] class method.
|
| 292 |
+
|
| 293 |
+
Arguments:
|
| 294 |
+
save_directory (`str` or `os.PathLike`):
|
| 295 |
+
Directory to save a model and its configuration file to. Will be created if it doesn't exist.
|
| 296 |
+
is_main_process (`bool`, *optional*, defaults to `True`):
|
| 297 |
+
Whether the process calling this is the main process or not. Useful during distributed training and you
|
| 298 |
+
need to call this function on all processes. In this case, set `is_main_process=True` only on the main
|
| 299 |
+
process to avoid race conditions.
|
| 300 |
+
save_function (`Callable`):
|
| 301 |
+
The function to use to save the state dictionary. Useful during distributed training when you need to
|
| 302 |
+
replace `torch.save` with another method. Can be configured with the environment variable
|
| 303 |
+
`DIFFUSERS_SAVE_MODE`.
|
| 304 |
+
safe_serialization (`bool`, *optional*, defaults to `True`):
|
| 305 |
+
Whether to save the model using `safetensors` or the traditional PyTorch way with `pickle`.
|
| 306 |
+
variant (`str`, *optional*):
|
| 307 |
+
If specified, weights are saved in the format `pytorch_model.<variant>.bin`.
|
| 308 |
+
max_shard_size (`int` or `str`, defaults to `"10GB"`):
|
| 309 |
+
The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size
|
| 310 |
+
lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5GB"`).
|
| 311 |
+
If expressed as an integer, the unit is bytes. Note that this limit will be decreased after a certain
|
| 312 |
+
period of time (starting from Oct 2024) to allow users to upgrade to the latest version of `diffusers`.
|
| 313 |
+
This is to establish a common default size for this argument across different libraries in the Hugging
|
| 314 |
+
Face ecosystem (`transformers`, and `accelerate`, for example).
|
| 315 |
+
push_to_hub (`bool`, *optional*, defaults to `False`):
|
| 316 |
+
Whether or not to push your model to the Hugging Face Hub after saving it. You can specify the
|
| 317 |
+
repository you want to push to with `repo_id` (will default to the name of `save_directory` in your
|
| 318 |
+
namespace).
|
| 319 |
+
kwargs (`Dict[str, Any]`, *optional*):
|
| 320 |
+
Additional keyword arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method.
|
| 321 |
+
"""
|
| 322 |
+
if os.path.isfile(save_directory):
|
| 323 |
+
logger.error(f"Provided path ({save_directory}) should be a directory, not a file")
|
| 324 |
+
return
|
| 325 |
+
|
| 326 |
+
weights_name = SAFETENSORS_WEIGHTS_NAME if safe_serialization else WEIGHTS_NAME
|
| 327 |
+
weights_name = _add_variant(weights_name, variant)
|
| 328 |
+
weight_name_split = weights_name.split(".")
|
| 329 |
+
if len(weight_name_split) in [2, 3]:
|
| 330 |
+
weights_name_pattern = weight_name_split[0] + "{suffix}." + ".".join(weight_name_split[1:])
|
| 331 |
+
else:
|
| 332 |
+
raise ValueError(f"Invalid {weights_name} provided.")
|
| 333 |
+
|
| 334 |
+
os.makedirs(save_directory, exist_ok=True)
|
| 335 |
+
|
| 336 |
+
if push_to_hub:
|
| 337 |
+
commit_message = kwargs.pop("commit_message", None)
|
| 338 |
+
private = kwargs.pop("private", False)
|
| 339 |
+
create_pr = kwargs.pop("create_pr", False)
|
| 340 |
+
token = kwargs.pop("token", None)
|
| 341 |
+
repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1])
|
| 342 |
+
repo_id = create_repo(repo_id, exist_ok=True, private=private, token=token).repo_id
|
| 343 |
+
|
| 344 |
+
# Only save the model itself if we are using distributed training
|
| 345 |
+
model_to_save = self
|
| 346 |
+
|
| 347 |
+
# Attach architecture to the config
|
| 348 |
+
# Save the config
|
| 349 |
+
if is_main_process:
|
| 350 |
+
model_to_save.save_config(save_directory)
|
| 351 |
+
|
| 352 |
+
# Save the model
|
| 353 |
+
state_dict = model_to_save.state_dict()
|
| 354 |
+
|
| 355 |
+
# Save the model
|
| 356 |
+
state_dict_split = split_torch_state_dict_into_shards(
|
| 357 |
+
state_dict, max_shard_size=max_shard_size, filename_pattern=weights_name_pattern
|
| 358 |
+
)
|
| 359 |
+
|
| 360 |
+
# Clean the folder from a previous save
|
| 361 |
+
if is_main_process:
|
| 362 |
+
for filename in os.listdir(save_directory):
|
| 363 |
+
if filename in state_dict_split.filename_to_tensors.keys():
|
| 364 |
+
continue
|
| 365 |
+
full_filename = os.path.join(save_directory, filename)
|
| 366 |
+
if not os.path.isfile(full_filename):
|
| 367 |
+
continue
|
| 368 |
+
weights_without_ext = weights_name_pattern.replace(".bin", "").replace(".safetensors", "")
|
| 369 |
+
weights_without_ext = weights_without_ext.replace("{suffix}", "")
|
| 370 |
+
filename_without_ext = filename.replace(".bin", "").replace(".safetensors", "")
|
| 371 |
+
# make sure that file to be deleted matches format of sharded file, e.g. pytorch_model-00001-of-00005
|
| 372 |
+
if (
|
| 373 |
+
filename.startswith(weights_without_ext)
|
| 374 |
+
and _REGEX_SHARD.fullmatch(filename_without_ext) is not None
|
| 375 |
+
):
|
| 376 |
+
os.remove(full_filename)
|
| 377 |
+
|
| 378 |
+
for filename, tensors in state_dict_split.filename_to_tensors.items():
|
| 379 |
+
shard = {tensor: state_dict[tensor] for tensor in tensors}
|
| 380 |
+
filepath = os.path.join(save_directory, filename)
|
| 381 |
+
if safe_serialization:
|
| 382 |
+
# At some point we will need to deal better with save_function (used for TPU and other distributed
|
| 383 |
+
# joyfulness), but for now this enough.
|
| 384 |
+
safetensors.torch.save_file(shard, filepath, metadata={"format": "pt"})
|
| 385 |
+
else:
|
| 386 |
+
torch.save(shard, filepath)
|
| 387 |
+
|
| 388 |
+
if state_dict_split.is_sharded:
|
| 389 |
+
index = {
|
| 390 |
+
"metadata": state_dict_split.metadata,
|
| 391 |
+
"weight_map": state_dict_split.tensor_to_filename,
|
| 392 |
+
}
|
| 393 |
+
save_index_file = SAFE_WEIGHTS_INDEX_NAME if safe_serialization else WEIGHTS_INDEX_NAME
|
| 394 |
+
save_index_file = os.path.join(save_directory, _add_variant(save_index_file, variant))
|
| 395 |
+
# Save the index as well
|
| 396 |
+
with open(save_index_file, "w", encoding="utf-8") as f:
|
| 397 |
+
content = json.dumps(index, indent=2, sort_keys=True) + "\n"
|
| 398 |
+
f.write(content)
|
| 399 |
+
logger.info(
|
| 400 |
+
f"The model is bigger than the maximum size per checkpoint ({max_shard_size}) and is going to be "
|
| 401 |
+
f"split in {len(state_dict_split.filename_to_tensors)} checkpoint shards. You can find where each "
|
| 402 |
+
f"parameters has been saved in the index located at {save_index_file}."
|
| 403 |
+
)
|
| 404 |
+
else:
|
| 405 |
+
path_to_weights = os.path.join(save_directory, weights_name)
|
| 406 |
+
logger.info(f"Model weights saved in {path_to_weights}")
|
| 407 |
+
|
| 408 |
+
if push_to_hub:
|
| 409 |
+
# Create a new empty model card and eventually tag it
|
| 410 |
+
model_card = load_or_create_model_card(repo_id, token=token)
|
| 411 |
+
model_card = populate_model_card(model_card)
|
| 412 |
+
model_card.save(Path(save_directory, "README.md").as_posix())
|
| 413 |
+
|
| 414 |
+
self._upload_folder(
|
| 415 |
+
save_directory,
|
| 416 |
+
repo_id,
|
| 417 |
+
token=token,
|
| 418 |
+
commit_message=commit_message,
|
| 419 |
+
create_pr=create_pr,
|
| 420 |
+
)
|
| 421 |
+
|
| 422 |
+
@classmethod
|
| 423 |
+
@validate_hf_hub_args
|
| 424 |
+
def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs):
|
| 425 |
+
r"""
|
| 426 |
+
Instantiate a pretrained PyTorch model from a pretrained model configuration.
|
| 427 |
+
|
| 428 |
+
The model is set in evaluation mode - `model.eval()` - by default, and dropout modules are deactivated. To
|
| 429 |
+
train the model, set it back in training mode with `model.train()`.
|
| 430 |
+
|
| 431 |
+
Parameters:
|
| 432 |
+
pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*):
|
| 433 |
+
Can be either:
|
| 434 |
+
|
| 435 |
+
- A string, the *model id* (for example `google/ddpm-celebahq-256`) of a pretrained model hosted on
|
| 436 |
+
the Hub.
|
| 437 |
+
- A path to a *directory* (for example `./my_model_directory`) containing the model weights saved
|
| 438 |
+
with [`~ModelMixin.save_pretrained`].
|
| 439 |
+
|
| 440 |
+
cache_dir (`Union[str, os.PathLike]`, *optional*):
|
| 441 |
+
Path to a directory where a downloaded pretrained model configuration is cached if the standard cache
|
| 442 |
+
is not used.
|
| 443 |
+
torch_dtype (`str` or `torch.dtype`, *optional*):
|
| 444 |
+
Override the default `torch.dtype` and load the model with another dtype. If `"auto"` is passed, the
|
| 445 |
+
dtype is automatically derived from the model's weights.
|
| 446 |
+
force_download (`bool`, *optional*, defaults to `False`):
|
| 447 |
+
Whether or not to force the (re-)download of the model weights and configuration files, overriding the
|
| 448 |
+
cached versions if they exist.
|
| 449 |
+
proxies (`Dict[str, str]`, *optional*):
|
| 450 |
+
A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128',
|
| 451 |
+
'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
|
| 452 |
+
output_loading_info (`bool`, *optional*, defaults to `False`):
|
| 453 |
+
Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages.
|
| 454 |
+
local_files_only(`bool`, *optional*, defaults to `False`):
|
| 455 |
+
Whether to only load local model weights and configuration files or not. If set to `True`, the model
|
| 456 |
+
won't be downloaded from the Hub.
|
| 457 |
+
token (`str` or *bool*, *optional*):
|
| 458 |
+
The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from
|
| 459 |
+
`diffusers-cli login` (stored in `~/.huggingface`) is used.
|
| 460 |
+
revision (`str`, *optional*, defaults to `"main"`):
|
| 461 |
+
The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier
|
| 462 |
+
allowed by Git.
|
| 463 |
+
from_flax (`bool`, *optional*, defaults to `False`):
|
| 464 |
+
Load the model weights from a Flax checkpoint save file.
|
| 465 |
+
subfolder (`str`, *optional*, defaults to `""`):
|
| 466 |
+
The subfolder location of a model file within a larger model repository on the Hub or locally.
|
| 467 |
+
mirror (`str`, *optional*):
|
| 468 |
+
Mirror source to resolve accessibility issues if you're downloading a model in China. We do not
|
| 469 |
+
guarantee the timeliness or safety of the source, and you should refer to the mirror site for more
|
| 470 |
+
information.
|
| 471 |
+
device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*):
|
| 472 |
+
A map that specifies where each submodule should go. It doesn't need to be defined for each
|
| 473 |
+
parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the
|
| 474 |
+
same device. Defaults to `None`, meaning that the model will be loaded on CPU.
|
| 475 |
+
|
| 476 |
+
Set `device_map="auto"` to have 🤗 Accelerate automatically compute the most optimized `device_map`. For
|
| 477 |
+
more information about each option see [designing a device
|
| 478 |
+
map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map).
|
| 479 |
+
max_memory (`Dict`, *optional*):
|
| 480 |
+
A dictionary device identifier for the maximum memory. Will default to the maximum memory available for
|
| 481 |
+
each GPU and the available CPU RAM if unset.
|
| 482 |
+
offload_folder (`str` or `os.PathLike`, *optional*):
|
| 483 |
+
The path to offload weights if `device_map` contains the value `"disk"`.
|
| 484 |
+
offload_state_dict (`bool`, *optional*):
|
| 485 |
+
If `True`, temporarily offloads the CPU state dict to the hard drive to avoid running out of CPU RAM if
|
| 486 |
+
the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to `True`
|
| 487 |
+
when there is some disk offload.
|
| 488 |
+
low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`):
|
| 489 |
+
Speed up model loading only loading the pretrained weights and not initializing the weights. This also
|
| 490 |
+
tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model.
|
| 491 |
+
Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this
|
| 492 |
+
argument to `True` will raise an error.
|
| 493 |
+
variant (`str`, *optional*):
|
| 494 |
+
Load weights from a specified `variant` filename such as `"fp16"` or `"ema"`. This is ignored when
|
| 495 |
+
loading `from_flax`.
|
| 496 |
+
use_safetensors (`bool`, *optional*, defaults to `None`):
|
| 497 |
+
If set to `None`, the `safetensors` weights are downloaded if they're available **and** if the
|
| 498 |
+
`safetensors` library is installed. If set to `True`, the model is forcibly loaded from `safetensors`
|
| 499 |
+
weights. If set to `False`, `safetensors` weights are not loaded.
|
| 500 |
+
|
| 501 |
+
<Tip>
|
| 502 |
+
|
| 503 |
+
To use private or [gated models](https://huggingface.co/docs/hub/models-gated#gated-models), log-in with
|
| 504 |
+
`huggingface-cli login`. You can also activate the special
|
| 505 |
+
["offline-mode"](https://huggingface.co/diffusers/installation.html#offline-mode) to use this method in a
|
| 506 |
+
firewalled environment.
|
| 507 |
+
|
| 508 |
+
</Tip>
|
| 509 |
+
|
| 510 |
+
Example:
|
| 511 |
+
|
| 512 |
+
```py
|
| 513 |
+
from diffusers import UNet2DConditionModel
|
| 514 |
+
|
| 515 |
+
unet = UNet2DConditionModel.from_pretrained("runwayml/stable-diffusion-v1-5", subfolder="unet")
|
| 516 |
+
```
|
| 517 |
+
|
| 518 |
+
If you get the error message below, you need to finetune the weights for your downstream task:
|
| 519 |
+
|
| 520 |
+
```bash
|
| 521 |
+
Some weights of UNet2DConditionModel were not initialized from the model checkpoint at
|
| 522 |
+
runwayml/stable-diffusion-v1-5 and are newly initialized because the shapes did not match:
|
| 523 |
+
- conv_in.weight: found shape torch.Size([320, 4, 3, 3]) in the checkpoint and torch.Size([320, 9, 3, 3])
|
| 524 |
+
in the model instantiated
|
| 525 |
+
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
|
| 526 |
+
```
|
| 527 |
+
"""
|
| 528 |
+
cache_dir = kwargs.pop("cache_dir", None)
|
| 529 |
+
ignore_mismatched_sizes = kwargs.pop("ignore_mismatched_sizes", False)
|
| 530 |
+
force_download = kwargs.pop("force_download", False)
|
| 531 |
+
from_flax = kwargs.pop("from_flax", False)
|
| 532 |
+
proxies = kwargs.pop("proxies", None)
|
| 533 |
+
output_loading_info = kwargs.pop("output_loading_info", False)
|
| 534 |
+
local_files_only = kwargs.pop("local_files_only", None)
|
| 535 |
+
token = kwargs.pop("token", None)
|
| 536 |
+
revision = kwargs.pop("revision", None)
|
| 537 |
+
torch_dtype = kwargs.pop("torch_dtype", None)
|
| 538 |
+
subfolder = kwargs.pop("subfolder", None)
|
| 539 |
+
device_map = kwargs.pop("device_map", None)
|
| 540 |
+
max_memory = kwargs.pop("max_memory", None)
|
| 541 |
+
offload_folder = kwargs.pop("offload_folder", None)
|
| 542 |
+
offload_state_dict = kwargs.pop("offload_state_dict", False)
|
| 543 |
+
low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", _LOW_CPU_MEM_USAGE_DEFAULT)
|
| 544 |
+
variant = kwargs.pop("variant", None)
|
| 545 |
+
use_safetensors = kwargs.pop("use_safetensors", None)
|
| 546 |
+
|
| 547 |
+
allow_pickle = False
|
| 548 |
+
if use_safetensors is None:
|
| 549 |
+
use_safetensors = True
|
| 550 |
+
allow_pickle = True
|
| 551 |
+
|
| 552 |
+
def _check_input(low_cpu_mem_usage, device_map):
|
| 553 |
+
if low_cpu_mem_usage and not is_accelerate_available():
|
| 554 |
+
low_cpu_mem_usage = False
|
| 555 |
+
logger.warning(
|
| 556 |
+
"Cannot initialize model with low cpu memory usage because `accelerate` was not found in the"
|
| 557 |
+
" environment. Defaulting to `low_cpu_mem_usage=False`. It is strongly recommended to install"
|
| 558 |
+
" `accelerate` for faster and less memory-intense model loading. You can do so with: \n```\npip"
|
| 559 |
+
" install accelerate\n```\n."
|
| 560 |
+
)
|
| 561 |
+
|
| 562 |
+
if device_map is not None and not is_accelerate_available():
|
| 563 |
+
raise NotImplementedError(
|
| 564 |
+
"Loading and dispatching requires `accelerate`. Please make sure to install accelerate or set"
|
| 565 |
+
" `device_map=None`. You can install accelerate with `pip install accelerate`."
|
| 566 |
+
)
|
| 567 |
+
|
| 568 |
+
# Check if we can handle device_map and dispatching the weights
|
| 569 |
+
if device_map is not None and not is_torch_version(">=", "1.9.0"):
|
| 570 |
+
raise NotImplementedError(
|
| 571 |
+
"Loading and dispatching requires torch >= 1.9.0. Please either update your PyTorch version or set"
|
| 572 |
+
" `device_map=None`."
|
| 573 |
+
)
|
| 574 |
+
|
| 575 |
+
if low_cpu_mem_usage is True and not is_torch_version(">=", "1.9.0"):
|
| 576 |
+
raise NotImplementedError(
|
| 577 |
+
"Low memory initialization requires torch >= 1.9.0. Please either update your PyTorch version or "
|
| 578 |
+
"set `low_cpu_mem_usage=False`."
|
| 579 |
+
)
|
| 580 |
+
|
| 581 |
+
if low_cpu_mem_usage is False and device_map is not None:
|
| 582 |
+
raise ValueError(
|
| 583 |
+
f"You cannot set `low_cpu_mem_usage` to `False` while using device_map={device_map} for loading and"
|
| 584 |
+
" dispatching. Please make sure to set `low_cpu_mem_usage=True`."
|
| 585 |
+
)
|
| 586 |
+
_check_input(low_cpu_mem_usage, device_map)
|
| 587 |
+
|
| 588 |
+
# change device_map into a map if we passed an int, a str or a torch.device
|
| 589 |
+
if isinstance(device_map, torch.device):
|
| 590 |
+
device_map = {"": device_map}
|
| 591 |
+
elif isinstance(device_map, str) and device_map not in ["auto", "balanced", "balanced_low_0", "sequential"]:
|
| 592 |
+
try:
|
| 593 |
+
device_map = {"": torch.device(device_map)}
|
| 594 |
+
except RuntimeError:
|
| 595 |
+
raise ValueError(
|
| 596 |
+
"When passing device_map as a string, the value needs to be a device name (e.g. cpu, cuda:0) or "
|
| 597 |
+
f"'auto', 'balanced', 'balanced_low_0', 'sequential' but found {device_map}."
|
| 598 |
+
)
|
| 599 |
+
elif isinstance(device_map, int):
|
| 600 |
+
if device_map < 0:
|
| 601 |
+
raise ValueError(
|
| 602 |
+
"You can't pass device_map as a negative int. " \
|
| 603 |
+
"If you want to put the model on the cpu, pass device_map = 'cpu' "
|
| 604 |
+
)
|
| 605 |
+
else:
|
| 606 |
+
device_map = {"": device_map}
|
| 607 |
+
|
| 608 |
+
if device_map is not None:
|
| 609 |
+
if low_cpu_mem_usage is None:
|
| 610 |
+
low_cpu_mem_usage = True
|
| 611 |
+
elif not low_cpu_mem_usage:
|
| 612 |
+
raise ValueError("Passing along a `device_map` requires `low_cpu_mem_usage=True`")
|
| 613 |
+
|
| 614 |
+
if low_cpu_mem_usage:
|
| 615 |
+
if device_map is not None and not is_torch_version(">=", "1.10"):
|
| 616 |
+
# The max memory utils require PyTorch >= 1.10 to have torch.cuda.mem_get_info.
|
| 617 |
+
raise ValueError("`low_cpu_mem_usage` and `device_map` require PyTorch >= 1.10.")
|
| 618 |
+
|
| 619 |
+
# Load config if we don't provide a configuration
|
| 620 |
+
config_path = pretrained_model_name_or_path
|
| 621 |
+
|
| 622 |
+
user_agent = {
|
| 623 |
+
"diffusers": __version__,
|
| 624 |
+
"file_type": "model",
|
| 625 |
+
"framework": "pytorch",
|
| 626 |
+
}
|
| 627 |
+
|
| 628 |
+
# load config
|
| 629 |
+
def _load_config(cls, config_path, **kwargs):
|
| 630 |
+
config, unused_kwargs, commit_hash = cls.load_config(
|
| 631 |
+
config_path,
|
| 632 |
+
cache_dir=cache_dir,
|
| 633 |
+
return_unused_kwargs=True,
|
| 634 |
+
return_commit_hash=True,
|
| 635 |
+
force_download=force_download,
|
| 636 |
+
proxies=proxies,
|
| 637 |
+
local_files_only=local_files_only,
|
| 638 |
+
token=token,
|
| 639 |
+
revision=revision,
|
| 640 |
+
subfolder=subfolder,
|
| 641 |
+
user_agent=user_agent,
|
| 642 |
+
**kwargs,
|
| 643 |
+
)
|
| 644 |
+
|
| 645 |
+
# Determine if we're loading from a directory of sharded checkpoints.
|
| 646 |
+
is_sharded = False
|
| 647 |
+
index_file = None
|
| 648 |
+
is_local = os.path.isdir(pretrained_model_name_or_path)
|
| 649 |
+
index_file = _fetch_index_file(
|
| 650 |
+
is_local=is_local,
|
| 651 |
+
pretrained_model_name_or_path=pretrained_model_name_or_path,
|
| 652 |
+
subfolder=subfolder or "",
|
| 653 |
+
use_safetensors=use_safetensors,
|
| 654 |
+
cache_dir=cache_dir,
|
| 655 |
+
variant=variant,
|
| 656 |
+
force_download=force_download,
|
| 657 |
+
proxies=proxies,
|
| 658 |
+
local_files_only=local_files_only,
|
| 659 |
+
token=token,
|
| 660 |
+
revision=revision,
|
| 661 |
+
user_agent=user_agent,
|
| 662 |
+
commit_hash=commit_hash,
|
| 663 |
+
)
|
| 664 |
+
if index_file is not None and index_file.is_file():
|
| 665 |
+
is_sharded = True
|
| 666 |
+
|
| 667 |
+
if is_sharded and from_flax:
|
| 668 |
+
raise ValueError("Loading of sharded checkpoints is not supported when `from_flax=True`.")
|
| 669 |
+
return config, unused_kwargs, commit_hash, is_sharded, index_file
|
| 670 |
+
|
| 671 |
+
config, unused_kwargs, commit_hash, is_sharded, index_file = _load_config(cls, config_path, **kwargs)
|
| 672 |
+
|
| 673 |
+
# load model
|
| 674 |
+
model_file = None
|
| 675 |
+
if from_flax:
|
| 676 |
+
model_file = _get_model_file(
|
| 677 |
+
pretrained_model_name_or_path,
|
| 678 |
+
weights_name=FLAX_WEIGHTS_NAME,
|
| 679 |
+
cache_dir=cache_dir,
|
| 680 |
+
force_download=force_download,
|
| 681 |
+
proxies=proxies,
|
| 682 |
+
local_files_only=local_files_only,
|
| 683 |
+
token=token,
|
| 684 |
+
revision=revision,
|
| 685 |
+
subfolder=subfolder,
|
| 686 |
+
user_agent=user_agent,
|
| 687 |
+
commit_hash=commit_hash,
|
| 688 |
+
)
|
| 689 |
+
model = cls.from_config(config, **unused_kwargs)
|
| 690 |
+
|
| 691 |
+
# Convert the weights
|
| 692 |
+
from .modeling_pytorch_flax_utils import load_flax_checkpoint_in_pytorch_model
|
| 693 |
+
|
| 694 |
+
model = load_flax_checkpoint_in_pytorch_model(model, model_file)
|
| 695 |
+
else:
|
| 696 |
+
if is_sharded:
|
| 697 |
+
sharded_ckpt_cached_folder, sharded_metadata = _get_checkpoint_shard_files(
|
| 698 |
+
pretrained_model_name_or_path,
|
| 699 |
+
index_file,
|
| 700 |
+
cache_dir=cache_dir,
|
| 701 |
+
proxies=proxies,
|
| 702 |
+
local_files_only=local_files_only,
|
| 703 |
+
token=token,
|
| 704 |
+
user_agent=user_agent,
|
| 705 |
+
revision=revision,
|
| 706 |
+
subfolder=subfolder or "",
|
| 707 |
+
)
|
| 708 |
+
|
| 709 |
+
elif use_safetensors and not is_sharded:
|
| 710 |
+
try:
|
| 711 |
+
model_file = _get_model_file(
|
| 712 |
+
pretrained_model_name_or_path,
|
| 713 |
+
weights_name=_add_variant(SAFETENSORS_WEIGHTS_NAME, variant),
|
| 714 |
+
cache_dir=cache_dir,
|
| 715 |
+
force_download=force_download,
|
| 716 |
+
proxies=proxies,
|
| 717 |
+
local_files_only=local_files_only,
|
| 718 |
+
token=token,
|
| 719 |
+
revision=revision,
|
| 720 |
+
subfolder=subfolder,
|
| 721 |
+
user_agent=user_agent,
|
| 722 |
+
commit_hash=commit_hash,
|
| 723 |
+
)
|
| 724 |
+
|
| 725 |
+
except IOError as e:
|
| 726 |
+
logger.error(f"An error occurred while trying to fetch {pretrained_model_name_or_path}: {e}")
|
| 727 |
+
if not allow_pickle:
|
| 728 |
+
raise
|
| 729 |
+
logger.warning(
|
| 730 |
+
"Defaulting to unsafe serialization. Pass `allow_pickle=False` to raise an error instead."
|
| 731 |
+
)
|
| 732 |
+
|
| 733 |
+
if model_file is None and not is_sharded:
|
| 734 |
+
model_file = _get_model_file(
|
| 735 |
+
pretrained_model_name_or_path,
|
| 736 |
+
weights_name=_add_variant(WEIGHTS_NAME, variant),
|
| 737 |
+
cache_dir=cache_dir,
|
| 738 |
+
force_download=force_download,
|
| 739 |
+
proxies=proxies,
|
| 740 |
+
local_files_only=local_files_only,
|
| 741 |
+
token=token,
|
| 742 |
+
revision=revision,
|
| 743 |
+
subfolder=subfolder,
|
| 744 |
+
user_agent=user_agent,
|
| 745 |
+
commit_hash=commit_hash,
|
| 746 |
+
)
|
| 747 |
+
|
| 748 |
+
if low_cpu_mem_usage:
|
| 749 |
+
# Instantiate model with empty weights
|
| 750 |
+
def load_model_in_low_cpu_mem_usage(config, device_map, unused_kwargs, commit_hash, is_sharded,
|
| 751 |
+
index_file):
|
| 752 |
+
with accelerate.init_empty_weights():
|
| 753 |
+
model = cls.from_config(config, **unused_kwargs)
|
| 754 |
+
|
| 755 |
+
# if device_map is None, load the state dict and move the params from meta device to the cpu
|
| 756 |
+
if device_map is None and not is_sharded:
|
| 757 |
+
param_device = "cpu"
|
| 758 |
+
state_dict = diffusers.models.model_loading_utils.load_state_dict(model_file, variant=variant)
|
| 759 |
+
model._convert_deprecated_attn(state_dict)
|
| 760 |
+
# move the params from meta device to cpu
|
| 761 |
+
missing_keys = set(model.state_dict().keys()) - set(state_dict.keys())
|
| 762 |
+
if len(missing_keys) > 0:
|
| 763 |
+
raise ValueError(
|
| 764 |
+
f"Cannot load {cls} from {pretrained_model_name_or_path} because the following keys are"
|
| 765 |
+
f" missing: \n {', '.join(missing_keys)}. \n Please make sure to pass"
|
| 766 |
+
" `low_cpu_mem_usage=False` and `device_map=None` if you want to randomly initialize"
|
| 767 |
+
" those weights or else make sure your checkpoint file is correct."
|
| 768 |
+
)
|
| 769 |
+
|
| 770 |
+
unexpected_keys = load_model_dict_into_meta(
|
| 771 |
+
model,
|
| 772 |
+
state_dict,
|
| 773 |
+
device=param_device,
|
| 774 |
+
dtype=torch_dtype,
|
| 775 |
+
model_name_or_path=pretrained_model_name_or_path,
|
| 776 |
+
)
|
| 777 |
+
|
| 778 |
+
if cls._keys_to_ignore_on_load_unexpected is not None:
|
| 779 |
+
for pat in cls._keys_to_ignore_on_load_unexpected:
|
| 780 |
+
unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None]
|
| 781 |
+
|
| 782 |
+
if len(unexpected_keys) > 0:
|
| 783 |
+
logger.warning(
|
| 784 |
+
f"Some weights of the model checkpoint were not used when initializing "
|
| 785 |
+
f" {cls.__name__}: \n {[', '.join(unexpected_keys)]}"
|
| 786 |
+
)
|
| 787 |
+
|
| 788 |
+
else: # else let accelerate handle loading and dispatching.
|
| 789 |
+
# Load weights and dispatch according to the device_map
|
| 790 |
+
# by default the device_map is None and the weights are loaded on the CPU
|
| 791 |
+
force_hook = True
|
| 792 |
+
device_map = _determine_device_map(model, device_map, max_memory, torch_dtype)
|
| 793 |
+
if device_map is None and is_sharded:
|
| 794 |
+
# we load the parameters on the cpu
|
| 795 |
+
device_map = {"": "cpu"}
|
| 796 |
+
force_hook = False
|
| 797 |
+
try:
|
| 798 |
+
accelerate.load_checkpoint_and_dispatch(
|
| 799 |
+
model,
|
| 800 |
+
model_file if not is_sharded else index_file,
|
| 801 |
+
device_map,
|
| 802 |
+
max_memory=max_memory,
|
| 803 |
+
offload_folder=offload_folder,
|
| 804 |
+
offload_state_dict=offload_state_dict,
|
| 805 |
+
dtype=torch_dtype,
|
| 806 |
+
force_hooks=force_hook,
|
| 807 |
+
strict=True,
|
| 808 |
+
)
|
| 809 |
+
except AttributeError as e:
|
| 810 |
+
# When using accelerate loading, we do not have the ability to load the state
|
| 811 |
+
# dict and rename the weight names manually. Additionally, accelerate skips
|
| 812 |
+
# torch loading conventions and directly writes into `module.{_buffers, _parameters}`
|
| 813 |
+
# (which look like they should be private variables?), so we can't use the standard hooks
|
| 814 |
+
# to rename parameters on load. We need to mimic the original weight names so the correct
|
| 815 |
+
# attributes are available. After we have loaded the weights, we convert the deprecated
|
| 816 |
+
# names to the new non-deprecated names. Then we _greatly encourage_ the user to convert
|
| 817 |
+
# the weights so we don't have to do this again.
|
| 818 |
+
|
| 819 |
+
if "'Attention' object has no attribute" in str(e):
|
| 820 |
+
logger.warning(
|
| 821 |
+
f"Taking `{str(e)}` while using `accelerate.load_checkpoint_and_dispatch` to mean "
|
| 822 |
+
f"{pretrained_model_name_or_path} was saved with deprecated attention block weight "
|
| 823 |
+
f"names. We will load it with the deprecated attention block names and convert them"
|
| 824 |
+
f" on the fly to the new attention block format. Please re-save the model after "
|
| 825 |
+
f"this conversion so we don't have to do the on the fly renaming in the future. "
|
| 826 |
+
f"If the model is from a hub checkpoint, please also re-upload it or open a PR on"
|
| 827 |
+
f" the original repository."
|
| 828 |
+
)
|
| 829 |
+
model._temp_convert_self_to_deprecated_attention_blocks()
|
| 830 |
+
accelerate.load_checkpoint_and_dispatch(
|
| 831 |
+
model,
|
| 832 |
+
model_file if not is_sharded else index_file,
|
| 833 |
+
device_map,
|
| 834 |
+
max_memory=max_memory,
|
| 835 |
+
offload_folder=offload_folder,
|
| 836 |
+
offload_state_dict=offload_state_dict,
|
| 837 |
+
dtype=torch_dtype,
|
| 838 |
+
force_hooks=force_hook,
|
| 839 |
+
strict=True,
|
| 840 |
+
)
|
| 841 |
+
model._undo_temp_convert_self_to_deprecated_attention_blocks()
|
| 842 |
+
else:
|
| 843 |
+
raise e
|
| 844 |
+
|
| 845 |
+
loading_info = {
|
| 846 |
+
"missing_keys": [],
|
| 847 |
+
"unexpected_keys": [],
|
| 848 |
+
"mismatched_keys": [],
|
| 849 |
+
"error_msgs": [],
|
| 850 |
+
}
|
| 851 |
+
return model, loading_info
|
| 852 |
+
model, loading_info = load_model_in_low_cpu_mem_usage(config, device_map, unused_kwargs, commit_hash,
|
| 853 |
+
is_sharded, index_file)
|
| 854 |
+
|
| 855 |
+
else:
|
| 856 |
+
model = cls.from_config(config, **unused_kwargs)
|
| 857 |
+
|
| 858 |
+
state_dict = load_state_dict(model_file, variant=variant)
|
| 859 |
+
model._convert_deprecated_attn(state_dict)
|
| 860 |
+
|
| 861 |
+
model, missing_keys, unexpected_keys, mismatched_keys, error_msgs = cls._load_pretrained_model(
|
| 862 |
+
model,
|
| 863 |
+
state_dict,
|
| 864 |
+
model_file,
|
| 865 |
+
pretrained_model_name_or_path,
|
| 866 |
+
ignore_mismatched_sizes=ignore_mismatched_sizes,
|
| 867 |
+
)
|
| 868 |
+
|
| 869 |
+
loading_info = {
|
| 870 |
+
"missing_keys": missing_keys,
|
| 871 |
+
"unexpected_keys": unexpected_keys,
|
| 872 |
+
"mismatched_keys": mismatched_keys,
|
| 873 |
+
"error_msgs": error_msgs,
|
| 874 |
+
}
|
| 875 |
+
|
| 876 |
+
if torch_dtype is not None and not isinstance(torch_dtype, torch.dtype):
|
| 877 |
+
raise ValueError(
|
| 878 |
+
f"{torch_dtype} needs to be of type `torch.dtype`, e.g. `torch.float16`, but is {type(torch_dtype)}."
|
| 879 |
+
)
|
| 880 |
+
elif torch_dtype is not None:
|
| 881 |
+
model = model.to(torch_dtype)
|
| 882 |
+
|
| 883 |
+
model.register_to_config(_name_or_path=pretrained_model_name_or_path)
|
| 884 |
+
|
| 885 |
+
# Set model in evaluation mode to deactivate DropOut modules by default
|
| 886 |
+
model.eval()
|
| 887 |
+
if output_loading_info:
|
| 888 |
+
return model, loading_info
|
| 889 |
+
|
| 890 |
+
return model
|
| 891 |
+
|
| 892 |
+
@classmethod
|
| 893 |
+
def _load_pretrained_model(
|
| 894 |
+
cls,
|
| 895 |
+
model,
|
| 896 |
+
state_dict: OrderedDict,
|
| 897 |
+
resolved_archive_file,
|
| 898 |
+
pretrained_model_name_or_path: Union[str, os.PathLike],
|
| 899 |
+
ignore_mismatched_sizes: bool = False,
|
| 900 |
+
):
|
| 901 |
+
# Retrieve missing & unexpected_keys
|
| 902 |
+
model_state_dict = model.state_dict()
|
| 903 |
+
loaded_keys = list(state_dict.keys())
|
| 904 |
+
|
| 905 |
+
expected_keys = list(model_state_dict.keys())
|
| 906 |
+
|
| 907 |
+
original_loaded_keys = loaded_keys
|
| 908 |
+
|
| 909 |
+
missing_keys = list(set(expected_keys) - set(loaded_keys))
|
| 910 |
+
unexpected_keys = list(set(loaded_keys) - set(expected_keys))
|
| 911 |
+
|
| 912 |
+
# Make sure we are able to load base models as well as derived models (with heads)
|
| 913 |
+
model_to_load = model
|
| 914 |
+
|
| 915 |
+
def _find_mismatched_keys(
|
| 916 |
+
state_dict,
|
| 917 |
+
model_state_dict,
|
| 918 |
+
loaded_keys,
|
| 919 |
+
ignore_mismatched_sizes,
|
| 920 |
+
):
|
| 921 |
+
mismatched_keys = []
|
| 922 |
+
if ignore_mismatched_sizes:
|
| 923 |
+
for checkpoint_key in loaded_keys:
|
| 924 |
+
model_key = checkpoint_key
|
| 925 |
+
|
| 926 |
+
if (
|
| 927 |
+
model_key in model_state_dict
|
| 928 |
+
and state_dict[checkpoint_key].shape != model_state_dict[model_key].shape
|
| 929 |
+
):
|
| 930 |
+
mismatched_keys.append(
|
| 931 |
+
(checkpoint_key, state_dict[checkpoint_key].shape, model_state_dict[model_key].shape)
|
| 932 |
+
)
|
| 933 |
+
del state_dict[checkpoint_key]
|
| 934 |
+
return mismatched_keys
|
| 935 |
+
|
| 936 |
+
if state_dict is not None:
|
| 937 |
+
# Whole checkpoint
|
| 938 |
+
mismatched_keys = _find_mismatched_keys(
|
| 939 |
+
state_dict,
|
| 940 |
+
model_state_dict,
|
| 941 |
+
original_loaded_keys,
|
| 942 |
+
ignore_mismatched_sizes,
|
| 943 |
+
)
|
| 944 |
+
error_msgs = _load_state_dict_into_model(model_to_load, state_dict)
|
| 945 |
+
|
| 946 |
+
if len(error_msgs) > 0:
|
| 947 |
+
error_msg = "\n\t".join(error_msgs)
|
| 948 |
+
if "size mismatch" in error_msg:
|
| 949 |
+
error_msg += (
|
| 950 |
+
"\n\tYou may consider adding `ignore_mismatched_sizes=True` in the model `from_pretrained` method."
|
| 951 |
+
)
|
| 952 |
+
raise RuntimeError(f"Error(s) in loading state_dict for {model.__class__.__name__}:\n\t{error_msg}")
|
| 953 |
+
|
| 954 |
+
if len(unexpected_keys) > 0:
|
| 955 |
+
logger.warning(
|
| 956 |
+
f"Some weights of the model checkpoint at {pretrained_model_name_or_path} were not used when"
|
| 957 |
+
f" initializing {model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are"
|
| 958 |
+
f" initializing {model.__class__.__name__} from the checkpoint of a model trained on another task"
|
| 959 |
+
" or with another architecture (e.g. initializing a BertForSequenceClassification model from a"
|
| 960 |
+
" BertForPreTraining model).\n- This IS NOT expected if you are initializing"
|
| 961 |
+
f" {model.__class__.__name__} from the checkpoint of a model that you expect to be exactly"
|
| 962 |
+
" identical (initializing a BertForSequenceClassification model from a"
|
| 963 |
+
" BertForSequenceClassification model)."
|
| 964 |
+
)
|
| 965 |
+
else:
|
| 966 |
+
logger.info(f"All model checkpoint weights were used when initializing {model.__class__.__name__}.\n")
|
| 967 |
+
if len(missing_keys) > 0:
|
| 968 |
+
logger.warning(
|
| 969 |
+
f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at"
|
| 970 |
+
f" {pretrained_model_name_or_path} and are newly initialized: {missing_keys}\nYou should probably"
|
| 971 |
+
" TRAIN this model on a down-stream task to be able to use it for predictions and inference."
|
| 972 |
+
)
|
| 973 |
+
elif len(mismatched_keys) == 0:
|
| 974 |
+
logger.info(
|
| 975 |
+
f"All the weights of {model.__class__.__name__} were initialized from the model checkpoint at"
|
| 976 |
+
f" {pretrained_model_name_or_path}.\nIf your task is similar to the task the model of the"
|
| 977 |
+
f" checkpoint was trained on, you can already use {model.__class__.__name__} for predictions"
|
| 978 |
+
" without further training."
|
| 979 |
+
)
|
| 980 |
+
if len(mismatched_keys) > 0:
|
| 981 |
+
mismatched_warning = "\n".join(
|
| 982 |
+
[
|
| 983 |
+
f"- {key}: found shape {shape1} in the checkpoint and {shape2} in the model instantiated"
|
| 984 |
+
for key, shape1, shape2 in mismatched_keys
|
| 985 |
+
]
|
| 986 |
+
)
|
| 987 |
+
logger.warning(
|
| 988 |
+
f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at"
|
| 989 |
+
f" {pretrained_model_name_or_path} and are newly initialized because the shapes did not"
|
| 990 |
+
f" match:\n{mismatched_warning}\nYou should probably TRAIN this model on a down-stream task to be"
|
| 991 |
+
" able to use it for predictions and inference."
|
| 992 |
+
)
|
| 993 |
+
|
| 994 |
+
return model, missing_keys, unexpected_keys, mismatched_keys, error_msgs
|
| 995 |
+
|
| 996 |
+
@classmethod
|
| 997 |
+
def _get_signature_keys(cls, obj):
|
| 998 |
+
parameters = inspect.signature(obj.__init__).parameters
|
| 999 |
+
required_parameters = {k: v for k, v in parameters.items() if v.default == inspect._empty}
|
| 1000 |
+
optional_parameters = set({k for k, v in parameters.items() if v.default != inspect._empty})
|
| 1001 |
+
expected_modules = set(required_parameters.keys()) - {"self"}
|
| 1002 |
+
|
| 1003 |
+
return expected_modules, optional_parameters
|
| 1004 |
+
|
| 1005 |
+
# Adapted from `transformers` modeling_utils.py
|
| 1006 |
+
def _get_no_split_modules(self, device_map: str):
|
| 1007 |
+
"""
|
| 1008 |
+
Get the modules of the model that should not be spit when using device_map. We iterate through the modules to
|
| 1009 |
+
get the underlying `_no_split_modules`.
|
| 1010 |
+
|
| 1011 |
+
Args:
|
| 1012 |
+
device_map (`str`):
|
| 1013 |
+
The device map value. Options are ["auto", "balanced", "balanced_low_0", "sequential"]
|
| 1014 |
+
|
| 1015 |
+
Returns:
|
| 1016 |
+
`List[str]`: List of modules that should not be split
|
| 1017 |
+
"""
|
| 1018 |
+
_no_split_modules = set()
|
| 1019 |
+
modules_to_check = [self]
|
| 1020 |
+
while len(modules_to_check) > 0:
|
| 1021 |
+
module = modules_to_check.pop(-1)
|
| 1022 |
+
# if the module does not appear in _no_split_modules, we also check the children
|
| 1023 |
+
if module.__class__.__name__ not in _no_split_modules:
|
| 1024 |
+
if isinstance(module, ModelMixin):
|
| 1025 |
+
if module._no_split_modules is None:
|
| 1026 |
+
raise ValueError(
|
| 1027 |
+
f"{module.__class__.__name__} does not support `device_map='{device_map}'`. To implement "
|
| 1028 |
+
f"support, the model class needs to implement the `_no_split_modules` attribute."
|
| 1029 |
+
)
|
| 1030 |
+
else:
|
| 1031 |
+
_no_split_modules = _no_split_modules | set(module._no_split_modules)
|
| 1032 |
+
modules_to_check += list(module.children())
|
| 1033 |
+
return list(_no_split_modules)
|
| 1034 |
+
|
| 1035 |
+
@property
|
| 1036 |
+
def device(self) -> torch.device:
|
| 1037 |
+
"""
|
| 1038 |
+
`torch.device`: The device on which the module is (assuming that all the module parameters are on the same
|
| 1039 |
+
device).
|
| 1040 |
+
"""
|
| 1041 |
+
return get_parameter_device(self)
|
| 1042 |
+
|
| 1043 |
+
@property
|
| 1044 |
+
def dtype(self) -> torch.dtype:
|
| 1045 |
+
"""
|
| 1046 |
+
`torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype).
|
| 1047 |
+
"""
|
| 1048 |
+
return get_parameter_dtype(self)
|
| 1049 |
+
|
| 1050 |
+
def num_parameters(self, only_trainable: bool = False, exclude_embeddings: bool = False) -> int:
|
| 1051 |
+
"""
|
| 1052 |
+
Get number of (trainable or non-embedding) parameters in the module.
|
| 1053 |
+
|
| 1054 |
+
Args:
|
| 1055 |
+
only_trainable (`bool`, *optional*, defaults to `False`):
|
| 1056 |
+
Whether or not to return only the number of trainable parameters.
|
| 1057 |
+
exclude_embeddings (`bool`, *optional*, defaults to `False`):
|
| 1058 |
+
Whether or not to return only the number of non-embedding parameters.
|
| 1059 |
+
|
| 1060 |
+
Returns:
|
| 1061 |
+
`int`: The number of parameters.
|
| 1062 |
+
|
| 1063 |
+
Example:
|
| 1064 |
+
|
| 1065 |
+
```py
|
| 1066 |
+
from diffusers import UNet2DConditionModel
|
| 1067 |
+
|
| 1068 |
+
model_id = "runwayml/stable-diffusion-v1-5"
|
| 1069 |
+
unet = UNet2DConditionModel.from_pretrained(model_id, subfolder="unet")
|
| 1070 |
+
unet.num_parameters(only_trainable=True)
|
| 1071 |
+
859520964
|
| 1072 |
+
```
|
| 1073 |
+
"""
|
| 1074 |
+
|
| 1075 |
+
if exclude_embeddings:
|
| 1076 |
+
embedding_param_names = [
|
| 1077 |
+
f"{name}.weight"
|
| 1078 |
+
for name, module_type in self.named_modules()
|
| 1079 |
+
if isinstance(module_type, torch.nn.Embedding)
|
| 1080 |
+
]
|
| 1081 |
+
non_embedding_parameters = [
|
| 1082 |
+
parameter for name, parameter in self.named_parameters() if name not in embedding_param_names
|
| 1083 |
+
]
|
| 1084 |
+
return sum(p.numel() for p in non_embedding_parameters if p.requires_grad or not only_trainable)
|
| 1085 |
+
else:
|
| 1086 |
+
return sum(p.numel() for p in self.parameters() if p.requires_grad or not only_trainable)
|
| 1087 |
+
|
| 1088 |
+
def _convert_deprecated_attn(self, state_dict: OrderedDict) -> None:
|
| 1089 |
+
# original function name: _convert_deprecated_attention_blocks
|
| 1090 |
+
deprecated_attention_block_paths = []
|
| 1091 |
+
|
| 1092 |
+
def recursive_find_attn_block(name, module):
|
| 1093 |
+
if hasattr(module, "_from_deprecated_attn_block") and module._from_deprecated_attn_block:
|
| 1094 |
+
deprecated_attention_block_paths.append(name)
|
| 1095 |
+
|
| 1096 |
+
for sub_name, sub_module in module.named_children():
|
| 1097 |
+
sub_name = sub_name if name == "" else f"{name}.{sub_name}"
|
| 1098 |
+
recursive_find_attn_block(sub_name, sub_module)
|
| 1099 |
+
|
| 1100 |
+
recursive_find_attn_block("", self)
|
| 1101 |
+
|
| 1102 |
+
# NOTE: we have to check if the deprecated parameters are in the state dict
|
| 1103 |
+
# because it is possible we are loading from a state dict that was already
|
| 1104 |
+
# converted
|
| 1105 |
+
|
| 1106 |
+
for path in deprecated_attention_block_paths:
|
| 1107 |
+
# group_norm path stays the same
|
| 1108 |
+
|
| 1109 |
+
# query -> to_q
|
| 1110 |
+
if f"{path}.query.weight" in state_dict:
|
| 1111 |
+
state_dict[f"{path}.to_q.weight"] = state_dict.pop(f"{path}.query.weight")
|
| 1112 |
+
if f"{path}.query.bias" in state_dict:
|
| 1113 |
+
state_dict[f"{path}.to_q.bias"] = state_dict.pop(f"{path}.query.bias")
|
| 1114 |
+
|
| 1115 |
+
# key -> to_k
|
| 1116 |
+
if f"{path}.key.weight" in state_dict:
|
| 1117 |
+
state_dict[f"{path}.to_k.weight"] = state_dict.pop(f"{path}.key.weight")
|
| 1118 |
+
if f"{path}.key.bias" in state_dict:
|
| 1119 |
+
state_dict[f"{path}.to_k.bias"] = state_dict.pop(f"{path}.key.bias")
|
| 1120 |
+
|
| 1121 |
+
# value -> to_v
|
| 1122 |
+
if f"{path}.value.weight" in state_dict:
|
| 1123 |
+
state_dict[f"{path}.to_v.weight"] = state_dict.pop(f"{path}.value.weight")
|
| 1124 |
+
if f"{path}.value.bias" in state_dict:
|
| 1125 |
+
state_dict[f"{path}.to_v.bias"] = state_dict.pop(f"{path}.value.bias")
|
| 1126 |
+
|
| 1127 |
+
# proj_attn -> to_out.0
|
| 1128 |
+
if f"{path}.proj_attn.weight" in state_dict:
|
| 1129 |
+
state_dict[f"{path}.to_out.0.weight"] = state_dict.pop(f"{path}.proj_attn.weight")
|
| 1130 |
+
if f"{path}.proj_attn.bias" in state_dict:
|
| 1131 |
+
state_dict[f"{path}.to_out.0.bias"] = state_dict.pop(f"{path}.proj_attn.bias")
|
| 1132 |
+
|
| 1133 |
+
|
| 1134 |
+
class LegacyModelMixin(ModelMixin):
|
| 1135 |
+
r"""
|
| 1136 |
+
A subclass of `ModelMixin` to resolve class mapping from legacy classes (like `Transformer2DModel`) to more
|
| 1137 |
+
pipeline-specific classes (like `DiTTransformer2DModel`).
|
| 1138 |
+
"""
|
| 1139 |
+
|
| 1140 |
+
@classmethod
|
| 1141 |
+
@validate_hf_hub_args
|
| 1142 |
+
def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs):
|
| 1143 |
+
# To prevent dependency import problem.
|
| 1144 |
+
from diffusers.models.model_loading_utils import _fetch_remapped_cls_from_config
|
| 1145 |
+
|
| 1146 |
+
# Create a copy of the kwargs so that we don't mess with the keyword arguments in the downstream calls.
|
| 1147 |
+
kwargs_copy = kwargs.copy()
|
| 1148 |
+
|
| 1149 |
+
cache_dir = kwargs.pop("cache_dir", None)
|
| 1150 |
+
force_download = kwargs.pop("force_download", False)
|
| 1151 |
+
proxies = kwargs.pop("proxies", None)
|
| 1152 |
+
local_files_only = kwargs.pop("local_files_only", None)
|
| 1153 |
+
token = kwargs.pop("token", None)
|
| 1154 |
+
revision = kwargs.pop("revision", None)
|
| 1155 |
+
subfolder = kwargs.pop("subfolder", None)
|
| 1156 |
+
|
| 1157 |
+
# Load config if we don't provide a configuration
|
| 1158 |
+
config_path = pretrained_model_name_or_path
|
| 1159 |
+
|
| 1160 |
+
user_agent = {
|
| 1161 |
+
"diffusers": __version__,
|
| 1162 |
+
"file_type": "model",
|
| 1163 |
+
"framework": "pytorch",
|
| 1164 |
+
}
|
| 1165 |
+
|
| 1166 |
+
# load config
|
| 1167 |
+
config, _, _ = cls.load_config(
|
| 1168 |
+
config_path,
|
| 1169 |
+
cache_dir=cache_dir,
|
| 1170 |
+
return_unused_kwargs=True,
|
| 1171 |
+
return_commit_hash=True,
|
| 1172 |
+
force_download=force_download,
|
| 1173 |
+
proxies=proxies,
|
| 1174 |
+
local_files_only=local_files_only,
|
| 1175 |
+
token=token,
|
| 1176 |
+
revision=revision,
|
| 1177 |
+
subfolder=subfolder,
|
| 1178 |
+
user_agent=user_agent,
|
| 1179 |
+
**kwargs,
|
| 1180 |
+
)
|
| 1181 |
+
# resolve remapping
|
| 1182 |
+
remapped_class = _fetch_remapped_cls_from_config(config, cls)
|
| 1183 |
+
|
| 1184 |
+
return remapped_class.from_pretrained(pretrained_model_name_or_path, **kwargs_copy)
|
omni_diffusion/tokenizer.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .constants import (
|
| 2 |
+
BOX_END_TOKEN,
|
| 3 |
+
BOX_START_TOKEN,
|
| 4 |
+
IMG_CONTEXT_TOKEN,
|
| 5 |
+
IMG_END_TOKEN,
|
| 6 |
+
IMG_START_TOKEN,
|
| 7 |
+
IMG_TAG_TOKEN,
|
| 8 |
+
PATCH_CONTEXT_TOKEN,
|
| 9 |
+
PATCH_END_TOKEN,
|
| 10 |
+
PATCH_START_TOKEN,
|
| 11 |
+
QUAD_END_TOKEN,
|
| 12 |
+
QUAD_START_TOKEN,
|
| 13 |
+
REF_END_TOKEN,
|
| 14 |
+
REF_START_TOKEN,
|
| 15 |
+
VID_CONTEXT_TOKEN,
|
| 16 |
+
VID_END_TOKEN,
|
| 17 |
+
VID_START_TOKEN,
|
| 18 |
+
VID_TAG_TOKEN,
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def update_tokenizer(tokenizer, audio_tokenizer_type=None):
|
| 23 |
+
"""
|
| 24 |
+
Adds special tokens related to multimodal tasks (Image, Video, Patch, Bounding Box, etc.)
|
| 25 |
+
to the tokenizer vocabulary.
|
| 26 |
+
Also handles specific tokenizer updates for audio models like SenseVoice/GLM4Voice.
|
| 27 |
+
"""
|
| 28 |
+
token_list = [
|
| 29 |
+
IMG_START_TOKEN,
|
| 30 |
+
IMG_END_TOKEN,
|
| 31 |
+
IMG_CONTEXT_TOKEN,
|
| 32 |
+
VID_START_TOKEN,
|
| 33 |
+
VID_END_TOKEN,
|
| 34 |
+
VID_CONTEXT_TOKEN,
|
| 35 |
+
PATCH_START_TOKEN,
|
| 36 |
+
PATCH_END_TOKEN,
|
| 37 |
+
PATCH_CONTEXT_TOKEN,
|
| 38 |
+
QUAD_START_TOKEN,
|
| 39 |
+
QUAD_END_TOKEN,
|
| 40 |
+
REF_START_TOKEN,
|
| 41 |
+
REF_END_TOKEN,
|
| 42 |
+
BOX_START_TOKEN,
|
| 43 |
+
BOX_END_TOKEN,
|
| 44 |
+
IMG_TAG_TOKEN,
|
| 45 |
+
VID_TAG_TOKEN,
|
| 46 |
+
]
|
| 47 |
+
num_new_tokens = tokenizer.add_tokens(token_list, special_tokens=True)
|
| 48 |
+
|
| 49 |
+
if audio_tokenizer_type == "sensevoice_glm4voice":
|
| 50 |
+
from .tokenizer_sensevoice_glm4voice import (
|
| 51 |
+
update_tokenizer_for_sensevoice_glm4voice,
|
| 52 |
+
SenseVoiceGLM4VoiceTokenizer,
|
| 53 |
+
)
|
| 54 |
+
return update_tokenizer_for_sensevoice_glm4voice(tokenizer)
|
| 55 |
+
|
| 56 |
+
raise NotImplementedError
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def get_audio_tokenizer(model_name_or_path, audio_tokenizer_type, flow_path=None, rank=None):
|
| 60 |
+
"""
|
| 61 |
+
Factory function to initialize and return the appropriate audio tokenizer instance
|
| 62 |
+
based on the provided type (e.g., 'sensevoice_glm4voice').
|
| 63 |
+
"""
|
| 64 |
+
if audio_tokenizer_type is None:
|
| 65 |
+
return None
|
| 66 |
+
|
| 67 |
+
if audio_tokenizer_type == "sensevoice_glm4voice":
|
| 68 |
+
from .tokenizer_sensevoice_glm4voice import (
|
| 69 |
+
update_tokenizer_for_sensevoice_glm4voice,
|
| 70 |
+
SenseVoiceGLM4VoiceTokenizer,
|
| 71 |
+
)
|
| 72 |
+
return SenseVoiceGLM4VoiceTokenizer(model_name_or_path, flow_path=flow_path, rank=rank)
|
| 73 |
+
|
| 74 |
+
raise NotImplementedError
|
omni_diffusion/tokenizer_magvitv2.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import glob
|
| 2 |
+
import io
|
| 3 |
+
import logging
|
| 4 |
+
import math
|
| 5 |
+
import os
|
| 6 |
+
import tarfile
|
| 7 |
+
import uuid
|
| 8 |
+
|
| 9 |
+
import safetensors
|
| 10 |
+
import torch
|
| 11 |
+
from transformers import WhisperFeatureExtractor, WhisperTokenizerFast
|
| 12 |
+
|
| 13 |
+
import torchaudio
|
| 14 |
+
|
| 15 |
+
from transformers import WhisperFeatureExtractor
|
| 16 |
+
from speech_tokenizer.modeling_whisper import WhisperVQEncoder
|
| 17 |
+
from flow_inference import AudioDecoder
|
| 18 |
+
from omni_diffusion.models.magvit.modeling_magvitv2 import MAGVITv2
|
| 19 |
+
|
| 20 |
+
from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank
|
| 21 |
+
from funasr.models.sense_voice.model import SenseVoiceSmall
|
| 22 |
+
|
| 23 |
+
logger = logging.getLogger(__name__)
|
| 24 |
+
logger.setLevel(logging.INFO)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class MagVITV2Tokenizer:
|
| 28 |
+
def __init__(self, model_path=None, rank=None):
|
| 29 |
+
if rank is None and torch.distributed.is_initialized():
|
| 30 |
+
rank = torch.distributed.get_rank()
|
| 31 |
+
self.rank = rank % 8
|
| 32 |
+
else:
|
| 33 |
+
self.rank = rank
|
| 34 |
+
logger.info(f"{self.rank=}")
|
| 35 |
+
self.model_path = model_path
|
| 36 |
+
|
| 37 |
+
def load_model(self):
|
| 38 |
+
|
| 39 |
+
if hasattr(self, "image_tokenizer"):
|
| 40 |
+
return
|
| 41 |
+
|
| 42 |
+
if self.rank is not None:
|
| 43 |
+
self.device = f"cuda:{self.rank}"
|
| 44 |
+
torch.cuda.set_device(self.rank)
|
| 45 |
+
else:
|
| 46 |
+
self.device = "cpu"
|
| 47 |
+
|
| 48 |
+
self.image_tokenizer = MAGVITv2()
|
| 49 |
+
logger.info(f"{self.device=} Loading MAGVITv2")
|
| 50 |
+
self.image_tokenizer = self.image_tokenizer.from_pretrained(self.model_path).to(self.device)
|
| 51 |
+
self.image_tokenizer.eval()
|
| 52 |
+
self.image_tokenizer.requires_grad_(False)
|
| 53 |
+
logger.info(f"{self.device=} Loading MAGVITv2 done")
|
| 54 |
+
|
| 55 |
+
def encode(self, image):
|
| 56 |
+
image = image.to(self.device)
|
| 57 |
+
image_tokens = self.image_tokenizer.get_code(image)
|
| 58 |
+
return image_tokens
|
| 59 |
+
|
| 60 |
+
def decode(self, image_tokens):
|
| 61 |
+
images = self.image_tokenizer.decode_code(image_tokens)
|
| 62 |
+
return images
|
| 63 |
+
|
| 64 |
+
def apply_to_role(self, role, **kwargs):
|
| 65 |
+
return True
|
omni_diffusion/tokenizer_sensevoice_glm4voice.py
ADDED
|
@@ -0,0 +1,280 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import glob
|
| 2 |
+
import io
|
| 3 |
+
import logging
|
| 4 |
+
import math
|
| 5 |
+
import os
|
| 6 |
+
import tarfile
|
| 7 |
+
import uuid
|
| 8 |
+
|
| 9 |
+
import safetensors
|
| 10 |
+
import torch
|
| 11 |
+
from transformers import WhisperFeatureExtractor, WhisperTokenizerFast
|
| 12 |
+
|
| 13 |
+
import torchaudio
|
| 14 |
+
|
| 15 |
+
from transformers import WhisperFeatureExtractor
|
| 16 |
+
from speech_tokenizer.modeling_whisper import WhisperVQEncoder
|
| 17 |
+
from flow_inference import AudioDecoder
|
| 18 |
+
|
| 19 |
+
from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank
|
| 20 |
+
from funasr.models.sense_voice.model import SenseVoiceSmall
|
| 21 |
+
import sys
|
| 22 |
+
|
| 23 |
+
from .constants import (
|
| 24 |
+
AUD_CONTEXT_TOKEN,
|
| 25 |
+
AUD_END_TOKEN,
|
| 26 |
+
AUD_START_TOKEN,
|
| 27 |
+
AUD_TAG_TOKEN,
|
| 28 |
+
BOX_END_TOKEN,
|
| 29 |
+
BOX_START_TOKEN,
|
| 30 |
+
IMG_CONTEXT_TOKEN,
|
| 31 |
+
IMG_END_TOKEN,
|
| 32 |
+
IMG_START_TOKEN,
|
| 33 |
+
IMG_TAG_TOKEN,
|
| 34 |
+
PATCH_CONTEXT_TOKEN,
|
| 35 |
+
PATCH_END_TOKEN,
|
| 36 |
+
PATCH_START_TOKEN,
|
| 37 |
+
QUAD_END_TOKEN,
|
| 38 |
+
QUAD_START_TOKEN,
|
| 39 |
+
REF_END_TOKEN,
|
| 40 |
+
REF_START_TOKEN,
|
| 41 |
+
VID_CONTEXT_TOKEN,
|
| 42 |
+
VID_END_TOKEN,
|
| 43 |
+
VID_START_TOKEN,
|
| 44 |
+
VID_TAG_TOKEN,
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
logger = logging.getLogger(__name__)
|
| 48 |
+
logger.setLevel(logging.INFO)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def update_tokenizer_for_sensevoice_glm4voice(tokenizer):
|
| 52 |
+
token_list = [
|
| 53 |
+
IMG_START_TOKEN,
|
| 54 |
+
IMG_END_TOKEN,
|
| 55 |
+
IMG_CONTEXT_TOKEN,
|
| 56 |
+
VID_START_TOKEN,
|
| 57 |
+
VID_END_TOKEN,
|
| 58 |
+
VID_CONTEXT_TOKEN,
|
| 59 |
+
PATCH_START_TOKEN,
|
| 60 |
+
PATCH_END_TOKEN,
|
| 61 |
+
PATCH_CONTEXT_TOKEN,
|
| 62 |
+
AUD_START_TOKEN,
|
| 63 |
+
AUD_END_TOKEN,
|
| 64 |
+
AUD_CONTEXT_TOKEN,
|
| 65 |
+
QUAD_START_TOKEN,
|
| 66 |
+
QUAD_END_TOKEN,
|
| 67 |
+
REF_START_TOKEN,
|
| 68 |
+
REF_END_TOKEN,
|
| 69 |
+
BOX_START_TOKEN,
|
| 70 |
+
BOX_END_TOKEN,
|
| 71 |
+
IMG_TAG_TOKEN,
|
| 72 |
+
VID_TAG_TOKEN,
|
| 73 |
+
AUD_TAG_TOKEN,
|
| 74 |
+
]
|
| 75 |
+
num_new_tokens = tokenizer.add_tokens(token_list, special_tokens=True)
|
| 76 |
+
|
| 77 |
+
token_list = [f"<|audio_{i}|>" for i in range(16384)]
|
| 78 |
+
num_new_tokens = tokenizer.add_tokens(token_list, special_tokens=False)
|
| 79 |
+
|
| 80 |
+
return tokenizer
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
class SenseVoiceGLM4VoiceTokenizer:
|
| 84 |
+
def __init__(self, model_name_or_path, flow_path=None, rank=None):
|
| 85 |
+
self.model_name_or_path = model_name_or_path
|
| 86 |
+
self.flow_path = flow_path
|
| 87 |
+
|
| 88 |
+
if rank is None and torch.distributed.is_initialized():
|
| 89 |
+
rank = torch.distributed.get_rank()
|
| 90 |
+
self.rank = rank % 8
|
| 91 |
+
else:
|
| 92 |
+
self.rank = rank
|
| 93 |
+
logger.info(f"{self.rank=}")
|
| 94 |
+
|
| 95 |
+
self.sample_rate = 16000
|
| 96 |
+
|
| 97 |
+
self.is_discrete = True
|
| 98 |
+
self.is_contiguous = True
|
| 99 |
+
|
| 100 |
+
self._resample_buffer: dict[int, torchaudio.transforms.Resample] = {}
|
| 101 |
+
for sample_rate in [8000, 22050, 24000, 32000, 44100, 48000]:
|
| 102 |
+
print(f"torchaudio.transforms.Resample {sample_rate=} {self.sample_rate=}")
|
| 103 |
+
self._resample_buffer[sample_rate] = torchaudio.transforms.Resample(
|
| 104 |
+
orig_freq=sample_rate, new_freq=self.sample_rate
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
def load_model(self):
|
| 108 |
+
|
| 109 |
+
if hasattr(self, "whisper_model"):
|
| 110 |
+
return
|
| 111 |
+
if self.rank is not None:
|
| 112 |
+
self.device = f"cuda:{self.rank}"
|
| 113 |
+
torch.cuda.set_device(self.rank)
|
| 114 |
+
else:
|
| 115 |
+
self.device = "cpu"
|
| 116 |
+
|
| 117 |
+
logger.info(f"{self.device=} Loading SenseVoiceSmall")
|
| 118 |
+
from huggingface_hub import snapshot_download
|
| 119 |
+
model_dir = snapshot_download(repo_id="FunAudioLLM/SenseVoiceSmall")
|
| 120 |
+
|
| 121 |
+
_, self.kwargs = SenseVoiceSmall.from_pretrained(model=model_dir, device=self.device)
|
| 122 |
+
logger.info(f"{self.device=} Loading SenseVoiceSmall Done")
|
| 123 |
+
|
| 124 |
+
logger.info(f"{self.device=} Loading GLM4VoiceTokenizer")
|
| 125 |
+
self.whisper_model = (
|
| 126 |
+
WhisperVQEncoder.from_pretrained(self.model_name_or_path).eval().to(self.device)
|
| 127 |
+
)
|
| 128 |
+
self.feature_extractor = WhisperFeatureExtractor.from_pretrained(self.model_name_or_path)
|
| 129 |
+
print(hasattr(self, "whisper_model"),'whisper_model')
|
| 130 |
+
|
| 131 |
+
if self.flow_path is not None:
|
| 132 |
+
flow_config = os.path.join(self.flow_path, "config.yaml")
|
| 133 |
+
flow_checkpoint = os.path.join(self.flow_path, "flow.pt")
|
| 134 |
+
hift_checkpoint = os.path.join(self.flow_path, "hift.pt")
|
| 135 |
+
|
| 136 |
+
# Flow & Hift
|
| 137 |
+
self.audio_decoder = AudioDecoder(
|
| 138 |
+
config_path=flow_config,
|
| 139 |
+
flow_ckpt_path=flow_checkpoint,
|
| 140 |
+
hift_ckpt_path=hift_checkpoint,
|
| 141 |
+
device=self.device,
|
| 142 |
+
)
|
| 143 |
+
logger.info(f"{self.device=} Loading GLM4VoiceTokenizer Done")
|
| 144 |
+
|
| 145 |
+
def encode(self, audio_path, is_discrete=False, is_contiguous=True, **kwargs):
|
| 146 |
+
|
| 147 |
+
import faulthandler
|
| 148 |
+
faulthandler.enable()
|
| 149 |
+
|
| 150 |
+
if not hasattr(self, "whisper_model"):
|
| 151 |
+
self.load_model()
|
| 152 |
+
|
| 153 |
+
assert not (is_discrete and is_contiguous)
|
| 154 |
+
assert is_discrete or is_contiguous
|
| 155 |
+
|
| 156 |
+
if is_discrete:
|
| 157 |
+
audio_tokens = self.extract_speech_token(
|
| 158 |
+
self.whisper_model, self.feature_extractor, [audio_path], device=self.device
|
| 159 |
+
)[0]
|
| 160 |
+
return audio_tokens
|
| 161 |
+
|
| 162 |
+
if is_contiguous:
|
| 163 |
+
|
| 164 |
+
audio, sample_rate = torchaudio.load(audio_path)
|
| 165 |
+
audio = audio.mean(0)
|
| 166 |
+
if sample_rate != self.sample_rate:
|
| 167 |
+
if sample_rate not in self._resample_buffer:
|
| 168 |
+
print(f"torchaudio.transforms.Resample {sample_rate=} {self.sample_rate=} {self.device=}", flush=True)
|
| 169 |
+
self._resample_buffer[sample_rate] = torchaudio.transforms.Resample(
|
| 170 |
+
orig_freq=sample_rate, new_freq=self.sample_rate
|
| 171 |
+
).to(self.device)
|
| 172 |
+
audio = audio.to(self.device)
|
| 173 |
+
self._resample_buffer[sample_rate].to(self.device)
|
| 174 |
+
audio = self._resample_buffer[sample_rate](audio[None, :])[0, :]
|
| 175 |
+
audio = audio.cpu()
|
| 176 |
+
|
| 177 |
+
frontend = self.kwargs["frontend"]
|
| 178 |
+
|
| 179 |
+
speech, speech_lengths = extract_fbank(audio, data_type="sound", frontend=frontend)
|
| 180 |
+
|
| 181 |
+
speech = speech[0]
|
| 182 |
+
|
| 183 |
+
return speech
|
| 184 |
+
|
| 185 |
+
def decode(self, audio_tokens, option_steps=10, **kwargs):
|
| 186 |
+
if not hasattr(self, "whisper_model"):
|
| 187 |
+
self.load_model()
|
| 188 |
+
|
| 189 |
+
this_uuid = str(uuid.uuid4())
|
| 190 |
+
this_uuid = "abc"
|
| 191 |
+
|
| 192 |
+
tts_token = torch.tensor(audio_tokens, device=self.device).unsqueeze(0)
|
| 193 |
+
|
| 194 |
+
flow_prompt_speech_token = torch.zeros(1, 0, dtype=torch.int64).to(self.device)
|
| 195 |
+
prompt_speech_feat = torch.zeros(1, 0, 80).to(self.device)
|
| 196 |
+
|
| 197 |
+
tts_speech, tts_mel = self.audio_decoder.token2wav(
|
| 198 |
+
tts_token,
|
| 199 |
+
uuid=this_uuid,
|
| 200 |
+
prompt_token=flow_prompt_speech_token.to(self.device),
|
| 201 |
+
prompt_feat=prompt_speech_feat.to(self.device),
|
| 202 |
+
finalize=True,
|
| 203 |
+
option_steps=option_steps,
|
| 204 |
+
)
|
| 205 |
+
tts_speechs = []
|
| 206 |
+
tts_speechs.append(tts_speech.squeeze())
|
| 207 |
+
tts_speech = torch.cat(tts_speechs, dim=-1).cpu()
|
| 208 |
+
|
| 209 |
+
return tts_speech
|
| 210 |
+
|
| 211 |
+
def apply_to_role(self, role, **kwargs):
|
| 212 |
+
is_discrete = kwargs.get("is_discrete", False)
|
| 213 |
+
if is_discrete and role in ["assistant", "gpt"]:
|
| 214 |
+
return True
|
| 215 |
+
|
| 216 |
+
is_contiguous = kwargs.get("is_contiguous", False)
|
| 217 |
+
if is_contiguous and role in ["user", "human"]:
|
| 218 |
+
return True
|
| 219 |
+
|
| 220 |
+
return False
|
| 221 |
+
|
| 222 |
+
def extract_speech_token(self, model, feature_extractor, utts, device="cuda"):
|
| 223 |
+
with torch.no_grad():
|
| 224 |
+
audios, indices = [], []
|
| 225 |
+
for idx, utt in enumerate(utts):
|
| 226 |
+
if isinstance(utt, tuple):
|
| 227 |
+
audio, sample_rate = utt
|
| 228 |
+
else:
|
| 229 |
+
audio, sample_rate = torchaudio.load(utt)
|
| 230 |
+
audio = audio.to(device)
|
| 231 |
+
if sample_rate != 16000:
|
| 232 |
+
if sample_rate not in self._resample_buffer:
|
| 233 |
+
print(f"torchaudio.transforms.Resample {sample_rate=} {self.sample_rate=} {device=}")
|
| 234 |
+
self._resample_buffer[sample_rate] = torchaudio.transforms.Resample(
|
| 235 |
+
orig_freq=sample_rate, new_freq=16000
|
| 236 |
+
).to(device)
|
| 237 |
+
self._resample_buffer[sample_rate].to(device)
|
| 238 |
+
audio = self._resample_buffer[sample_rate](audio)
|
| 239 |
+
|
| 240 |
+
audio = audio[0]
|
| 241 |
+
audio = audio.cpu().numpy()
|
| 242 |
+
time_step = 0
|
| 243 |
+
while time_step * 16000 < audio.shape[0]:
|
| 244 |
+
audio_segment = audio[time_step * 16000 : (time_step + 30) * 16000]
|
| 245 |
+
audios.append(audio_segment)
|
| 246 |
+
indices.append(idx)
|
| 247 |
+
time_step += 30
|
| 248 |
+
pooling_kernel_size = model.config.pooling_kernel_size or 1
|
| 249 |
+
stride = (
|
| 250 |
+
model.conv1.stride[0]
|
| 251 |
+
* model.conv2.stride[0]
|
| 252 |
+
* pooling_kernel_size
|
| 253 |
+
* feature_extractor.hop_length
|
| 254 |
+
)
|
| 255 |
+
all_speech_tokens = [[] for _ in range(len(utts))]
|
| 256 |
+
batch_size = 128
|
| 257 |
+
for start in range(0, len(audios), batch_size):
|
| 258 |
+
features = feature_extractor(
|
| 259 |
+
audios[start : start + batch_size],
|
| 260 |
+
sampling_rate=16000,
|
| 261 |
+
return_attention_mask=True,
|
| 262 |
+
return_tensors="pt",
|
| 263 |
+
device=device,
|
| 264 |
+
padding="longest",
|
| 265 |
+
pad_to_multiple_of=stride,
|
| 266 |
+
)
|
| 267 |
+
features = features.to(device=device)
|
| 268 |
+
outputs = model(**features)
|
| 269 |
+
speech_tokens = outputs.quantized_token_ids
|
| 270 |
+
attention_mask = features.attention_mask[
|
| 271 |
+
:, :: model.conv1.stride[0] * model.conv2.stride[0]
|
| 272 |
+
]
|
| 273 |
+
attention_mask = attention_mask[:, :: model.config.pooling_kernel_size]
|
| 274 |
+
assert attention_mask.shape == speech_tokens.shape
|
| 275 |
+
for i in range(len(speech_tokens)):
|
| 276 |
+
idx = indices[start + i]
|
| 277 |
+
speech_token = speech_tokens[i][attention_mask[i].bool()].tolist()
|
| 278 |
+
all_speech_tokens[idx].extend(speech_token)
|
| 279 |
+
|
| 280 |
+
return all_speech_tokens
|
requirements.txt
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
transformers==4.51.3
|
| 2 |
+
accelerate>=1.1.1
|
| 3 |
+
sentencepiece
|
| 4 |
+
einops
|
| 5 |
+
safetensors
|
| 6 |
+
pillow
|
| 7 |
+
numpy
|
| 8 |
+
opencv-python-headless
|
| 9 |
+
torchaudio
|
| 10 |
+
torchvision
|
| 11 |
+
timm
|
| 12 |
+
decord
|
| 13 |
+
natsort
|
| 14 |
+
tiktoken
|
| 15 |
+
funasr
|
| 16 |
+
hyperpyyaml
|
| 17 |
+
https://huggingface.co/datasets/multimodalart/zerogpu-blackwell-wheels/resolve/main/wheels/pt210-cu128-cp310/flash_attn-2.8.3-cp310-cp310-linux_x86_64.whl
|
s2i_0.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9b8bd63844e968ffd84e083d34411ea6ccfcbdf101df3d8f0853cb557428be2f
|
| 3 |
+
size 593358
|
svqa_0.jpg
ADDED
|
Git LFS Details
|
svqa_0.wav
ADDED
|
Binary file (48.1 kB). View file
|
|
|
third_party/GLM-4-Voice/.gitignore
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*venv
|
| 2 |
+
*.DS_Store
|
| 3 |
+
*.idea/
|
| 4 |
+
test*
|
third_party/GLM-4-Voice/LICENSE
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Apache License
|
| 2 |
+
Version 2.0, January 2004
|
| 3 |
+
http://www.apache.org/licenses/
|
| 4 |
+
|
| 5 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 6 |
+
|
| 7 |
+
1. Definitions.
|
| 8 |
+
|
| 9 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 10 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 11 |
+
|
| 12 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 13 |
+
the copyright owner that is granting the License.
|
| 14 |
+
|
| 15 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 16 |
+
other entities that control, are controlled by, or are under common
|
| 17 |
+
control with that entity. For the purposes of this definition,
|
| 18 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 19 |
+
direction or management of such entity, whether by contract or
|
| 20 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 21 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 22 |
+
|
| 23 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 24 |
+
exercising permissions granted by this License.
|
| 25 |
+
|
| 26 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 27 |
+
including but not limited to software source code, documentation
|
| 28 |
+
source, and configuration files.
|
| 29 |
+
|
| 30 |
+
"Object" form shall mean any form resulting from mechanical
|
| 31 |
+
transformation or translation of a Source form, including but
|
| 32 |
+
not limited to compiled object code, generated documentation,
|
| 33 |
+
and conversions to other media types.
|
| 34 |
+
|
| 35 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 36 |
+
Object form, made available under the License, as indicated by a
|
| 37 |
+
copyright notice that is included in or attached to the work
|
| 38 |
+
(an example is provided in the Appendix below).
|
| 39 |
+
|
| 40 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 41 |
+
form, that is based on (or derived from) the Work and for which the
|
| 42 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 43 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 44 |
+
of this License, Derivative Works shall not include works that remain
|
| 45 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 46 |
+
the Work and Derivative Works thereof.
|
| 47 |
+
|
| 48 |
+
"Contribution" shall mean any work of authorship, including
|
| 49 |
+
the original version of the Work and any modifications or additions
|
| 50 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 51 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 52 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 53 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 54 |
+
means any form of electronic, verbal, or written communication sent
|
| 55 |
+
to the Licensor or its representatives, including but not limited to
|
| 56 |
+
communication on electronic mailing lists, source code control systems,
|
| 57 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 58 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 59 |
+
excluding communication that is conspicuously marked or otherwise
|
| 60 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 61 |
+
|
| 62 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 63 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 64 |
+
subsequently incorporated within the Work.
|
| 65 |
+
|
| 66 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 67 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 68 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 69 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 70 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 71 |
+
Work and such Derivative Works in Source or Object form.
|
| 72 |
+
|
| 73 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 74 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 75 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 76 |
+
(except as stated in this section) patent license to make, have made,
|
| 77 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 78 |
+
where such license applies only to those patent claims licensable
|
| 79 |
+
by such Contributor that are necessarily infringed by their
|
| 80 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 81 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 82 |
+
institute patent litigation against any entity (including a
|
| 83 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 84 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 85 |
+
or contributory patent infringement, then any patent licenses
|
| 86 |
+
granted to You under this License for that Work shall terminate
|
| 87 |
+
as of the date such litigation is filed.
|
| 88 |
+
|
| 89 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 90 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 91 |
+
modifications, and in Source or Object form, provided that You
|
| 92 |
+
meet the following conditions:
|
| 93 |
+
|
| 94 |
+
(a) You must give any other recipients of the Work or
|
| 95 |
+
Derivative Works a copy of this License; and
|
| 96 |
+
|
| 97 |
+
(b) You must cause any modified files to carry prominent notices
|
| 98 |
+
stating that You changed the files; and
|
| 99 |
+
|
| 100 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 101 |
+
that You distribute, all copyright, patent, trademark, and
|
| 102 |
+
attribution notices from the Source form of the Work,
|
| 103 |
+
excluding those notices that do not pertain to any part of
|
| 104 |
+
the Derivative Works; and
|
| 105 |
+
|
| 106 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 107 |
+
distribution, then any Derivative Works that You distribute must
|
| 108 |
+
include a readable copy of the attribution notices contained
|
| 109 |
+
within such NOTICE file, excluding those notices that do not
|
| 110 |
+
pertain to any part of the Derivative Works, in at least one
|
| 111 |
+
of the following places: within a NOTICE text file distributed
|
| 112 |
+
as part of the Derivative Works; within the Source form or
|
| 113 |
+
documentation, if provided along with the Derivative Works; or,
|
| 114 |
+
within a display generated by the Derivative Works, if and
|
| 115 |
+
wherever such third-party notices normally appear. The contents
|
| 116 |
+
of the NOTICE file are for informational purposes only and
|
| 117 |
+
do not modify the License. You may add Your own attribution
|
| 118 |
+
notices within Derivative Works that You distribute, alongside
|
| 119 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 120 |
+
that such additional attribution notices cannot be construed
|
| 121 |
+
as modifying the License.
|
| 122 |
+
|
| 123 |
+
You may add Your own copyright statement to Your modifications and
|
| 124 |
+
may provide additional or different license terms and conditions
|
| 125 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 126 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 127 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 128 |
+
the conditions stated in this License.
|
| 129 |
+
|
| 130 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 131 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 132 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 133 |
+
this License, without any additional terms or conditions.
|
| 134 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 135 |
+
the terms of any separate license agreement you may have executed
|
| 136 |
+
with Licensor regarding such Contributions.
|
| 137 |
+
|
| 138 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 139 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 140 |
+
except as required for reasonable and customary use in describing the
|
| 141 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 142 |
+
|
| 143 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 144 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 145 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 146 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 147 |
+
implied, including, without limitation, any warranties or conditions
|
| 148 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 149 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 150 |
+
appropriateness of using or redistributing the Work and assume any
|
| 151 |
+
risks associated with Your exercise of permissions under this License.
|
| 152 |
+
|
| 153 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 154 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 155 |
+
unless required by applicable law (such as deliberate and grossly
|
| 156 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 157 |
+
liable to You for damages, including any direct, indirect, special,
|
| 158 |
+
incidental, or consequential damages of any character arising as a
|
| 159 |
+
result of this License or out of the use or inability to use the
|
| 160 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 161 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 162 |
+
other commercial damages or losses), even if such Contributor
|
| 163 |
+
has been advised of the possibility of such damages.
|
| 164 |
+
|
| 165 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 166 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 167 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 168 |
+
or other liability obligations and/or rights consistent with this
|
| 169 |
+
License. However, in accepting such obligations, You may act only
|
| 170 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 171 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 172 |
+
defend, and hold each Contributor harmless for any liability
|
| 173 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 174 |
+
of your accepting any such warranty or additional liability.
|
| 175 |
+
|
| 176 |
+
END OF TERMS AND CONDITIONS
|
| 177 |
+
|
| 178 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 179 |
+
|
| 180 |
+
To apply the Apache License to your work, attach the following
|
| 181 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 182 |
+
replaced with your own identifying information. (Don't include
|
| 183 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 184 |
+
comment syntax for the file format. We also recommend that a
|
| 185 |
+
file or class name and description of purpose be included on the
|
| 186 |
+
same "printed page" as the copyright notice for easier
|
| 187 |
+
identification within third-party archives.
|
| 188 |
+
|
| 189 |
+
Copyright 2024 GLM-4-Voice Model Team @ Zhipu AI
|
| 190 |
+
|
| 191 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 192 |
+
you may not use this file except in compliance with the License.
|
| 193 |
+
You may obtain a copy of the License at
|
| 194 |
+
|
| 195 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 196 |
+
|
| 197 |
+
Unless required by applicable law or agreed to in writing, software
|
| 198 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 199 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 200 |
+
See the License for the specific language governing permissions and
|
| 201 |
+
limitations under the License.
|
third_party/GLM-4-Voice/README.md
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# GLM-4-Voice
|
| 2 |
+
<p align="center">
|
| 3 |
+
📄<a href="https://arxiv.org/abs/2412.02612" target="_blank"> Report </a> • 🤗 <a href="https://huggingface.co/THUDM/glm-4-voice-9b" target="_blank">HF Repo</a> • 🤖 <a href="https://modelscope.cn/studios/ZhipuAI/GLM-4-Voice-Demo" target="_blank">Demo</a> • 🐦 <a href="https://twitter.com/thukeg" target="_blank">Twitter</a>
|
| 4 |
+
</p>
|
| 5 |
+
|
| 6 |
+
Read this in [English](./README_en.md)
|
| 7 |
+
|
| 8 |
+
GLM-4-Voice 是智谱 AI 推出的端到端语音模型。GLM-4-Voice 能够直接理解和生成中英文语音,进行实时语音对话,并且能够遵循用户的指令要求改变语音的情感、语调、语速、方言等属性。
|
| 9 |
+
|
| 10 |
+
## Model Architecture
|
| 11 |
+

|
| 12 |
+
|
| 13 |
+
GLM-4-Voice 由三个部分组成:
|
| 14 |
+
* GLM-4-Voice-Tokenizer: 通过在 [Whisper](https://github.com/openai/whisper) 的 Encoder 部分增加 Vector Quantization 并在 ASR 数据上有监督训练,将连续的语音输入转化为离散的 token。每秒音频平均只需要用 12.5 个离散 token 表示。
|
| 15 |
+
* GLM-4-Voice-Decoder: 基于 [CosyVoice](https://github.com/FunAudioLLM/CosyVoice) 的 Flow Matching 模型结构训练的支持流式推理的语音解码器,将离散化的语音 token 转化为连续的语音输出。最少只需要 10 个语音 token 即可开始生成,降低端到端对话延迟。
|
| 16 |
+
* GLM-4-Voice-9B: 在 [GLM-4-9B](https://github.com/THUDM/GLM-4) 的基础上进行语音模态的预训练和对齐,从而能够理解和生成离散化的语音 token。
|
| 17 |
+
|
| 18 |
+
预训练方面,为了攻克模型在语音模态下的智商和合成表现力两个难关,我们将 Speech2Speech 任务解耦合为“根据用户音频做出文本回复”和“根据文本回复和用户语音合成回复语音”两个任务,并设计两种预训练目标,分别基于文本预训练数据和无监督音频数据合成语音-文本交错数据以适配这两种任务形式。GLM-4-Voice-9B 在 GLM-4-9B 的基座模型基础之上,经过了数百万小时音频和数千亿 token 的音频文本交错数据预训练,拥有很强的音频理解和建模能力。
|
| 19 |
+
|
| 20 |
+
对齐方面,为了支持高质量的语音对话,我们设计了一套流式思考架构:根据用户语音,GLM-4-Voice 可以流式交替输出文本和语音两个模态的内容,其中语音模态以文本作为参照保证回复内容的高质量,并根据用户的语音指令要求做出相应的声音变化,在最大程度保留语言模型智商的情况下仍然具有端到端建模的能力,同时具备低延迟性,最低只需要输出 20 个 token 便可以合成语音。
|
| 21 |
+
|
| 22 |
+
## Model List
|
| 23 |
+
|
| 24 |
+
| Model | Type | Download |
|
| 25 |
+
|:---------------------:|:----------------:|:------------------------------------------------------------------------------------------------------------------------------------------------:|
|
| 26 |
+
| GLM-4-Voice-Tokenizer | Speech Tokenizer | [🤗 Huggingface](https://huggingface.co/THUDM/glm-4-voice-tokenizer) [🤖 ModelScope](https://modelscope.cn/models/ZhipuAI/glm-4-voice-tokenizer) |
|
| 27 |
+
| GLM-4-Voice-9B | Chat Model | [🤗 Huggingface](https://huggingface.co/THUDM/glm-4-voice-9b) [🤖 ModelScope](https://modelscope.cn/models/ZhipuAI/glm-4-voice-9b) |
|
| 28 |
+
| GLM-4-Voice-Decoder | Speech Decoder | [🤗 Huggingface](https://huggingface.co/THUDM/glm-4-voice-decoder) [🤖 ModelScope](https://modelscope.cn/models/ZhipuAI/glm-4-voice-decoder) |
|
| 29 |
+
|
| 30 |
+
## Usage
|
| 31 |
+
我们提供了可以直接启动的 Web Demo。用户可以输入语音或文本,模型会同时给出语音和文字回复。
|
| 32 |
+
|
| 33 |
+

|
| 34 |
+
|
| 35 |
+
### Preparation
|
| 36 |
+
|
| 37 |
+
首先下载仓库
|
| 38 |
+
```shell
|
| 39 |
+
git clone --recurse-submodules https://github.com/THUDM/GLM-4-Voice
|
| 40 |
+
cd GLM-4-Voice
|
| 41 |
+
```
|
| 42 |
+
然后安装依赖。也可以使用我们提供的镜像 `zhipuai/glm-4-voice:0.1` 以跳过这一步。
|
| 43 |
+
```shell
|
| 44 |
+
pip install -r requirements.txt
|
| 45 |
+
```
|
| 46 |
+
由于 Decoder 模型不支持通过 `transformers` 初始化,因此 checkpoint 需要单独下载。
|
| 47 |
+
|
| 48 |
+
```shell
|
| 49 |
+
# git 模型下载,请确保已安装 git-lfs
|
| 50 |
+
git lfs install
|
| 51 |
+
git clone https://huggingface.co/THUDM/glm-4-voice-decoder
|
| 52 |
+
```
|
| 53 |
+
|
| 54 |
+
### Launch Web Demo
|
| 55 |
+
|
| 56 |
+
1. 启动模型服务
|
| 57 |
+
|
| 58 |
+
```shell
|
| 59 |
+
python model_server.py --host localhost --model-path THUDM/glm-4-voice-9b --port 10000 --dtype bfloat16 --device cuda:0
|
| 60 |
+
```
|
| 61 |
+
|
| 62 |
+
如果你需要使用 Int4 精度启动,请运行
|
| 63 |
+
|
| 64 |
+
```shell
|
| 65 |
+
python model_server.py --host localhost --model-path THUDM/glm-4-voice-9b --port 10000 --dtype int4 --device cuda:0
|
| 66 |
+
```
|
| 67 |
+
|
| 68 |
+
此命令会自动下载 `glm-4-voice-9b`。如果网络条件不好,也手动下载之后通过 `--model-path` 指定本地的路径。
|
| 69 |
+
|
| 70 |
+
2. 启动 web 服务
|
| 71 |
+
|
| 72 |
+
```shell
|
| 73 |
+
python web_demo.py --tokenizer-path THUDM/glm-4-voice-tokenizer --model-path THUDM/glm-4-voice-9b --flow-path ./glm-4-voice-decoder
|
| 74 |
+
```
|
| 75 |
+
|
| 76 |
+
即可在 http://127.0.0.1:8888 访问 web demo。
|
| 77 |
+
|
| 78 |
+
此命令会自动下载 `glm-4-voice-tokenizer` 和 `glm-4-voice-9b`。 请注意,`glm-4-voice-decoder` 需要手动下载。
|
| 79 |
+
|
| 80 |
+
如果网络条件不好,可以手动下载这三个模型之后通过 `--tokenizer-path`, `--flow-path` 和 `--model-path` 指定本地的路径。
|
| 81 |
+
|
| 82 |
+
### Known Issues
|
| 83 |
+
|
| 84 |
+
* Gradio 的流式音频播放效果不稳定。在生成完成后点击对话框中的音频质量会更高。
|
| 85 |
+
|
| 86 |
+
## Cases
|
| 87 |
+
|
| 88 |
+
我们提供了 GLM-4-Voice 的部分对话案例,包括控制情绪、改变语速、生成方言等。
|
| 89 |
+
|
| 90 |
+
* 用轻柔的声音引导我放松
|
| 91 |
+
|
| 92 |
+
https://github.com/user-attachments/assets/4e3d9200-076d-4c28-a641-99df3af38eb0
|
| 93 |
+
|
| 94 |
+
* 用激动的声音解说足球比赛
|
| 95 |
+
|
| 96 |
+
https://github.com/user-attachments/assets/0163de2d-e876-4999-b1bc-bbfa364b799b
|
| 97 |
+
|
| 98 |
+
* 用哀怨的声音讲一个鬼故事
|
| 99 |
+
|
| 100 |
+
https://github.com/user-attachments/assets/a75b2087-d7bc-49fa-a0c5-e8c99935b39a
|
| 101 |
+
|
| 102 |
+
* 用东北话介绍一下冬天有多冷
|
| 103 |
+
|
| 104 |
+
https://github.com/user-attachments/assets/91ba54a1-8f5c-4cfe-8e87-16ed1ecf4037
|
| 105 |
+
|
| 106 |
+
* 用重庆话念“吃葡萄不吐葡萄皮”
|
| 107 |
+
|
| 108 |
+
https://github.com/user-attachments/assets/7eb72461-9e84-4d8e-9c58-1809cf6a8a9b
|
| 109 |
+
|
| 110 |
+
* 用北京话念一句绕口令
|
| 111 |
+
|
| 112 |
+
https://github.com/user-attachments/assets/a9bb223e-9c0a-440d-8537-0a7f16e31651
|
| 113 |
+
|
| 114 |
+
* 加快语速
|
| 115 |
+
|
| 116 |
+
https://github.com/user-attachments/assets/c98a4604-366b-4304-917f-3c850a82fe9f
|
| 117 |
+
|
| 118 |
+
* 再快一点
|
| 119 |
+
|
| 120 |
+
https://github.com/user-attachments/assets/d5ff0815-74f8-4738-b0f1-477cfc8dcc2d
|
| 121 |
+
|
| 122 |
+
## Acknowledgements
|
| 123 |
+
|
| 124 |
+
本项目的部分代码来自:
|
| 125 |
+
* [CosyVoice](https://github.com/FunAudioLLM/CosyVoice)
|
| 126 |
+
* [transformers](https://github.com/huggingface/transformers)
|
| 127 |
+
* [GLM-4](https://github.com/THUDM/GLM-4)
|
| 128 |
+
|
| 129 |
+
## 协议
|
| 130 |
+
|
| 131 |
+
+ GLM-4 模型的权重的使用则需要遵循 [模型协议](https://huggingface.co/THUDM/glm-4-voice-9b/blob/main/LICENSE)。
|
| 132 |
+
|
| 133 |
+
+ 本开源仓库的代码则遵循 [Apache 2.0](LICENSE) 协议。
|
| 134 |
+
|
| 135 |
+
## 引用
|
| 136 |
+
|
| 137 |
+
```
|
| 138 |
+
@misc{zeng2024glm4,
|
| 139 |
+
title={GLM-4-Voice: Towards Intelligent and Human-Like End-to-End Spoken Chatbot},
|
| 140 |
+
author={Aohan Zeng and Zhengxiao Du and Mingdao Liu and Kedong Wang and Shengmin Jiang and Lei Zhao and Yuxiao Dong and Jie Tang},
|
| 141 |
+
year={2024},
|
| 142 |
+
eprint={2412.02612},
|
| 143 |
+
archivePrefix={arXiv},
|
| 144 |
+
primaryClass={cs.CL},
|
| 145 |
+
url={https://arxiv.org/abs/2412.02612},
|
| 146 |
+
}
|
| 147 |
+
```
|
| 148 |
+
|
| 149 |
+
```
|
| 150 |
+
@misc{zeng2024scaling,
|
| 151 |
+
title={Scaling Speech-Text Pre-training with Synthetic Interleaved Data},
|
| 152 |
+
author={Aohan Zeng and Zhengxiao Du and Mingdao Liu and Lei Zhang and Shengmin Jiang and Yuxiao Dong and Jie Tang},
|
| 153 |
+
year={2024},
|
| 154 |
+
eprint={2411.17607},
|
| 155 |
+
archivePrefix={arXiv},
|
| 156 |
+
primaryClass={cs.CL},
|
| 157 |
+
url={https://arxiv.org/abs/2411.17607},
|
| 158 |
+
}
|
| 159 |
+
```
|
third_party/GLM-4-Voice/README_en.md
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# GLM-4-Voice
|
| 2 |
+
<p align="center">
|
| 3 |
+
📄<a href="https://arxiv.org/abs/2412.02612" target="_blank"> Report </a> • 🤗 <a href="https://huggingface.co/THUDM/glm-4-voice-9b" target="_blank">HF Repo</a> • 🤖 <a href="https://modelscope.cn/studios/ZhipuAI/GLM-4-Voice-Demo" target="_blank">Demo</a> • 🐦 <a href="https://twitter.com/thukeg" target="_blank">Twitter</a>
|
| 4 |
+
</p>
|
| 5 |
+
|
| 6 |
+
GLM-4-Voice is an end-to-end voice model launched by Zhipu AI. GLM-4-Voice can directly understand and generate Chinese and English speech, engage in real-time voice conversations, and change attributes such as emotion, intonation, speech rate, and dialect based on user instructions.
|
| 7 |
+
|
| 8 |
+
## Model Architecture
|
| 9 |
+
|
| 10 |
+

|
| 11 |
+
We provide the three components of GLM-4-Voice:
|
| 12 |
+
* GLM-4-Voice-Tokenizer: Trained by adding vector quantization to the encoder part of [Whisper](https://github.com/openai/whisper), converting continuous speech input into discrete tokens. Each second of audio is converted into 12.5 discrete tokens.
|
| 13 |
+
* GLM-4-Voice-9B: Pre-trained and aligned on speech modality based on [GLM-4-9B](https://github.com/THUDM/GLM-4), enabling understanding and generation of discretized speech.
|
| 14 |
+
* GLM-4-Voice-Decoder: A speech decoder supporting streaming inference, retrained based on [CosyVoice](https://github.com/FunAudioLLM/CosyVoice), converting discrete speech tokens into continuous speech output. Generation can start with as few as 10 audio tokens, reducing conversation latency.
|
| 15 |
+
|
| 16 |
+
## Model List
|
| 17 |
+
|
| 18 |
+
| Model | Type | Download |
|
| 19 |
+
|:---------------------:|:----------------:|:--------------------------------------------------------------------:|
|
| 20 |
+
| GLM-4-Voice-Tokenizer | Speech Tokenizer | [🤗 Huggingface](https://huggingface.co/THUDM/glm-4-voice-tokenizer) |
|
| 21 |
+
| GLM-4-Voice-9B | Chat Model | [🤗 Huggingface](https://huggingface.co/THUDM/glm-4-voice-9b) |
|
| 22 |
+
| GLM-4-Voice-Decoder | Speech Decoder | [🤗 Huggingface](https://huggingface.co/THUDM/glm-4-voice-decoder) |
|
| 23 |
+
|
| 24 |
+
## Usage
|
| 25 |
+
We provide a Web Demo that can be launched directly. Users can input speech or text, and the model will respond with both speech and text.
|
| 26 |
+
|
| 27 |
+

|
| 28 |
+
|
| 29 |
+
### Preparation
|
| 30 |
+
|
| 31 |
+
First, download the repository
|
| 32 |
+
```shell
|
| 33 |
+
git clone --recurse-submodules https://github.com/THUDM/GLM-4-Voice
|
| 34 |
+
cd GLM-4-Voice
|
| 35 |
+
```
|
| 36 |
+
Then, install the dependencies. You can also use our pre-built docker image `zhipuai/glm-4-voice:0.1` to skip the step.
|
| 37 |
+
```shell
|
| 38 |
+
pip install -r requirements.txt
|
| 39 |
+
```
|
| 40 |
+
Since the Decoder model does not support initialization via `transformers`, the checkpoint needs to be downloaded separately.
|
| 41 |
+
|
| 42 |
+
```shell
|
| 43 |
+
# Git model download, please ensure git-lfs is installed
|
| 44 |
+
git clone https://huggingface.co/THUDM/glm-4-voice-decoder
|
| 45 |
+
```
|
| 46 |
+
|
| 47 |
+
### Launch Web Demo
|
| 48 |
+
|
| 49 |
+
1. Start the model server
|
| 50 |
+
|
| 51 |
+
```shell
|
| 52 |
+
python model_server.py --host localhost --model-path THUDM/glm-4-voice-9b --port 10000 --dtype bfloat16 --device cuda:0
|
| 53 |
+
```
|
| 54 |
+
|
| 55 |
+
If you need to launch with Int4 precision, run
|
| 56 |
+
|
| 57 |
+
```shell
|
| 58 |
+
python model_server.py --host localhost --model-path THUDM/glm-4-voice-9b --port 10000 --dtype int4 --device cuda:0
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
This command will automatically download `glm-4-voice-9b`. If network conditions are poor, you can manually download it and specify the local path using `--model-path`.
|
| 62 |
+
|
| 63 |
+
2. Start the web service
|
| 64 |
+
|
| 65 |
+
```shell
|
| 66 |
+
python web_demo.py --tokenizer-path THUDM/glm-4-voice-tokenizer --model-path THUDM/glm-4-voice-9b --flow-path ./glm-4-voice-decoder
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
You can access the web demo at [http://127.0.0.1:8888](http://127.0.0.1:8888).
|
| 70 |
+
This command will automatically download `glm-4-voice-tokenizer` and `glm-4-voice-9b`. Please note that `glm-4-voice-decoder` needs to be downloaded manually.
|
| 71 |
+
If the network connection is poor, you can manually download these three models and specify the local paths using `--tokenizer-path`, `--flow-path`, and `--model-path`.
|
| 72 |
+
|
| 73 |
+
### Known Issues
|
| 74 |
+
* Gradio’s streaming audio playback can be unstable. The audio quality will be higher when clicking on the audio in the dialogue box after generation is complete.
|
| 75 |
+
|
| 76 |
+
## Examples
|
| 77 |
+
We provide some dialogue cases for GLM-4-Voice, including emotion control, speech rate alteration, dialect generation, etc. (The examples are in Chinese.)
|
| 78 |
+
|
| 79 |
+
* Use a gentle voice to guide me to relax
|
| 80 |
+
|
| 81 |
+
https://github.com/user-attachments/assets/4e3d9200-076d-4c28-a641-99df3af38eb0
|
| 82 |
+
|
| 83 |
+
* Use an excited voice to commentate a football match
|
| 84 |
+
|
| 85 |
+
https://github.com/user-attachments/assets/0163de2d-e876-4999-b1bc-bbfa364b799b
|
| 86 |
+
|
| 87 |
+
* Tell a ghost story with a mournful voice
|
| 88 |
+
|
| 89 |
+
https://github.com/user-attachments/assets/a75b2087-d7bc-49fa-a0c5-e8c99935b39a
|
| 90 |
+
|
| 91 |
+
* Introduce how cold winter is with a Northeastern dialect
|
| 92 |
+
|
| 93 |
+
https://github.com/user-attachments/assets/91ba54a1-8f5c-4cfe-8e87-16ed1ecf4037
|
| 94 |
+
|
| 95 |
+
* Say "Eat grapes without spitting out the skins" in Chongqing dialect
|
| 96 |
+
|
| 97 |
+
https://github.com/user-attachments/assets/7eb72461-9e84-4d8e-9c58-1809cf6a8a9b
|
| 98 |
+
|
| 99 |
+
* Recite a tongue twister with a Beijing accent
|
| 100 |
+
|
| 101 |
+
https://github.com/user-attachments/assets/a9bb223e-9c0a-440d-8537-0a7f16e31651
|
| 102 |
+
|
| 103 |
+
* Increase the speech rate
|
| 104 |
+
|
| 105 |
+
https://github.com/user-attachments/assets/c98a4604-366b-4304-917f-3c850a82fe9f
|
| 106 |
+
|
| 107 |
+
* Even faster
|
| 108 |
+
|
| 109 |
+
https://github.com/user-attachments/assets/d5ff0815-74f8-4738-b0f1-477cfc8dcc2d
|
| 110 |
+
|
| 111 |
+
## Acknowledgements
|
| 112 |
+
|
| 113 |
+
Some code in this project is from:
|
| 114 |
+
* [CosyVoice](https://github.com/FunAudioLLM/CosyVoice)
|
| 115 |
+
* [transformers](https://github.com/huggingface/transformers)
|
| 116 |
+
* [GLM-4](https://github.com/THUDM/GLM-4)
|
| 117 |
+
|
| 118 |
+
## License Agreement
|
| 119 |
+
|
| 120 |
+
+ The use of GLM-4 model weights must follow the [Model License Agreement](https://huggingface.co/THUDM/glm-4-voice-9b/blob/main/LICENSE).
|
| 121 |
+
|
| 122 |
+
+ The code in this open-source repository is licensed under the [Apache 2.0](LICENSE) License.
|
| 123 |
+
|
| 124 |
+
## Citation
|
| 125 |
+
|
| 126 |
+
```
|
| 127 |
+
@misc{zeng2024glm4,
|
| 128 |
+
title={GLM-4-Voice: Towards Intelligent and Human-Like End-to-End Spoken Chatbot},
|
| 129 |
+
author={Aohan Zeng and Zhengxiao Du and Mingdao Liu and Kedong Wang and Shengmin Jiang and Lei Zhao and Yuxiao Dong and Jie Tang},
|
| 130 |
+
year={2024},
|
| 131 |
+
eprint={2412.02612},
|
| 132 |
+
archivePrefix={arXiv},
|
| 133 |
+
primaryClass={cs.CL},
|
| 134 |
+
url={https://arxiv.org/abs/2412.02612},
|
| 135 |
+
}
|
| 136 |
+
```
|
| 137 |
+
|
| 138 |
+
```
|
| 139 |
+
@misc{zeng2024scaling,
|
| 140 |
+
title={Scaling Speech-Text Pre-training with Synthetic Interleaved Data},
|
| 141 |
+
author={Aohan Zeng and Zhengxiao Du and Mingdao Liu and Lei Zhang and Shengmin Jiang and Yuxiao Dong and Jie Tang},
|
| 142 |
+
year={2024},
|
| 143 |
+
eprint={2411.17607},
|
| 144 |
+
archivePrefix={arXiv},
|
| 145 |
+
primaryClass={cs.CL},
|
| 146 |
+
url={https://arxiv.org/abs/2411.17607},
|
| 147 |
+
}
|
| 148 |
+
```
|
third_party/GLM-4-Voice/audio_process.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import librosa
|
| 3 |
+
import soundfile as sf
|
| 4 |
+
import numpy as np
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
import io
|
| 7 |
+
|
| 8 |
+
# Split audio stream at silence points to prevent playback stuttering issues
|
| 9 |
+
# caused by AAC encoder frame padding when streaming audio through Gradio audio components.
|
| 10 |
+
class AudioStreamProcessor:
|
| 11 |
+
def __init__(self, sr=22050, min_silence_duration=0.1, threshold_db=-40):
|
| 12 |
+
self.sr = sr
|
| 13 |
+
self.min_silence_duration = min_silence_duration
|
| 14 |
+
self.threshold_db = threshold_db
|
| 15 |
+
self.buffer = np.array([])
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def process(self, audio_data, last=False):
|
| 19 |
+
"""
|
| 20 |
+
Add audio data and process it
|
| 21 |
+
params:
|
| 22 |
+
audio_data: audio data in numpy array
|
| 23 |
+
last: whether this is the last chunk of data
|
| 24 |
+
returns:
|
| 25 |
+
Processed audio data, returns None if no split point is found
|
| 26 |
+
"""
|
| 27 |
+
|
| 28 |
+
# Add new data to buffer
|
| 29 |
+
self.buffer = np.concatenate([self.buffer, audio_data]) if len(self.buffer) > 0 else audio_data
|
| 30 |
+
|
| 31 |
+
if last:
|
| 32 |
+
result = self.buffer
|
| 33 |
+
self.buffer = np.array([])
|
| 34 |
+
return self._to_wav_bytes(result)
|
| 35 |
+
|
| 36 |
+
# Find silence boundary
|
| 37 |
+
split_point = self._find_silence_boundary(self.buffer)
|
| 38 |
+
|
| 39 |
+
if split_point is not None:
|
| 40 |
+
# Modified: Extend split point to the end of silence
|
| 41 |
+
silence_end = self._find_silence_end(split_point)
|
| 42 |
+
result = self.buffer[:silence_end]
|
| 43 |
+
self.buffer = self.buffer[silence_end:]
|
| 44 |
+
return self._to_wav_bytes(result)
|
| 45 |
+
|
| 46 |
+
return None
|
| 47 |
+
|
| 48 |
+
def _find_silence_boundary(self, audio):
|
| 49 |
+
"""
|
| 50 |
+
Find the starting point of silence boundary in audio
|
| 51 |
+
"""
|
| 52 |
+
# Convert audio to decibels
|
| 53 |
+
db = librosa.amplitude_to_db(np.abs(audio), ref=np.max)
|
| 54 |
+
|
| 55 |
+
# Find points below threshold
|
| 56 |
+
silence_points = np.where(db < self.threshold_db)[0]
|
| 57 |
+
|
| 58 |
+
if len(silence_points) == 0:
|
| 59 |
+
return None
|
| 60 |
+
|
| 61 |
+
# Calculate minimum silence samples
|
| 62 |
+
min_silence_samples = int(self.min_silence_duration * self.sr)
|
| 63 |
+
|
| 64 |
+
# Search backwards for continuous silence segment starting point
|
| 65 |
+
for i in range(len(silence_points) - min_silence_samples, -1, -1):
|
| 66 |
+
if i < 0:
|
| 67 |
+
break
|
| 68 |
+
if np.all(np.diff(silence_points[i:i+min_silence_samples]) == 1):
|
| 69 |
+
return silence_points[i]
|
| 70 |
+
|
| 71 |
+
return None
|
| 72 |
+
|
| 73 |
+
def _find_silence_end(self, start_point):
|
| 74 |
+
"""
|
| 75 |
+
Find the end point of silence segment
|
| 76 |
+
"""
|
| 77 |
+
db = librosa.amplitude_to_db(np.abs(self.buffer[start_point:]), ref=np.max)
|
| 78 |
+
silence_points = np.where(db >= self.threshold_db)[0]
|
| 79 |
+
|
| 80 |
+
if len(silence_points) == 0:
|
| 81 |
+
return len(self.buffer)
|
| 82 |
+
|
| 83 |
+
return start_point + silence_points[0]
|
| 84 |
+
|
| 85 |
+
def _to_wav_bytes(self, audio_data):
|
| 86 |
+
"""
|
| 87 |
+
trans_to_wav_bytes
|
| 88 |
+
"""
|
| 89 |
+
wav_buffer = io.BytesIO()
|
| 90 |
+
sf.write(wav_buffer, audio_data, self.sr, format='WAV')
|
| 91 |
+
return wav_buffer.getvalue()
|
| 92 |
+
|
| 93 |
+
|
third_party/GLM-4-Voice/cosyvoice/__init__.py
ADDED
|
File without changes
|
third_party/GLM-4-Voice/cosyvoice/bin/inference.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu)
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
from __future__ import print_function
|
| 16 |
+
|
| 17 |
+
import argparse
|
| 18 |
+
import logging
|
| 19 |
+
logging.getLogger('matplotlib').setLevel(logging.WARNING)
|
| 20 |
+
import os
|
| 21 |
+
|
| 22 |
+
import torch
|
| 23 |
+
from torch.utils.data import DataLoader
|
| 24 |
+
import torchaudio
|
| 25 |
+
from hyperpyyaml import load_hyperpyyaml
|
| 26 |
+
from tqdm import tqdm
|
| 27 |
+
from cosyvoice.cli.model import CosyVoiceModel
|
| 28 |
+
|
| 29 |
+
from cosyvoice.dataset.dataset import Dataset
|
| 30 |
+
|
| 31 |
+
def get_args():
|
| 32 |
+
parser = argparse.ArgumentParser(description='inference with your model')
|
| 33 |
+
parser.add_argument('--config', required=True, help='config file')
|
| 34 |
+
parser.add_argument('--prompt_data', required=True, help='prompt data file')
|
| 35 |
+
parser.add_argument('--prompt_utt2data', required=True, help='prompt data file')
|
| 36 |
+
parser.add_argument('--tts_text', required=True, help='tts input file')
|
| 37 |
+
parser.add_argument('--llm_model', required=True, help='llm model file')
|
| 38 |
+
parser.add_argument('--flow_model', required=True, help='flow model file')
|
| 39 |
+
parser.add_argument('--hifigan_model', required=True, help='hifigan model file')
|
| 40 |
+
parser.add_argument('--gpu',
|
| 41 |
+
type=int,
|
| 42 |
+
default=-1,
|
| 43 |
+
help='gpu id for this rank, -1 for cpu')
|
| 44 |
+
parser.add_argument('--mode',
|
| 45 |
+
default='sft',
|
| 46 |
+
choices=['sft', 'zero_shot'],
|
| 47 |
+
help='inference mode')
|
| 48 |
+
parser.add_argument('--result_dir', required=True, help='asr result file')
|
| 49 |
+
args = parser.parse_args()
|
| 50 |
+
print(args)
|
| 51 |
+
return args
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def main():
|
| 55 |
+
args = get_args()
|
| 56 |
+
logging.basicConfig(level=logging.DEBUG,
|
| 57 |
+
format='%(asctime)s %(levelname)s %(message)s')
|
| 58 |
+
os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu)
|
| 59 |
+
|
| 60 |
+
# Init cosyvoice models from configs
|
| 61 |
+
use_cuda = args.gpu >= 0 and torch.cuda.is_available()
|
| 62 |
+
device = torch.device('cuda' if use_cuda else 'cpu')
|
| 63 |
+
with open(args.config, 'r') as f:
|
| 64 |
+
configs = load_hyperpyyaml(f)
|
| 65 |
+
|
| 66 |
+
model = CosyVoiceModel(configs['llm'], configs['flow'], configs['hift'])
|
| 67 |
+
model.load(args.llm_model, args.flow_model, args.hifigan_model)
|
| 68 |
+
|
| 69 |
+
test_dataset = Dataset(args.prompt_data, data_pipeline=configs['data_pipeline'], mode='inference', shuffle=False, partition=False, tts_file=args.tts_text, prompt_utt2data=args.prompt_utt2data)
|
| 70 |
+
test_data_loader = DataLoader(test_dataset, batch_size=None, num_workers=0)
|
| 71 |
+
|
| 72 |
+
del configs
|
| 73 |
+
os.makedirs(args.result_dir, exist_ok=True)
|
| 74 |
+
fn = os.path.join(args.result_dir, 'wav.scp')
|
| 75 |
+
f = open(fn, 'w')
|
| 76 |
+
with torch.no_grad():
|
| 77 |
+
for batch_idx, batch in tqdm(enumerate(test_data_loader)):
|
| 78 |
+
utts = batch["utts"]
|
| 79 |
+
assert len(utts) == 1, "inference mode only support batchsize 1"
|
| 80 |
+
text = batch["text"]
|
| 81 |
+
text_token = batch["text_token"].to(device)
|
| 82 |
+
text_token_len = batch["text_token_len"].to(device)
|
| 83 |
+
tts_text = batch["tts_text"]
|
| 84 |
+
tts_index = batch["tts_index"]
|
| 85 |
+
tts_text_token = batch["tts_text_token"].to(device)
|
| 86 |
+
tts_text_token_len = batch["tts_text_token_len"].to(device)
|
| 87 |
+
speech_token = batch["speech_token"].to(device)
|
| 88 |
+
speech_token_len = batch["speech_token_len"].to(device)
|
| 89 |
+
speech_feat = batch["speech_feat"].to(device)
|
| 90 |
+
speech_feat_len = batch["speech_feat_len"].to(device)
|
| 91 |
+
utt_embedding = batch["utt_embedding"].to(device)
|
| 92 |
+
spk_embedding = batch["spk_embedding"].to(device)
|
| 93 |
+
if args.mode == 'sft':
|
| 94 |
+
model_input = {'text': tts_text_token, 'text_len': tts_text_token_len,
|
| 95 |
+
'llm_embedding': spk_embedding, 'flow_embedding': spk_embedding}
|
| 96 |
+
else:
|
| 97 |
+
model_input = {'text': tts_text_token, 'text_len': tts_text_token_len,
|
| 98 |
+
'prompt_text': text_token, 'prompt_text_len': text_token_len,
|
| 99 |
+
'llm_prompt_speech_token': speech_token, 'llm_prompt_speech_token_len': speech_token_len,
|
| 100 |
+
'flow_prompt_speech_token': speech_token, 'flow_prompt_speech_token_len': speech_token_len,
|
| 101 |
+
'prompt_speech_feat': speech_feat, 'prompt_speech_feat_len': speech_feat_len,
|
| 102 |
+
'llm_embedding': utt_embedding, 'flow_embedding': utt_embedding}
|
| 103 |
+
model_output = model.inference(**model_input)
|
| 104 |
+
tts_key = '{}_{}'.format(utts[0], tts_index[0])
|
| 105 |
+
tts_fn = os.path.join(args.result_dir, '{}.wav'.format(tts_key))
|
| 106 |
+
torchaudio.save(tts_fn, model_output['tts_speech'], sample_rate=22050)
|
| 107 |
+
f.write('{} {}\n'.format(tts_key, tts_fn))
|
| 108 |
+
f.flush()
|
| 109 |
+
f.close()
|
| 110 |
+
logging.info('Result wav.scp saved in {}'.format(fn))
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
if __name__ == '__main__':
|
| 114 |
+
main()
|
third_party/GLM-4-Voice/cosyvoice/bin/train.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu)
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
from __future__ import print_function
|
| 16 |
+
import argparse
|
| 17 |
+
import datetime
|
| 18 |
+
import logging
|
| 19 |
+
logging.getLogger('matplotlib').setLevel(logging.WARNING)
|
| 20 |
+
from copy import deepcopy
|
| 21 |
+
import torch
|
| 22 |
+
import torch.distributed as dist
|
| 23 |
+
# import deepspeed
|
| 24 |
+
import pdb
|
| 25 |
+
from hyperpyyaml import load_hyperpyyaml
|
| 26 |
+
|
| 27 |
+
from torch.distributed.elastic.multiprocessing.errors import record
|
| 28 |
+
|
| 29 |
+
from cosyvoice.utils.executor import Executor
|
| 30 |
+
from cosyvoice.utils.train_utils import (
|
| 31 |
+
init_distributed,
|
| 32 |
+
init_dataset_and_dataloader,
|
| 33 |
+
init_optimizer_and_scheduler,
|
| 34 |
+
init_summarywriter, save_model,
|
| 35 |
+
wrap_cuda_model, check_modify_and_save_config)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def get_args():
|
| 39 |
+
parser = argparse.ArgumentParser(description='training your network')
|
| 40 |
+
parser.add_argument('--train_engine',
|
| 41 |
+
default='torch_ddp',
|
| 42 |
+
choices=['torch_ddp', 'deepspeed'],
|
| 43 |
+
help='Engine for paralleled training')
|
| 44 |
+
parser.add_argument('--model', required=True, help='model which will be trained')
|
| 45 |
+
parser.add_argument('--config', required=True, help='config file')
|
| 46 |
+
parser.add_argument('--train_data', required=True, help='train data file')
|
| 47 |
+
parser.add_argument('--cv_data', required=True, help='cv data file')
|
| 48 |
+
parser.add_argument('--checkpoint', help='checkpoint model')
|
| 49 |
+
parser.add_argument('--model_dir', required=True, help='save model dir')
|
| 50 |
+
parser.add_argument('--tensorboard_dir',
|
| 51 |
+
default='tensorboard',
|
| 52 |
+
help='tensorboard log dir')
|
| 53 |
+
parser.add_argument('--ddp.dist_backend',
|
| 54 |
+
dest='dist_backend',
|
| 55 |
+
default='nccl',
|
| 56 |
+
choices=['nccl', 'gloo'],
|
| 57 |
+
help='distributed backend')
|
| 58 |
+
parser.add_argument('--num_workers',
|
| 59 |
+
default=0,
|
| 60 |
+
type=int,
|
| 61 |
+
help='num of subprocess workers for reading')
|
| 62 |
+
parser.add_argument('--prefetch',
|
| 63 |
+
default=100,
|
| 64 |
+
type=int,
|
| 65 |
+
help='prefetch number')
|
| 66 |
+
parser.add_argument('--pin_memory',
|
| 67 |
+
action='store_true',
|
| 68 |
+
default=False,
|
| 69 |
+
help='Use pinned memory buffers used for reading')
|
| 70 |
+
parser.add_argument('--deepspeed.save_states',
|
| 71 |
+
dest='save_states',
|
| 72 |
+
default='model_only',
|
| 73 |
+
choices=['model_only', 'model+optimizer'],
|
| 74 |
+
help='save model/optimizer states')
|
| 75 |
+
parser.add_argument('--timeout',
|
| 76 |
+
default=30,
|
| 77 |
+
type=int,
|
| 78 |
+
help='timeout (in seconds) of cosyvoice_join.')
|
| 79 |
+
# parser = deepspeed.add_config_arguments(parser)
|
| 80 |
+
args = parser.parse_args()
|
| 81 |
+
return args
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
@record
|
| 85 |
+
def main():
|
| 86 |
+
args = get_args()
|
| 87 |
+
logging.basicConfig(level=logging.DEBUG,
|
| 88 |
+
format='%(asctime)s %(levelname)s %(message)s')
|
| 89 |
+
|
| 90 |
+
override_dict = {k: None for k in ['llm', 'flow', 'hift'] if k != args.model}
|
| 91 |
+
with open(args.config, 'r') as f:
|
| 92 |
+
configs = load_hyperpyyaml(f, overrides=override_dict)
|
| 93 |
+
configs['train_conf'].update(vars(args))
|
| 94 |
+
|
| 95 |
+
# Init env for ddp
|
| 96 |
+
init_distributed(args)
|
| 97 |
+
|
| 98 |
+
# Get dataset & dataloader
|
| 99 |
+
train_dataset, cv_dataset, train_data_loader, cv_data_loader = \
|
| 100 |
+
init_dataset_and_dataloader(args, configs)
|
| 101 |
+
|
| 102 |
+
# Do some sanity checks and save config to arsg.model_dir
|
| 103 |
+
configs = check_modify_and_save_config(args, configs)
|
| 104 |
+
|
| 105 |
+
# Tensorboard summary
|
| 106 |
+
writer = init_summarywriter(args)
|
| 107 |
+
|
| 108 |
+
# load checkpoint
|
| 109 |
+
model = configs[args.model]
|
| 110 |
+
if args.checkpoint is not None:
|
| 111 |
+
model.load_state_dict(torch.load(args.checkpoint, map_location='cpu'))
|
| 112 |
+
|
| 113 |
+
# Dispatch model from cpu to gpu
|
| 114 |
+
model = wrap_cuda_model(args, model)
|
| 115 |
+
|
| 116 |
+
# Get optimizer & scheduler
|
| 117 |
+
model, optimizer, scheduler = init_optimizer_and_scheduler(args, configs, model)
|
| 118 |
+
# pdb.set_trace()
|
| 119 |
+
# Save init checkpoints
|
| 120 |
+
info_dict = deepcopy(configs['train_conf'])
|
| 121 |
+
save_model(model, 'init', info_dict)
|
| 122 |
+
|
| 123 |
+
# Get executor
|
| 124 |
+
executor = Executor()
|
| 125 |
+
|
| 126 |
+
# Start training loop
|
| 127 |
+
for epoch in range(info_dict['max_epoch']):
|
| 128 |
+
executor.epoch = epoch
|
| 129 |
+
train_dataset.set_epoch(epoch)
|
| 130 |
+
dist.barrier()
|
| 131 |
+
# try:
|
| 132 |
+
# dist.barrier()
|
| 133 |
+
# except RuntimeError as e:
|
| 134 |
+
# logging.info('except RuntimeError as e: {}'.format(e))
|
| 135 |
+
group_join = dist.new_group(backend="gloo", timeout=datetime.timedelta(seconds=args.timeout))
|
| 136 |
+
executor.train_one_epoc(model, optimizer, scheduler, train_data_loader, cv_data_loader, writer, info_dict, group_join)
|
| 137 |
+
dist.destroy_process_group(group_join)
|
| 138 |
+
|
| 139 |
+
if __name__ == '__main__':
|
| 140 |
+
main()
|
third_party/GLM-4-Voice/cosyvoice/cli/__init__.py
ADDED
|
File without changes
|
third_party/GLM-4-Voice/cosyvoice/cli/cosyvoice.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu)
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
import os
|
| 15 |
+
import torch
|
| 16 |
+
from hyperpyyaml import load_hyperpyyaml
|
| 17 |
+
from modelscope import snapshot_download
|
| 18 |
+
from cosyvoice.cli.frontend import CosyVoiceFrontEnd
|
| 19 |
+
from cosyvoice.cli.model import CosyVoiceModel
|
| 20 |
+
|
| 21 |
+
class CosyVoice:
|
| 22 |
+
|
| 23 |
+
def __init__(self, model_dir):
|
| 24 |
+
instruct = True if '-Instruct' in model_dir else False
|
| 25 |
+
self.model_dir = model_dir
|
| 26 |
+
if not os.path.exists(model_dir):
|
| 27 |
+
model_dir = snapshot_download(model_dir)
|
| 28 |
+
with open('{}/cosyvoice.yaml'.format(model_dir), 'r') as f:
|
| 29 |
+
configs = load_hyperpyyaml(f)
|
| 30 |
+
self.frontend = CosyVoiceFrontEnd(configs['get_tokenizer'],
|
| 31 |
+
configs['feat_extractor'],
|
| 32 |
+
'{}/campplus.onnx'.format(model_dir),
|
| 33 |
+
'{}/speech_tokenizer_v1.onnx'.format(model_dir),
|
| 34 |
+
'{}/spk2info.pt'.format(model_dir),
|
| 35 |
+
instruct,
|
| 36 |
+
configs['allowed_special'])
|
| 37 |
+
self.model = CosyVoiceModel(configs['llm'], configs['flow'], configs['hift'])
|
| 38 |
+
self.model.load('{}/llm.pt'.format(model_dir),
|
| 39 |
+
'{}/flow.pt'.format(model_dir),
|
| 40 |
+
'{}/hift.pt'.format(model_dir))
|
| 41 |
+
del configs
|
| 42 |
+
|
| 43 |
+
def list_avaliable_spks(self):
|
| 44 |
+
spks = list(self.frontend.spk2info.keys())
|
| 45 |
+
return spks
|
| 46 |
+
|
| 47 |
+
def inference_sft(self, tts_text, spk_id):
|
| 48 |
+
tts_speeches = []
|
| 49 |
+
for i in self.frontend.text_normalize(tts_text, split=True):
|
| 50 |
+
model_input = self.frontend.frontend_sft(i, spk_id)
|
| 51 |
+
model_output = self.model.inference(**model_input)
|
| 52 |
+
tts_speeches.append(model_output['tts_speech'])
|
| 53 |
+
return {'tts_speech': torch.concat(tts_speeches, dim=1)}
|
| 54 |
+
|
| 55 |
+
def inference_zero_shot(self, tts_text, prompt_text, prompt_speech_16k):
|
| 56 |
+
prompt_text = self.frontend.text_normalize(prompt_text, split=False)
|
| 57 |
+
tts_speeches = []
|
| 58 |
+
for i in self.frontend.text_normalize(tts_text, split=True):
|
| 59 |
+
model_input = self.frontend.frontend_zero_shot(i, prompt_text, prompt_speech_16k)
|
| 60 |
+
model_output = self.model.inference(**model_input)
|
| 61 |
+
tts_speeches.append(model_output['tts_speech'])
|
| 62 |
+
return {'tts_speech': torch.concat(tts_speeches, dim=1)}
|
| 63 |
+
|
| 64 |
+
def inference_cross_lingual(self, tts_text, prompt_speech_16k):
|
| 65 |
+
if self.frontend.instruct is True:
|
| 66 |
+
raise ValueError('{} do not support cross_lingual inference'.format(self.model_dir))
|
| 67 |
+
tts_speeches = []
|
| 68 |
+
for i in self.frontend.text_normalize(tts_text, split=True):
|
| 69 |
+
model_input = self.frontend.frontend_cross_lingual(i, prompt_speech_16k)
|
| 70 |
+
model_output = self.model.inference(**model_input)
|
| 71 |
+
tts_speeches.append(model_output['tts_speech'])
|
| 72 |
+
return {'tts_speech': torch.concat(tts_speeches, dim=1)}
|
| 73 |
+
|
| 74 |
+
def inference_instruct(self, tts_text, spk_id, instruct_text):
|
| 75 |
+
if self.frontend.instruct is False:
|
| 76 |
+
raise ValueError('{} do not support instruct inference'.format(self.model_dir))
|
| 77 |
+
instruct_text = self.frontend.text_normalize(instruct_text, split=False)
|
| 78 |
+
tts_speeches = []
|
| 79 |
+
for i in self.frontend.text_normalize(tts_text, split=True):
|
| 80 |
+
model_input = self.frontend.frontend_instruct(i, spk_id, instruct_text)
|
| 81 |
+
model_output = self.model.inference(**model_input)
|
| 82 |
+
tts_speeches.append(model_output['tts_speech'])
|
| 83 |
+
return {'tts_speech': torch.concat(tts_speeches, dim=1)}
|
third_party/GLM-4-Voice/cosyvoice/cli/frontend.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu)
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
from functools import partial
|
| 15 |
+
import onnxruntime
|
| 16 |
+
import torch
|
| 17 |
+
import numpy as np
|
| 18 |
+
import whisper
|
| 19 |
+
from typing import Callable
|
| 20 |
+
import torchaudio.compliance.kaldi as kaldi
|
| 21 |
+
import torchaudio
|
| 22 |
+
import os
|
| 23 |
+
import re
|
| 24 |
+
import inflect
|
| 25 |
+
try:
|
| 26 |
+
import ttsfrd
|
| 27 |
+
use_ttsfrd = True
|
| 28 |
+
except ImportError:
|
| 29 |
+
print("failed to import ttsfrd, use WeTextProcessing instead")
|
| 30 |
+
from tn.chinese.normalizer import Normalizer as ZhNormalizer
|
| 31 |
+
from tn.english.normalizer import Normalizer as EnNormalizer
|
| 32 |
+
use_ttsfrd = False
|
| 33 |
+
from cosyvoice.utils.frontend_utils import contains_chinese, replace_blank, replace_corner_mark, remove_bracket, spell_out_number, split_paragraph
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class CosyVoiceFrontEnd:
|
| 37 |
+
|
| 38 |
+
def __init__(self,
|
| 39 |
+
get_tokenizer: Callable,
|
| 40 |
+
feat_extractor: Callable,
|
| 41 |
+
campplus_model: str,
|
| 42 |
+
speech_tokenizer_model: str,
|
| 43 |
+
spk2info: str = '',
|
| 44 |
+
instruct: bool = False,
|
| 45 |
+
allowed_special: str = 'all'):
|
| 46 |
+
self.tokenizer = get_tokenizer()
|
| 47 |
+
self.feat_extractor = feat_extractor
|
| 48 |
+
self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
| 49 |
+
option = onnxruntime.SessionOptions()
|
| 50 |
+
option.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
| 51 |
+
option.intra_op_num_threads = 1
|
| 52 |
+
self.campplus_session = onnxruntime.InferenceSession(campplus_model, sess_options=option, providers=["CPUExecutionProvider"])
|
| 53 |
+
self.speech_tokenizer_session = onnxruntime.InferenceSession(speech_tokenizer_model, sess_options=option, providers=["CUDAExecutionProvider"if torch.cuda.is_available() else "CPUExecutionProvider"])
|
| 54 |
+
if os.path.exists(spk2info):
|
| 55 |
+
self.spk2info = torch.load(spk2info, map_location=self.device)
|
| 56 |
+
self.instruct = instruct
|
| 57 |
+
self.allowed_special = allowed_special
|
| 58 |
+
self.inflect_parser = inflect.engine()
|
| 59 |
+
self.use_ttsfrd = use_ttsfrd
|
| 60 |
+
if self.use_ttsfrd:
|
| 61 |
+
self.frd = ttsfrd.TtsFrontendEngine()
|
| 62 |
+
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 63 |
+
assert self.frd.initialize('{}/../../pretrained_models/CosyVoice-ttsfrd/resource'.format(ROOT_DIR)) is True, 'failed to initialize ttsfrd resource'
|
| 64 |
+
self.frd.set_lang_type('pinyin')
|
| 65 |
+
self.frd.enable_pinyin_mix(True)
|
| 66 |
+
self.frd.set_breakmodel_index(1)
|
| 67 |
+
else:
|
| 68 |
+
self.zh_tn_model = ZhNormalizer(remove_erhua=False, full_to_half=False)
|
| 69 |
+
self.en_tn_model = EnNormalizer()
|
| 70 |
+
|
| 71 |
+
def _extract_text_token(self, text):
|
| 72 |
+
text_token = self.tokenizer.encode(text, allowed_special=self.allowed_special)
|
| 73 |
+
text_token = torch.tensor([text_token], dtype=torch.int32).to(self.device)
|
| 74 |
+
text_token_len = torch.tensor([text_token.shape[1]], dtype=torch.int32).to(self.device)
|
| 75 |
+
return text_token, text_token_len
|
| 76 |
+
|
| 77 |
+
def _extract_speech_token(self, speech):
|
| 78 |
+
feat = whisper.log_mel_spectrogram(speech, n_mels=128)
|
| 79 |
+
speech_token = self.speech_tokenizer_session.run(None, {self.speech_tokenizer_session.get_inputs()[0].name: feat.detach().cpu().numpy(),
|
| 80 |
+
self.speech_tokenizer_session.get_inputs()[1].name: np.array([feat.shape[2]], dtype=np.int32)})[0].flatten().tolist()
|
| 81 |
+
speech_token = torch.tensor([speech_token], dtype=torch.int32).to(self.device)
|
| 82 |
+
speech_token_len = torch.tensor([speech_token.shape[1]], dtype=torch.int32).to(self.device)
|
| 83 |
+
return speech_token, speech_token_len
|
| 84 |
+
|
| 85 |
+
def _extract_spk_embedding(self, speech):
|
| 86 |
+
feat = kaldi.fbank(speech,
|
| 87 |
+
num_mel_bins=80,
|
| 88 |
+
dither=0,
|
| 89 |
+
sample_frequency=16000)
|
| 90 |
+
feat = feat - feat.mean(dim=0, keepdim=True)
|
| 91 |
+
embedding = self.campplus_session.run(None, {self.campplus_session.get_inputs()[0].name: feat.unsqueeze(dim=0).cpu().numpy()})[0].flatten().tolist()
|
| 92 |
+
embedding = torch.tensor([embedding]).to(self.device)
|
| 93 |
+
return embedding
|
| 94 |
+
|
| 95 |
+
def _extract_speech_feat(self, speech):
|
| 96 |
+
speech_feat = self.feat_extractor(speech).squeeze(dim=0).transpose(0, 1).to(self.device)
|
| 97 |
+
speech_feat = speech_feat.unsqueeze(dim=0)
|
| 98 |
+
speech_feat_len = torch.tensor([speech_feat.shape[1]], dtype=torch.int32).to(self.device)
|
| 99 |
+
return speech_feat, speech_feat_len
|
| 100 |
+
|
| 101 |
+
def text_normalize(self, text, split=True):
|
| 102 |
+
text = text.strip()
|
| 103 |
+
if contains_chinese(text):
|
| 104 |
+
if self.use_ttsfrd:
|
| 105 |
+
text = self.frd.get_frd_extra_info(text, 'input')
|
| 106 |
+
else:
|
| 107 |
+
text = self.zh_tn_model.normalize(text)
|
| 108 |
+
text = text.replace("\n", "")
|
| 109 |
+
text = replace_blank(text)
|
| 110 |
+
text = replace_corner_mark(text)
|
| 111 |
+
text = text.replace(".", "、")
|
| 112 |
+
text = text.replace(" - ", ",")
|
| 113 |
+
text = remove_bracket(text)
|
| 114 |
+
text = re.sub(r'[,,]+$', '。', text)
|
| 115 |
+
texts = [i for i in split_paragraph(text, partial(self.tokenizer.encode, allowed_special=self.allowed_special), "zh", token_max_n=80,
|
| 116 |
+
token_min_n=60, merge_len=20,
|
| 117 |
+
comma_split=False)]
|
| 118 |
+
else:
|
| 119 |
+
if self.use_ttsfrd:
|
| 120 |
+
text = self.frd.get_frd_extra_info(text, 'input')
|
| 121 |
+
else:
|
| 122 |
+
text = self.en_tn_model.normalize(text)
|
| 123 |
+
text = spell_out_number(text, self.inflect_parser)
|
| 124 |
+
texts = [i for i in split_paragraph(text, partial(self.tokenizer.encode, allowed_special=self.allowed_special), "en", token_max_n=80,
|
| 125 |
+
token_min_n=60, merge_len=20,
|
| 126 |
+
comma_split=False)]
|
| 127 |
+
if split is False:
|
| 128 |
+
return text
|
| 129 |
+
return texts
|
| 130 |
+
|
| 131 |
+
def frontend_sft(self, tts_text, spk_id):
|
| 132 |
+
tts_text_token, tts_text_token_len = self._extract_text_token(tts_text)
|
| 133 |
+
embedding = self.spk2info[spk_id]['embedding']
|
| 134 |
+
model_input = {'text': tts_text_token, 'text_len': tts_text_token_len, 'llm_embedding': embedding, 'flow_embedding': embedding}
|
| 135 |
+
return model_input
|
| 136 |
+
|
| 137 |
+
def frontend_zero_shot(self, tts_text, prompt_text, prompt_speech_16k):
|
| 138 |
+
tts_text_token, tts_text_token_len = self._extract_text_token(tts_text)
|
| 139 |
+
prompt_text_token, prompt_text_token_len = self._extract_text_token(prompt_text)
|
| 140 |
+
prompt_speech_22050 = torchaudio.transforms.Resample(orig_freq=16000, new_freq=22050)(prompt_speech_16k)
|
| 141 |
+
speech_feat, speech_feat_len = self._extract_speech_feat(prompt_speech_22050)
|
| 142 |
+
speech_token, speech_token_len = self._extract_speech_token(prompt_speech_16k)
|
| 143 |
+
embedding = self._extract_spk_embedding(prompt_speech_16k)
|
| 144 |
+
model_input = {'text': tts_text_token, 'text_len': tts_text_token_len,
|
| 145 |
+
'prompt_text': prompt_text_token, 'prompt_text_len': prompt_text_token_len,
|
| 146 |
+
'llm_prompt_speech_token': speech_token, 'llm_prompt_speech_token_len': speech_token_len,
|
| 147 |
+
'flow_prompt_speech_token': speech_token, 'flow_prompt_speech_token_len': speech_token_len,
|
| 148 |
+
'prompt_speech_feat': speech_feat, 'prompt_speech_feat_len': speech_feat_len,
|
| 149 |
+
'llm_embedding': embedding, 'flow_embedding': embedding}
|
| 150 |
+
return model_input
|
| 151 |
+
|
| 152 |
+
def frontend_cross_lingual(self, tts_text, prompt_speech_16k):
|
| 153 |
+
model_input = self.frontend_zero_shot(tts_text, '', prompt_speech_16k)
|
| 154 |
+
# in cross lingual mode, we remove prompt in llm
|
| 155 |
+
del model_input['prompt_text']
|
| 156 |
+
del model_input['prompt_text_len']
|
| 157 |
+
del model_input['llm_prompt_speech_token']
|
| 158 |
+
del model_input['llm_prompt_speech_token_len']
|
| 159 |
+
return model_input
|
| 160 |
+
|
| 161 |
+
def frontend_instruct(self, tts_text, spk_id, instruct_text):
|
| 162 |
+
model_input = self.frontend_sft(tts_text, spk_id)
|
| 163 |
+
# in instruct mode, we remove spk_embedding in llm due to information leakage
|
| 164 |
+
del model_input['llm_embedding']
|
| 165 |
+
instruct_text_token, instruct_text_token_len = self._extract_text_token(instruct_text + '<endofprompt>')
|
| 166 |
+
model_input['prompt_text'] = instruct_text_token
|
| 167 |
+
model_input['prompt_text_len'] = instruct_text_token_len
|
| 168 |
+
return model_input
|