diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..022d99b2a3019e8ac486582766b1129528ea3034 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +asr_0.wav filter=lfs diff=lfs merge=lfs -text +s2i_0.wav filter=lfs diff=lfs merge=lfs -text +svqa_0.jpg filter=lfs diff=lfs merge=lfs -text +third_party/GLM-4-Voice/resources/architecture.jpeg filter=lfs diff=lfs merge=lfs -text +third_party/GLM-4-Voice/resources/web_demo.png filter=lfs diff=lfs merge=lfs -text +vqa_0.png filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md index b4bba362fa2d0e66db2ef3fc801ebd6564d633fa..7731763179bc6e4d6aed4ecb3c1f1a9c9f1de0f9 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,19 @@ --- -title: Omni Diffusion -emoji: 🏆 -colorFrom: pink -colorTo: yellow +title: Omni-Diffusion +emoji: 🌐 +colorFrom: yellow +colorTo: purple sdk: gradio -sdk_version: 6.20.0 -python_version: '3.12' +sdk_version: 6.15.1 app_file: app.py -pinned: false +short_description: Any-to-any multimodal model with discrete diffusion +python_version: "3.10" +startup_duration_timeout: 1h --- -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +# Omni-Diffusion + +Interactive demo for **Omni-Diffusion**, a unified any-to-any multimodal model based on masked discrete diffusion. +Supports text-to-image, speech-to-image, text-to-speech, ASR, visual question answering, and spoken VQA. + +Paper: [Omni-Diffusion: Unified Multimodal Understanding and Generation with Masked Discrete Diffusion](https://arxiv.org/abs/2603.06577) \ No newline at end of file diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..e597aad498d918ba2e27df99cd2cc5b28927a908 --- /dev/null +++ b/app.py @@ -0,0 +1,532 @@ +import os +os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") +import spaces +import torch +import gradio as gr +import numpy as np +import re +import sys +import random +import tempfile +import torchaudio +import cv2 +from transformers import AutoConfig, AutoModel, AutoTokenizer +from transformers.generation import GenerationConfig + +# Add third_party paths for GLM-4-Voice +_HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(_HERE, "third_party", "GLM-4-Voice")) +sys.path.insert(0, os.path.join(_HERE, "third_party", "GLM-4-Voice", "cosyvoice")) +sys.path.insert(0, os.path.join(_HERE, "third_party", "GLM-4-Voice", "third_party", "Matcha-TTS")) + +from omni_diffusion.data.processor.audio_processor import add_audio_input_contiguous +from omni_diffusion.tokenizer import get_audio_tokenizer +from omni_diffusion.data.processor.image_processor import ImageProcessor + +MODEL_ID = "lijiang/Omni-Diffusion" +AUDIO_TOKENIZER_PATH = "THUDM/glm-4-voice-tokenizer" +AUDIO_DECODER_PATH = "THUDM/glm-4-voice-decoder" +IMAGE_TOKENIZER_PATH = "showlab/magvitv2" +AUDIO_TOKENIZER_TYPE = "sensevoice_glm4voice" + +# Load chat template from file to avoid escaping issues +_TEMPLATE_PATH = os.path.join(_HERE, "chat_template.txt") +with open(_TEMPLATE_PATH, "r") as _f: + qwen2_chat_template = _f.read() + +def set_seed(seed=42): + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + +set_seed() + +print("Loading model...", flush=True) + +config = AutoConfig.from_pretrained(MODEL_ID, trust_remote_code=True) + +tokenizer = AutoTokenizer.from_pretrained( + MODEL_ID, trust_remote_code=True, chat_template=qwen2_chat_template +) + +model = AutoModel.from_pretrained( + MODEL_ID, + trust_remote_code=True, + torch_dtype=torch.bfloat16, + attn_implementation="sdpa", +).to("cuda").eval() + +model.generation_config = GenerationConfig.from_pretrained(MODEL_ID, trust_remote_code=True) +model.generation_config.max_new_tokens = 8192 +model.generation_config.chat_format = "chatml" +model.generation_config.max_window_size = 8192 +model.generation_config.use_cache = True +model.generation_config.do_sample = False +model.generation_config.temperature = 1.0 +model.generation_config.top_k = 50 +model.generation_config.top_p = 1.0 +model.generation_config.num_beams = 1 +model.generation_config.pad_token_id = tokenizer.pad_token_id + +print("Loading audio tokenizer...", flush=True) +audio_tokenizer = get_audio_tokenizer( + AUDIO_TOKENIZER_PATH, + AUDIO_TOKENIZER_TYPE, + flow_path=AUDIO_DECODER_PATH, + rank=0, +) + +print("Loading image processor...", flush=True) +image_processor = ImageProcessor( + IMAGE_TOKENIZER_PATH, + "dynamic", + image_size=512, + normalize_type="imagenet", + min_patch_grid=1, + max_patch_grid=12, +) +image_processor.image_tokenizer.rank = 0 +image_processor.load_model() + +print("All models loaded successfully!", flush=True) + + +def find_audio_segments_regex(text): + pattern = re.compile(r"<\|begin_of_audio\|>(.*?)<\|end_of_audio\|>", re.DOTALL) + segments = pattern.findall(text) + return [segment.strip() for segment in segments] + + +def run_inference( + task, + text_prompt="", + audio_path=None, + image_path=None, + max_tokens=128, + steps=128, + alg="entropy", + cfg=0.0, + max_position_penalty=1.0, + repeat_penalty=1.0, + add_boa_token=0, + output_text_only=False, +): + """Run Omni-Diffusion inference for a given task. + + Args: + task: One of T2I, S2I, TTS, ASR, VQA, SVQA. + text_prompt: Text input for the task. + audio_path: Path to input audio file (for ASR, S2I, SVQA). + image_path: Path to input image file (for VQA, SVQA). + max_tokens: Maximum number of new tokens to generate. + steps: Number of diffusion steps. + alg: Algorithm for generation ("entropy", "entropy-penalty", "origin"). + cfg: Classifier-free guidance scale. + max_position_penalty: Position penalty for generation. + repeat_penalty: Repeat penalty for generation. + add_boa_token: Whether to add beginning-of-audio token. + output_text_only: Whether to output text only. + + Returns: + Tuple of (text_output, audio_output, image_output). + """ + system_message = [] + + if task == "S2I": + system_message = [{"role": "system", "content": "Please generate an image based on the input audio."}] + elif task == "SQA": + system_message = [{"role": "system", "content": "Please response the input audio."}] + elif task == "SVQA": + system_message = [{"role": "system", "content": "Please response the input audio based on the given image."}] + + if audio_path is not None and image_path is not None: + messages = system_message + [{"role": "user", "content": "<|audio|>\n<|image|>"}] + elif audio_path is not None: + messages = system_message + [{"role": "user", "content": text_prompt + "\n<|audio|>"}] + elif image_path is not None: + messages = system_message + [{"role": "user", "content": text_prompt + "\n<|image|>"}] + else: + messages = system_message + [{"role": "user", "content": text_prompt}] + + if audio_path is not None and audio_tokenizer.apply_to_role("user", is_discrete=True): + audio_tokens = audio_tokenizer.encode(audio_path) + audio_tokens_str = "".join(f"<|audio_{i}|>" for i in audio_tokens) + messages[-1]["content"] = messages[-1]["content"].replace("<|audio|>", audio_tokens_str) + + if image_path is not None: + image_tokens = image_processor.process_images_with_subpatch(image_path, 512) + image_tokens = image_processor.get_image_token(image_tokens) + image_tokens = image_tokens[0].tolist() + image_tokens_str = "".join(f"<|image_{i}|>" for i in image_tokens) + messages[-1]["content"] = messages[-1]["content"].replace("<|image|>", image_tokens_str) + + input_ids = tokenizer.apply_chat_template( + messages, tokenize=True, add_generation_prompt=True + ) + + audios = None + audio_indices = None + if audio_path is not None and audio_tokenizer.apply_to_role("user", is_contiguous=True): + audio_paths = [audio_path] + input_ids, audios, audio_indices = add_audio_input_contiguous( + input_ids, audio_paths, tokenizer, audio_tokenizer + ) + + input_ids_tensor = torch.tensor([input_ids], dtype=torch.long).to("cuda") + + with torch.no_grad(): + outputs, histories = model.generate( + input_ids_tensor, + audios=audios, + audio_indices=audio_indices, + temperature=0.0, + top_p=0.9, + steps=steps, + max_new_tokens=max_tokens, + alg=alg, + cfg=cfg, + tokenizer=tokenizer, + add_boa_token=add_boa_token, + max_position_penalty=max_position_penalty, + repeat_penalty=repeat_penalty, + output_text_only=output_text_only, + task=task, + ) + + audio_offset = tokenizer.convert_tokens_to_ids("<|audio_0|>") + image_offset = tokenizer.convert_tokens_to_ids("<|image_0|>") + audio_tokens_out = [] + image_tokens_out = [] + text_tokens_out = [] + + for token_id in outputs[0][input_ids_tensor.shape[1]:]: + tid = token_id.item() + if tid >= audio_offset and tid < audio_offset + 16384: + audio_tokens_out.append(tid - audio_offset) + elif tid >= image_offset: + image_tokens_out.append(tid - image_offset) + else: + text_tokens_out.append(tid) + + # Decode audio + tts_speech = None + if len(audio_tokens_out) > 0: + tts_speech = audio_tokenizer.decode(audio_tokens_out, source_speech_16k=None) + + # Decode image + image_out = None + if len(image_tokens_out) < 256 and len(image_tokens_out) > 0: + image_tokens_out += [image_tokens_out[-1]] * (256 - len(image_tokens_out)) + + if len(image_tokens_out) > 0: + gen_token_ids = torch.stack([torch.tensor(t) for t in image_tokens_out], dim=0).unsqueeze(0) + gen_token_ids = torch.clamp(gen_token_ids, max=8192 - 1, min=0) + with torch.no_grad(): + image_out = image_processor.image_tokenizer.image_tokenizer.decode_code( + gen_token_ids[:, :256] + ) + image_out = torch.clamp((image_out + 1.0) / 2.0, min=0.0, max=1.0) + image_out *= 255.0 + image_out = image_out.permute(0, 2, 3, 1).cpu().numpy().astype(np.uint8) + image_out = image_out[:, :, :, [2, 1, 0]][0] + + # Decode text + text_out = tokenizer.decode(text_tokens_out, skip_special_tokens=True) + + return text_out, tts_speech, image_out + + +@spaces.GPU(duration=300) +def generate_t2i(prompt, steps=260, alg="entropy-penalty", repeat_penalty=1.2): + """Generate an image from a text prompt. + + Args: + prompt: Text description of the image to generate. + steps: Number of diffusion denoising steps. + alg: Generation algorithm. + repeat_penalty: Penalty for repeated tokens. + """ + message = "Generate an image based on the provided text description.\n" + prompt + text_out, audio_out, image_out = run_inference( + task="T2I", + text_prompt=message, + max_tokens=260, + steps=steps, + alg=alg, + repeat_penalty=repeat_penalty, + max_position_penalty=2.0, + ) + return image_out, text_out + + +@spaces.GPU(duration=300) +def generate_tts(text, steps=25, alg="entropy", repeat_penalty=1.0): + """Convert text to speech. + + Args: + text: Text to synthesize. + steps: Number of diffusion steps. + alg: Generation algorithm. + repeat_penalty: Penalty for repeated tokens. + """ + message = "Convert the text to speech.\n" + text + text_out, audio_out, image_out = run_inference( + task="TTS", + text_prompt=message, + max_tokens=50, + steps=steps, + alg=alg, + repeat_penalty=repeat_penalty, + ) + if audio_out is not None: + audio_path = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name + torchaudio.save(audio_path, audio_out.unsqueeze(0), 22050, format="wav") + return audio_path, text_out + return None, text_out + + +@spaces.GPU(duration=300) +def generate_asr(audio_file, steps=25, alg="entropy", repeat_penalty=1.0): + """Convert speech to text (ASR). + + Args: + audio_file: Input audio file path. + steps: Number of diffusion steps. + alg: Generation algorithm. + repeat_penalty: Penalty for repeated tokens. + """ + text_out, audio_out, image_out = run_inference( + task="ASR", + text_prompt="Convert the speech to text.", + audio_path=audio_file, + max_tokens=50, + steps=steps, + alg=alg, + repeat_penalty=repeat_penalty, + ) + return text_out + + +@spaces.GPU(duration=300) +def generate_vqa(image_file, question, steps=64, alg="entropy", repeat_penalty=1.0): + """Answer a question about an image (Visual QA). + + Args: + image_file: Input image file path. + question: Question about the image. + steps: Number of diffusion steps. + alg: Generation algorithm. + repeat_penalty: Penalty for repeated tokens. + """ + text_out, audio_out, image_out = run_inference( + task="VQA", + text_prompt=question, + image_path=image_file, + max_tokens=64, + steps=steps, + alg=alg, + repeat_penalty=repeat_penalty, + ) + return text_out + + +@spaces.GPU(duration=300) +def generate_s2i(audio_file, steps=260, alg="entropy-penalty", repeat_penalty=1.5, cfg=2.0): + """Generate an image from an audio input (Speech-to-Image). + + Args: + audio_file: Input audio file path. + steps: Number of diffusion steps. + alg: Generation algorithm. + repeat_penalty: Penalty for repeated tokens. + cfg: Classifier-free guidance scale. + """ + text_out, audio_out, image_out = run_inference( + task="S2I", + audio_path=audio_file, + max_tokens=260, + steps=steps, + alg=alg, + repeat_penalty=repeat_penalty, + max_position_penalty=2.0, + cfg=cfg, + ) + return image_out, text_out + + +@spaces.GPU(duration=300) +def generate_svqa(image_file, audio_file, steps=128, alg="entropy-penalty", repeat_penalty=1.2): + """Answer a spoken question about an image (Spoken Visual QA). + + Args: + image_file: Input image file path. + audio_file: Input audio (spoken question) file path. + steps: Number of diffusion steps. + alg: Generation algorithm. + repeat_penalty: Penalty for repeated tokens. + """ + text_out, audio_out, image_out = run_inference( + task="SVQA", + audio_path=audio_file, + image_path=image_file, + add_boa_token=1, + max_tokens=128, + steps=steps, + alg=alg, + repeat_penalty=repeat_penalty, + ) + result_audio = None + if audio_out is not None: + result_audio = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name + torchaudio.save(result_audio, audio_out.unsqueeze(0), 22050, format="wav") + return text_out, result_audio + + +CSS = """ +#col-container { max-width: 1100px; margin: 0 auto; } +.dark .gradio-container { color: var(--body-text-color); } +""" + +with gr.Blocks(theme=gr.themes.Citrus(), css=CSS) as demo: + gr.Markdown("# Omni-Diffusion: Unified Multimodal Understanding and Generation") + gr.Markdown("Interactive demo for any-to-any multimodal generation with masked discrete diffusion. Supports text-to-image, speech-to-image, TTS, ASR, visual QA, and spoken visual QA.") + gr.Markdown("[Paper](https://arxiv.org/abs/2603.06577) | [Model](https://huggingface.co/lijiang/Omni-Diffusion) | [GitHub](https://github.com/VITA-MLLM/Omni-Diffusion)") + + with gr.Tabs(): + with gr.Tab("Text-to-Image"): + with gr.Row(): + with gr.Column(scale=1): + t2i_prompt = gr.Textbox(label="Prompt", placeholder="Describe the image to generate...", lines=3) + t2i_btn = gr.Button("Generate Image", variant="primary") + with gr.Accordion("Advanced", open=False): + t2i_steps = gr.Slider(50, 512, value=260, step=10, label="Steps") + t2i_alg = gr.Dropdown(["entropy-penalty", "entropy", "origin"], value="entropy-penalty", label="Algorithm") + t2i_rp = gr.Slider(0.5, 3.0, value=1.2, step=0.1, label="Repeat Penalty") + with gr.Column(scale=1): + t2i_image = gr.Image(label="Generated Image", type="numpy") + t2i_text = gr.Textbox(label="Output Text", lines=2) + gr.Examples( + examples=[["The image shows a landscape background with double exposure glasses of wine, displaying a hyperealistic and detailed view of the subject."]], + inputs=[t2i_prompt], + outputs=[t2i_image, t2i_text], + fn=generate_t2i, + cache_examples=False, + run_on_click=True, + ) + t2i_btn.click(generate_t2i, inputs=[t2i_prompt, t2i_steps, t2i_alg, t2i_rp], outputs=[t2i_image, t2i_text]) + + with gr.Tab("Text-to-Speech"): + with gr.Row(): + with gr.Column(scale=1): + tts_text = gr.Textbox(label="Text", placeholder="Text to synthesize...", lines=3) + tts_btn = gr.Button("Generate Speech", variant="primary") + with gr.Accordion("Advanced", open=False): + tts_steps = gr.Slider(10, 100, value=25, step=5, label="Steps") + tts_alg = gr.Dropdown(["entropy", "entropy-penalty", "origin"], value="entropy", label="Algorithm") + tts_rp = gr.Slider(0.5, 3.0, value=1.0, step=0.1, label="Repeat Penalty") + with gr.Column(scale=1): + tts_audio = gr.Audio(label="Generated Audio") + tts_text_out = gr.Textbox(label="Output Text", lines=2) + gr.Examples( + examples=[["Get the trust fund to the bank early."], ["It is a nice walk though, with trees and a river and stuff."]], + inputs=[tts_text], + outputs=[tts_audio, tts_text_out], + fn=generate_tts, + cache_examples=False, + run_on_click=True, + ) + tts_btn.click(generate_tts, inputs=[tts_text, tts_steps, tts_alg, tts_rp], outputs=[tts_audio, tts_text_out]) + + with gr.Tab("Speech-to-Text (ASR)"): + with gr.Row(): + with gr.Column(scale=1): + asr_audio = gr.Audio(label="Input Audio", type="filepath") + asr_btn = gr.Button("Transcribe", variant="primary") + with gr.Accordion("Advanced", open=False): + asr_steps = gr.Slider(10, 100, value=25, step=5, label="Steps") + asr_alg = gr.Dropdown(["entropy", "entropy-penalty", "origin"], value="entropy", label="Algorithm") + asr_rp = gr.Slider(0.5, 3.0, value=1.0, step=0.1, label="Repeat Penalty") + with gr.Column(scale=1): + asr_text = gr.Textbox(label="Transcription", lines=5) + gr.Examples( + examples=[["asr_0.wav"]], + inputs=[asr_audio], + outputs=[asr_text], + fn=generate_asr, + cache_examples=False, + run_on_click=True, + ) + asr_btn.click(generate_asr, inputs=[asr_audio, asr_steps, asr_alg, asr_rp], outputs=[asr_text]) + + with gr.Tab("Visual QA"): + with gr.Row(): + with gr.Column(scale=1): + vqa_image = gr.Image(label="Input Image", type="filepath") + vqa_question = gr.Textbox(label="Question", placeholder="Ask about the image...", lines=2) + vqa_btn = gr.Button("Answer", variant="primary") + with gr.Accordion("Advanced", open=False): + vqa_steps = gr.Slider(20, 200, value=64, step=10, label="Steps") + vqa_alg = gr.Dropdown(["entropy", "entropy-penalty", "origin"], value="entropy", label="Algorithm") + vqa_rp = gr.Slider(0.5, 3.0, value=1.0, step=0.1, label="Repeat Penalty") + with gr.Column(scale=1): + vqa_answer = gr.Textbox(label="Answer", lines=5) + gr.Examples( + examples=[["vqa_0.png", "Is the glass of orange juice half empty or half full?"]], + inputs=[vqa_image, vqa_question], + outputs=[vqa_answer], + fn=generate_vqa, + cache_examples=False, + run_on_click=True, + ) + vqa_btn.click(generate_vqa, inputs=[vqa_image, vqa_question, vqa_steps, vqa_alg, vqa_rp], outputs=[vqa_answer]) + + with gr.Tab("Speech-to-Image"): + with gr.Row(): + with gr.Column(scale=1): + s2i_audio = gr.Audio(label="Input Audio", type="filepath") + s2i_btn = gr.Button("Generate Image", variant="primary") + with gr.Accordion("Advanced", open=False): + s2i_steps = gr.Slider(50, 512, value=260, step=10, label="Steps") + s2i_alg = gr.Dropdown(["entropy-penalty", "entropy", "origin"], value="entropy-penalty", label="Algorithm") + s2i_rp = gr.Slider(0.5, 3.0, value=1.5, step=0.1, label="Repeat Penalty") + s2i_cfg = gr.Slider(0.0, 5.0, value=2.0, step=0.5, label="CFG Scale") + with gr.Column(scale=1): + s2i_image = gr.Image(label="Generated Image", type="numpy") + s2i_text = gr.Textbox(label="Output Text", lines=2) + gr.Examples( + examples=[["s2i_0.wav"]], + inputs=[s2i_audio], + outputs=[s2i_image, s2i_text], + fn=generate_s2i, + cache_examples=False, + run_on_click=True, + ) + s2i_btn.click(generate_s2i, inputs=[s2i_audio, s2i_steps, s2i_alg, s2i_rp, s2i_cfg], outputs=[s2i_image, s2i_text]) + + with gr.Tab("Spoken Visual QA"): + with gr.Row(): + with gr.Column(scale=1): + svqa_image = gr.Image(label="Input Image", type="filepath") + svqa_audio = gr.Audio(label="Input Audio (spoken question)", type="filepath") + svqa_btn = gr.Button("Answer", variant="primary") + with gr.Accordion("Advanced", open=False): + svqa_steps = gr.Slider(20, 300, value=128, step=10, label="Steps") + svqa_alg = gr.Dropdown(["entropy-penalty", "entropy", "origin"], value="entropy-penalty", label="Algorithm") + svqa_rp = gr.Slider(0.5, 3.0, value=1.2, step=0.1, label="Repeat Penalty") + with gr.Column(scale=1): + svqa_text = gr.Textbox(label="Answer (Text)", lines=3) + svqa_audio_out = gr.Audio(label="Answer (Audio)") + gr.Examples( + examples=[["svqa_0.jpg", "svqa_0.wav"]], + inputs=[svqa_image, svqa_audio], + outputs=[svqa_text, svqa_audio_out], + fn=generate_svqa, + cache_examples=False, + run_on_click=True, + ) + svqa_btn.click(generate_svqa, inputs=[svqa_image, svqa_audio, svqa_steps, svqa_alg, svqa_rp], outputs=[svqa_text, svqa_audio_out]) + +demo.launch(mcp_server=True) diff --git a/asr_0.wav b/asr_0.wav new file mode 100644 index 0000000000000000000000000000000000000000..544e9373f0e217aded51d68d992cc92f67a9ae2f --- /dev/null +++ b/asr_0.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cc5488d43dadd87e8fa93f8c2a418b6771b7a9201c87ef39fff7b10e5650803 +size 1108218 diff --git a/chat_template.txt b/chat_template.txt new file mode 100644 index 0000000000000000000000000000000000000000..62eaa1ebacbf9bf9ab2d51b4e7d23e0d3daa697a --- /dev/null +++ b/chat_template.txt @@ -0,0 +1,2 @@ + +{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0]['role'] == 'system' %}\n {{- messages[0]['content'] }}\n {%- endif %}\n {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role }}\n {%- if message.content %}\n {{- '\\n' + message.content }}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- '}\\n' }}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- message.content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n diff --git a/omni_diffusion/__init__.py b/omni_diffusion/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..20e620005d746005fb0eccee967f07f03ab4f046 --- /dev/null +++ b/omni_diffusion/__init__.py @@ -0,0 +1,2 @@ +# from .models import * +from .data import * diff --git a/omni_diffusion/constants.py b/omni_diffusion/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..a9130aa2941ff34b066ba464dfb43368b02f7e81 --- /dev/null +++ b/omni_diffusion/constants.py @@ -0,0 +1,105 @@ +import logging + +logger = logging.getLogger(__name__) + + +if False: + IMG_TAG_TOKEN = "" + IMG_CONTEXT_TOKEN = "" + IMG_START_TOKEN = "" + IMG_END_TOKEN = "" + + VID_TAG_TOKEN = "