"""UEmo-D1 — unified multimodal emotion recognition (ZeroGPU demo). Reference implementation ported 1:1 from the authors' `inference.py` shipped in the model repo `wudq/UEmo-D1` (same preprocessing, same attention hook, same calibrated softmax), with the file-path I/O replaced by in-memory inputs. """ import spaces # noqa: F401 -- MUST precede torch / transformers / CUDA imports import time import av import gradio as gr import numpy as np import soxr import torch from PIL import Image from transformers import AutoModel, Qwen2_5OmniProcessor from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize MODEL_ID = "wudq/UEmo-D1" SYSTEM = "You are a helpful assistant. Classify the target using the supplied media and text." LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" EMOTIONS = "happy, sad, angry, surprised, disgusted, fearful, neutral" SENTIMENTS = "positive, negative, neutral" DEFAULT_INSTRUCTION = "Classify the emotion expressed by the target." # --------------------------------------------------------------------------- # # Model — loaded at module scope, moved to the GPU eagerly (ZeroGPU pattern). # --------------------------------------------------------------------------- # processor = Qwen2_5OmniProcessor.from_pretrained(MODEL_ID) processor.tokenizer.padding_side = "left" model, loading_info = AutoModel.from_pretrained( MODEL_ID, trust_remote_code=True, torch_dtype=torch.bfloat16, attn_implementation="flash_attention_2", output_loading_info=True, ) if loading_info["missing_keys"] or loading_info["unexpected_keys"] or loading_info["mismatched_keys"]: raise RuntimeError(f"Incomplete checkpoint: {loading_info}") model.eval() model.requires_grad_(False) model.to("cuda") print(f"[boot] UEmo-D1 ready on {model.device}, temperature={model.config.uemo_temperature}", flush=True) # --------------------------------------------------------------------------- # # Media helpers — identical to the reference `read_audio` / `read_video`. # --------------------------------------------------------------------------- # def read_audio_file(path) -> np.ndarray: import soundfile as sf a, sr = sf.read(str(path), dtype="float32", always_2d=True) a = a.mean(axis=1) if sr != 16000: a = soxr.resample(a, sr, 16000) if not len(a): raise ValueError("Empty audio") return np.asarray(a, dtype=np.float32) def read_video_frames(path) -> np.ndarray: """0.5 s frame sampling at 224x224-max resolution, exactly like the authors' code.""" c = av.open(str(path)) try: st = c.streams.video[0] h, w = smart_resize(st.height, st.width, factor=28, min_pixels=56 * 56, max_pixels=224 * 224) frames, target, start = [], 0.0, None for f in c.decode(st): t = float(f.pts * f.time_base) if f.pts is not None else (float(start or 0) + len(frames) * 0.5) if start is None: start = t if t - start + 1e-6 >= target: frames.append(f.reformat(width=w, height=h, format="rgb24").to_ndarray()) target += 0.5 finally: c.close() if not frames: raise ValueError("Empty video") if len(frames) == 1: frames.append(frames[-1]) return np.stack(frames).transpose(0, 3, 1, 2) def to_audio(x): """Normalise whatever the Audio component hands us into 16 kHz mono float32.""" if x is None: return None if isinstance(x, tuple): # (sample_rate, ndarray) for type="numpy" sr, arr = x arr = np.asarray(arr, dtype=np.float32) if arr.ndim > 1: arr = arr.mean(axis=1) if sr and int(sr) != 16000: arr = soxr.resample(arr, int(sr), 16000) return np.ascontiguousarray(arr, dtype=np.float32) return read_audio_file(x) def to_video(x): if x is None: return None if isinstance(x, dict): x = x.get("video") or x.get("path") return read_video_frames(x) # --------------------------------------------------------------------------- # # Prompt assembly — a direct port of the reference `prepare()`. # --------------------------------------------------------------------------- # def prepare(instruction, opts, target_text, context_text, image, audio, video): audios, images, videos, content = [], [], [], [] groups = [] if context_text and context_text.strip(): groups.append(("context", context_text, None, None, None)) groups.append(("target", target_text, image, audio, video)) for kind, txt, im, au, vi in groups: content.append({"type": "text", "text": "Preceding context:" if kind == "context" else "Target to classify:"}) if txt and txt.strip(): content.append({"type": "text", "text": txt}) if im is not None: images.append(Image.open(im).convert("RGB") if isinstance(im, str) else im.convert("RGB")) content.append({"type": "image", "image": "provided image"}) if vi is not None: videos.append(vi) # The authors' pipeline requires audio aligned with every video. Our # example clips carry no audio track, so we supply matched silence. audios.append(au if au is not None else np.zeros(16000, dtype=np.float32)) content.append({"type": "video", "video": "provided audiovisual clip"}) elif au is not None: audios.append(au) content.append({"type": "audio", "audio": "provided audio"}) prompt = ( instruction + "\nChoose your answer from:\n" + "\n".join(f"{LETTERS[i]}. {v}" for i, v in enumerate(opts)) + "\nAnswer with only the single option letter, without explanation." ) content.append({"type": "text", "text": prompt}) conversation = [ {"role": "system", "content": [{"type": "text", "text": SYSTEM}]}, {"role": "user", "content": content}, ] return { "text": processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False), "audio": audios, "images": images, "videos": videos, } def batch_prepare(prepared): aa = [a for p in prepared for a in p["audio"]] ii = [a for p in prepared for a in p["images"]] vv = [a for p in prepared for a in p["videos"]] return processor( text=[p["text"] for p in prepared], audio=aa or None, images=ii or None, videos=vv or None, return_tensors="pt", padding=True, use_audio_in_video=True, images_kwargs={"min_pixels": 56 * 56, "max_pixels": 448 * 448}, videos_kwargs={"fps": 2.0, "do_sample_frames": False, "min_pixels": 56 * 56, "max_pixels": 224 * 224}, audio_kwargs={"max_length": max(len(a) for a in aa) if aa else 16000, "truncation": False}, ) # --------------------------------------------------------------------------- # # Inference # --------------------------------------------------------------------------- # @spaces.GPU(duration=20) def decide(instruction, choices, target_text="", image=None, audio=None, video=None, context_text=""): """Classify one target (text, image, audio or video+audio) into the supplied options. Returns the chosen option and the model's calibrated probability distribution over all options. `choices` is a comma-separated list of 1..26 alternatives. Args: instruction: The classification question, e.g. "Classify the emotion expressed by the target." choices: Comma-separated candidate answers, e.g. "happy, sad, angry, neutral". target_text: Text of the target to classify (optional if another modality is given). image: Image of the target to classify (optional). audio: Audio of the target to classify (optional). video: Video of the target to classify (optional). context_text: Optional preceding context shown to the model before the target. """ opts = [c.strip() for c in (choices or "").split(",") if c.strip()] if not 1 <= len(opts) <= 26 or len(set(opts)) != len(opts): raise gr.Error("Provide 1..26 unique, non-empty comma-separated options.") if not (instruction or "").strip(): raise gr.Error("Provide a non-empty instruction.") im = Image.open(image).convert("RGB") if isinstance(image, str) else image au = to_audio(audio) vi = to_video(video) if im is None and au is None and vi is None and not (target_text or "").strip(): raise gr.Error("Provide a target: text, image, audio or video.") t0 = time.time() sample = prepare(instruction, opts, target_text or "", context_text or "", im, au, vi) inputs = batch_prepare([sample]).to(model.device).to(torch.bfloat16) tokens = [processor.tokenizer.encode(c, add_special_tokens=False) for c in LETTERS[: len(opts)]] if any(len(x) != 1 for x in tokens): raise gr.Error("Each option letter must be a single token.") model.rope_deltas = None with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16): logits = model( **inputs, use_audio_in_video=True, use_cache=False, mer_prefix_tokens=inputs["input_ids"].shape[-1], ).logits[0, -1].float() scores = logits[[x[0] for x in tokens]] probabilities = (scores / model.config.uemo_temperature).softmax(-1) if not torch.isfinite(probabilities).all(): raise gr.Error("Non-finite decision probabilities.") index = int(scores.argmax()) label = f"{LETTERS[index]}. {opts[index]}" distribution = {f"{LETTERS[i]}. {opts[i]}": float(p) for i, p in enumerate(probabilities.cpu().tolist())} print(f"[decide] opts={len(opts)} tokens={inputs['input_ids'].shape[-1]} elapsed={time.time() - t0:.2f}s", flush=True) return label, distribution def _core(instruction, choices, target_text="", image=None, audio=None, video=None, context_text=""): return decide(instruction, choices, target_text, image, audio, video, context_text) # Wrappers whose signatures match each gr.Examples block exactly (no None cells). def fn_text(instruction, choices, target_text): return _core(instruction, choices, target_text=target_text) def fn_context(instruction, choices, target_text, context_text): return _core(instruction, choices, target_text=target_text, context_text=context_text) def fn_image(instruction, choices, image): return _core(instruction, choices, image=image) def fn_audio(instruction, choices, audio): return _core(instruction, choices, audio=audio) def fn_video(instruction, choices, video): return _core(instruction, choices, video=video) # --------------------------------------------------------------------------- # # UI # --------------------------------------------------------------------------- # CSS = """ #col-container { max-width: 1150px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } """ with gr.Blocks(title="UEmo-D1 — Multimodal Emotion Recognition") as demo: gr.Markdown( "# 🎭 UEmo-D1 — Unified Multimodal Emotion Recognition\n" "**UEmo-D1** is a calibrated decision model built on `Qwen/Qwen2.5-Omni-3B`. Give it an " "instruction, a list of candidate answers and a **target** — text, image, audio, or " "video+audio — and it returns the chosen option plus its calibrated probability " "distribution over *all* options (a bidirectional-prefix / causal-answer attention pass " "over the option letters)." ) with gr.Row(): with gr.Column(scale=3): instruction = gr.Textbox( label="Instruction", value=DEFAULT_INSTRUCTION, lines=2, info="What should the model decide about the target?", ) choices = gr.Textbox( label="Candidate answers (comma-separated, 1–26)", value=EMOTIONS, lines=1, info="Each becomes one option letter A, B, C …", ) with gr.Column(scale=4): with gr.Tabs(): with gr.Tab("📝 Text"): target_text = gr.Textbox(label="Target text", lines=4, placeholder="Type the text to classify…") with gr.Tab("🖼️ Image"): image = gr.Image(label="Target image", type="filepath") with gr.Tab("🔊 Audio"): audio = gr.Audio(label="Target audio", type="numpy") with gr.Tab("🎬 Video"): video = gr.Video(label="Target video") gr.Markdown( "*Combine modalities freely — e.g. an image **and** text in one call. " "Video examples carry no audio track, so matched silence is supplied.*", elem_classes="note", ) with gr.Accordion("⚙️ Advanced: preceding context", open=False): context_text = gr.Textbox( label="Preceding context (optional)", lines=3, placeholder="Optional context shown to the model before the target…", ) run_btn = gr.Button("Decide", variant="primary", size="lg") with gr.Row(): answer = gr.Textbox(label="Decision", interactive=False, lines=1) distribution = gr.Label(label="Calibrated distribution", num_top_classes=26) run_btn.click( decide, inputs=[instruction, choices, target_text, image, audio, video, context_text], outputs=[answer, distribution], api_name="decide", ) gr.Markdown("### Examples — click one to run it") with gr.Accordion("📝 Text", open=True): gr.Examples( examples=[ [DEFAULT_INSTRUCTION, EMOTIONS, "I can't believe you remembered my birthday — this is the happiest day of my life!"], ["Classify the sentiment of the target text.", SENTIMENTS, "The soup was cold and the waiter was rude to us."], ], inputs=[instruction, choices, target_text], outputs=[answer, distribution], fn=fn_text, example_labels=["text · emotion", "text · sentiment"], cache_examples=True, cache_mode="lazy", ) with gr.Accordion("🧩 Text + preceding context", open=False): gr.Examples( examples=[ ["Classify the emotion of the target reply, given the preceding context.", EMOTIONS, "Fine. Do whatever you want.", "I told him I had been offered a job in another city."], ], inputs=[instruction, choices, target_text, context_text], outputs=[answer, distribution], fn=fn_context, example_labels=["reply emotion · with context"], cache_examples=True, cache_mode="lazy", ) with gr.Accordion("🖼️ Image", open=False): gr.Examples( examples=[ [DEFAULT_INSTRUCTION, EMOTIONS, "examples/woman_sad.jpg"], [DEFAULT_INSTRUCTION, EMOTIONS, "examples/girl_with_dog.jpg"], ], inputs=[instruction, choices, image], outputs=[answer, distribution], fn=fn_image, example_labels=["woman", "girl with dog"], cache_examples=True, cache_mode="lazy", ) with gr.Accordion("🔊 Audio", open=False): gr.Examples( examples=[ ["Classify the emotion expressed by the speaker in the target audio.", EMOTIONS, "examples/amused.wav"], ["Classify the emotion expressed by the speaker in the target audio.", EMOTIONS, "examples/angry.wav"], ["Classify the emotion expressed by the speaker in the target audio.", EMOTIONS, "examples/surprised.wav"], ], inputs=[instruction, choices, audio], outputs=[answer, distribution], fn=fn_audio, example_labels=["amused (CC0)", "angry (CC0)", "surprised (CC0)"], cache_examples=True, cache_mode="lazy", ) with gr.Accordion("🎬 Video", open=False): gr.Examples( examples=[ ["Classify the emotion expressed by the people in the target video.", EMOTIONS, "examples/2_women_arguing.mp4"], ["Classify the emotion expressed by the person in the target video.", EMOTIONS, "examples/elderly_woman_laughing.mp4"], ], inputs=[instruction, choices, video], outputs=[answer, distribution], fn=fn_video, example_labels=["two women arguing", "elderly woman laughing"], cache_examples=True, cache_mode="lazy", ) gr.Markdown( "Weights: [UEmo-D1](https://huggingface.co/wudq/UEmo-D1) · Code: " "[wdqqdw/UEmo-D1](https://github.com/wdqqdw/UEmo-D1) · Base: Qwen2.5-Omni-3B " "(Qwen Research License). Audio examples: Thorsten-Voice TV-44kHz-Full " "(CC0-1.0). Image/video examples: linoyts example assets (CC0-1.0)." ) demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)