Spaces:
Running on Zero
Running on Zero
Download app.py from hugging-apps/uemo-d1: direct link, hf CLI and curl.
- Browser
- Download file 17.1 kB
-
https://huggingface.co/spaces/hugging-apps/uemo-d1/resolve/main/app.py
- Command line
-
hf download hf://spaces/hugging-apps/uemo-d1/app.py
-
curl -L -o app.py https://huggingface.co/spaces/hugging-apps/uemo-d1/resolve/main/app.py
17.1 kB
| """UEmo-D1 — unified multimodal emotion recognition (ZeroGPU demo). | |
| Reference implementation ported 1:1 from the authors' `inference.py` shipped in | |
| the model repo `wudq/UEmo-D1` (same preprocessing, same attention hook, same | |
| calibrated softmax), with the file-path I/O replaced by in-memory inputs. | |
| """ | |
| import spaces # noqa: F401 -- MUST precede torch / transformers / CUDA imports | |
| import time | |
| import av | |
| import gradio as gr | |
| import numpy as np | |
| import soxr | |
| import torch | |
| from PIL import Image | |
| from transformers import AutoModel, Qwen2_5OmniProcessor | |
| from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize | |
| MODEL_ID = "wudq/UEmo-D1" | |
| SYSTEM = "You are a helpful assistant. Classify the target using the supplied media and text." | |
| LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" | |
| EMOTIONS = "happy, sad, angry, surprised, disgusted, fearful, neutral" | |
| SENTIMENTS = "positive, negative, neutral" | |
| DEFAULT_INSTRUCTION = "Classify the emotion expressed by the target." | |
| # --------------------------------------------------------------------------- # | |
| # Model — loaded at module scope, moved to the GPU eagerly (ZeroGPU pattern). | |
| # --------------------------------------------------------------------------- # | |
| processor = Qwen2_5OmniProcessor.from_pretrained(MODEL_ID) | |
| processor.tokenizer.padding_side = "left" | |
| model, loading_info = AutoModel.from_pretrained( | |
| MODEL_ID, | |
| trust_remote_code=True, | |
| torch_dtype=torch.bfloat16, | |
| attn_implementation="flash_attention_2", | |
| output_loading_info=True, | |
| ) | |
| if loading_info["missing_keys"] or loading_info["unexpected_keys"] or loading_info["mismatched_keys"]: | |
| raise RuntimeError(f"Incomplete checkpoint: {loading_info}") | |
| model.eval() | |
| model.requires_grad_(False) | |
| model.to("cuda") | |
| print(f"[boot] UEmo-D1 ready on {model.device}, temperature={model.config.uemo_temperature}", flush=True) | |
| # --------------------------------------------------------------------------- # | |
| # Media helpers — identical to the reference `read_audio` / `read_video`. | |
| # --------------------------------------------------------------------------- # | |
| def read_audio_file(path) -> np.ndarray: | |
| import soundfile as sf | |
| a, sr = sf.read(str(path), dtype="float32", always_2d=True) | |
| a = a.mean(axis=1) | |
| if sr != 16000: | |
| a = soxr.resample(a, sr, 16000) | |
| if not len(a): | |
| raise ValueError("Empty audio") | |
| return np.asarray(a, dtype=np.float32) | |
| def read_video_frames(path) -> np.ndarray: | |
| """0.5 s frame sampling at 224x224-max resolution, exactly like the authors' code.""" | |
| c = av.open(str(path)) | |
| try: | |
| st = c.streams.video[0] | |
| h, w = smart_resize(st.height, st.width, factor=28, min_pixels=56 * 56, max_pixels=224 * 224) | |
| frames, target, start = [], 0.0, None | |
| for f in c.decode(st): | |
| t = float(f.pts * f.time_base) if f.pts is not None else (float(start or 0) + len(frames) * 0.5) | |
| if start is None: | |
| start = t | |
| if t - start + 1e-6 >= target: | |
| frames.append(f.reformat(width=w, height=h, format="rgb24").to_ndarray()) | |
| target += 0.5 | |
| finally: | |
| c.close() | |
| if not frames: | |
| raise ValueError("Empty video") | |
| if len(frames) == 1: | |
| frames.append(frames[-1]) | |
| return np.stack(frames).transpose(0, 3, 1, 2) | |
| def to_audio(x): | |
| """Normalise whatever the Audio component hands us into 16 kHz mono float32.""" | |
| if x is None: | |
| return None | |
| if isinstance(x, tuple): # (sample_rate, ndarray) for type="numpy" | |
| sr, arr = x | |
| arr = np.asarray(arr, dtype=np.float32) | |
| if arr.ndim > 1: | |
| arr = arr.mean(axis=1) | |
| if sr and int(sr) != 16000: | |
| arr = soxr.resample(arr, int(sr), 16000) | |
| return np.ascontiguousarray(arr, dtype=np.float32) | |
| return read_audio_file(x) | |
| def to_video(x): | |
| if x is None: | |
| return None | |
| if isinstance(x, dict): | |
| x = x.get("video") or x.get("path") | |
| return read_video_frames(x) | |
| # --------------------------------------------------------------------------- # | |
| # Prompt assembly — a direct port of the reference `prepare()`. | |
| # --------------------------------------------------------------------------- # | |
| def prepare(instruction, opts, target_text, context_text, image, audio, video): | |
| audios, images, videos, content = [], [], [], [] | |
| groups = [] | |
| if context_text and context_text.strip(): | |
| groups.append(("context", context_text, None, None, None)) | |
| groups.append(("target", target_text, image, audio, video)) | |
| for kind, txt, im, au, vi in groups: | |
| content.append({"type": "text", "text": "Preceding context:" if kind == "context" else "Target to classify:"}) | |
| if txt and txt.strip(): | |
| content.append({"type": "text", "text": txt}) | |
| if im is not None: | |
| images.append(Image.open(im).convert("RGB") if isinstance(im, str) else im.convert("RGB")) | |
| content.append({"type": "image", "image": "provided image"}) | |
| if vi is not None: | |
| videos.append(vi) | |
| # The authors' pipeline requires audio aligned with every video. Our | |
| # example clips carry no audio track, so we supply matched silence. | |
| audios.append(au if au is not None else np.zeros(16000, dtype=np.float32)) | |
| content.append({"type": "video", "video": "provided audiovisual clip"}) | |
| elif au is not None: | |
| audios.append(au) | |
| content.append({"type": "audio", "audio": "provided audio"}) | |
| prompt = ( | |
| instruction | |
| + "\nChoose your answer from:\n" | |
| + "\n".join(f"{LETTERS[i]}. {v}" for i, v in enumerate(opts)) | |
| + "\nAnswer with only the single option letter, without explanation." | |
| ) | |
| content.append({"type": "text", "text": prompt}) | |
| conversation = [ | |
| {"role": "system", "content": [{"type": "text", "text": SYSTEM}]}, | |
| {"role": "user", "content": content}, | |
| ] | |
| return { | |
| "text": processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False), | |
| "audio": audios, | |
| "images": images, | |
| "videos": videos, | |
| } | |
| def batch_prepare(prepared): | |
| aa = [a for p in prepared for a in p["audio"]] | |
| ii = [a for p in prepared for a in p["images"]] | |
| vv = [a for p in prepared for a in p["videos"]] | |
| return processor( | |
| text=[p["text"] for p in prepared], | |
| audio=aa or None, | |
| images=ii or None, | |
| videos=vv or None, | |
| return_tensors="pt", | |
| padding=True, | |
| use_audio_in_video=True, | |
| images_kwargs={"min_pixels": 56 * 56, "max_pixels": 448 * 448}, | |
| videos_kwargs={"fps": 2.0, "do_sample_frames": False, "min_pixels": 56 * 56, "max_pixels": 224 * 224}, | |
| audio_kwargs={"max_length": max(len(a) for a in aa) if aa else 16000, "truncation": False}, | |
| ) | |
| # --------------------------------------------------------------------------- # | |
| # Inference | |
| # --------------------------------------------------------------------------- # | |
| def decide(instruction, choices, target_text="", image=None, audio=None, video=None, context_text=""): | |
| """Classify one target (text, image, audio or video+audio) into the supplied options. | |
| Returns the chosen option and the model's calibrated probability distribution | |
| over all options. `choices` is a comma-separated list of 1..26 alternatives. | |
| Args: | |
| instruction: The classification question, e.g. "Classify the emotion expressed by the target." | |
| choices: Comma-separated candidate answers, e.g. "happy, sad, angry, neutral". | |
| target_text: Text of the target to classify (optional if another modality is given). | |
| image: Image of the target to classify (optional). | |
| audio: Audio of the target to classify (optional). | |
| video: Video of the target to classify (optional). | |
| context_text: Optional preceding context shown to the model before the target. | |
| """ | |
| opts = [c.strip() for c in (choices or "").split(",") if c.strip()] | |
| if not 1 <= len(opts) <= 26 or len(set(opts)) != len(opts): | |
| raise gr.Error("Provide 1..26 unique, non-empty comma-separated options.") | |
| if not (instruction or "").strip(): | |
| raise gr.Error("Provide a non-empty instruction.") | |
| im = Image.open(image).convert("RGB") if isinstance(image, str) else image | |
| au = to_audio(audio) | |
| vi = to_video(video) | |
| if im is None and au is None and vi is None and not (target_text or "").strip(): | |
| raise gr.Error("Provide a target: text, image, audio or video.") | |
| t0 = time.time() | |
| sample = prepare(instruction, opts, target_text or "", context_text or "", im, au, vi) | |
| inputs = batch_prepare([sample]).to(model.device).to(torch.bfloat16) | |
| tokens = [processor.tokenizer.encode(c, add_special_tokens=False) for c in LETTERS[: len(opts)]] | |
| if any(len(x) != 1 for x in tokens): | |
| raise gr.Error("Each option letter must be a single token.") | |
| model.rope_deltas = None | |
| with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16): | |
| logits = model( | |
| **inputs, | |
| use_audio_in_video=True, | |
| use_cache=False, | |
| mer_prefix_tokens=inputs["input_ids"].shape[-1], | |
| ).logits[0, -1].float() | |
| scores = logits[[x[0] for x in tokens]] | |
| probabilities = (scores / model.config.uemo_temperature).softmax(-1) | |
| if not torch.isfinite(probabilities).all(): | |
| raise gr.Error("Non-finite decision probabilities.") | |
| index = int(scores.argmax()) | |
| label = f"{LETTERS[index]}. {opts[index]}" | |
| distribution = {f"{LETTERS[i]}. {opts[i]}": float(p) for i, p in enumerate(probabilities.cpu().tolist())} | |
| print(f"[decide] opts={len(opts)} tokens={inputs['input_ids'].shape[-1]} elapsed={time.time() - t0:.2f}s", flush=True) | |
| return label, distribution | |
| def _core(instruction, choices, target_text="", image=None, audio=None, video=None, context_text=""): | |
| return decide(instruction, choices, target_text, image, audio, video, context_text) | |
| # Wrappers whose signatures match each gr.Examples block exactly (no None cells). | |
| def fn_text(instruction, choices, target_text): | |
| return _core(instruction, choices, target_text=target_text) | |
| def fn_context(instruction, choices, target_text, context_text): | |
| return _core(instruction, choices, target_text=target_text, context_text=context_text) | |
| def fn_image(instruction, choices, image): | |
| return _core(instruction, choices, image=image) | |
| def fn_audio(instruction, choices, audio): | |
| return _core(instruction, choices, audio=audio) | |
| def fn_video(instruction, choices, video): | |
| return _core(instruction, choices, video=video) | |
| # --------------------------------------------------------------------------- # | |
| # UI | |
| # --------------------------------------------------------------------------- # | |
| CSS = """ | |
| #col-container { max-width: 1150px; margin: 0 auto; } | |
| .dark .gradio-container { color: var(--body-text-color); } | |
| """ | |
| with gr.Blocks(title="UEmo-D1 — Multimodal Emotion Recognition") as demo: | |
| gr.Markdown( | |
| "# 🎭 UEmo-D1 — Unified Multimodal Emotion Recognition\n" | |
| "**UEmo-D1** is a calibrated decision model built on `Qwen/Qwen2.5-Omni-3B`. Give it an " | |
| "instruction, a list of candidate answers and a **target** — text, image, audio, or " | |
| "video+audio — and it returns the chosen option plus its calibrated probability " | |
| "distribution over *all* options (a bidirectional-prefix / causal-answer attention pass " | |
| "over the option letters)." | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=3): | |
| instruction = gr.Textbox( | |
| label="Instruction", value=DEFAULT_INSTRUCTION, lines=2, | |
| info="What should the model decide about the target?", | |
| ) | |
| choices = gr.Textbox( | |
| label="Candidate answers (comma-separated, 1–26)", value=EMOTIONS, lines=1, | |
| info="Each becomes one option letter A, B, C …", | |
| ) | |
| with gr.Column(scale=4): | |
| with gr.Tabs(): | |
| with gr.Tab("📝 Text"): | |
| target_text = gr.Textbox(label="Target text", lines=4, placeholder="Type the text to classify…") | |
| with gr.Tab("🖼️ Image"): | |
| image = gr.Image(label="Target image", type="filepath") | |
| with gr.Tab("🔊 Audio"): | |
| audio = gr.Audio(label="Target audio", type="numpy") | |
| with gr.Tab("🎬 Video"): | |
| video = gr.Video(label="Target video") | |
| gr.Markdown( | |
| "*Combine modalities freely — e.g. an image **and** text in one call. " | |
| "Video examples carry no audio track, so matched silence is supplied.*", | |
| elem_classes="note", | |
| ) | |
| with gr.Accordion("⚙️ Advanced: preceding context", open=False): | |
| context_text = gr.Textbox( | |
| label="Preceding context (optional)", lines=3, | |
| placeholder="Optional context shown to the model before the target…", | |
| ) | |
| run_btn = gr.Button("Decide", variant="primary", size="lg") | |
| with gr.Row(): | |
| answer = gr.Textbox(label="Decision", interactive=False, lines=1) | |
| distribution = gr.Label(label="Calibrated distribution", num_top_classes=26) | |
| run_btn.click( | |
| decide, | |
| inputs=[instruction, choices, target_text, image, audio, video, context_text], | |
| outputs=[answer, distribution], | |
| api_name="decide", | |
| ) | |
| gr.Markdown("### Examples — click one to run it") | |
| with gr.Accordion("📝 Text", open=True): | |
| gr.Examples( | |
| examples=[ | |
| [DEFAULT_INSTRUCTION, EMOTIONS, | |
| "I can't believe you remembered my birthday — this is the happiest day of my life!"], | |
| ["Classify the sentiment of the target text.", SENTIMENTS, | |
| "The soup was cold and the waiter was rude to us."], | |
| ], | |
| inputs=[instruction, choices, target_text], | |
| outputs=[answer, distribution], fn=fn_text, | |
| example_labels=["text · emotion", "text · sentiment"], | |
| cache_examples=True, cache_mode="lazy", | |
| ) | |
| with gr.Accordion("🧩 Text + preceding context", open=False): | |
| gr.Examples( | |
| examples=[ | |
| ["Classify the emotion of the target reply, given the preceding context.", EMOTIONS, | |
| "Fine. Do whatever you want.", "I told him I had been offered a job in another city."], | |
| ], | |
| inputs=[instruction, choices, target_text, context_text], | |
| outputs=[answer, distribution], fn=fn_context, | |
| example_labels=["reply emotion · with context"], | |
| cache_examples=True, cache_mode="lazy", | |
| ) | |
| with gr.Accordion("🖼️ Image", open=False): | |
| gr.Examples( | |
| examples=[ | |
| [DEFAULT_INSTRUCTION, EMOTIONS, "examples/woman_sad.jpg"], | |
| [DEFAULT_INSTRUCTION, EMOTIONS, "examples/girl_with_dog.jpg"], | |
| ], | |
| inputs=[instruction, choices, image], | |
| outputs=[answer, distribution], fn=fn_image, | |
| example_labels=["woman", "girl with dog"], | |
| cache_examples=True, cache_mode="lazy", | |
| ) | |
| with gr.Accordion("🔊 Audio", open=False): | |
| gr.Examples( | |
| examples=[ | |
| ["Classify the emotion expressed by the speaker in the target audio.", EMOTIONS, "examples/amused.wav"], | |
| ["Classify the emotion expressed by the speaker in the target audio.", EMOTIONS, "examples/angry.wav"], | |
| ["Classify the emotion expressed by the speaker in the target audio.", EMOTIONS, "examples/surprised.wav"], | |
| ], | |
| inputs=[instruction, choices, audio], | |
| outputs=[answer, distribution], fn=fn_audio, | |
| example_labels=["amused (CC0)", "angry (CC0)", "surprised (CC0)"], | |
| cache_examples=True, cache_mode="lazy", | |
| ) | |
| with gr.Accordion("🎬 Video", open=False): | |
| gr.Examples( | |
| examples=[ | |
| ["Classify the emotion expressed by the people in the target video.", EMOTIONS, "examples/2_women_arguing.mp4"], | |
| ["Classify the emotion expressed by the person in the target video.", EMOTIONS, "examples/elderly_woman_laughing.mp4"], | |
| ], | |
| inputs=[instruction, choices, video], | |
| outputs=[answer, distribution], fn=fn_video, | |
| example_labels=["two women arguing", "elderly woman laughing"], | |
| cache_examples=True, cache_mode="lazy", | |
| ) | |
| gr.Markdown( | |
| "<small>Weights: [UEmo-D1](https://huggingface.co/wudq/UEmo-D1) · Code: " | |
| "[wdqqdw/UEmo-D1](https://github.com/wdqqdw/UEmo-D1) · Base: Qwen2.5-Omni-3B " | |
| "(Qwen Research License). Audio examples: Thorsten-Voice TV-44kHz-Full " | |
| "(CC0-1.0). Image/video examples: linoyts example assets (CC0-1.0).</small>" | |
| ) | |
| demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True) | |