uemo-d1 / app.py
multimodalart's picture
multimodalart HF Staff
Upload app.py with huggingface_hub
44327c3 verified
Raw History Blame Contribute Delete
17.1 kB
"""UEmo-D1 — unified multimodal emotion recognition (ZeroGPU demo).
Reference implementation ported 1:1 from the authors' `inference.py` shipped in
the model repo `wudq/UEmo-D1` (same preprocessing, same attention hook, same
calibrated softmax), with the file-path I/O replaced by in-memory inputs.
"""
import spaces # noqa: F401 -- MUST precede torch / transformers / CUDA imports
import time
import av
import gradio as gr
import numpy as np
import soxr
import torch
from PIL import Image
from transformers import AutoModel, Qwen2_5OmniProcessor
from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
MODEL_ID = "wudq/UEmo-D1"
SYSTEM = "You are a helpful assistant. Classify the target using the supplied media and text."
LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
EMOTIONS = "happy, sad, angry, surprised, disgusted, fearful, neutral"
SENTIMENTS = "positive, negative, neutral"
DEFAULT_INSTRUCTION = "Classify the emotion expressed by the target."
# --------------------------------------------------------------------------- #
# Model — loaded at module scope, moved to the GPU eagerly (ZeroGPU pattern).
# --------------------------------------------------------------------------- #
processor = Qwen2_5OmniProcessor.from_pretrained(MODEL_ID)
processor.tokenizer.padding_side = "left"
model, loading_info = AutoModel.from_pretrained(
MODEL_ID,
trust_remote_code=True,
torch_dtype=torch.bfloat16,
attn_implementation="flash_attention_2",
output_loading_info=True,
)
if loading_info["missing_keys"] or loading_info["unexpected_keys"] or loading_info["mismatched_keys"]:
raise RuntimeError(f"Incomplete checkpoint: {loading_info}")
model.eval()
model.requires_grad_(False)
model.to("cuda")
print(f"[boot] UEmo-D1 ready on {model.device}, temperature={model.config.uemo_temperature}", flush=True)
# --------------------------------------------------------------------------- #
# Media helpers — identical to the reference `read_audio` / `read_video`.
# --------------------------------------------------------------------------- #
def read_audio_file(path) -> np.ndarray:
import soundfile as sf
a, sr = sf.read(str(path), dtype="float32", always_2d=True)
a = a.mean(axis=1)
if sr != 16000:
a = soxr.resample(a, sr, 16000)
if not len(a):
raise ValueError("Empty audio")
return np.asarray(a, dtype=np.float32)
def read_video_frames(path) -> np.ndarray:
"""0.5 s frame sampling at 224x224-max resolution, exactly like the authors' code."""
c = av.open(str(path))
try:
st = c.streams.video[0]
h, w = smart_resize(st.height, st.width, factor=28, min_pixels=56 * 56, max_pixels=224 * 224)
frames, target, start = [], 0.0, None
for f in c.decode(st):
t = float(f.pts * f.time_base) if f.pts is not None else (float(start or 0) + len(frames) * 0.5)
if start is None:
start = t
if t - start + 1e-6 >= target:
frames.append(f.reformat(width=w, height=h, format="rgb24").to_ndarray())
target += 0.5
finally:
c.close()
if not frames:
raise ValueError("Empty video")
if len(frames) == 1:
frames.append(frames[-1])
return np.stack(frames).transpose(0, 3, 1, 2)
def to_audio(x):
"""Normalise whatever the Audio component hands us into 16 kHz mono float32."""
if x is None:
return None
if isinstance(x, tuple): # (sample_rate, ndarray) for type="numpy"
sr, arr = x
arr = np.asarray(arr, dtype=np.float32)
if arr.ndim > 1:
arr = arr.mean(axis=1)
if sr and int(sr) != 16000:
arr = soxr.resample(arr, int(sr), 16000)
return np.ascontiguousarray(arr, dtype=np.float32)
return read_audio_file(x)
def to_video(x):
if x is None:
return None
if isinstance(x, dict):
x = x.get("video") or x.get("path")
return read_video_frames(x)
# --------------------------------------------------------------------------- #
# Prompt assembly — a direct port of the reference `prepare()`.
# --------------------------------------------------------------------------- #
def prepare(instruction, opts, target_text, context_text, image, audio, video):
audios, images, videos, content = [], [], [], []
groups = []
if context_text and context_text.strip():
groups.append(("context", context_text, None, None, None))
groups.append(("target", target_text, image, audio, video))
for kind, txt, im, au, vi in groups:
content.append({"type": "text", "text": "Preceding context:" if kind == "context" else "Target to classify:"})
if txt and txt.strip():
content.append({"type": "text", "text": txt})
if im is not None:
images.append(Image.open(im).convert("RGB") if isinstance(im, str) else im.convert("RGB"))
content.append({"type": "image", "image": "provided image"})
if vi is not None:
videos.append(vi)
# The authors' pipeline requires audio aligned with every video. Our
# example clips carry no audio track, so we supply matched silence.
audios.append(au if au is not None else np.zeros(16000, dtype=np.float32))
content.append({"type": "video", "video": "provided audiovisual clip"})
elif au is not None:
audios.append(au)
content.append({"type": "audio", "audio": "provided audio"})
prompt = (
instruction
+ "\nChoose your answer from:\n"
+ "\n".join(f"{LETTERS[i]}. {v}" for i, v in enumerate(opts))
+ "\nAnswer with only the single option letter, without explanation."
)
content.append({"type": "text", "text": prompt})
conversation = [
{"role": "system", "content": [{"type": "text", "text": SYSTEM}]},
{"role": "user", "content": content},
]
return {
"text": processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False),
"audio": audios,
"images": images,
"videos": videos,
}
def batch_prepare(prepared):
aa = [a for p in prepared for a in p["audio"]]
ii = [a for p in prepared for a in p["images"]]
vv = [a for p in prepared for a in p["videos"]]
return processor(
text=[p["text"] for p in prepared],
audio=aa or None,
images=ii or None,
videos=vv or None,
return_tensors="pt",
padding=True,
use_audio_in_video=True,
images_kwargs={"min_pixels": 56 * 56, "max_pixels": 448 * 448},
videos_kwargs={"fps": 2.0, "do_sample_frames": False, "min_pixels": 56 * 56, "max_pixels": 224 * 224},
audio_kwargs={"max_length": max(len(a) for a in aa) if aa else 16000, "truncation": False},
)
# --------------------------------------------------------------------------- #
# Inference
# --------------------------------------------------------------------------- #
@spaces.GPU(duration=20)
def decide(instruction, choices, target_text="", image=None, audio=None, video=None, context_text=""):
"""Classify one target (text, image, audio or video+audio) into the supplied options.
Returns the chosen option and the model's calibrated probability distribution
over all options. `choices` is a comma-separated list of 1..26 alternatives.
Args:
instruction: The classification question, e.g. "Classify the emotion expressed by the target."
choices: Comma-separated candidate answers, e.g. "happy, sad, angry, neutral".
target_text: Text of the target to classify (optional if another modality is given).
image: Image of the target to classify (optional).
audio: Audio of the target to classify (optional).
video: Video of the target to classify (optional).
context_text: Optional preceding context shown to the model before the target.
"""
opts = [c.strip() for c in (choices or "").split(",") if c.strip()]
if not 1 <= len(opts) <= 26 or len(set(opts)) != len(opts):
raise gr.Error("Provide 1..26 unique, non-empty comma-separated options.")
if not (instruction or "").strip():
raise gr.Error("Provide a non-empty instruction.")
im = Image.open(image).convert("RGB") if isinstance(image, str) else image
au = to_audio(audio)
vi = to_video(video)
if im is None and au is None and vi is None and not (target_text or "").strip():
raise gr.Error("Provide a target: text, image, audio or video.")
t0 = time.time()
sample = prepare(instruction, opts, target_text or "", context_text or "", im, au, vi)
inputs = batch_prepare([sample]).to(model.device).to(torch.bfloat16)
tokens = [processor.tokenizer.encode(c, add_special_tokens=False) for c in LETTERS[: len(opts)]]
if any(len(x) != 1 for x in tokens):
raise gr.Error("Each option letter must be a single token.")
model.rope_deltas = None
with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16):
logits = model(
**inputs,
use_audio_in_video=True,
use_cache=False,
mer_prefix_tokens=inputs["input_ids"].shape[-1],
).logits[0, -1].float()
scores = logits[[x[0] for x in tokens]]
probabilities = (scores / model.config.uemo_temperature).softmax(-1)
if not torch.isfinite(probabilities).all():
raise gr.Error("Non-finite decision probabilities.")
index = int(scores.argmax())
label = f"{LETTERS[index]}. {opts[index]}"
distribution = {f"{LETTERS[i]}. {opts[i]}": float(p) for i, p in enumerate(probabilities.cpu().tolist())}
print(f"[decide] opts={len(opts)} tokens={inputs['input_ids'].shape[-1]} elapsed={time.time() - t0:.2f}s", flush=True)
return label, distribution
def _core(instruction, choices, target_text="", image=None, audio=None, video=None, context_text=""):
return decide(instruction, choices, target_text, image, audio, video, context_text)
# Wrappers whose signatures match each gr.Examples block exactly (no None cells).
def fn_text(instruction, choices, target_text):
return _core(instruction, choices, target_text=target_text)
def fn_context(instruction, choices, target_text, context_text):
return _core(instruction, choices, target_text=target_text, context_text=context_text)
def fn_image(instruction, choices, image):
return _core(instruction, choices, image=image)
def fn_audio(instruction, choices, audio):
return _core(instruction, choices, audio=audio)
def fn_video(instruction, choices, video):
return _core(instruction, choices, video=video)
# --------------------------------------------------------------------------- #
# UI
# --------------------------------------------------------------------------- #
CSS = """
#col-container { max-width: 1150px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""
with gr.Blocks(title="UEmo-D1 — Multimodal Emotion Recognition") as demo:
gr.Markdown(
"# 🎭 UEmo-D1 — Unified Multimodal Emotion Recognition\n"
"**UEmo-D1** is a calibrated decision model built on `Qwen/Qwen2.5-Omni-3B`. Give it an "
"instruction, a list of candidate answers and a **target** — text, image, audio, or "
"video+audio — and it returns the chosen option plus its calibrated probability "
"distribution over *all* options (a bidirectional-prefix / causal-answer attention pass "
"over the option letters)."
)
with gr.Row():
with gr.Column(scale=3):
instruction = gr.Textbox(
label="Instruction", value=DEFAULT_INSTRUCTION, lines=2,
info="What should the model decide about the target?",
)
choices = gr.Textbox(
label="Candidate answers (comma-separated, 1–26)", value=EMOTIONS, lines=1,
info="Each becomes one option letter A, B, C …",
)
with gr.Column(scale=4):
with gr.Tabs():
with gr.Tab("📝 Text"):
target_text = gr.Textbox(label="Target text", lines=4, placeholder="Type the text to classify…")
with gr.Tab("🖼️ Image"):
image = gr.Image(label="Target image", type="filepath")
with gr.Tab("🔊 Audio"):
audio = gr.Audio(label="Target audio", type="numpy")
with gr.Tab("🎬 Video"):
video = gr.Video(label="Target video")
gr.Markdown(
"*Combine modalities freely — e.g. an image **and** text in one call. "
"Video examples carry no audio track, so matched silence is supplied.*",
elem_classes="note",
)
with gr.Accordion("⚙️ Advanced: preceding context", open=False):
context_text = gr.Textbox(
label="Preceding context (optional)", lines=3,
placeholder="Optional context shown to the model before the target…",
)
run_btn = gr.Button("Decide", variant="primary", size="lg")
with gr.Row():
answer = gr.Textbox(label="Decision", interactive=False, lines=1)
distribution = gr.Label(label="Calibrated distribution", num_top_classes=26)
run_btn.click(
decide,
inputs=[instruction, choices, target_text, image, audio, video, context_text],
outputs=[answer, distribution],
api_name="decide",
)
gr.Markdown("### Examples — click one to run it")
with gr.Accordion("📝 Text", open=True):
gr.Examples(
examples=[
[DEFAULT_INSTRUCTION, EMOTIONS,
"I can't believe you remembered my birthday — this is the happiest day of my life!"],
["Classify the sentiment of the target text.", SENTIMENTS,
"The soup was cold and the waiter was rude to us."],
],
inputs=[instruction, choices, target_text],
outputs=[answer, distribution], fn=fn_text,
example_labels=["text · emotion", "text · sentiment"],
cache_examples=True, cache_mode="lazy",
)
with gr.Accordion("🧩 Text + preceding context", open=False):
gr.Examples(
examples=[
["Classify the emotion of the target reply, given the preceding context.", EMOTIONS,
"Fine. Do whatever you want.", "I told him I had been offered a job in another city."],
],
inputs=[instruction, choices, target_text, context_text],
outputs=[answer, distribution], fn=fn_context,
example_labels=["reply emotion · with context"],
cache_examples=True, cache_mode="lazy",
)
with gr.Accordion("🖼️ Image", open=False):
gr.Examples(
examples=[
[DEFAULT_INSTRUCTION, EMOTIONS, "examples/woman_sad.jpg"],
[DEFAULT_INSTRUCTION, EMOTIONS, "examples/girl_with_dog.jpg"],
],
inputs=[instruction, choices, image],
outputs=[answer, distribution], fn=fn_image,
example_labels=["woman", "girl with dog"],
cache_examples=True, cache_mode="lazy",
)
with gr.Accordion("🔊 Audio", open=False):
gr.Examples(
examples=[
["Classify the emotion expressed by the speaker in the target audio.", EMOTIONS, "examples/amused.wav"],
["Classify the emotion expressed by the speaker in the target audio.", EMOTIONS, "examples/angry.wav"],
["Classify the emotion expressed by the speaker in the target audio.", EMOTIONS, "examples/surprised.wav"],
],
inputs=[instruction, choices, audio],
outputs=[answer, distribution], fn=fn_audio,
example_labels=["amused (CC0)", "angry (CC0)", "surprised (CC0)"],
cache_examples=True, cache_mode="lazy",
)
with gr.Accordion("🎬 Video", open=False):
gr.Examples(
examples=[
["Classify the emotion expressed by the people in the target video.", EMOTIONS, "examples/2_women_arguing.mp4"],
["Classify the emotion expressed by the person in the target video.", EMOTIONS, "examples/elderly_woman_laughing.mp4"],
],
inputs=[instruction, choices, video],
outputs=[answer, distribution], fn=fn_video,
example_labels=["two women arguing", "elderly woman laughing"],
cache_examples=True, cache_mode="lazy",
)
gr.Markdown(
"<small>Weights: [UEmo-D1](https://huggingface.co/wudq/UEmo-D1) · Code: "
"[wdqqdw/UEmo-D1](https://github.com/wdqqdw/UEmo-D1) · Base: Qwen2.5-Omni-3B "
"(Qwen Research License). Audio examples: Thorsten-Voice TV-44kHz-Full "
"(CC0-1.0). Image/video examples: linoyts example assets (CC0-1.0).</small>"
)
demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)