Feature Extraction
Transformers
Safetensors
English
Chinese
uemo_d1
emotion-recognition
multimodal
decision-model
custom_code
Instructions to use wudq/UEmo-D1 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use wudq/UEmo-D1 with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("feature-extraction", model="wudq/UEmo-D1", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("wudq/UEmo-D1", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download inference.py from wudq/UEmo-D1: direct link, hf CLI and curl.
- Browser
- Download file 7.5 kB
-
https://huggingface.co/wudq/UEmo-D1/resolve/main/inference.py
- Command line
-
hf download hf://wudq/UEmo-D1/inference.py
-
curl -L -o inference.py https://huggingface.co/wudq/UEmo-D1/resolve/main/inference.py
7.5 kB
| """Local UEmo-D1 decision inference; no training or evaluation dataset dependencies.""" | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| import av | |
| import numpy as np | |
| import soundfile as sf | |
| import soxr | |
| import torch | |
| from PIL import Image | |
| from transformers import AutoModel, Qwen2_5OmniProcessor | |
| from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize | |
| SYSTEM = 'You are a helpful assistant. Classify the target using the supplied media and text.' | |
| LETTERS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' | |
| def read_audio(path): | |
| a, sr = sf.read(path, dtype='float32', always_2d=True) | |
| a = a.mean(axis=1) | |
| if sr != 16000: | |
| a = soxr.resample(a, sr, 16000) | |
| if not len(a): | |
| raise ValueError('Empty audio') | |
| return a | |
| def read_video(path): | |
| c = av.open(str(path)) | |
| try: | |
| st = c.streams.video[0] | |
| h, w = smart_resize(st.height, st.width, factor=28, min_pixels=56*56, max_pixels=224*224) | |
| frames, target, start = [], 0.0, None | |
| for f in c.decode(st): | |
| t = float(f.pts*f.time_base) if f.pts is not None else (float(start or 0)+len(frames)*0.5) | |
| if start is None: | |
| start = t | |
| if t-start+1e-6 >= target: | |
| frames.append(f.reformat(width=w, height=h, format='rgb24').to_ndarray()) | |
| target += 0.5 | |
| finally: | |
| c.close() | |
| if not frames: | |
| raise ValueError('Empty video') | |
| if len(frames) == 1: | |
| frames.append(frames[-1]) | |
| return np.stack(frames).transpose(0, 3, 1, 2) | |
| def prepare(processor, sample, root=Path('.')): | |
| """The text modality contains UTF-8 file paths, like the other modalities. | |
| Paired audio paths explicitly provide the audio for each video clip. | |
| """ | |
| root = Path(root) | |
| opts = sample['choices'] | |
| if not isinstance(opts, list) or not 1 <= len(opts) <= 26 or any(not isinstance(x, str) or not x for x in opts) or len(set(opts)) != len(opts): | |
| raise ValueError('Provide 1..26 unique nonempty candidate strings') | |
| if not isinstance(sample.get('instruction'), str) or not sample['instruction'].strip(): | |
| raise ValueError('Provide a nonempty instruction') | |
| audios, images, videos, content = [], [], [], [] | |
| mm = sample.get('multimodal', {}) | |
| for kind in ['context', 'target']: | |
| if kind == 'context' and 'context' not in mm: | |
| continue | |
| group = mm['context'] if kind == 'context' else mm | |
| content.append({'type': 'text', 'text': 'Preceding context:' if kind == 'context' else 'Target to classify:'}) | |
| for path in group.get('text', []): | |
| content.append({'type': 'text', 'text': (root/path).read_text(encoding='utf-8')}) | |
| for path in group.get('image', []): | |
| with Image.open(root/path) as im: | |
| images.append(im.convert('RGB')) | |
| content.append({'type': 'image', 'image': 'provided image'}) | |
| vp, ap = group.get('video', []), group.get('audio', []) | |
| if vp: | |
| if len(vp) != len(ap): | |
| raise ValueError('Each video needs its aligned audio file in the audio list') | |
| for video, audio in zip(vp, ap): | |
| videos.append(read_video(root/video)); audios.append(read_audio(root/audio)) | |
| content.append({'type': 'video', 'video': 'provided audiovisual clip'}) | |
| else: | |
| for path in ap: | |
| audios.append(read_audio(root/path)) | |
| content.append({'type': 'audio', 'audio': 'provided audio'}) | |
| prompt = sample['instruction']+'\nChoose your answer from:\n'+'\n'.join(f'{LETTERS[i]}. {v}' for i, v in enumerate(opts))+'\nAnswer with only the single option letter, without explanation.' | |
| content.append({'type': 'text', 'text': prompt}) | |
| conversation = [{'role': 'system', 'content': [{'type': 'text', 'text': SYSTEM}]}, {'role': 'user', 'content': content}] | |
| return {'text': processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False), 'audio': audios, 'images': images, 'videos': videos} | |
| def batch_prepare(processor, prepared): | |
| aa = [a for p in prepared for a in p['audio']] | |
| ii = [a for p in prepared for a in p['images']] | |
| vv = [a for p in prepared for a in p['videos']] | |
| return processor(text=[p['text'] for p in prepared], audio=aa or None, | |
| images=ii or None, videos=vv or None, return_tensors='pt', | |
| padding=True, use_audio_in_video=True, | |
| images_kwargs={'min_pixels': 56*56, 'max_pixels': 448*448}, | |
| videos_kwargs={'fps': 2.0, 'do_sample_frames': False, 'min_pixels': 56*56, 'max_pixels': 224*224}, | |
| audio_kwargs={'max_length': max(len(a) for a in aa) if aa else 16000, 'truncation': False}) | |
| def load_model(path, device='cuda:0'): | |
| if not torch.cuda.is_available(): | |
| raise RuntimeError('This validated implementation requires CUDA and FlashAttention 2') | |
| processor = Qwen2_5OmniProcessor.from_pretrained(path, local_files_only=True) | |
| processor.tokenizer.padding_side = 'left' | |
| model, info = AutoModel.from_pretrained(path, trust_remote_code=True, local_files_only=True, | |
| torch_dtype=torch.bfloat16, device_map={'': device}, attn_implementation='flash_attention_2', output_loading_info=True) | |
| if info['missing_keys'] or info['unexpected_keys'] or info['mismatched_keys']: | |
| raise RuntimeError(f'Incomplete checkpoint: {info}') | |
| model.eval(); model.requires_grad_(False) | |
| return model, processor | |
| def decide(model, processor, sample, root=Path('.')): | |
| inputs = batch_prepare(processor, [prepare(processor, sample, root)]).to(model.device).to(torch.bfloat16) | |
| tokens = [processor.tokenizer.encode(c, add_special_tokens=False) for c in LETTERS[:len(sample['choices'])]] | |
| if any(len(x) != 1 for x in tokens): | |
| raise ValueError('Each option letter must be one token') | |
| model.rope_deltas = None | |
| with torch.inference_mode(), torch.autocast('cuda', dtype=torch.bfloat16): | |
| logits = model(**inputs, use_audio_in_video=True, use_cache=False, | |
| mer_prefix_tokens=inputs['input_ids'].shape[-1]).logits[0, -1].float() | |
| scores = logits[[x[0] for x in tokens]] | |
| probabilities = (scores / model.config.uemo_temperature).softmax(-1) | |
| if not torch.isfinite(probabilities).all(): | |
| raise RuntimeError('Non-finite decision probabilities') | |
| index = int(scores.argmax()) | |
| return {'letter': LETTERS[index], 'label': sample['choices'][index], | |
| 'probabilities': {LETTERS[i]: p for i, p in enumerate(probabilities.cpu().tolist())}} | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('--model', default=str(Path(__file__).resolve().parent)) | |
| parser.add_argument('--input', required=True, help='JSON with multimodal, instruction, choices') | |
| parser.add_argument('--media-root', default=None, help='Default: directory containing the input JSON') | |
| parser.add_argument('--device', default='cuda:0') | |
| parser.add_argument('--probabilities', action='store_true') | |
| args = parser.parse_args() | |
| torch.set_num_threads(4) | |
| sample = json.loads(Path(args.input).read_text()) | |
| model, processor = load_model(args.model, args.device) | |
| result = decide(model, processor, sample, Path(args.media_root) if args.media_root else Path(args.input).resolve().parent) | |
| print(json.dumps(result, ensure_ascii=False) if args.probabilities else result['letter']) | |
| if __name__ == '__main__': | |
| main() | |