"""Local UEmo-D1 decision inference; no training or evaluation dataset dependencies.""" import argparse import json from pathlib import Path import av import numpy as np import soundfile as sf import soxr import torch from PIL import Image from transformers import AutoModel, Qwen2_5OmniProcessor from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize SYSTEM = 'You are a helpful assistant. Classify the target using the supplied media and text.' LETTERS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' def read_audio(path): a, sr = sf.read(path, dtype='float32', always_2d=True) a = a.mean(axis=1) if sr != 16000: a = soxr.resample(a, sr, 16000) if not len(a): raise ValueError('Empty audio') return a def read_video(path): c = av.open(str(path)) try: st = c.streams.video[0] h, w = smart_resize(st.height, st.width, factor=28, min_pixels=56*56, max_pixels=224*224) frames, target, start = [], 0.0, None for f in c.decode(st): t = float(f.pts*f.time_base) if f.pts is not None else (float(start or 0)+len(frames)*0.5) if start is None: start = t if t-start+1e-6 >= target: frames.append(f.reformat(width=w, height=h, format='rgb24').to_ndarray()) target += 0.5 finally: c.close() if not frames: raise ValueError('Empty video') if len(frames) == 1: frames.append(frames[-1]) return np.stack(frames).transpose(0, 3, 1, 2) def prepare(processor, sample, root=Path('.')): """The text modality contains UTF-8 file paths, like the other modalities. Paired audio paths explicitly provide the audio for each video clip. """ root = Path(root) opts = sample['choices'] if not isinstance(opts, list) or not 1 <= len(opts) <= 26 or any(not isinstance(x, str) or not x for x in opts) or len(set(opts)) != len(opts): raise ValueError('Provide 1..26 unique nonempty candidate strings') if not isinstance(sample.get('instruction'), str) or not sample['instruction'].strip(): raise ValueError('Provide a nonempty instruction') audios, images, videos, content = [], [], [], [] mm = sample.get('multimodal', {}) for kind in ['context', 'target']: if kind == 'context' and 'context' not in mm: continue group = mm['context'] if kind == 'context' else mm content.append({'type': 'text', 'text': 'Preceding context:' if kind == 'context' else 'Target to classify:'}) for path in group.get('text', []): content.append({'type': 'text', 'text': (root/path).read_text(encoding='utf-8')}) for path in group.get('image', []): with Image.open(root/path) as im: images.append(im.convert('RGB')) content.append({'type': 'image', 'image': 'provided image'}) vp, ap = group.get('video', []), group.get('audio', []) if vp: if len(vp) != len(ap): raise ValueError('Each video needs its aligned audio file in the audio list') for video, audio in zip(vp, ap): videos.append(read_video(root/video)); audios.append(read_audio(root/audio)) content.append({'type': 'video', 'video': 'provided audiovisual clip'}) else: for path in ap: audios.append(read_audio(root/path)) content.append({'type': 'audio', 'audio': 'provided audio'}) prompt = sample['instruction']+'\nChoose your answer from:\n'+'\n'.join(f'{LETTERS[i]}. {v}' for i, v in enumerate(opts))+'\nAnswer with only the single option letter, without explanation.' content.append({'type': 'text', 'text': prompt}) conversation = [{'role': 'system', 'content': [{'type': 'text', 'text': SYSTEM}]}, {'role': 'user', 'content': content}] return {'text': processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False), 'audio': audios, 'images': images, 'videos': videos} def batch_prepare(processor, prepared): aa = [a for p in prepared for a in p['audio']] ii = [a for p in prepared for a in p['images']] vv = [a for p in prepared for a in p['videos']] return processor(text=[p['text'] for p in prepared], audio=aa or None, images=ii or None, videos=vv or None, return_tensors='pt', padding=True, use_audio_in_video=True, images_kwargs={'min_pixels': 56*56, 'max_pixels': 448*448}, videos_kwargs={'fps': 2.0, 'do_sample_frames': False, 'min_pixels': 56*56, 'max_pixels': 224*224}, audio_kwargs={'max_length': max(len(a) for a in aa) if aa else 16000, 'truncation': False}) def load_model(path, device='cuda:0'): if not torch.cuda.is_available(): raise RuntimeError('This validated implementation requires CUDA and FlashAttention 2') processor = Qwen2_5OmniProcessor.from_pretrained(path, local_files_only=True) processor.tokenizer.padding_side = 'left' model, info = AutoModel.from_pretrained(path, trust_remote_code=True, local_files_only=True, torch_dtype=torch.bfloat16, device_map={'': device}, attn_implementation='flash_attention_2', output_loading_info=True) if info['missing_keys'] or info['unexpected_keys'] or info['mismatched_keys']: raise RuntimeError(f'Incomplete checkpoint: {info}') model.eval(); model.requires_grad_(False) return model, processor def decide(model, processor, sample, root=Path('.')): inputs = batch_prepare(processor, [prepare(processor, sample, root)]).to(model.device).to(torch.bfloat16) tokens = [processor.tokenizer.encode(c, add_special_tokens=False) for c in LETTERS[:len(sample['choices'])]] if any(len(x) != 1 for x in tokens): raise ValueError('Each option letter must be one token') model.rope_deltas = None with torch.inference_mode(), torch.autocast('cuda', dtype=torch.bfloat16): logits = model(**inputs, use_audio_in_video=True, use_cache=False, mer_prefix_tokens=inputs['input_ids'].shape[-1]).logits[0, -1].float() scores = logits[[x[0] for x in tokens]] probabilities = (scores / model.config.uemo_temperature).softmax(-1) if not torch.isfinite(probabilities).all(): raise RuntimeError('Non-finite decision probabilities') index = int(scores.argmax()) return {'letter': LETTERS[index], 'label': sample['choices'][index], 'probabilities': {LETTERS[i]: p for i, p in enumerate(probabilities.cpu().tolist())}} def main(): parser = argparse.ArgumentParser() parser.add_argument('--model', default=str(Path(__file__).resolve().parent)) parser.add_argument('--input', required=True, help='JSON with multimodal, instruction, choices') parser.add_argument('--media-root', default=None, help='Default: directory containing the input JSON') parser.add_argument('--device', default='cuda:0') parser.add_argument('--probabilities', action='store_true') args = parser.parse_args() torch.set_num_threads(4) sample = json.loads(Path(args.input).read_text()) model, processor = load_model(args.model, args.device) result = decide(model, processor, sample, Path(args.media_root) if args.media_root else Path(args.input).resolve().parent) print(json.dumps(result, ensure_ascii=False) if args.probabilities else result['letter']) if __name__ == '__main__': main()