UEmo-D1 / inference.py
wudq's picture
Release validated complete UEmo-D1 inference checkpoint
cf17117 verified
Raw History Blame Contribute Delete
7.5 kB
"""Local UEmo-D1 decision inference; no training or evaluation dataset dependencies."""
import argparse
import json
from pathlib import Path
import av
import numpy as np
import soundfile as sf
import soxr
import torch
from PIL import Image
from transformers import AutoModel, Qwen2_5OmniProcessor
from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
SYSTEM = 'You are a helpful assistant. Classify the target using the supplied media and text.'
LETTERS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
def read_audio(path):
a, sr = sf.read(path, dtype='float32', always_2d=True)
a = a.mean(axis=1)
if sr != 16000:
a = soxr.resample(a, sr, 16000)
if not len(a):
raise ValueError('Empty audio')
return a
def read_video(path):
c = av.open(str(path))
try:
st = c.streams.video[0]
h, w = smart_resize(st.height, st.width, factor=28, min_pixels=56*56, max_pixels=224*224)
frames, target, start = [], 0.0, None
for f in c.decode(st):
t = float(f.pts*f.time_base) if f.pts is not None else (float(start or 0)+len(frames)*0.5)
if start is None:
start = t
if t-start+1e-6 >= target:
frames.append(f.reformat(width=w, height=h, format='rgb24').to_ndarray())
target += 0.5
finally:
c.close()
if not frames:
raise ValueError('Empty video')
if len(frames) == 1:
frames.append(frames[-1])
return np.stack(frames).transpose(0, 3, 1, 2)
def prepare(processor, sample, root=Path('.')):
"""The text modality contains UTF-8 file paths, like the other modalities.
Paired audio paths explicitly provide the audio for each video clip.
"""
root = Path(root)
opts = sample['choices']
if not isinstance(opts, list) or not 1 <= len(opts) <= 26 or any(not isinstance(x, str) or not x for x in opts) or len(set(opts)) != len(opts):
raise ValueError('Provide 1..26 unique nonempty candidate strings')
if not isinstance(sample.get('instruction'), str) or not sample['instruction'].strip():
raise ValueError('Provide a nonempty instruction')
audios, images, videos, content = [], [], [], []
mm = sample.get('multimodal', {})
for kind in ['context', 'target']:
if kind == 'context' and 'context' not in mm:
continue
group = mm['context'] if kind == 'context' else mm
content.append({'type': 'text', 'text': 'Preceding context:' if kind == 'context' else 'Target to classify:'})
for path in group.get('text', []):
content.append({'type': 'text', 'text': (root/path).read_text(encoding='utf-8')})
for path in group.get('image', []):
with Image.open(root/path) as im:
images.append(im.convert('RGB'))
content.append({'type': 'image', 'image': 'provided image'})
vp, ap = group.get('video', []), group.get('audio', [])
if vp:
if len(vp) != len(ap):
raise ValueError('Each video needs its aligned audio file in the audio list')
for video, audio in zip(vp, ap):
videos.append(read_video(root/video)); audios.append(read_audio(root/audio))
content.append({'type': 'video', 'video': 'provided audiovisual clip'})
else:
for path in ap:
audios.append(read_audio(root/path))
content.append({'type': 'audio', 'audio': 'provided audio'})
prompt = sample['instruction']+'\nChoose your answer from:\n'+'\n'.join(f'{LETTERS[i]}. {v}' for i, v in enumerate(opts))+'\nAnswer with only the single option letter, without explanation.'
content.append({'type': 'text', 'text': prompt})
conversation = [{'role': 'system', 'content': [{'type': 'text', 'text': SYSTEM}]}, {'role': 'user', 'content': content}]
return {'text': processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False), 'audio': audios, 'images': images, 'videos': videos}
def batch_prepare(processor, prepared):
aa = [a for p in prepared for a in p['audio']]
ii = [a for p in prepared for a in p['images']]
vv = [a for p in prepared for a in p['videos']]
return processor(text=[p['text'] for p in prepared], audio=aa or None,
images=ii or None, videos=vv or None, return_tensors='pt',
padding=True, use_audio_in_video=True,
images_kwargs={'min_pixels': 56*56, 'max_pixels': 448*448},
videos_kwargs={'fps': 2.0, 'do_sample_frames': False, 'min_pixels': 56*56, 'max_pixels': 224*224},
audio_kwargs={'max_length': max(len(a) for a in aa) if aa else 16000, 'truncation': False})
def load_model(path, device='cuda:0'):
if not torch.cuda.is_available():
raise RuntimeError('This validated implementation requires CUDA and FlashAttention 2')
processor = Qwen2_5OmniProcessor.from_pretrained(path, local_files_only=True)
processor.tokenizer.padding_side = 'left'
model, info = AutoModel.from_pretrained(path, trust_remote_code=True, local_files_only=True,
torch_dtype=torch.bfloat16, device_map={'': device}, attn_implementation='flash_attention_2', output_loading_info=True)
if info['missing_keys'] or info['unexpected_keys'] or info['mismatched_keys']:
raise RuntimeError(f'Incomplete checkpoint: {info}')
model.eval(); model.requires_grad_(False)
return model, processor
def decide(model, processor, sample, root=Path('.')):
inputs = batch_prepare(processor, [prepare(processor, sample, root)]).to(model.device).to(torch.bfloat16)
tokens = [processor.tokenizer.encode(c, add_special_tokens=False) for c in LETTERS[:len(sample['choices'])]]
if any(len(x) != 1 for x in tokens):
raise ValueError('Each option letter must be one token')
model.rope_deltas = None
with torch.inference_mode(), torch.autocast('cuda', dtype=torch.bfloat16):
logits = model(**inputs, use_audio_in_video=True, use_cache=False,
mer_prefix_tokens=inputs['input_ids'].shape[-1]).logits[0, -1].float()
scores = logits[[x[0] for x in tokens]]
probabilities = (scores / model.config.uemo_temperature).softmax(-1)
if not torch.isfinite(probabilities).all():
raise RuntimeError('Non-finite decision probabilities')
index = int(scores.argmax())
return {'letter': LETTERS[index], 'label': sample['choices'][index],
'probabilities': {LETTERS[i]: p for i, p in enumerate(probabilities.cpu().tolist())}}
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--model', default=str(Path(__file__).resolve().parent))
parser.add_argument('--input', required=True, help='JSON with multimodal, instruction, choices')
parser.add_argument('--media-root', default=None, help='Default: directory containing the input JSON')
parser.add_argument('--device', default='cuda:0')
parser.add_argument('--probabilities', action='store_true')
args = parser.parse_args()
torch.set_num_threads(4)
sample = json.loads(Path(args.input).read_text())
model, processor = load_model(args.model, args.device)
result = decide(model, processor, sample, Path(args.media_root) if args.media_root else Path(args.input).resolve().parent)
print(json.dumps(result, ensure_ascii=False) if args.probabilities else result['letter'])
if __name__ == '__main__':
main()