File size: 4,020 Bytes
a64600b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | """Supplementary quality diagnostics; not FID/FVD/HumanML3D retrieval evaluation."""
import json
from pathlib import Path
import numpy as np
from .results import atomic_json
def evaluate(run,weights,device='cuda',all_jobs=False):
import torch
from transformers import CLIPModel,CLIPProcessor
from huggingface_hub import HfApi,snapshot_download
from PIL import Image
import lpips
root=Path(run);weights=Path(weights);lockpath=weights/'evaluation.lock.json'
lock=json.loads(lockpath.read_text()) if lockpath.exists() else {}
repo='openai/clip-vit-large-patch14'
if repo not in lock:lock[repo]=HfApi().model_info(repo).sha;atomic_json(lockpath,lock)
path=snapshot_download(repo,revision=lock[repo]);clip=CLIPModel.from_pretrained(path).to(device).eval();processor=CLIPProcessor.from_pretrained(path)
perceptual=lpips.LPIPS(net='alex').to(device).eval();rows=[]
def frames(folder,model):
if model=='self_forcing':
import imageio.v3 as iio
arr=iio.imread(folder/'output.mp4');return [Image.fromarray(arr[i]) for i in np.linspace(0,len(arr)-1,8).astype(int)]
return [Image.open(folder/'output.png').convert('RGB')]
for file in sorted((root/'jobs').glob('*/result.json')):
record=json.loads(file.read_text())
if record.get('status')!='complete':continue
j=record['job'];model=j['model']
if not all_jobs:
if j.get('study')=='control_v2':
midpoint=j.get('calibration_mode') in ('raw','mapped') and j.get('requested_effect')==.5
endpoint=j.get('calibration_mode') in ('anchor','endpoint') and j['alpha']==1.
if j['seed']!=3 or not (midpoint or endpoint):continue
elif j['experiment']!='main' or j['seed']!=0 or j['alpha'] not in (0.,.5,1.):continue
row=dict(id=j['id'],model=model,task=j['task']['id'],seed=j['seed'],
method_id=j.get('method_id'),calibration_mode=j.get('calibration_mode'),
requested_effect=j.get('requested_effect'))
if model=='momask':
a=np.load(file.parent/'output.joints.npy');velocity=np.diff(a,axis=0);acceleration=np.diff(velocity,axis=0);jerk=np.diff(acceleration,axis=0)
row.update(mean_joint_speed=float(np.linalg.norm(velocity,axis=-1).mean()*20),mean_joint_acceleration=float(np.linalg.norm(acceleration,axis=-1).mean()*20**2),mean_joint_jerk=float(np.linalg.norm(jerk,axis=-1).mean()*20**3))
else:
images=frames(file.parent,model);inputs=processor(text=[j['task']['prompt']],images=images,return_tensors='pt',padding=True).to(device)
with torch.no_grad():
output=clip(**inputs);similarity=(output.image_embeds@output.text_embeds.T).mean();row['clip_cosine']=float(similarity)
if j.get('study')=='control_v2' and j['task'].get('reference_prompt'):
target_inputs=processor(text=[j['task']['reference_prompt']],return_tensors='pt',padding=True).to(device)
target_embed=clip.get_text_features(**target_inputs)
target_embed=target_embed/target_embed.norm(dim=-1,keepdim=True)
row['clip_reference_prompt_cosine']=float((output.image_embeds@target_embed.T).mean())
if model!='self_forcing':
a=np.load(file.parent/'output.npy').astype('float32');ref=np.load(root/'references'/model/j['task']['id']/'output.npy').astype('float32')
with torch.no_grad():row['lpips_reference']=float(perceptual(torch.from_numpy(a).to(device)*2-1,torch.from_numpy(ref).to(device)*2-1).mean())
rows.append(row)
atomic_json(root/'summary/quality_diagnostics.json',dict(scope='all jobs' if all_jobs else 'legacy main seed=0 alpha=0/0.5/1; control_v2 seed=3 endpoints alpha1 and requested effect .5 raw/mapped',model_lock=lock,video_note='CLIP evaluated on decoded MP4; compression is part of this diagnostic',records=rows))
return dict(evaluated=len(rows))
|