"""Supplementary quality diagnostics; not FID/FVD/HumanML3D retrieval evaluation.""" import json from pathlib import Path import numpy as np from .results import atomic_json def evaluate(run,weights,device='cuda',all_jobs=False): import torch from transformers import CLIPModel,CLIPProcessor from huggingface_hub import HfApi,snapshot_download from PIL import Image import lpips root=Path(run);weights=Path(weights);lockpath=weights/'evaluation.lock.json' lock=json.loads(lockpath.read_text()) if lockpath.exists() else {} repo='openai/clip-vit-large-patch14' if repo not in lock:lock[repo]=HfApi().model_info(repo).sha;atomic_json(lockpath,lock) path=snapshot_download(repo,revision=lock[repo]);clip=CLIPModel.from_pretrained(path).to(device).eval();processor=CLIPProcessor.from_pretrained(path) perceptual=lpips.LPIPS(net='alex').to(device).eval();rows=[] def frames(folder,model): if model=='self_forcing': import imageio.v3 as iio arr=iio.imread(folder/'output.mp4');return [Image.fromarray(arr[i]) for i in np.linspace(0,len(arr)-1,8).astype(int)] return [Image.open(folder/'output.png').convert('RGB')] for file in sorted((root/'jobs').glob('*/result.json')): record=json.loads(file.read_text()) if record.get('status')!='complete':continue j=record['job'];model=j['model'] if not all_jobs: if j.get('study')=='control_v2': midpoint=j.get('calibration_mode') in ('raw','mapped') and j.get('requested_effect')==.5 endpoint=j.get('calibration_mode') in ('anchor','endpoint') and j['alpha']==1. if j['seed']!=3 or not (midpoint or endpoint):continue elif j['experiment']!='main' or j['seed']!=0 or j['alpha'] not in (0.,.5,1.):continue row=dict(id=j['id'],model=model,task=j['task']['id'],seed=j['seed'], method_id=j.get('method_id'),calibration_mode=j.get('calibration_mode'), requested_effect=j.get('requested_effect')) if model=='momask': a=np.load(file.parent/'output.joints.npy');velocity=np.diff(a,axis=0);acceleration=np.diff(velocity,axis=0);jerk=np.diff(acceleration,axis=0) row.update(mean_joint_speed=float(np.linalg.norm(velocity,axis=-1).mean()*20),mean_joint_acceleration=float(np.linalg.norm(acceleration,axis=-1).mean()*20**2),mean_joint_jerk=float(np.linalg.norm(jerk,axis=-1).mean()*20**3)) else: images=frames(file.parent,model);inputs=processor(text=[j['task']['prompt']],images=images,return_tensors='pt',padding=True).to(device) with torch.no_grad(): output=clip(**inputs);similarity=(output.image_embeds@output.text_embeds.T).mean();row['clip_cosine']=float(similarity) if j.get('study')=='control_v2' and j['task'].get('reference_prompt'): target_inputs=processor(text=[j['task']['reference_prompt']],return_tensors='pt',padding=True).to(device) target_embed=clip.get_text_features(**target_inputs) target_embed=target_embed/target_embed.norm(dim=-1,keepdim=True) row['clip_reference_prompt_cosine']=float((output.image_embeds@target_embed.T).mean()) if model!='self_forcing': a=np.load(file.parent/'output.npy').astype('float32');ref=np.load(root/'references'/model/j['task']['id']/'output.npy').astype('float32') with torch.no_grad():row['lpips_reference']=float(perceptual(torch.from_numpy(a).to(device)*2-1,torch.from_numpy(ref).to(device)*2-1).mean()) rows.append(row) atomic_json(root/'summary/quality_diagnostics.json',dict(scope='all jobs' if all_jobs else 'legacy main seed=0 alpha=0/0.5/1; control_v2 seed=3 endpoints alpha1 and requested effect .5 raw/mapped',model_lock=lock,video_note='CLIP evaluated on decoded MP4; compression is part of this diagnostic',records=rows)) return dict(evaluated=len(rows))