File size: 4,020 Bytes
a64600b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
"""Supplementary quality diagnostics; not FID/FVD/HumanML3D retrieval evaluation."""
import json
from pathlib import Path
import numpy as np
from .results import atomic_json

def evaluate(run,weights,device='cuda',all_jobs=False):
    import torch
    from transformers import CLIPModel,CLIPProcessor
    from huggingface_hub import HfApi,snapshot_download
    from PIL import Image
    import lpips
    root=Path(run);weights=Path(weights);lockpath=weights/'evaluation.lock.json'
    lock=json.loads(lockpath.read_text()) if lockpath.exists() else {}
    repo='openai/clip-vit-large-patch14'
    if repo not in lock:lock[repo]=HfApi().model_info(repo).sha;atomic_json(lockpath,lock)
    path=snapshot_download(repo,revision=lock[repo]);clip=CLIPModel.from_pretrained(path).to(device).eval();processor=CLIPProcessor.from_pretrained(path)
    perceptual=lpips.LPIPS(net='alex').to(device).eval();rows=[]
    def frames(folder,model):
        if model=='self_forcing':
            import imageio.v3 as iio
            arr=iio.imread(folder/'output.mp4');return [Image.fromarray(arr[i]) for i in np.linspace(0,len(arr)-1,8).astype(int)]
        return [Image.open(folder/'output.png').convert('RGB')]
    for file in sorted((root/'jobs').glob('*/result.json')):
        record=json.loads(file.read_text())
        if record.get('status')!='complete':continue
        j=record['job'];model=j['model']
        if not all_jobs:
            if j.get('study')=='control_v2':
                midpoint=j.get('calibration_mode') in ('raw','mapped') and j.get('requested_effect')==.5
                endpoint=j.get('calibration_mode') in ('anchor','endpoint') and j['alpha']==1.
                if j['seed']!=3 or not (midpoint or endpoint):continue
            elif j['experiment']!='main' or j['seed']!=0 or j['alpha'] not in (0.,.5,1.):continue
        row=dict(id=j['id'],model=model,task=j['task']['id'],seed=j['seed'],
                 method_id=j.get('method_id'),calibration_mode=j.get('calibration_mode'),
                 requested_effect=j.get('requested_effect'))
        if model=='momask':
            a=np.load(file.parent/'output.joints.npy');velocity=np.diff(a,axis=0);acceleration=np.diff(velocity,axis=0);jerk=np.diff(acceleration,axis=0)
            row.update(mean_joint_speed=float(np.linalg.norm(velocity,axis=-1).mean()*20),mean_joint_acceleration=float(np.linalg.norm(acceleration,axis=-1).mean()*20**2),mean_joint_jerk=float(np.linalg.norm(jerk,axis=-1).mean()*20**3))
        else:
            images=frames(file.parent,model);inputs=processor(text=[j['task']['prompt']],images=images,return_tensors='pt',padding=True).to(device)
            with torch.no_grad():
                output=clip(**inputs);similarity=(output.image_embeds@output.text_embeds.T).mean();row['clip_cosine']=float(similarity)
                if j.get('study')=='control_v2' and j['task'].get('reference_prompt'):
                    target_inputs=processor(text=[j['task']['reference_prompt']],return_tensors='pt',padding=True).to(device)
                    target_embed=clip.get_text_features(**target_inputs)
                    target_embed=target_embed/target_embed.norm(dim=-1,keepdim=True)
                    row['clip_reference_prompt_cosine']=float((output.image_embeds@target_embed.T).mean())
            if model!='self_forcing':
                a=np.load(file.parent/'output.npy').astype('float32');ref=np.load(root/'references'/model/j['task']['id']/'output.npy').astype('float32')
                with torch.no_grad():row['lpips_reference']=float(perceptual(torch.from_numpy(a).to(device)*2-1,torch.from_numpy(ref).to(device)*2-1).mean())
        rows.append(row)
    atomic_json(root/'summary/quality_diagnostics.json',dict(scope='all jobs' if all_jobs else 'legacy main seed=0 alpha=0/0.5/1; control_v2 seed=3 endpoints alpha1 and requested effect .5 raw/mapped',model_lock=lock,video_note='CLIP evaluated on decoded MP4; compression is part of this diagnostic',records=rows))
    return dict(evaluated=len(rows))