Add Control v2 incremental experiment results (evidence, visuals, 42 fixed samples)
a64600b verified Download method_v2/evidence/code/suite/evaluation.py from CharlesZhang-USTC/snf_experiments: direct link, hf CLI and curl.
- Browser
- Download file 4.02 kB
-
https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/method_v2/evidence/code/suite/evaluation.py
- Command line
-
hf download hf://CharlesZhang-USTC/snf_experiments/method_v2/evidence/code/suite/evaluation.py
-
curl -L -o evaluation.py https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/method_v2/evidence/code/suite/evaluation.py
4.02 kB
| """Supplementary quality diagnostics; not FID/FVD/HumanML3D retrieval evaluation.""" | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| from .results import atomic_json | |
| def evaluate(run,weights,device='cuda',all_jobs=False): | |
| import torch | |
| from transformers import CLIPModel,CLIPProcessor | |
| from huggingface_hub import HfApi,snapshot_download | |
| from PIL import Image | |
| import lpips | |
| root=Path(run);weights=Path(weights);lockpath=weights/'evaluation.lock.json' | |
| lock=json.loads(lockpath.read_text()) if lockpath.exists() else {} | |
| repo='openai/clip-vit-large-patch14' | |
| if repo not in lock:lock[repo]=HfApi().model_info(repo).sha;atomic_json(lockpath,lock) | |
| path=snapshot_download(repo,revision=lock[repo]);clip=CLIPModel.from_pretrained(path).to(device).eval();processor=CLIPProcessor.from_pretrained(path) | |
| perceptual=lpips.LPIPS(net='alex').to(device).eval();rows=[] | |
| def frames(folder,model): | |
| if model=='self_forcing': | |
| import imageio.v3 as iio | |
| arr=iio.imread(folder/'output.mp4');return [Image.fromarray(arr[i]) for i in np.linspace(0,len(arr)-1,8).astype(int)] | |
| return [Image.open(folder/'output.png').convert('RGB')] | |
| for file in sorted((root/'jobs').glob('*/result.json')): | |
| record=json.loads(file.read_text()) | |
| if record.get('status')!='complete':continue | |
| j=record['job'];model=j['model'] | |
| if not all_jobs: | |
| if j.get('study')=='control_v2': | |
| midpoint=j.get('calibration_mode') in ('raw','mapped') and j.get('requested_effect')==.5 | |
| endpoint=j.get('calibration_mode') in ('anchor','endpoint') and j['alpha']==1. | |
| if j['seed']!=3 or not (midpoint or endpoint):continue | |
| elif j['experiment']!='main' or j['seed']!=0 or j['alpha'] not in (0.,.5,1.):continue | |
| row=dict(id=j['id'],model=model,task=j['task']['id'],seed=j['seed'], | |
| method_id=j.get('method_id'),calibration_mode=j.get('calibration_mode'), | |
| requested_effect=j.get('requested_effect')) | |
| if model=='momask': | |
| a=np.load(file.parent/'output.joints.npy');velocity=np.diff(a,axis=0);acceleration=np.diff(velocity,axis=0);jerk=np.diff(acceleration,axis=0) | |
| row.update(mean_joint_speed=float(np.linalg.norm(velocity,axis=-1).mean()*20),mean_joint_acceleration=float(np.linalg.norm(acceleration,axis=-1).mean()*20**2),mean_joint_jerk=float(np.linalg.norm(jerk,axis=-1).mean()*20**3)) | |
| else: | |
| images=frames(file.parent,model);inputs=processor(text=[j['task']['prompt']],images=images,return_tensors='pt',padding=True).to(device) | |
| with torch.no_grad(): | |
| output=clip(**inputs);similarity=(output.image_embeds@output.text_embeds.T).mean();row['clip_cosine']=float(similarity) | |
| if j.get('study')=='control_v2' and j['task'].get('reference_prompt'): | |
| target_inputs=processor(text=[j['task']['reference_prompt']],return_tensors='pt',padding=True).to(device) | |
| target_embed=clip.get_text_features(**target_inputs) | |
| target_embed=target_embed/target_embed.norm(dim=-1,keepdim=True) | |
| row['clip_reference_prompt_cosine']=float((output.image_embeds@target_embed.T).mean()) | |
| if model!='self_forcing': | |
| a=np.load(file.parent/'output.npy').astype('float32');ref=np.load(root/'references'/model/j['task']['id']/'output.npy').astype('float32') | |
| with torch.no_grad():row['lpips_reference']=float(perceptual(torch.from_numpy(a).to(device)*2-1,torch.from_numpy(ref).to(device)*2-1).mean()) | |
| rows.append(row) | |
| atomic_json(root/'summary/quality_diagnostics.json',dict(scope='all jobs' if all_jobs else 'legacy main seed=0 alpha=0/0.5/1; control_v2 seed=3 endpoints alpha1 and requested effect .5 raw/mapped',model_lock=lock,video_note='CLIP evaluated on decoded MP4; compression is part of this diagnostic',records=rows)) | |
| return dict(evaluated=len(rows)) | |