Add Control v2 incremental experiment results (evidence, visuals, 42 fixed samples)
a64600b verified Download method_v2/evidence/code/suite/worker.py from CharlesZhang-USTC/snf_experiments: direct link, hf CLI and curl.
- Browser
- Download file 13 kB
-
https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/method_v2/evidence/code/suite/worker.py
- Command line
-
hf download hf://CharlesZhang-USTC/snf_experiments/method_v2/evidence/code/suite/worker.py
-
curl -L -o worker.py https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/method_v2/evidence/code/suite/worker.py
13 kB
| """Persistent per-model worker. Spawn with the model's own environment.""" | |
| import argparse | |
| import json | |
| import os | |
| import time | |
| import traceback | |
| from pathlib import Path | |
| import numpy as np | |
| from .design import ROOT,job_id | |
| from .results import atomic_json,distances,save_media,plot_trace | |
| def v2_baseline_key(job): | |
| """Include every sampler option; exclude only presentation/calibration metadata.""" | |
| ignored = {'id','experiment','requested_effect','requested_beta','calibration_mode', | |
| 'calibration_mapping','calibration_mapping_sha256','serves_as_endpoint', | |
| 'calibration_provenance','planning_status','slot_id'} | |
| identity = {k:v for k,v in job.items() if k not in ignored} | |
| identity['alpha'] = 0. | |
| return job_id(identity) | |
| def common_v2_job(job): | |
| """One no-control confidence sampler shared by every compared method.""" | |
| # A common anchor must not inherit resampling, a guide, or method-specific | |
| # knobs. Only task, seed, and the shared model-evaluation budget vary. | |
| result = {k:job[k] for k in ('study','study_version','model','task','seed','steps') if k in job} | |
| result.update(alpha=0., arm='confidence', method_id='confidence_duration', | |
| control_schedule='duration', reference_copy=True, authority_weight=.5, | |
| soft_release=True, context_fraction=1., commitment_delay=0.) | |
| result['id']=job_id(result) | |
| return result | |
| def can_reuse_common_v2_baseline(job): | |
| """Check sampler settings, rather than trusting a method label alone.""" | |
| return (job.get('method_id')=='confidence_duration' and job.get('arm')=='confidence' | |
| and job.get('control_schedule','duration')=='duration' | |
| and job.get('reference_copy',True) is True | |
| and job.get('soft_release',True) is True | |
| and job.get('context_fraction',1.)==1. and job.get('commitment_delay',0.)==0. | |
| and job.get('resamples',1)==1) | |
| def audit_control(trace): | |
| if not trace or 'target_count' not in trace[0]: return {} | |
| total = int(trace[0]['target_count']) | |
| direct = sum(t.get('target_direct_copy_count', 0) for t in trace) | |
| exposure = sum(t.get('target_temporary_count', 0) for t in trace) | |
| weighted = sum(t.get('target_newly_count', 0) * t['progress'] for t in trace) | |
| return dict(target_count=total, target_direct_copy_count=direct, | |
| direct_copy_fraction=direct/total if total else 0., | |
| reference_exposure_count=exposure, | |
| exposure_per_target_step=exposure/(total*len(trace)) if total else 0., | |
| mean_target_commit_progress=weighted/total if total else 0., | |
| target_conflict_count=sum(t.get('target_conflict_count', 0) for t in trace), | |
| outside_direct_copy_count=sum(t.get('outside_direct_copy_count', 0) for t in trace), | |
| denoiser_calls=len(trace)) | |
| def main(): | |
| parser=argparse.ArgumentParser();parser.add_argument('--shard',required=True);parser.add_argument('--run',required=True);parser.add_argument('--weights',required=True) | |
| parser.add_argument('--render-motion',action='store_true');args=parser.parse_args() | |
| jobs=[json.loads(x) for x in Path(args.shard).read_text().splitlines() if x.strip()];run=Path(args.run) | |
| if not jobs:return | |
| import torch | |
| from .adapters import LOADERS | |
| model=jobs[0]['model'];cfg=json.loads((ROOT/'suite/models.json').read_text())[model] | |
| lockfile=Path(args.weights)/'weights.lock.json' | |
| if model not in ('self_forcing','momask'): | |
| if not lockfile.exists():raise FileNotFoundError('Run fetch first, or mount weights with weights.lock.json') | |
| from huggingface_hub import snapshot_download | |
| lock=json.loads(lockfile.read_text()) | |
| for key,value in list(cfg.items()): | |
| if key.endswith('repo'): | |
| cfg[key]=snapshot_download(value,revision=lock[value],local_files_only=True) | |
| # Only these trusted, user-mounted official MoMask checkpoints require old pickle support. | |
| if model=='momask': | |
| original=torch.load | |
| def load(*a,**kw):kw.setdefault('weights_only',False);return original(*a,**kw) | |
| torch.load=load | |
| torch.backends.cuda.matmul.allow_tf32=False | |
| instance=LOADERS[model](cfg,args.weights) | |
| cache={};baselines={};common_baselines={};native_saved=set();failed=0 | |
| env=dict(torch=torch.__version__,cuda=torch.version.cuda,gpu=torch.cuda.get_device_name(),visible=os.getenv('CUDA_VISIBLE_DEVICES'), | |
| attention_backend='explicit SDPA' if model=='self_forcing' else 'upstream default') | |
| for job in jobs: | |
| if (run/'STOP').exists():break | |
| folder=run/'jobs'/job['id'];result=folder/'result.json' | |
| if result.exists() and json.loads(result.read_text()).get('status')=='complete':continue | |
| started=time.monotonic();trace=[];all_traces=[];auxiliary=dict(reference=0,method_baseline=0,common_baseline=0,native=0) | |
| try: | |
| task=job['task'];tid=task['id'] | |
| if tid not in cache: | |
| pixels,tokens=instance.reference(task);cache[tid]=(pixels,tokens);auxiliary['reference']=1 | |
| refdir=run/'references'/model/tid;refdir.mkdir(parents=True,exist_ok=True) | |
| np.save(refdir/'output.npy',pixels.float().cpu().numpy(),allow_pickle=False) | |
| save_media(refdir,'reference',pixels.float().cpu().numpy(),model,args.render_motion) | |
| atomic_json(refdir/'task.json',task) | |
| pixels,ref=cache[tid];basekey=(tid,job['seed'],job.get('steps'),job['arm'],job.get('context_fraction'),job.get('commitment_delay'),job.get('block_frames'),job.get('rollback_blocks'),job.get('resamples')) | |
| is_v2=job.get('study')=='control_v2' | |
| if is_v2: basekey=v2_baseline_key(job) | |
| if basekey not in baselines: | |
| basejob=dict(job,alpha=0.);bt=[] | |
| baseline_candidates=[] | |
| for attempt in range(job.get('resamples',1)): | |
| baseline,*_=instance.generate(dict(basejob,seed=job['seed']+attempt),ref,bt) | |
| auxiliary['method_baseline']+=1 | |
| bn=baseline.float().cpu().numpy() | |
| bm=distances(bn,pixels.float().cpu().numpy(),bn,task,model) | |
| baseline_candidates.append((bm['target_reference_l1'],bn)) | |
| baselines[basekey]=min(baseline_candidates,key=lambda x:x[0])[1] | |
| basedir=run/'baselines'/model/job_id(dict(baseline_identity=basekey)) | |
| basedir.mkdir(parents=True,exist_ok=True) | |
| save_media(basedir,'research_alpha0',baselines[basekey],model,args.render_motion) | |
| # Native sampler is saved separately; order-permutation loop is a research sampler. | |
| native_key=(tid,job['seed']) | |
| if not is_v2 or native_key not in native_saved: | |
| native,_=instance.native(task['prompt'],job['seed']);auxiliary['native']=1 | |
| native_dir=(run/'baselines'/model/('native_'+job_id(dict(task=task,seed=job['seed'])))) if is_v2 else basedir | |
| save_media(native_dir,'upstream_native',native.float().cpu().numpy(),model,args.render_motion) | |
| if is_v2: atomic_json(native_dir/'identity.json',dict(task=task,seed=job['seed'],role='upstream_native_quality_baseline')) | |
| native_saved.add(native_key) | |
| atomic_json(basedir/'identity.json',dict(baseline_identity=basekey,job=job)) | |
| common_key=v2_baseline_key(common_v2_job(job)) if is_v2 else None | |
| if is_v2 and common_key not in common_baselines: | |
| if can_reuse_common_v2_baseline(job): | |
| common_baselines[common_key]=baselines[basekey] | |
| else: | |
| anchor,*_=instance.generate(common_v2_job(job),ref,[]);auxiliary['common_baseline']=1 | |
| common_baselines[common_key]=anchor.float().cpu().numpy() | |
| common_dir=run/'baselines'/model/('common_'+common_key) | |
| save_media(common_dir,'common_alpha0',common_baselines[common_key],model,args.render_motion) | |
| atomic_json(common_dir/'identity.json',dict(role='common_confidence_duration_alpha0',job=common_v2_job(job))) | |
| torch.cuda.reset_peak_memory_stats();torch.cuda.synchronize() | |
| infer_start=time.monotonic() | |
| attempts=job.get('resamples',1);best=None;all_traces=[] | |
| for attempt in range(attempts): | |
| candidate=dict(job,seed=job['seed']+attempt);ct=[] | |
| # Append the live trace before generation, so a failure still | |
| # reports model calls that completed before the exception. | |
| all_traces.append(dict(attempt=attempt,seed=candidate['seed'],trace=ct)) | |
| output,state,commits,mask=instance.generate(candidate,ref,ct) | |
| cm=distances(output.float().cpu().numpy(),pixels.float().cpu().numpy(),baselines[basekey],task,model) | |
| if best is None or cm['target_reference_l1']<best[0]:best=(cm['target_reference_l1'],output,state,commits,mask,ct,attempt) | |
| _,output,state,commits,mask,trace,selected_attempt=best;torch.cuda.synchronize() | |
| seconds=time.monotonic()-infer_start | |
| arr=output.float().cpu().numpy();reference=pixels.float().cpu().numpy() | |
| folder.mkdir(parents=True,exist_ok=True) | |
| # Float16 arrays avoid storing uncompressed float32 video for every sweep. | |
| if model!='self_forcing':np.save(folder/'output.npy',arr.astype('float16'),allow_pickle=False) | |
| cpu_state=state.detach().cpu() | |
| if cpu_state.is_floating_point():cpu_state=cpu_state.float() | |
| np.save(folder/'state.npy',cpu_state.numpy(),allow_pickle=False) | |
| np.save(folder/'mask.npy',mask.detach().float().cpu().numpy(),allow_pickle=False) | |
| if commits is not None:np.save(folder/'commit.npy',commits.cpu().numpy(),allow_pickle=False) | |
| (folder/'trace.jsonl').write_text(''.join(json.dumps(t)+'\n' for t in trace),encoding='utf-8') | |
| if attempts>1:atomic_json(folder/'resampling.json',dict(selection='minimum target reference L1',selected_attempt=selected_attempt,attempts=all_traces)) | |
| media=save_media(folder,'output',arr,model,args.render_motion);plot_trace(folder,trace,None if commits is None else commits.cpu().numpy(),spatial=model in ('meissonic','showo')) | |
| metrics=distances(arr,reference,baselines[basekey],task,model) | |
| if is_v2: | |
| shared=distances(arr,reference,common_baselines[common_key],task,model) | |
| metrics.update(common_collateral_l1=shared['collateral_l1'], | |
| common_target_baseline_l1=shared['target_baseline_l1'], | |
| common_anchor_target_reference_l1=distances(common_baselines[common_key],reference,common_baselines[common_key],task,model)['target_reference_l1'], | |
| control_audit=audit_control(trace)) | |
| if 'requested_beta' in job: | |
| metrics['requested_beta_error']=abs(-metrics['target_reference_l1']-job['requested_beta']) | |
| atomic_json(result,dict(status='complete',job=job,metrics=metrics,media=media,inference_seconds=seconds, | |
| total_seconds=time.monotonic()-started,control_loop_model_evaluations=sum(len(x['trace']) for x in all_traces), | |
| nfe_scope='Control denoiser calls; Self Forcing includes cache refresh. Tokenizer, text encoder and MoMask residual decoder are excluded; total_seconds includes full measured job work.', | |
| peak_memory_gib=torch.cuda.max_memory_allocated()/2**30,environment=env, | |
| auxiliary_generations=auxiliary)) | |
| print(json.dumps(dict(id=job['id'],status='complete',seconds=round(seconds,3))),flush=True) | |
| except Exception: | |
| failed+=1;atomic_json(result,dict(status='failed',job=job,error=traceback.format_exc(),environment=env, | |
| total_seconds=time.monotonic()-started,auxiliary_generations=auxiliary, | |
| control_loop_model_evaluations=sum(len(x['trace']) for x in all_traces), | |
| nfe_scope='Completed traced control calls before failure; the failed call and untraced auxiliary work are not counted. Total seconds includes all elapsed failed-job work.')) | |
| print(traceback.format_exc(),flush=True) | |
| # References and baselines live on CPU except the current reference; release old tasks. | |
| if len(cache)>2: | |
| oldest=next(iter(cache));del cache[oldest] | |
| baseline_limit=128 if job.get('study')=='control_v2' else 4 | |
| if len(baselines)>baseline_limit:baselines.pop(next(iter(baselines))) | |
| if len(common_baselines)>12:common_baselines.pop(next(iter(common_baselines))) | |
| if failed:raise SystemExit(1) | |
| if __name__=='__main__':main() | |