File size: 12,978 Bytes
a64600b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 | """Persistent per-model worker. Spawn with the model's own environment."""
import argparse
import json
import os
import time
import traceback
from pathlib import Path
import numpy as np
from .design import ROOT,job_id
from .results import atomic_json,distances,save_media,plot_trace
def v2_baseline_key(job):
"""Include every sampler option; exclude only presentation/calibration metadata."""
ignored = {'id','experiment','requested_effect','requested_beta','calibration_mode',
'calibration_mapping','calibration_mapping_sha256','serves_as_endpoint',
'calibration_provenance','planning_status','slot_id'}
identity = {k:v for k,v in job.items() if k not in ignored}
identity['alpha'] = 0.
return job_id(identity)
def common_v2_job(job):
"""One no-control confidence sampler shared by every compared method."""
# A common anchor must not inherit resampling, a guide, or method-specific
# knobs. Only task, seed, and the shared model-evaluation budget vary.
result = {k:job[k] for k in ('study','study_version','model','task','seed','steps') if k in job}
result.update(alpha=0., arm='confidence', method_id='confidence_duration',
control_schedule='duration', reference_copy=True, authority_weight=.5,
soft_release=True, context_fraction=1., commitment_delay=0.)
result['id']=job_id(result)
return result
def can_reuse_common_v2_baseline(job):
"""Check sampler settings, rather than trusting a method label alone."""
return (job.get('method_id')=='confidence_duration' and job.get('arm')=='confidence'
and job.get('control_schedule','duration')=='duration'
and job.get('reference_copy',True) is True
and job.get('soft_release',True) is True
and job.get('context_fraction',1.)==1. and job.get('commitment_delay',0.)==0.
and job.get('resamples',1)==1)
def audit_control(trace):
if not trace or 'target_count' not in trace[0]: return {}
total = int(trace[0]['target_count'])
direct = sum(t.get('target_direct_copy_count', 0) for t in trace)
exposure = sum(t.get('target_temporary_count', 0) for t in trace)
weighted = sum(t.get('target_newly_count', 0) * t['progress'] for t in trace)
return dict(target_count=total, target_direct_copy_count=direct,
direct_copy_fraction=direct/total if total else 0.,
reference_exposure_count=exposure,
exposure_per_target_step=exposure/(total*len(trace)) if total else 0.,
mean_target_commit_progress=weighted/total if total else 0.,
target_conflict_count=sum(t.get('target_conflict_count', 0) for t in trace),
outside_direct_copy_count=sum(t.get('outside_direct_copy_count', 0) for t in trace),
denoiser_calls=len(trace))
def main():
parser=argparse.ArgumentParser();parser.add_argument('--shard',required=True);parser.add_argument('--run',required=True);parser.add_argument('--weights',required=True)
parser.add_argument('--render-motion',action='store_true');args=parser.parse_args()
jobs=[json.loads(x) for x in Path(args.shard).read_text().splitlines() if x.strip()];run=Path(args.run)
if not jobs:return
import torch
from .adapters import LOADERS
model=jobs[0]['model'];cfg=json.loads((ROOT/'suite/models.json').read_text())[model]
lockfile=Path(args.weights)/'weights.lock.json'
if model not in ('self_forcing','momask'):
if not lockfile.exists():raise FileNotFoundError('Run fetch first, or mount weights with weights.lock.json')
from huggingface_hub import snapshot_download
lock=json.loads(lockfile.read_text())
for key,value in list(cfg.items()):
if key.endswith('repo'):
cfg[key]=snapshot_download(value,revision=lock[value],local_files_only=True)
# Only these trusted, user-mounted official MoMask checkpoints require old pickle support.
if model=='momask':
original=torch.load
def load(*a,**kw):kw.setdefault('weights_only',False);return original(*a,**kw)
torch.load=load
torch.backends.cuda.matmul.allow_tf32=False
instance=LOADERS[model](cfg,args.weights)
cache={};baselines={};common_baselines={};native_saved=set();failed=0
env=dict(torch=torch.__version__,cuda=torch.version.cuda,gpu=torch.cuda.get_device_name(),visible=os.getenv('CUDA_VISIBLE_DEVICES'),
attention_backend='explicit SDPA' if model=='self_forcing' else 'upstream default')
for job in jobs:
if (run/'STOP').exists():break
folder=run/'jobs'/job['id'];result=folder/'result.json'
if result.exists() and json.loads(result.read_text()).get('status')=='complete':continue
started=time.monotonic();trace=[];all_traces=[];auxiliary=dict(reference=0,method_baseline=0,common_baseline=0,native=0)
try:
task=job['task'];tid=task['id']
if tid not in cache:
pixels,tokens=instance.reference(task);cache[tid]=(pixels,tokens);auxiliary['reference']=1
refdir=run/'references'/model/tid;refdir.mkdir(parents=True,exist_ok=True)
np.save(refdir/'output.npy',pixels.float().cpu().numpy(),allow_pickle=False)
save_media(refdir,'reference',pixels.float().cpu().numpy(),model,args.render_motion)
atomic_json(refdir/'task.json',task)
pixels,ref=cache[tid];basekey=(tid,job['seed'],job.get('steps'),job['arm'],job.get('context_fraction'),job.get('commitment_delay'),job.get('block_frames'),job.get('rollback_blocks'),job.get('resamples'))
is_v2=job.get('study')=='control_v2'
if is_v2: basekey=v2_baseline_key(job)
if basekey not in baselines:
basejob=dict(job,alpha=0.);bt=[]
baseline_candidates=[]
for attempt in range(job.get('resamples',1)):
baseline,*_=instance.generate(dict(basejob,seed=job['seed']+attempt),ref,bt)
auxiliary['method_baseline']+=1
bn=baseline.float().cpu().numpy()
bm=distances(bn,pixels.float().cpu().numpy(),bn,task,model)
baseline_candidates.append((bm['target_reference_l1'],bn))
baselines[basekey]=min(baseline_candidates,key=lambda x:x[0])[1]
basedir=run/'baselines'/model/job_id(dict(baseline_identity=basekey))
basedir.mkdir(parents=True,exist_ok=True)
save_media(basedir,'research_alpha0',baselines[basekey],model,args.render_motion)
# Native sampler is saved separately; order-permutation loop is a research sampler.
native_key=(tid,job['seed'])
if not is_v2 or native_key not in native_saved:
native,_=instance.native(task['prompt'],job['seed']);auxiliary['native']=1
native_dir=(run/'baselines'/model/('native_'+job_id(dict(task=task,seed=job['seed'])))) if is_v2 else basedir
save_media(native_dir,'upstream_native',native.float().cpu().numpy(),model,args.render_motion)
if is_v2: atomic_json(native_dir/'identity.json',dict(task=task,seed=job['seed'],role='upstream_native_quality_baseline'))
native_saved.add(native_key)
atomic_json(basedir/'identity.json',dict(baseline_identity=basekey,job=job))
common_key=v2_baseline_key(common_v2_job(job)) if is_v2 else None
if is_v2 and common_key not in common_baselines:
if can_reuse_common_v2_baseline(job):
common_baselines[common_key]=baselines[basekey]
else:
anchor,*_=instance.generate(common_v2_job(job),ref,[]);auxiliary['common_baseline']=1
common_baselines[common_key]=anchor.float().cpu().numpy()
common_dir=run/'baselines'/model/('common_'+common_key)
save_media(common_dir,'common_alpha0',common_baselines[common_key],model,args.render_motion)
atomic_json(common_dir/'identity.json',dict(role='common_confidence_duration_alpha0',job=common_v2_job(job)))
torch.cuda.reset_peak_memory_stats();torch.cuda.synchronize()
infer_start=time.monotonic()
attempts=job.get('resamples',1);best=None;all_traces=[]
for attempt in range(attempts):
candidate=dict(job,seed=job['seed']+attempt);ct=[]
# Append the live trace before generation, so a failure still
# reports model calls that completed before the exception.
all_traces.append(dict(attempt=attempt,seed=candidate['seed'],trace=ct))
output,state,commits,mask=instance.generate(candidate,ref,ct)
cm=distances(output.float().cpu().numpy(),pixels.float().cpu().numpy(),baselines[basekey],task,model)
if best is None or cm['target_reference_l1']<best[0]:best=(cm['target_reference_l1'],output,state,commits,mask,ct,attempt)
_,output,state,commits,mask,trace,selected_attempt=best;torch.cuda.synchronize()
seconds=time.monotonic()-infer_start
arr=output.float().cpu().numpy();reference=pixels.float().cpu().numpy()
folder.mkdir(parents=True,exist_ok=True)
# Float16 arrays avoid storing uncompressed float32 video for every sweep.
if model!='self_forcing':np.save(folder/'output.npy',arr.astype('float16'),allow_pickle=False)
cpu_state=state.detach().cpu()
if cpu_state.is_floating_point():cpu_state=cpu_state.float()
np.save(folder/'state.npy',cpu_state.numpy(),allow_pickle=False)
np.save(folder/'mask.npy',mask.detach().float().cpu().numpy(),allow_pickle=False)
if commits is not None:np.save(folder/'commit.npy',commits.cpu().numpy(),allow_pickle=False)
(folder/'trace.jsonl').write_text(''.join(json.dumps(t)+'\n' for t in trace),encoding='utf-8')
if attempts>1:atomic_json(folder/'resampling.json',dict(selection='minimum target reference L1',selected_attempt=selected_attempt,attempts=all_traces))
media=save_media(folder,'output',arr,model,args.render_motion);plot_trace(folder,trace,None if commits is None else commits.cpu().numpy(),spatial=model in ('meissonic','showo'))
metrics=distances(arr,reference,baselines[basekey],task,model)
if is_v2:
shared=distances(arr,reference,common_baselines[common_key],task,model)
metrics.update(common_collateral_l1=shared['collateral_l1'],
common_target_baseline_l1=shared['target_baseline_l1'],
common_anchor_target_reference_l1=distances(common_baselines[common_key],reference,common_baselines[common_key],task,model)['target_reference_l1'],
control_audit=audit_control(trace))
if 'requested_beta' in job:
metrics['requested_beta_error']=abs(-metrics['target_reference_l1']-job['requested_beta'])
atomic_json(result,dict(status='complete',job=job,metrics=metrics,media=media,inference_seconds=seconds,
total_seconds=time.monotonic()-started,control_loop_model_evaluations=sum(len(x['trace']) for x in all_traces),
nfe_scope='Control denoiser calls; Self Forcing includes cache refresh. Tokenizer, text encoder and MoMask residual decoder are excluded; total_seconds includes full measured job work.',
peak_memory_gib=torch.cuda.max_memory_allocated()/2**30,environment=env,
auxiliary_generations=auxiliary))
print(json.dumps(dict(id=job['id'],status='complete',seconds=round(seconds,3))),flush=True)
except Exception:
failed+=1;atomic_json(result,dict(status='failed',job=job,error=traceback.format_exc(),environment=env,
total_seconds=time.monotonic()-started,auxiliary_generations=auxiliary,
control_loop_model_evaluations=sum(len(x['trace']) for x in all_traces),
nfe_scope='Completed traced control calls before failure; the failed call and untraced auxiliary work are not counted. Total seconds includes all elapsed failed-job work.'))
print(traceback.format_exc(),flush=True)
# References and baselines live on CPU except the current reference; release old tasks.
if len(cache)>2:
oldest=next(iter(cache));del cache[oldest]
baseline_limit=128 if job.get('study')=='control_v2' else 4
if len(baselines)>baseline_limit:baselines.pop(next(iter(baselines)))
if len(common_baselines)>12:common_baselines.pop(next(iter(common_baselines)))
if failed:raise SystemExit(1)
if __name__=='__main__':main()
|