CharlesZhang-USTC's picture
Add files using upload-large-folder tool
ea1b5d0 verified
Raw History Blame Contribute Delete
7.56 kB
"""Persistent per-model worker. Spawn with the model's own environment."""
import argparse
import json
import os
import time
import traceback
from pathlib import Path
import numpy as np
from .design import ROOT,job_id
from .results import atomic_json,distances,save_media,plot_trace
def main():
parser=argparse.ArgumentParser();parser.add_argument('--shard',required=True);parser.add_argument('--run',required=True);parser.add_argument('--weights',required=True)
parser.add_argument('--render-motion',action='store_true');args=parser.parse_args()
jobs=[json.loads(x) for x in Path(args.shard).read_text().splitlines() if x.strip()];run=Path(args.run)
if not jobs:return
import torch
from .adapters import LOADERS
model=jobs[0]['model'];cfg=json.loads((ROOT/'suite/models.json').read_text())[model]
lockfile=Path(args.weights)/'weights.lock.json'
if model not in ('self_forcing','momask'):
if not lockfile.exists():raise FileNotFoundError('Run fetch first, or mount weights with weights.lock.json')
from huggingface_hub import snapshot_download
lock=json.loads(lockfile.read_text())
for key,value in list(cfg.items()):
if key.endswith('repo'):
cfg[key]=snapshot_download(value,revision=lock[value],local_files_only=True)
# Only these trusted, user-mounted official MoMask checkpoints require old pickle support.
if model=='momask':
original=torch.load
def load(*a,**kw):kw.setdefault('weights_only',False);return original(*a,**kw)
torch.load=load
torch.backends.cuda.matmul.allow_tf32=False
instance=LOADERS[model](cfg,args.weights)
cache={};baselines={};failed=0
env=dict(torch=torch.__version__,cuda=torch.version.cuda,gpu=torch.cuda.get_device_name(),visible=os.getenv('CUDA_VISIBLE_DEVICES'),
attention_backend='explicit SDPA' if model=='self_forcing' else 'upstream default')
for job in jobs:
if (run/'STOP').exists():break
folder=run/'jobs'/job['id'];result=folder/'result.json'
if result.exists() and json.loads(result.read_text()).get('status')=='complete':continue
started=time.monotonic();trace=[]
try:
task=job['task'];tid=task['id']
if tid not in cache:
pixels,tokens=instance.reference(task);cache[tid]=(pixels,tokens)
refdir=run/'references'/model/tid;refdir.mkdir(parents=True,exist_ok=True)
np.save(refdir/'output.npy',pixels.float().cpu().numpy(),allow_pickle=False)
save_media(refdir,'reference',pixels.float().cpu().numpy(),model,args.render_motion)
atomic_json(refdir/'task.json',task)
pixels,ref=cache[tid];basekey=(tid,job['seed'],job.get('steps'),job['arm'],job.get('context_fraction'),job.get('commitment_delay'),job.get('block_frames'),job.get('rollback_blocks'),job.get('resamples'))
if basekey not in baselines:
basejob=dict(job,alpha=0.);bt=[]
baseline_candidates=[]
for attempt in range(job.get('resamples',1)):
baseline,*_=instance.generate(dict(basejob,seed=job['seed']+attempt),ref,bt)
bn=baseline.float().cpu().numpy()
bm=distances(bn,pixels.float().cpu().numpy(),bn,task,model)
baseline_candidates.append((bm['target_reference_l1'],bn))
baselines[basekey]=min(baseline_candidates,key=lambda x:x[0])[1]
basedir=run/'baselines'/model/job_id(dict(baseline_identity=basekey))
basedir.mkdir(parents=True,exist_ok=True)
save_media(basedir,'research_alpha0',baselines[basekey],model,args.render_motion)
# Native sampler is saved separately; order-permutation loop is a research sampler.
native,_=instance.native(task['prompt'],job['seed'])
save_media(basedir,'upstream_native',native.float().cpu().numpy(),model,args.render_motion)
atomic_json(basedir/'identity.json',dict(baseline_identity=basekey,job=job))
torch.cuda.reset_peak_memory_stats();torch.cuda.synchronize()
infer_start=time.monotonic()
attempts=job.get('resamples',1);best=None;all_traces=[]
for attempt in range(attempts):
candidate=dict(job,seed=job['seed']+attempt);ct=[]
output,state,commits,mask=instance.generate(candidate,ref,ct)
cm=distances(output.float().cpu().numpy(),pixels.float().cpu().numpy(),baselines[basekey],task,model)
all_traces.append(dict(attempt=attempt,seed=candidate['seed'],trace=ct))
if best is None or cm['target_reference_l1']<best[0]:best=(cm['target_reference_l1'],output,state,commits,mask,ct,attempt)
_,output,state,commits,mask,trace,selected_attempt=best;torch.cuda.synchronize()
seconds=time.monotonic()-infer_start
arr=output.float().cpu().numpy();reference=pixels.float().cpu().numpy()
folder.mkdir(parents=True,exist_ok=True)
# Float16 arrays avoid storing uncompressed float32 video for every sweep.
if model!='self_forcing':np.save(folder/'output.npy',arr.astype('float16'),allow_pickle=False)
cpu_state=state.detach().cpu()
if cpu_state.is_floating_point():cpu_state=cpu_state.float()
np.save(folder/'state.npy',cpu_state.numpy(),allow_pickle=False)
np.save(folder/'mask.npy',mask.detach().float().cpu().numpy(),allow_pickle=False)
if commits is not None:np.save(folder/'commit.npy',commits.cpu().numpy(),allow_pickle=False)
(folder/'trace.jsonl').write_text(''.join(json.dumps(t)+'\n' for t in trace),encoding='utf-8')
if attempts>1:atomic_json(folder/'resampling.json',dict(selection='minimum target reference L1',selected_attempt=selected_attempt,attempts=all_traces))
media=save_media(folder,'output',arr,model,args.render_motion);plot_trace(folder,trace,None if commits is None else commits.cpu().numpy(),spatial=model in ('meissonic','showo'))
metrics=distances(arr,reference,baselines[basekey],task,model)
if 'requested_beta' in job:
metrics['requested_beta_error']=abs(-metrics['target_reference_l1']-job['requested_beta'])
atomic_json(result,dict(status='complete',job=job,metrics=metrics,media=media,inference_seconds=seconds,
total_seconds=time.monotonic()-started,control_loop_model_evaluations=sum(len(x['trace']) for x in all_traces),
nfe_scope='Control denoiser calls; Self Forcing includes cache refresh. Tokenizer, text encoder and MoMask residual decoder are excluded; total_seconds includes full measured job work.',
peak_memory_gib=torch.cuda.max_memory_allocated()/2**30,environment=env))
print(json.dumps(dict(id=job['id'],status='complete',seconds=round(seconds,3))),flush=True)
except Exception:
failed+=1;atomic_json(result,dict(status='failed',job=job,error=traceback.format_exc(),environment=env))
print(traceback.format_exc(),flush=True)
# References and baselines live on CPU except the current reference; release old tasks.
if len(cache)>2:
oldest=next(iter(cache));del cache[oldest]
if len(baselines)>4:baselines.pop(next(iter(baselines)))
if failed:raise SystemExit(1)
if __name__=='__main__':main()