File size: 12,978 Bytes
a64600b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
"""Persistent per-model worker. Spawn with the model's own environment."""
import argparse
import json
import os
import time
import traceback
from pathlib import Path
import numpy as np
from .design import ROOT,job_id
from .results import atomic_json,distances,save_media,plot_trace

def v2_baseline_key(job):
    """Include every sampler option; exclude only presentation/calibration metadata."""
    ignored = {'id','experiment','requested_effect','requested_beta','calibration_mode',
               'calibration_mapping','calibration_mapping_sha256','serves_as_endpoint',
               'calibration_provenance','planning_status','slot_id'}
    identity = {k:v for k,v in job.items() if k not in ignored}
    identity['alpha'] = 0.
    return job_id(identity)


def common_v2_job(job):
    """One no-control confidence sampler shared by every compared method."""
    # A common anchor must not inherit resampling, a guide, or method-specific
    # knobs. Only task, seed, and the shared model-evaluation budget vary.
    result = {k:job[k] for k in ('study','study_version','model','task','seed','steps') if k in job}
    result.update(alpha=0., arm='confidence', method_id='confidence_duration',
                  control_schedule='duration', reference_copy=True, authority_weight=.5,
                  soft_release=True, context_fraction=1., commitment_delay=0.)
    result['id']=job_id(result)
    return result


def can_reuse_common_v2_baseline(job):
    """Check sampler settings, rather than trusting a method label alone."""
    return (job.get('method_id')=='confidence_duration' and job.get('arm')=='confidence'
            and job.get('control_schedule','duration')=='duration'
            and job.get('reference_copy',True) is True
            and job.get('soft_release',True) is True
            and job.get('context_fraction',1.)==1. and job.get('commitment_delay',0.)==0.
            and job.get('resamples',1)==1)


def audit_control(trace):
    if not trace or 'target_count' not in trace[0]: return {}
    total = int(trace[0]['target_count'])
    direct = sum(t.get('target_direct_copy_count', 0) for t in trace)
    exposure = sum(t.get('target_temporary_count', 0) for t in trace)
    weighted = sum(t.get('target_newly_count', 0) * t['progress'] for t in trace)
    return dict(target_count=total, target_direct_copy_count=direct,
                direct_copy_fraction=direct/total if total else 0.,
                reference_exposure_count=exposure,
                exposure_per_target_step=exposure/(total*len(trace)) if total else 0.,
                mean_target_commit_progress=weighted/total if total else 0.,
                target_conflict_count=sum(t.get('target_conflict_count', 0) for t in trace),
                outside_direct_copy_count=sum(t.get('outside_direct_copy_count', 0) for t in trace),
                denoiser_calls=len(trace))

def main():
    parser=argparse.ArgumentParser();parser.add_argument('--shard',required=True);parser.add_argument('--run',required=True);parser.add_argument('--weights',required=True)
    parser.add_argument('--render-motion',action='store_true');args=parser.parse_args()
    jobs=[json.loads(x) for x in Path(args.shard).read_text().splitlines() if x.strip()];run=Path(args.run)
    if not jobs:return
    import torch
    from .adapters import LOADERS
    model=jobs[0]['model'];cfg=json.loads((ROOT/'suite/models.json').read_text())[model]
    lockfile=Path(args.weights)/'weights.lock.json'
    if model not in ('self_forcing','momask'):
        if not lockfile.exists():raise FileNotFoundError('Run fetch first, or mount weights with weights.lock.json')
        from huggingface_hub import snapshot_download
        lock=json.loads(lockfile.read_text())
        for key,value in list(cfg.items()):
            if key.endswith('repo'):
                cfg[key]=snapshot_download(value,revision=lock[value],local_files_only=True)
    # Only these trusted, user-mounted official MoMask checkpoints require old pickle support.
    if model=='momask':
        original=torch.load
        def load(*a,**kw):kw.setdefault('weights_only',False);return original(*a,**kw)
        torch.load=load
    torch.backends.cuda.matmul.allow_tf32=False
    instance=LOADERS[model](cfg,args.weights)
    cache={};baselines={};common_baselines={};native_saved=set();failed=0
    env=dict(torch=torch.__version__,cuda=torch.version.cuda,gpu=torch.cuda.get_device_name(),visible=os.getenv('CUDA_VISIBLE_DEVICES'),
             attention_backend='explicit SDPA' if model=='self_forcing' else 'upstream default')
    for job in jobs:
        if (run/'STOP').exists():break
        folder=run/'jobs'/job['id'];result=folder/'result.json'
        if result.exists() and json.loads(result.read_text()).get('status')=='complete':continue
        started=time.monotonic();trace=[];all_traces=[];auxiliary=dict(reference=0,method_baseline=0,common_baseline=0,native=0)
        try:
            task=job['task'];tid=task['id']
            if tid not in cache:
                pixels,tokens=instance.reference(task);cache[tid]=(pixels,tokens);auxiliary['reference']=1
                refdir=run/'references'/model/tid;refdir.mkdir(parents=True,exist_ok=True)
                np.save(refdir/'output.npy',pixels.float().cpu().numpy(),allow_pickle=False)
                save_media(refdir,'reference',pixels.float().cpu().numpy(),model,args.render_motion)
                atomic_json(refdir/'task.json',task)
            pixels,ref=cache[tid];basekey=(tid,job['seed'],job.get('steps'),job['arm'],job.get('context_fraction'),job.get('commitment_delay'),job.get('block_frames'),job.get('rollback_blocks'),job.get('resamples'))
            is_v2=job.get('study')=='control_v2'
            if is_v2: basekey=v2_baseline_key(job)
            if basekey not in baselines:
                basejob=dict(job,alpha=0.);bt=[]
                baseline_candidates=[]
                for attempt in range(job.get('resamples',1)):
                    baseline,*_=instance.generate(dict(basejob,seed=job['seed']+attempt),ref,bt)
                    auxiliary['method_baseline']+=1
                    bn=baseline.float().cpu().numpy()
                    bm=distances(bn,pixels.float().cpu().numpy(),bn,task,model)
                    baseline_candidates.append((bm['target_reference_l1'],bn))
                baselines[basekey]=min(baseline_candidates,key=lambda x:x[0])[1]
                basedir=run/'baselines'/model/job_id(dict(baseline_identity=basekey))
                basedir.mkdir(parents=True,exist_ok=True)
                save_media(basedir,'research_alpha0',baselines[basekey],model,args.render_motion)
                # Native sampler is saved separately; order-permutation loop is a research sampler.
                native_key=(tid,job['seed'])
                if not is_v2 or native_key not in native_saved:
                    native,_=instance.native(task['prompt'],job['seed']);auxiliary['native']=1
                    native_dir=(run/'baselines'/model/('native_'+job_id(dict(task=task,seed=job['seed'])))) if is_v2 else basedir
                    save_media(native_dir,'upstream_native',native.float().cpu().numpy(),model,args.render_motion)
                    if is_v2: atomic_json(native_dir/'identity.json',dict(task=task,seed=job['seed'],role='upstream_native_quality_baseline'))
                    native_saved.add(native_key)
                atomic_json(basedir/'identity.json',dict(baseline_identity=basekey,job=job))
            common_key=v2_baseline_key(common_v2_job(job)) if is_v2 else None
            if is_v2 and common_key not in common_baselines:
                if can_reuse_common_v2_baseline(job):
                    common_baselines[common_key]=baselines[basekey]
                else:
                    anchor,*_=instance.generate(common_v2_job(job),ref,[]);auxiliary['common_baseline']=1
                    common_baselines[common_key]=anchor.float().cpu().numpy()
                common_dir=run/'baselines'/model/('common_'+common_key)
                save_media(common_dir,'common_alpha0',common_baselines[common_key],model,args.render_motion)
                atomic_json(common_dir/'identity.json',dict(role='common_confidence_duration_alpha0',job=common_v2_job(job)))
            torch.cuda.reset_peak_memory_stats();torch.cuda.synchronize()
            infer_start=time.monotonic()
            attempts=job.get('resamples',1);best=None;all_traces=[]
            for attempt in range(attempts):
                candidate=dict(job,seed=job['seed']+attempt);ct=[]
                # Append the live trace before generation, so a failure still
                # reports model calls that completed before the exception.
                all_traces.append(dict(attempt=attempt,seed=candidate['seed'],trace=ct))
                output,state,commits,mask=instance.generate(candidate,ref,ct)
                cm=distances(output.float().cpu().numpy(),pixels.float().cpu().numpy(),baselines[basekey],task,model)
                if best is None or cm['target_reference_l1']<best[0]:best=(cm['target_reference_l1'],output,state,commits,mask,ct,attempt)
            _,output,state,commits,mask,trace,selected_attempt=best;torch.cuda.synchronize()
            seconds=time.monotonic()-infer_start
            arr=output.float().cpu().numpy();reference=pixels.float().cpu().numpy()
            folder.mkdir(parents=True,exist_ok=True)
            # Float16 arrays avoid storing uncompressed float32 video for every sweep.
            if model!='self_forcing':np.save(folder/'output.npy',arr.astype('float16'),allow_pickle=False)
            cpu_state=state.detach().cpu()
            if cpu_state.is_floating_point():cpu_state=cpu_state.float()
            np.save(folder/'state.npy',cpu_state.numpy(),allow_pickle=False)
            np.save(folder/'mask.npy',mask.detach().float().cpu().numpy(),allow_pickle=False)
            if commits is not None:np.save(folder/'commit.npy',commits.cpu().numpy(),allow_pickle=False)
            (folder/'trace.jsonl').write_text(''.join(json.dumps(t)+'\n' for t in trace),encoding='utf-8')
            if attempts>1:atomic_json(folder/'resampling.json',dict(selection='minimum target reference L1',selected_attempt=selected_attempt,attempts=all_traces))
            media=save_media(folder,'output',arr,model,args.render_motion);plot_trace(folder,trace,None if commits is None else commits.cpu().numpy(),spatial=model in ('meissonic','showo'))
            metrics=distances(arr,reference,baselines[basekey],task,model)
            if is_v2:
                shared=distances(arr,reference,common_baselines[common_key],task,model)
                metrics.update(common_collateral_l1=shared['collateral_l1'],
                               common_target_baseline_l1=shared['target_baseline_l1'],
                               common_anchor_target_reference_l1=distances(common_baselines[common_key],reference,common_baselines[common_key],task,model)['target_reference_l1'],
                               control_audit=audit_control(trace))
            if 'requested_beta' in job:
                metrics['requested_beta_error']=abs(-metrics['target_reference_l1']-job['requested_beta'])
            atomic_json(result,dict(status='complete',job=job,metrics=metrics,media=media,inference_seconds=seconds,
                        total_seconds=time.monotonic()-started,control_loop_model_evaluations=sum(len(x['trace']) for x in all_traces),
                        nfe_scope='Control denoiser calls; Self Forcing includes cache refresh. Tokenizer, text encoder and MoMask residual decoder are excluded; total_seconds includes full measured job work.',
                        peak_memory_gib=torch.cuda.max_memory_allocated()/2**30,environment=env,
                        auxiliary_generations=auxiliary))
            print(json.dumps(dict(id=job['id'],status='complete',seconds=round(seconds,3))),flush=True)
        except Exception:
            failed+=1;atomic_json(result,dict(status='failed',job=job,error=traceback.format_exc(),environment=env,
                        total_seconds=time.monotonic()-started,auxiliary_generations=auxiliary,
                        control_loop_model_evaluations=sum(len(x['trace']) for x in all_traces),
                        nfe_scope='Completed traced control calls before failure; the failed call and untraced auxiliary work are not counted. Total seconds includes all elapsed failed-job work.'))
            print(traceback.format_exc(),flush=True)
        # References and baselines live on CPU except the current reference; release old tasks.
        if len(cache)>2:
            oldest=next(iter(cache));del cache[oldest]
        baseline_limit=128 if job.get('study')=='control_v2' else 4
        if len(baselines)>baseline_limit:baselines.pop(next(iter(baselines)))
        if len(common_baselines)>12:common_baselines.pop(next(iter(common_baselines)))
    if failed:raise SystemExit(1)

if __name__=='__main__':main()