File size: 17,283 Bytes
a64600b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
"""Container entry point; GPU models are imported only inside their environments."""
import argparse
import concurrent.futures
import hashlib
import importlib
import json
import os
import subprocess
import sys
import time
from pathlib import Path
from .design import ROOT,MODELS,read_tasks,plan

def selected(value):
    result=MODELS if value=='all' else tuple(value.split(','))
    if not result or not set(result)<=set(MODELS):raise ValueError('Unknown model selection')
    return result

def validate_external_jobs(jobs):
    """Reject manifests that would inflate planned counts or overwrite outputs."""
    from .design import job_id
    if not jobs:raise ValueError('Empty external job manifest')
    ids=[j['id'] for j in jobs]
    if len(ids)!=len(set(ids)):raise ValueError('Duplicate external job IDs')
    models=tuple(dict.fromkeys(j['model'] for j in jobs))
    if not set(models)<=set(MODELS):raise ValueError('Unknown external model')
    for j in jobs:
        if j['id']!=job_id({k:v for k,v in j.items() if k!='id'}):raise ValueError('External job ID mismatch')
        if j.get('planning_status')=='unreachable' or j.get('alpha') is None:
            raise ValueError('Unreachable slots belong in expected_slots.json, not executable jobs.jsonl')
    return models

def validate_expected_slots(jobs, metadata):
    """The sidecar denominator must describe exactly this executable plan."""
    expected=metadata.get('expected_slots')
    if not isinstance(expected,list):raise ValueError('Invalid expected_slots.json schema')
    ids=[j['id'] for j in expected]
    if len(ids)!=len(set(ids)):raise ValueError('Duplicate expected slot IDs')
    executable={j['id']:j for j in expected if j.get('planning_status')!='unreachable'}
    if executable!={j['id']:j for j in jobs}:
        raise ValueError('Executable jobs differ from expected_slots.json; regenerate matching plan metadata')

def source_check():
    lock=json.loads((ROOT/'packaging/sources.lock.json').read_text())
    manifest=json.loads((ROOT/'packaging/sources.manifest.json').read_text())
    for relative,expected in manifest.items():
        path=ROOT/relative
        if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest()!=expected:
            raise ValueError(f'Vendored source checksum mismatch: {relative}')
    return dict(verified_files=len(manifest),revisions=lock)

def import_check(model):
    import torch
    from .adapters import activate,sdpa_flash_fallback
    if model=='sd35':
        from diffusers import StableDiffusion3Pipeline
    else:
        activate(model)
        if model=='meissonic':
            from src.pipeline import Pipeline
            from src.transformer import Transformer2DModel
        elif model=='self_forcing':
            # Upstream memory helper queries current_device at import time.
            # Docker builds have no GPU; this only bypasses that device lookup.
            from unittest.mock import patch
            with patch('torch.cuda.current_device',return_value=0):
                sdpa_flash_fallback()
                from pipeline.causal_inference import CausalInferencePipeline
        elif model=='momask':
            from .momask_loading import loading_functions
            loading_functions()
        elif model=='showo':
            from models import Showo,MAGVITv2
            from training.prompting_utils import UniversalPrompting
    return dict(model=model,imports='passed',torch=torch.__version__,cuda=torch.version.cuda)

def doctor(weights):
    import torch
    result=dict(python=sys.version,torch=torch.__version__,cuda_runtime=torch.version.cuda,
                cuda_available=torch.cuda.is_available(),weights=str(weights),sources=source_check(),models={})
    if not torch.cuda.is_available():raise RuntimeError('CUDA unavailable: use docker run --gpus all and a compatible host driver')
    result['gpus']=[dict(name=torch.cuda.get_device_name(i),capability=torch.cuda.get_device_capability(i),memory_gib=torch.cuda.get_device_properties(i).total_memory/2**30) for i in range(torch.cuda.device_count())]
    # Exercise a real CUDA kernel; availability alone does not prove Blackwell support.
    x=torch.randn(256,256,device='cuda',dtype=torch.bfloat16);result['kernel_finite']=bool(torch.isfinite(x@x).all());torch.cuda.synchronize()
    for model in MODELS:
        p=subprocess.run([str(Path('/opt/envs')/model/'bin/python'),'-m','suite.cli','import-check','--model',model],capture_output=True,text=True)
        result['models'][model]=dict(returncode=p.returncode,output=p.stdout[-2500:],error=p.stderr[-2500:])
    print(json.dumps(result,indent=2,ensure_ascii=False))
    if any(v['returncode'] for v in result['models'].values()):raise SystemExit(1)

def fetch(models,weights,offline=False):
    from huggingface_hub import HfApi,snapshot_download,hf_hub_download
    from .results import atomic_json
    weights=Path(weights);weights.mkdir(parents=True,exist_ok=True)
    path=weights/'weights.lock.json';lock=json.loads(path.read_text()) if path.exists() else {}
    cfg=json.loads((ROOT/'suite/models.json').read_text())
    for model in models:
        if model=='momask':
            destination=weights/'momask/checkpoints/t2m';destination.mkdir(parents=True,exist_ok=True)
            if not (destination/cfg[model]['name']/'opt.txt').exists():
                if offline:raise FileNotFoundError('Missing mounted MoMask checkpoints')
                import gdown,zipfile
                archive=weights/'momask/humanml3d_models.zip'
                gdown.download(id='1vXS7SHJBgWPt59wupQ5UUzhFObrnGkQ0',output=str(archive),quiet=False)
                with zipfile.ZipFile(archive) as z:
                    for member in z.infolist():
                        target=(destination/member.filename).resolve()
                        if not target.is_relative_to(destination.resolve()):raise ValueError('Unsafe checkpoint archive path')
                    z.extractall(destination)
                lock['momask_archive_sha256']=hashlib.sha256(archive.read_bytes()).hexdigest();atomic_json(path,lock)
            continue
        repositories=[v for k,v in cfg[model].items() if k.endswith('repo') and k!='student_repo']
        if model=='self_forcing':repositories.append(cfg[model]['student_repo'])
        for repo in repositories:
            if repo not in lock:
                if offline:raise ValueError(f'No pinned weight revision for {repo}')
                lock[repo]=HfApi().model_info(repo).sha;atomic_json(path,lock)
            rev=lock[repo]
            if model=='self_forcing':
                if repo==cfg[model]['student_repo']:
                    downloaded=hf_hub_download(repo,filename=cfg[model]['student_file'],revision=rev,local_files_only=offline)
                    dest=weights/'self_forcing'/cfg[model]['student_file'];dest.parent.mkdir(parents=True,exist_ok=True)
                    if not dest.exists():dest.symlink_to(Path(downloaded).resolve())
                else:
                    dest=weights/'self_forcing/wan_models/Wan2.1-T2V-1.3B'
                    snapshot_download(repo,revision=rev,local_dir=dest,local_files_only=offline)
            else:snapshot_download(repo,revision=rev,local_files_only=offline)
    print(json.dumps(dict(weights=str(weights),lock=lock),indent=2))

def run(args):
    from .results import atomic_json,analyze
    source_check()
    tasks=read_tasks(args.tasks);models=selected(args.models)
    jobs=plan(tasks,args.profile,models,args.extras) if not args.jobs else [json.loads(line) for line in Path(args.jobs).read_text().splitlines() if line.strip()]
    if args.jobs:
        models=validate_external_jobs(jobs)
    out=Path(args.output).resolve();out.mkdir(parents=True,exist_ok=True)
    # A run directory is immutable across code/config changes; refuse silent mixing.
    code={str(p.relative_to(ROOT)):hashlib.sha256(p.read_bytes()).hexdigest() for p in (ROOT/'suite').glob('*.py')}
    weightlock=Path(args.weights)/'weights.lock.json'
    configuration_files={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in (ROOT/'suite').glob('*.json')}
    external_metadata={}
    if args.jobs:
        for name in ('expected_slots.json','protocol.json'):
            sidecar=Path(args.jobs).parent/name
            if sidecar.exists():external_metadata[name]=dict(sha256=hashlib.sha256(sidecar.read_bytes()).hexdigest(),data=json.loads(sidecar.read_text(encoding='utf8')))
        if 'expected_slots.json' in external_metadata:
            validate_expected_slots(jobs,external_metadata['expected_slots.json']['data'])
    freezes={m:hashlib.sha256(p.read_bytes()).hexdigest() for m in models if (p:=Path('/opt/envs')/m/'installed.freeze.txt').exists()}
    identity=dict(profile=args.profile,models=models,jobs=jobs,code=code,configuration_files=configuration_files,environment_freezes=freezes,source_manifest_sha256=hashlib.sha256((ROOT/'packaging/sources.manifest.json').read_bytes()).hexdigest(),
                  weights=json.loads(weightlock.read_text()) if weightlock.exists() else {},render_motion=args.render_motion,
                  external_plan_metadata={k:v['sha256'] for k,v in external_metadata.items()})
    digest=hashlib.sha256(json.dumps(identity,sort_keys=True).encode()).hexdigest()
    manifest=out/'run.json'
    if manifest.exists():
        previous=json.loads(manifest.read_text())
        if previous['digest']!=digest:raise ValueError('Run identity changed. Use a new --output directory.')
        if not args.resume:raise ValueError('Existing run; pass --resume')
    else:atomic_json(manifest,dict(digest=digest,configuration=identity,created_utc=time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())))
    for name,metadata in external_metadata.items():atomic_json(out/name,metadata['data'])
    (out/'plan.jsonl').write_text(''.join(json.dumps(j,ensure_ascii=False)+'\n' for j in jobs),encoding='utf-8')
    counts={model:sum(j['model']==model for j in jobs) for model in models};print(json.dumps(dict(job_counts=counts,output=str(out)),indent=2),flush=True)
    if args.dry_run:return
    import torch
    if not torch.cuda.is_available():raise RuntimeError('CUDA required. Use --dry-run for CPU planning.')
    gpus=args.gpus.split(',') if args.gpus else [str(i) for i in range(torch.cuda.device_count())]
    if len(gpus)!=len(set(gpus)):raise ValueError('GPU list must be unique')
    # Jobs for one task remain together, minimizing duplicate reference generation.
    shards=[]
    for model in models:
        bytask={}
        for j in jobs:
            if j['model']==model:bytask.setdefault(j['task']['id'],[]).append(j)
        buckets=[[] for _ in range(min(len(gpus),len(bytask)))]
        for group in sorted(bytask.values(),key=len,reverse=True):min(buckets,key=len).extend(group)
        for i,bucket in enumerate(buckets):
            p=out/'shards'/f'{model}_{i}.jsonl';p.parent.mkdir(exist_ok=True)
            p.write_text(''.join(json.dumps(j)+'\n' for j in bucket));shards.append((model,p))
    import queue
    work=queue.Queue()
    for item in sorted(shards,key=lambda x:x[1].stat().st_size,reverse=True):work.put(item)
    def gpu_worker(gpu):
        errors=[]
        while not work.empty() and not (out/'STOP').exists():
            try:model,shard=work.get_nowait()
            except queue.Empty:break
            env=dict(os.environ,CUDA_VISIBLE_DEVICES=gpu,PYTHONPATH=str(ROOT),HF_HOME=str(Path(args.weights)/'huggingface'),TORCH_HOME=str(Path(args.weights)/'torch'),XDG_CACHE_HOME=str(Path(args.weights)/'cache'))
            cmd=[str(Path('/opt/envs')/model/'bin/python'),'-m','suite.worker','--shard',str(shard),'--run',str(out),'--weights',str(Path(args.weights).resolve())]
            if args.render_motion:cmd.append('--render-motion')
            logs=out/'logs';logs.mkdir(exist_ok=True)
            with (logs/f'{shard.stem}.log').open('a') as f:
                result=subprocess.run(cmd,env=env,stdout=f,stderr=subprocess.STDOUT)
            if result.returncode:errors.append(dict(model=model,shard=str(shard),returncode=result.returncode))
        return errors
    with concurrent.futures.ThreadPoolExecutor(max_workers=len(gpus)) as pool:
        failures=[error for batch in pool.map(gpu_worker,gpus) for error in batch]
    summary=analyze(out);summary['worker_failures']=failures
    summary['planned']=len(jobs);summary['incomplete']=len(jobs)-summary['completed'];atomic_json(out/'summary/status.json',summary)
    print(json.dumps(summary,ensure_ascii=False,indent=2))
    if failures or summary['incomplete']:raise SystemExit(1)

def main():
    p=argparse.ArgumentParser(prog='snf-suite');sub=p.add_subparsers(dest='command',required=True)
    sub.add_parser('source-check')
    c=sub.add_parser('create-tasks');c.add_argument('--output',default='/data/tasks.curated.jsonl')
    c=sub.add_parser('import-check');c.add_argument('--model',choices=MODELS,required=True)
    c=sub.add_parser('doctor');c.add_argument('--weights',default='/weights')
    c=sub.add_parser('fetch');c.add_argument('--models',default='all');c.add_argument('--weights',default='/weights');c.add_argument('--offline',action='store_true')
    for name in ('run','plan'):
        c=sub.add_parser(name);c.add_argument('--tasks',default=str(ROOT/'suite/tasks.example.jsonl'));c.add_argument('--models',default='all');c.add_argument('--profile',choices=('smoke','pilot','full'),default='smoke')
        c.add_argument('--output',default='/outputs/smoke');c.add_argument('--weights',default='/weights');c.add_argument('--gpus');c.add_argument('--resume',action='store_true');c.add_argument('--dry-run',action='store_true')
        c.add_argument('--extras',action=argparse.BooleanOptionalAction,default=None);c.add_argument('--render-motion',action='store_true')
        c.add_argument('--jobs',help='Validated external jobs.jsonl, e.g. held-out calibration')
    c=sub.add_parser('analyze');c.add_argument('--output',required=True)
    c=sub.add_parser('calibrate');c.add_argument('--run',required=True);c.add_argument('--tasks',required=True);c.add_argument('--targets',required=True,help='Common negative-L1 targets, e.g. --targets=-0.3,-0.2');c.add_argument('--output',required=True)
    c=sub.add_parser('analytic');c.add_argument('--output',default='/outputs/analytic');c.add_argument('--scripts',default='all')
    c=sub.add_parser('recover');c.add_argument('--run',required=True);c.add_argument('--tasks',required=True);c.add_argument('--output',required=True);c.add_argument('--penalty',type=float,default=.5)
    c=sub.add_parser('evaluate');c.add_argument('--run',required=True);c.add_argument('--weights',default='/weights');c.add_argument('--device',default='cuda');c.add_argument('--all-jobs',action='store_true')
    c=sub.add_parser('probe');c.add_argument('--model',choices=('meissonic','momask','showo'),required=True);c.add_argument('--weights',default='/weights');c.add_argument('--tasks',required=True);c.add_argument('--output',required=True);c.add_argument('--groups',type=int,default=16);c.add_argument('--limit',type=int,default=8)
    args=p.parse_args()
    if args.command=='source-check':print(json.dumps(source_check(),indent=2))
    elif args.command=='create-tasks':
        from .create_tasks import create
        path=Path(args.output);path.parent.mkdir(parents=True,exist_ok=True);path.write_text(''.join(json.dumps(t)+'\n' for t in create()),encoding='utf-8');print(path)
    elif args.command=='import-check':print(json.dumps(import_check(args.model)))
    elif args.command=='doctor':doctor(args.weights)
    elif args.command=='fetch':fetch(selected(args.models),args.weights,args.offline)
    elif args.command in ('run','plan'):
        if args.command=='plan':args.dry_run=True
        run(args)
    elif args.command=='analyze':
        from .results import analyze
        print(json.dumps(analyze(args.output),indent=2))
    elif args.command=='calibrate':
        from .calibration import fit_and_plan
        print(json.dumps(fit_and_plan(args.run,args.tasks,[float(x) for x in args.targets.split(',')],args.output),indent=2))
    elif args.command=='analytic':
        from .analytic import ORDER
        chosen=ORDER if args.scripts=='all' else args.scripts.split(',');out=Path(args.output).resolve();out.mkdir(parents=True,exist_ok=True)
        if not set(chosen)<=set(ORDER):raise ValueError('Unknown analytic script')
        for script in chosen:
            with (out/f'{script}.log').open('w') as f:subprocess.run([sys.executable,'-m','suite.analytic','--script',script,'--output',str(out)],stdout=f,stderr=subprocess.STDOUT,check=True)
    elif args.command=='recover':
        from .recovery import choose_and_plan
        print(json.dumps(choose_and_plan(args.run,args.tasks,args.output,args.penalty),indent=2))
    elif args.command=='evaluate':
        from .evaluation import evaluate
        print(json.dumps(evaluate(args.run,args.weights,args.device,args.all_jobs),indent=2))
    elif args.command=='probe':
        subprocess.run([f'/opt/envs/{args.model}/bin/python','-m','suite.probe','--model',args.model,'--weights',args.weights,'--tasks',args.tasks,'--output',args.output,'--groups',str(args.groups),'--limit',str(args.limit)],check=True)

if __name__=='__main__':main()