File size: 15,559 Bytes
ea1b5d0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 | """Container entry point; GPU models are imported only inside their environments."""
import argparse
import concurrent.futures
import hashlib
import importlib
import json
import os
import subprocess
import sys
import time
from pathlib import Path
from .design import ROOT,MODELS,read_tasks,plan
def selected(value):
result=MODELS if value=='all' else tuple(value.split(','))
if not result or not set(result)<=set(MODELS):raise ValueError('Unknown model selection')
return result
def source_check():
lock=json.loads((ROOT/'packaging/sources.lock.json').read_text())
manifest=json.loads((ROOT/'packaging/sources.manifest.json').read_text())
for relative,expected in manifest.items():
path=ROOT/relative
if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest()!=expected:
raise ValueError(f'Vendored source checksum mismatch: {relative}')
return dict(verified_files=len(manifest),revisions=lock)
def import_check(model):
import torch
from .adapters import activate,sdpa_flash_fallback
if model=='sd35':
from diffusers import StableDiffusion3Pipeline
else:
activate(model)
if model=='meissonic':
from src.pipeline import Pipeline
from src.transformer import Transformer2DModel
elif model=='self_forcing':
# Upstream memory helper queries current_device at import time.
# Docker builds have no GPU; this only bypasses that device lookup.
from unittest.mock import patch
with patch('torch.cuda.current_device',return_value=0):
sdpa_flash_fallback()
from pipeline.causal_inference import CausalInferencePipeline
elif model=='momask':
from .momask_loading import loading_functions
loading_functions()
elif model=='showo':
from models import Showo,MAGVITv2
from training.prompting_utils import UniversalPrompting
return dict(model=model,imports='passed',torch=torch.__version__,cuda=torch.version.cuda)
def doctor(weights):
import torch
result=dict(python=sys.version,torch=torch.__version__,cuda_runtime=torch.version.cuda,
cuda_available=torch.cuda.is_available(),weights=str(weights),sources=source_check(),models={})
if not torch.cuda.is_available():raise RuntimeError('CUDA unavailable: use docker run --gpus all and a compatible host driver')
result['gpus']=[dict(name=torch.cuda.get_device_name(i),capability=torch.cuda.get_device_capability(i),memory_gib=torch.cuda.get_device_properties(i).total_memory/2**30) for i in range(torch.cuda.device_count())]
# Exercise a real CUDA kernel; availability alone does not prove Blackwell support.
x=torch.randn(256,256,device='cuda',dtype=torch.bfloat16);result['kernel_finite']=bool(torch.isfinite(x@x).all());torch.cuda.synchronize()
for model in MODELS:
p=subprocess.run([str(Path('/opt/envs')/model/'bin/python'),'-m','suite.cli','import-check','--model',model],capture_output=True,text=True)
result['models'][model]=dict(returncode=p.returncode,output=p.stdout[-2500:],error=p.stderr[-2500:])
print(json.dumps(result,indent=2,ensure_ascii=False))
if any(v['returncode'] for v in result['models'].values()):raise SystemExit(1)
def fetch(models,weights,offline=False):
from huggingface_hub import HfApi,snapshot_download,hf_hub_download
from .results import atomic_json
weights=Path(weights);weights.mkdir(parents=True,exist_ok=True)
path=weights/'weights.lock.json';lock=json.loads(path.read_text()) if path.exists() else {}
cfg=json.loads((ROOT/'suite/models.json').read_text())
for model in models:
if model=='momask':
destination=weights/'momask/checkpoints/t2m';destination.mkdir(parents=True,exist_ok=True)
if not (destination/cfg[model]['name']/'opt.txt').exists():
if offline:raise FileNotFoundError('Missing mounted MoMask checkpoints')
import gdown,zipfile
archive=weights/'momask/humanml3d_models.zip'
gdown.download(id='1vXS7SHJBgWPt59wupQ5UUzhFObrnGkQ0',output=str(archive),quiet=False)
with zipfile.ZipFile(archive) as z:
for member in z.infolist():
target=(destination/member.filename).resolve()
if not target.is_relative_to(destination.resolve()):raise ValueError('Unsafe checkpoint archive path')
z.extractall(destination)
lock['momask_archive_sha256']=hashlib.sha256(archive.read_bytes()).hexdigest();atomic_json(path,lock)
continue
repositories=[v for k,v in cfg[model].items() if k.endswith('repo') and k!='student_repo']
if model=='self_forcing':repositories.append(cfg[model]['student_repo'])
for repo in repositories:
if repo not in lock:
if offline:raise ValueError(f'No pinned weight revision for {repo}')
lock[repo]=HfApi().model_info(repo).sha;atomic_json(path,lock)
rev=lock[repo]
if model=='self_forcing':
if repo==cfg[model]['student_repo']:
downloaded=hf_hub_download(repo,filename=cfg[model]['student_file'],revision=rev,local_files_only=offline)
dest=weights/'self_forcing'/cfg[model]['student_file'];dest.parent.mkdir(parents=True,exist_ok=True)
if not dest.exists():dest.symlink_to(Path(downloaded).resolve())
else:
dest=weights/'self_forcing/wan_models/Wan2.1-T2V-1.3B'
snapshot_download(repo,revision=rev,local_dir=dest,local_files_only=offline)
else:snapshot_download(repo,revision=rev,local_files_only=offline)
print(json.dumps(dict(weights=str(weights),lock=lock),indent=2))
def run(args):
from .results import atomic_json,analyze
source_check()
tasks=read_tasks(args.tasks);models=selected(args.models)
jobs=plan(tasks,args.profile,models,args.extras) if not args.jobs else [json.loads(line) for line in Path(args.jobs).read_text().splitlines() if line.strip()]
if args.jobs:
from .design import job_id
if not jobs:raise ValueError('Empty external job manifest')
models=tuple(dict.fromkeys(j['model'] for j in jobs))
if not set(models)<=set(MODELS):raise ValueError('Unknown external model')
for j in jobs:
if j['id']!=job_id({k:v for k,v in j.items() if k!='id'}):raise ValueError('External job ID mismatch')
out=Path(args.output).resolve();out.mkdir(parents=True,exist_ok=True)
# A run directory is immutable across code/config changes; refuse silent mixing.
code={str(p.relative_to(ROOT)):hashlib.sha256(p.read_bytes()).hexdigest() for p in (ROOT/'suite').glob('*.py')}
weightlock=Path(args.weights)/'weights.lock.json'
configuration_files={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in (ROOT/'suite').glob('*.json')}
freezes={m:hashlib.sha256(p.read_bytes()).hexdigest() for m in models if (p:=Path('/opt/envs')/m/'installed.freeze.txt').exists()}
identity=dict(profile=args.profile,models=models,jobs=jobs,code=code,configuration_files=configuration_files,environment_freezes=freezes,source_manifest_sha256=hashlib.sha256((ROOT/'packaging/sources.manifest.json').read_bytes()).hexdigest(),
weights=json.loads(weightlock.read_text()) if weightlock.exists() else {},render_motion=args.render_motion)
digest=hashlib.sha256(json.dumps(identity,sort_keys=True).encode()).hexdigest()
manifest=out/'run.json'
if manifest.exists():
previous=json.loads(manifest.read_text())
if previous['digest']!=digest:raise ValueError('Run identity changed. Use a new --output directory.')
if not args.resume:raise ValueError('Existing run; pass --resume')
else:atomic_json(manifest,dict(digest=digest,configuration=identity,created_utc=time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())))
(out/'plan.jsonl').write_text(''.join(json.dumps(j,ensure_ascii=False)+'\n' for j in jobs),encoding='utf-8')
counts={model:sum(j['model']==model for j in jobs) for model in models};print(json.dumps(dict(job_counts=counts,output=str(out)),indent=2),flush=True)
if args.dry_run:return
import torch
if not torch.cuda.is_available():raise RuntimeError('CUDA required. Use --dry-run for CPU planning.')
gpus=args.gpus.split(',') if args.gpus else [str(i) for i in range(torch.cuda.device_count())]
if len(gpus)!=len(set(gpus)):raise ValueError('GPU list must be unique')
# Jobs for one task remain together, minimizing duplicate reference generation.
shards=[]
for model in models:
bytask={}
for j in jobs:
if j['model']==model:bytask.setdefault(j['task']['id'],[]).append(j)
buckets=[[] for _ in range(min(len(gpus),len(bytask)))]
for group in sorted(bytask.values(),key=len,reverse=True):min(buckets,key=len).extend(group)
for i,bucket in enumerate(buckets):
p=out/'shards'/f'{model}_{i}.jsonl';p.parent.mkdir(exist_ok=True)
p.write_text(''.join(json.dumps(j)+'\n' for j in bucket));shards.append((model,p))
import queue
work=queue.Queue()
for item in sorted(shards,key=lambda x:x[1].stat().st_size,reverse=True):work.put(item)
def gpu_worker(gpu):
errors=[]
while not work.empty() and not (out/'STOP').exists():
try:model,shard=work.get_nowait()
except queue.Empty:break
env=dict(os.environ,CUDA_VISIBLE_DEVICES=gpu,PYTHONPATH=str(ROOT),HF_HOME=str(Path(args.weights)/'huggingface'),TORCH_HOME=str(Path(args.weights)/'torch'),XDG_CACHE_HOME=str(Path(args.weights)/'cache'))
cmd=[str(Path('/opt/envs')/model/'bin/python'),'-m','suite.worker','--shard',str(shard),'--run',str(out),'--weights',str(Path(args.weights).resolve())]
if args.render_motion:cmd.append('--render-motion')
logs=out/'logs';logs.mkdir(exist_ok=True)
with (logs/f'{shard.stem}.log').open('a') as f:
result=subprocess.run(cmd,env=env,stdout=f,stderr=subprocess.STDOUT)
if result.returncode:errors.append(dict(model=model,shard=str(shard),returncode=result.returncode))
return errors
with concurrent.futures.ThreadPoolExecutor(max_workers=len(gpus)) as pool:
failures=[error for batch in pool.map(gpu_worker,gpus) for error in batch]
summary=analyze(out);summary['worker_failures']=failures
summary['planned']=len(jobs);summary['incomplete']=len(jobs)-summary['completed'];atomic_json(out/'summary/status.json',summary)
print(json.dumps(summary,ensure_ascii=False,indent=2))
if failures or summary['incomplete']:raise SystemExit(1)
def main():
p=argparse.ArgumentParser(prog='snf-suite');sub=p.add_subparsers(dest='command',required=True)
sub.add_parser('source-check')
c=sub.add_parser('create-tasks');c.add_argument('--output',default='/data/tasks.curated.jsonl')
c=sub.add_parser('import-check');c.add_argument('--model',choices=MODELS,required=True)
c=sub.add_parser('doctor');c.add_argument('--weights',default='/weights')
c=sub.add_parser('fetch');c.add_argument('--models',default='all');c.add_argument('--weights',default='/weights');c.add_argument('--offline',action='store_true')
for name in ('run','plan'):
c=sub.add_parser(name);c.add_argument('--tasks',default=str(ROOT/'suite/tasks.example.jsonl'));c.add_argument('--models',default='all');c.add_argument('--profile',choices=('smoke','pilot','full'),default='smoke')
c.add_argument('--output',default='/outputs/smoke');c.add_argument('--weights',default='/weights');c.add_argument('--gpus');c.add_argument('--resume',action='store_true');c.add_argument('--dry-run',action='store_true')
c.add_argument('--extras',action=argparse.BooleanOptionalAction,default=None);c.add_argument('--render-motion',action='store_true')
c.add_argument('--jobs',help='Validated external jobs.jsonl, e.g. held-out calibration')
c=sub.add_parser('analyze');c.add_argument('--output',required=True)
c=sub.add_parser('calibrate');c.add_argument('--run',required=True);c.add_argument('--tasks',required=True);c.add_argument('--targets',required=True,help='Common negative-L1 targets, e.g. --targets=-0.3,-0.2');c.add_argument('--output',required=True)
c=sub.add_parser('analytic');c.add_argument('--output',default='/outputs/analytic');c.add_argument('--scripts',default='all')
c=sub.add_parser('recover');c.add_argument('--run',required=True);c.add_argument('--tasks',required=True);c.add_argument('--output',required=True);c.add_argument('--penalty',type=float,default=.5)
c=sub.add_parser('evaluate');c.add_argument('--run',required=True);c.add_argument('--weights',default='/weights');c.add_argument('--device',default='cuda');c.add_argument('--all-jobs',action='store_true')
c=sub.add_parser('probe');c.add_argument('--model',choices=('meissonic','momask','showo'),required=True);c.add_argument('--weights',default='/weights');c.add_argument('--tasks',required=True);c.add_argument('--output',required=True);c.add_argument('--groups',type=int,default=16);c.add_argument('--limit',type=int,default=8)
args=p.parse_args()
if args.command=='source-check':print(json.dumps(source_check(),indent=2))
elif args.command=='create-tasks':
from .create_tasks import create
path=Path(args.output);path.parent.mkdir(parents=True,exist_ok=True);path.write_text(''.join(json.dumps(t)+'\n' for t in create()),encoding='utf-8');print(path)
elif args.command=='import-check':print(json.dumps(import_check(args.model)))
elif args.command=='doctor':doctor(args.weights)
elif args.command=='fetch':fetch(selected(args.models),args.weights,args.offline)
elif args.command in ('run','plan'):
if args.command=='plan':args.dry_run=True
run(args)
elif args.command=='analyze':
from .results import analyze
print(json.dumps(analyze(args.output),indent=2))
elif args.command=='calibrate':
from .calibration import fit_and_plan
print(json.dumps(fit_and_plan(args.run,args.tasks,[float(x) for x in args.targets.split(',')],args.output),indent=2))
elif args.command=='analytic':
from .analytic import ORDER
chosen=ORDER if args.scripts=='all' else args.scripts.split(',');out=Path(args.output).resolve();out.mkdir(parents=True,exist_ok=True)
if not set(chosen)<=set(ORDER):raise ValueError('Unknown analytic script')
for script in chosen:
with (out/f'{script}.log').open('w') as f:subprocess.run([sys.executable,'-m','suite.analytic','--script',script,'--output',str(out)],stdout=f,stderr=subprocess.STDOUT,check=True)
elif args.command=='recover':
from .recovery import choose_and_plan
print(json.dumps(choose_and_plan(args.run,args.tasks,args.output,args.penalty),indent=2))
elif args.command=='evaluate':
from .evaluation import evaluate
print(json.dumps(evaluate(args.run,args.weights,args.device,args.all_jobs),indent=2))
elif args.command=='probe':
subprocess.run([f'/opt/envs/{args.model}/bin/python','-m','suite.probe','--model',args.model,'--weights',args.weights,'--tasks',args.tasks,'--output',args.output,'--groups',str(args.groups),'--limit',str(args.limit)],check=True)
if __name__=='__main__':main()
|