"""Container entry point; GPU models are imported only inside their environments.""" import argparse import concurrent.futures import hashlib import importlib import json import os import subprocess import sys import time from pathlib import Path from .design import ROOT,MODELS,read_tasks,plan def selected(value): result=MODELS if value=='all' else tuple(value.split(',')) if not result or not set(result)<=set(MODELS):raise ValueError('Unknown model selection') return result def source_check(): lock=json.loads((ROOT/'packaging/sources.lock.json').read_text()) manifest=json.loads((ROOT/'packaging/sources.manifest.json').read_text()) for relative,expected in manifest.items(): path=ROOT/relative if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest()!=expected: raise ValueError(f'Vendored source checksum mismatch: {relative}') return dict(verified_files=len(manifest),revisions=lock) def import_check(model): import torch from .adapters import activate,sdpa_flash_fallback if model=='sd35': from diffusers import StableDiffusion3Pipeline else: activate(model) if model=='meissonic': from src.pipeline import Pipeline from src.transformer import Transformer2DModel elif model=='self_forcing': # Upstream memory helper queries current_device at import time. # Docker builds have no GPU; this only bypasses that device lookup. from unittest.mock import patch with patch('torch.cuda.current_device',return_value=0): sdpa_flash_fallback() from pipeline.causal_inference import CausalInferencePipeline elif model=='momask': from .momask_loading import loading_functions loading_functions() elif model=='showo': from models import Showo,MAGVITv2 from training.prompting_utils import UniversalPrompting return dict(model=model,imports='passed',torch=torch.__version__,cuda=torch.version.cuda) def doctor(weights): import torch result=dict(python=sys.version,torch=torch.__version__,cuda_runtime=torch.version.cuda, cuda_available=torch.cuda.is_available(),weights=str(weights),sources=source_check(),models={}) if not torch.cuda.is_available():raise RuntimeError('CUDA unavailable: use docker run --gpus all and a compatible host driver') result['gpus']=[dict(name=torch.cuda.get_device_name(i),capability=torch.cuda.get_device_capability(i),memory_gib=torch.cuda.get_device_properties(i).total_memory/2**30) for i in range(torch.cuda.device_count())] # Exercise a real CUDA kernel; availability alone does not prove Blackwell support. x=torch.randn(256,256,device='cuda',dtype=torch.bfloat16);result['kernel_finite']=bool(torch.isfinite(x@x).all());torch.cuda.synchronize() for model in MODELS: p=subprocess.run([str(Path('/opt/envs')/model/'bin/python'),'-m','suite.cli','import-check','--model',model],capture_output=True,text=True) result['models'][model]=dict(returncode=p.returncode,output=p.stdout[-2500:],error=p.stderr[-2500:]) print(json.dumps(result,indent=2,ensure_ascii=False)) if any(v['returncode'] for v in result['models'].values()):raise SystemExit(1) def fetch(models,weights,offline=False): from huggingface_hub import HfApi,snapshot_download,hf_hub_download from .results import atomic_json weights=Path(weights);weights.mkdir(parents=True,exist_ok=True) path=weights/'weights.lock.json';lock=json.loads(path.read_text()) if path.exists() else {} cfg=json.loads((ROOT/'suite/models.json').read_text()) for model in models: if model=='momask': destination=weights/'momask/checkpoints/t2m';destination.mkdir(parents=True,exist_ok=True) if not (destination/cfg[model]['name']/'opt.txt').exists(): if offline:raise FileNotFoundError('Missing mounted MoMask checkpoints') import gdown,zipfile archive=weights/'momask/humanml3d_models.zip' gdown.download(id='1vXS7SHJBgWPt59wupQ5UUzhFObrnGkQ0',output=str(archive),quiet=False) with zipfile.ZipFile(archive) as z: for member in z.infolist(): target=(destination/member.filename).resolve() if not target.is_relative_to(destination.resolve()):raise ValueError('Unsafe checkpoint archive path') z.extractall(destination) lock['momask_archive_sha256']=hashlib.sha256(archive.read_bytes()).hexdigest();atomic_json(path,lock) continue repositories=[v for k,v in cfg[model].items() if k.endswith('repo') and k!='student_repo'] if model=='self_forcing':repositories.append(cfg[model]['student_repo']) for repo in repositories: if repo not in lock: if offline:raise ValueError(f'No pinned weight revision for {repo}') lock[repo]=HfApi().model_info(repo).sha;atomic_json(path,lock) rev=lock[repo] if model=='self_forcing': if repo==cfg[model]['student_repo']: downloaded=hf_hub_download(repo,filename=cfg[model]['student_file'],revision=rev,local_files_only=offline) dest=weights/'self_forcing'/cfg[model]['student_file'];dest.parent.mkdir(parents=True,exist_ok=True) if not dest.exists():dest.symlink_to(Path(downloaded).resolve()) else: dest=weights/'self_forcing/wan_models/Wan2.1-T2V-1.3B' snapshot_download(repo,revision=rev,local_dir=dest,local_files_only=offline) else:snapshot_download(repo,revision=rev,local_files_only=offline) print(json.dumps(dict(weights=str(weights),lock=lock),indent=2)) def run(args): from .results import atomic_json,analyze source_check() tasks=read_tasks(args.tasks);models=selected(args.models) jobs=plan(tasks,args.profile,models,args.extras) if not args.jobs else [json.loads(line) for line in Path(args.jobs).read_text().splitlines() if line.strip()] if args.jobs: from .design import job_id if not jobs:raise ValueError('Empty external job manifest') models=tuple(dict.fromkeys(j['model'] for j in jobs)) if not set(models)<=set(MODELS):raise ValueError('Unknown external model') for j in jobs: if j['id']!=job_id({k:v for k,v in j.items() if k!='id'}):raise ValueError('External job ID mismatch') out=Path(args.output).resolve();out.mkdir(parents=True,exist_ok=True) # A run directory is immutable across code/config changes; refuse silent mixing. code={str(p.relative_to(ROOT)):hashlib.sha256(p.read_bytes()).hexdigest() for p in (ROOT/'suite').glob('*.py')} weightlock=Path(args.weights)/'weights.lock.json' configuration_files={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in (ROOT/'suite').glob('*.json')} freezes={m:hashlib.sha256(p.read_bytes()).hexdigest() for m in models if (p:=Path('/opt/envs')/m/'installed.freeze.txt').exists()} identity=dict(profile=args.profile,models=models,jobs=jobs,code=code,configuration_files=configuration_files,environment_freezes=freezes,source_manifest_sha256=hashlib.sha256((ROOT/'packaging/sources.manifest.json').read_bytes()).hexdigest(), weights=json.loads(weightlock.read_text()) if weightlock.exists() else {},render_motion=args.render_motion) digest=hashlib.sha256(json.dumps(identity,sort_keys=True).encode()).hexdigest() manifest=out/'run.json' if manifest.exists(): previous=json.loads(manifest.read_text()) if previous['digest']!=digest:raise ValueError('Run identity changed. Use a new --output directory.') if not args.resume:raise ValueError('Existing run; pass --resume') else:atomic_json(manifest,dict(digest=digest,configuration=identity,created_utc=time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()))) (out/'plan.jsonl').write_text(''.join(json.dumps(j,ensure_ascii=False)+'\n' for j in jobs),encoding='utf-8') counts={model:sum(j['model']==model for j in jobs) for model in models};print(json.dumps(dict(job_counts=counts,output=str(out)),indent=2),flush=True) if args.dry_run:return import torch if not torch.cuda.is_available():raise RuntimeError('CUDA required. Use --dry-run for CPU planning.') gpus=args.gpus.split(',') if args.gpus else [str(i) for i in range(torch.cuda.device_count())] if len(gpus)!=len(set(gpus)):raise ValueError('GPU list must be unique') # Jobs for one task remain together, minimizing duplicate reference generation. shards=[] for model in models: bytask={} for j in jobs: if j['model']==model:bytask.setdefault(j['task']['id'],[]).append(j) buckets=[[] for _ in range(min(len(gpus),len(bytask)))] for group in sorted(bytask.values(),key=len,reverse=True):min(buckets,key=len).extend(group) for i,bucket in enumerate(buckets): p=out/'shards'/f'{model}_{i}.jsonl';p.parent.mkdir(exist_ok=True) p.write_text(''.join(json.dumps(j)+'\n' for j in bucket));shards.append((model,p)) import queue work=queue.Queue() for item in sorted(shards,key=lambda x:x[1].stat().st_size,reverse=True):work.put(item) def gpu_worker(gpu): errors=[] while not work.empty() and not (out/'STOP').exists(): try:model,shard=work.get_nowait() except queue.Empty:break env=dict(os.environ,CUDA_VISIBLE_DEVICES=gpu,PYTHONPATH=str(ROOT),HF_HOME=str(Path(args.weights)/'huggingface'),TORCH_HOME=str(Path(args.weights)/'torch'),XDG_CACHE_HOME=str(Path(args.weights)/'cache')) cmd=[str(Path('/opt/envs')/model/'bin/python'),'-m','suite.worker','--shard',str(shard),'--run',str(out),'--weights',str(Path(args.weights).resolve())] if args.render_motion:cmd.append('--render-motion') logs=out/'logs';logs.mkdir(exist_ok=True) with (logs/f'{shard.stem}.log').open('a') as f: result=subprocess.run(cmd,env=env,stdout=f,stderr=subprocess.STDOUT) if result.returncode:errors.append(dict(model=model,shard=str(shard),returncode=result.returncode)) return errors with concurrent.futures.ThreadPoolExecutor(max_workers=len(gpus)) as pool: failures=[error for batch in pool.map(gpu_worker,gpus) for error in batch] summary=analyze(out);summary['worker_failures']=failures summary['planned']=len(jobs);summary['incomplete']=len(jobs)-summary['completed'];atomic_json(out/'summary/status.json',summary) print(json.dumps(summary,ensure_ascii=False,indent=2)) if failures or summary['incomplete']:raise SystemExit(1) def main(): p=argparse.ArgumentParser(prog='snf-suite');sub=p.add_subparsers(dest='command',required=True) sub.add_parser('source-check') c=sub.add_parser('create-tasks');c.add_argument('--output',default='/data/tasks.curated.jsonl') c=sub.add_parser('import-check');c.add_argument('--model',choices=MODELS,required=True) c=sub.add_parser('doctor');c.add_argument('--weights',default='/weights') c=sub.add_parser('fetch');c.add_argument('--models',default='all');c.add_argument('--weights',default='/weights');c.add_argument('--offline',action='store_true') for name in ('run','plan'): c=sub.add_parser(name);c.add_argument('--tasks',default=str(ROOT/'suite/tasks.example.jsonl'));c.add_argument('--models',default='all');c.add_argument('--profile',choices=('smoke','pilot','full'),default='smoke') c.add_argument('--output',default='/outputs/smoke');c.add_argument('--weights',default='/weights');c.add_argument('--gpus');c.add_argument('--resume',action='store_true');c.add_argument('--dry-run',action='store_true') c.add_argument('--extras',action=argparse.BooleanOptionalAction,default=None);c.add_argument('--render-motion',action='store_true') c.add_argument('--jobs',help='Validated external jobs.jsonl, e.g. held-out calibration') c=sub.add_parser('analyze');c.add_argument('--output',required=True) c=sub.add_parser('calibrate');c.add_argument('--run',required=True);c.add_argument('--tasks',required=True);c.add_argument('--targets',required=True,help='Common negative-L1 targets, e.g. --targets=-0.3,-0.2');c.add_argument('--output',required=True) c=sub.add_parser('analytic');c.add_argument('--output',default='/outputs/analytic');c.add_argument('--scripts',default='all') c=sub.add_parser('recover');c.add_argument('--run',required=True);c.add_argument('--tasks',required=True);c.add_argument('--output',required=True);c.add_argument('--penalty',type=float,default=.5) c=sub.add_parser('evaluate');c.add_argument('--run',required=True);c.add_argument('--weights',default='/weights');c.add_argument('--device',default='cuda');c.add_argument('--all-jobs',action='store_true') c=sub.add_parser('probe');c.add_argument('--model',choices=('meissonic','momask','showo'),required=True);c.add_argument('--weights',default='/weights');c.add_argument('--tasks',required=True);c.add_argument('--output',required=True);c.add_argument('--groups',type=int,default=16);c.add_argument('--limit',type=int,default=8) args=p.parse_args() if args.command=='source-check':print(json.dumps(source_check(),indent=2)) elif args.command=='create-tasks': from .create_tasks import create path=Path(args.output);path.parent.mkdir(parents=True,exist_ok=True);path.write_text(''.join(json.dumps(t)+'\n' for t in create()),encoding='utf-8');print(path) elif args.command=='import-check':print(json.dumps(import_check(args.model))) elif args.command=='doctor':doctor(args.weights) elif args.command=='fetch':fetch(selected(args.models),args.weights,args.offline) elif args.command in ('run','plan'): if args.command=='plan':args.dry_run=True run(args) elif args.command=='analyze': from .results import analyze print(json.dumps(analyze(args.output),indent=2)) elif args.command=='calibrate': from .calibration import fit_and_plan print(json.dumps(fit_and_plan(args.run,args.tasks,[float(x) for x in args.targets.split(',')],args.output),indent=2)) elif args.command=='analytic': from .analytic import ORDER chosen=ORDER if args.scripts=='all' else args.scripts.split(',');out=Path(args.output).resolve();out.mkdir(parents=True,exist_ok=True) if not set(chosen)<=set(ORDER):raise ValueError('Unknown analytic script') for script in chosen: with (out/f'{script}.log').open('w') as f:subprocess.run([sys.executable,'-m','suite.analytic','--script',script,'--output',str(out)],stdout=f,stderr=subprocess.STDOUT,check=True) elif args.command=='recover': from .recovery import choose_and_plan print(json.dumps(choose_and_plan(args.run,args.tasks,args.output,args.penalty),indent=2)) elif args.command=='evaluate': from .evaluation import evaluate print(json.dumps(evaluate(args.run,args.weights,args.device,args.all_jobs),indent=2)) elif args.command=='probe': subprocess.run([f'/opt/envs/{args.model}/bin/python','-m','suite.probe','--model',args.model,'--weights',args.weights,'--tasks',args.tasks,'--output',args.output,'--groups',str(args.groups),'--limit',str(args.limit)],check=True) if __name__=='__main__':main()