CharlesZhang-USTC's picture
Add files using upload-large-folder tool
ea1b5d0 verified
Raw History Blame Contribute Delete
15.6 kB
"""Container entry point; GPU models are imported only inside their environments."""
import argparse
import concurrent.futures
import hashlib
import importlib
import json
import os
import subprocess
import sys
import time
from pathlib import Path
from .design import ROOT,MODELS,read_tasks,plan
def selected(value):
result=MODELS if value=='all' else tuple(value.split(','))
if not result or not set(result)<=set(MODELS):raise ValueError('Unknown model selection')
return result
def source_check():
lock=json.loads((ROOT/'packaging/sources.lock.json').read_text())
manifest=json.loads((ROOT/'packaging/sources.manifest.json').read_text())
for relative,expected in manifest.items():
path=ROOT/relative
if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest()!=expected:
raise ValueError(f'Vendored source checksum mismatch: {relative}')
return dict(verified_files=len(manifest),revisions=lock)
def import_check(model):
import torch
from .adapters import activate,sdpa_flash_fallback
if model=='sd35':
from diffusers import StableDiffusion3Pipeline
else:
activate(model)
if model=='meissonic':
from src.pipeline import Pipeline
from src.transformer import Transformer2DModel
elif model=='self_forcing':
# Upstream memory helper queries current_device at import time.
# Docker builds have no GPU; this only bypasses that device lookup.
from unittest.mock import patch
with patch('torch.cuda.current_device',return_value=0):
sdpa_flash_fallback()
from pipeline.causal_inference import CausalInferencePipeline
elif model=='momask':
from .momask_loading import loading_functions
loading_functions()
elif model=='showo':
from models import Showo,MAGVITv2
from training.prompting_utils import UniversalPrompting
return dict(model=model,imports='passed',torch=torch.__version__,cuda=torch.version.cuda)
def doctor(weights):
import torch
result=dict(python=sys.version,torch=torch.__version__,cuda_runtime=torch.version.cuda,
cuda_available=torch.cuda.is_available(),weights=str(weights),sources=source_check(),models={})
if not torch.cuda.is_available():raise RuntimeError('CUDA unavailable: use docker run --gpus all and a compatible host driver')
result['gpus']=[dict(name=torch.cuda.get_device_name(i),capability=torch.cuda.get_device_capability(i),memory_gib=torch.cuda.get_device_properties(i).total_memory/2**30) for i in range(torch.cuda.device_count())]
# Exercise a real CUDA kernel; availability alone does not prove Blackwell support.
x=torch.randn(256,256,device='cuda',dtype=torch.bfloat16);result['kernel_finite']=bool(torch.isfinite(x@x).all());torch.cuda.synchronize()
for model in MODELS:
p=subprocess.run([str(Path('/opt/envs')/model/'bin/python'),'-m','suite.cli','import-check','--model',model],capture_output=True,text=True)
result['models'][model]=dict(returncode=p.returncode,output=p.stdout[-2500:],error=p.stderr[-2500:])
print(json.dumps(result,indent=2,ensure_ascii=False))
if any(v['returncode'] for v in result['models'].values()):raise SystemExit(1)
def fetch(models,weights,offline=False):
from huggingface_hub import HfApi,snapshot_download,hf_hub_download
from .results import atomic_json
weights=Path(weights);weights.mkdir(parents=True,exist_ok=True)
path=weights/'weights.lock.json';lock=json.loads(path.read_text()) if path.exists() else {}
cfg=json.loads((ROOT/'suite/models.json').read_text())
for model in models:
if model=='momask':
destination=weights/'momask/checkpoints/t2m';destination.mkdir(parents=True,exist_ok=True)
if not (destination/cfg[model]['name']/'opt.txt').exists():
if offline:raise FileNotFoundError('Missing mounted MoMask checkpoints')
import gdown,zipfile
archive=weights/'momask/humanml3d_models.zip'
gdown.download(id='1vXS7SHJBgWPt59wupQ5UUzhFObrnGkQ0',output=str(archive),quiet=False)
with zipfile.ZipFile(archive) as z:
for member in z.infolist():
target=(destination/member.filename).resolve()
if not target.is_relative_to(destination.resolve()):raise ValueError('Unsafe checkpoint archive path')
z.extractall(destination)
lock['momask_archive_sha256']=hashlib.sha256(archive.read_bytes()).hexdigest();atomic_json(path,lock)
continue
repositories=[v for k,v in cfg[model].items() if k.endswith('repo') and k!='student_repo']
if model=='self_forcing':repositories.append(cfg[model]['student_repo'])
for repo in repositories:
if repo not in lock:
if offline:raise ValueError(f'No pinned weight revision for {repo}')
lock[repo]=HfApi().model_info(repo).sha;atomic_json(path,lock)
rev=lock[repo]
if model=='self_forcing':
if repo==cfg[model]['student_repo']:
downloaded=hf_hub_download(repo,filename=cfg[model]['student_file'],revision=rev,local_files_only=offline)
dest=weights/'self_forcing'/cfg[model]['student_file'];dest.parent.mkdir(parents=True,exist_ok=True)
if not dest.exists():dest.symlink_to(Path(downloaded).resolve())
else:
dest=weights/'self_forcing/wan_models/Wan2.1-T2V-1.3B'
snapshot_download(repo,revision=rev,local_dir=dest,local_files_only=offline)
else:snapshot_download(repo,revision=rev,local_files_only=offline)
print(json.dumps(dict(weights=str(weights),lock=lock),indent=2))
def run(args):
from .results import atomic_json,analyze
source_check()
tasks=read_tasks(args.tasks);models=selected(args.models)
jobs=plan(tasks,args.profile,models,args.extras) if not args.jobs else [json.loads(line) for line in Path(args.jobs).read_text().splitlines() if line.strip()]
if args.jobs:
from .design import job_id
if not jobs:raise ValueError('Empty external job manifest')
models=tuple(dict.fromkeys(j['model'] for j in jobs))
if not set(models)<=set(MODELS):raise ValueError('Unknown external model')
for j in jobs:
if j['id']!=job_id({k:v for k,v in j.items() if k!='id'}):raise ValueError('External job ID mismatch')
out=Path(args.output).resolve();out.mkdir(parents=True,exist_ok=True)
# A run directory is immutable across code/config changes; refuse silent mixing.
code={str(p.relative_to(ROOT)):hashlib.sha256(p.read_bytes()).hexdigest() for p in (ROOT/'suite').glob('*.py')}
weightlock=Path(args.weights)/'weights.lock.json'
configuration_files={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in (ROOT/'suite').glob('*.json')}
freezes={m:hashlib.sha256(p.read_bytes()).hexdigest() for m in models if (p:=Path('/opt/envs')/m/'installed.freeze.txt').exists()}
identity=dict(profile=args.profile,models=models,jobs=jobs,code=code,configuration_files=configuration_files,environment_freezes=freezes,source_manifest_sha256=hashlib.sha256((ROOT/'packaging/sources.manifest.json').read_bytes()).hexdigest(),
weights=json.loads(weightlock.read_text()) if weightlock.exists() else {},render_motion=args.render_motion)
digest=hashlib.sha256(json.dumps(identity,sort_keys=True).encode()).hexdigest()
manifest=out/'run.json'
if manifest.exists():
previous=json.loads(manifest.read_text())
if previous['digest']!=digest:raise ValueError('Run identity changed. Use a new --output directory.')
if not args.resume:raise ValueError('Existing run; pass --resume')
else:atomic_json(manifest,dict(digest=digest,configuration=identity,created_utc=time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())))
(out/'plan.jsonl').write_text(''.join(json.dumps(j,ensure_ascii=False)+'\n' for j in jobs),encoding='utf-8')
counts={model:sum(j['model']==model for j in jobs) for model in models};print(json.dumps(dict(job_counts=counts,output=str(out)),indent=2),flush=True)
if args.dry_run:return
import torch
if not torch.cuda.is_available():raise RuntimeError('CUDA required. Use --dry-run for CPU planning.')
gpus=args.gpus.split(',') if args.gpus else [str(i) for i in range(torch.cuda.device_count())]
if len(gpus)!=len(set(gpus)):raise ValueError('GPU list must be unique')
# Jobs for one task remain together, minimizing duplicate reference generation.
shards=[]
for model in models:
bytask={}
for j in jobs:
if j['model']==model:bytask.setdefault(j['task']['id'],[]).append(j)
buckets=[[] for _ in range(min(len(gpus),len(bytask)))]
for group in sorted(bytask.values(),key=len,reverse=True):min(buckets,key=len).extend(group)
for i,bucket in enumerate(buckets):
p=out/'shards'/f'{model}_{i}.jsonl';p.parent.mkdir(exist_ok=True)
p.write_text(''.join(json.dumps(j)+'\n' for j in bucket));shards.append((model,p))
import queue
work=queue.Queue()
for item in sorted(shards,key=lambda x:x[1].stat().st_size,reverse=True):work.put(item)
def gpu_worker(gpu):
errors=[]
while not work.empty() and not (out/'STOP').exists():
try:model,shard=work.get_nowait()
except queue.Empty:break
env=dict(os.environ,CUDA_VISIBLE_DEVICES=gpu,PYTHONPATH=str(ROOT),HF_HOME=str(Path(args.weights)/'huggingface'),TORCH_HOME=str(Path(args.weights)/'torch'),XDG_CACHE_HOME=str(Path(args.weights)/'cache'))
cmd=[str(Path('/opt/envs')/model/'bin/python'),'-m','suite.worker','--shard',str(shard),'--run',str(out),'--weights',str(Path(args.weights).resolve())]
if args.render_motion:cmd.append('--render-motion')
logs=out/'logs';logs.mkdir(exist_ok=True)
with (logs/f'{shard.stem}.log').open('a') as f:
result=subprocess.run(cmd,env=env,stdout=f,stderr=subprocess.STDOUT)
if result.returncode:errors.append(dict(model=model,shard=str(shard),returncode=result.returncode))
return errors
with concurrent.futures.ThreadPoolExecutor(max_workers=len(gpus)) as pool:
failures=[error for batch in pool.map(gpu_worker,gpus) for error in batch]
summary=analyze(out);summary['worker_failures']=failures
summary['planned']=len(jobs);summary['incomplete']=len(jobs)-summary['completed'];atomic_json(out/'summary/status.json',summary)
print(json.dumps(summary,ensure_ascii=False,indent=2))
if failures or summary['incomplete']:raise SystemExit(1)
def main():
p=argparse.ArgumentParser(prog='snf-suite');sub=p.add_subparsers(dest='command',required=True)
sub.add_parser('source-check')
c=sub.add_parser('create-tasks');c.add_argument('--output',default='/data/tasks.curated.jsonl')
c=sub.add_parser('import-check');c.add_argument('--model',choices=MODELS,required=True)
c=sub.add_parser('doctor');c.add_argument('--weights',default='/weights')
c=sub.add_parser('fetch');c.add_argument('--models',default='all');c.add_argument('--weights',default='/weights');c.add_argument('--offline',action='store_true')
for name in ('run','plan'):
c=sub.add_parser(name);c.add_argument('--tasks',default=str(ROOT/'suite/tasks.example.jsonl'));c.add_argument('--models',default='all');c.add_argument('--profile',choices=('smoke','pilot','full'),default='smoke')
c.add_argument('--output',default='/outputs/smoke');c.add_argument('--weights',default='/weights');c.add_argument('--gpus');c.add_argument('--resume',action='store_true');c.add_argument('--dry-run',action='store_true')
c.add_argument('--extras',action=argparse.BooleanOptionalAction,default=None);c.add_argument('--render-motion',action='store_true')
c.add_argument('--jobs',help='Validated external jobs.jsonl, e.g. held-out calibration')
c=sub.add_parser('analyze');c.add_argument('--output',required=True)
c=sub.add_parser('calibrate');c.add_argument('--run',required=True);c.add_argument('--tasks',required=True);c.add_argument('--targets',required=True,help='Common negative-L1 targets, e.g. --targets=-0.3,-0.2');c.add_argument('--output',required=True)
c=sub.add_parser('analytic');c.add_argument('--output',default='/outputs/analytic');c.add_argument('--scripts',default='all')
c=sub.add_parser('recover');c.add_argument('--run',required=True);c.add_argument('--tasks',required=True);c.add_argument('--output',required=True);c.add_argument('--penalty',type=float,default=.5)
c=sub.add_parser('evaluate');c.add_argument('--run',required=True);c.add_argument('--weights',default='/weights');c.add_argument('--device',default='cuda');c.add_argument('--all-jobs',action='store_true')
c=sub.add_parser('probe');c.add_argument('--model',choices=('meissonic','momask','showo'),required=True);c.add_argument('--weights',default='/weights');c.add_argument('--tasks',required=True);c.add_argument('--output',required=True);c.add_argument('--groups',type=int,default=16);c.add_argument('--limit',type=int,default=8)
args=p.parse_args()
if args.command=='source-check':print(json.dumps(source_check(),indent=2))
elif args.command=='create-tasks':
from .create_tasks import create
path=Path(args.output);path.parent.mkdir(parents=True,exist_ok=True);path.write_text(''.join(json.dumps(t)+'\n' for t in create()),encoding='utf-8');print(path)
elif args.command=='import-check':print(json.dumps(import_check(args.model)))
elif args.command=='doctor':doctor(args.weights)
elif args.command=='fetch':fetch(selected(args.models),args.weights,args.offline)
elif args.command in ('run','plan'):
if args.command=='plan':args.dry_run=True
run(args)
elif args.command=='analyze':
from .results import analyze
print(json.dumps(analyze(args.output),indent=2))
elif args.command=='calibrate':
from .calibration import fit_and_plan
print(json.dumps(fit_and_plan(args.run,args.tasks,[float(x) for x in args.targets.split(',')],args.output),indent=2))
elif args.command=='analytic':
from .analytic import ORDER
chosen=ORDER if args.scripts=='all' else args.scripts.split(',');out=Path(args.output).resolve();out.mkdir(parents=True,exist_ok=True)
if not set(chosen)<=set(ORDER):raise ValueError('Unknown analytic script')
for script in chosen:
with (out/f'{script}.log').open('w') as f:subprocess.run([sys.executable,'-m','suite.analytic','--script',script,'--output',str(out)],stdout=f,stderr=subprocess.STDOUT,check=True)
elif args.command=='recover':
from .recovery import choose_and_plan
print(json.dumps(choose_and_plan(args.run,args.tasks,args.output,args.penalty),indent=2))
elif args.command=='evaluate':
from .evaluation import evaluate
print(json.dumps(evaluate(args.run,args.weights,args.device,args.all_jobs),indent=2))
elif args.command=='probe':
subprocess.run([f'/opt/envs/{args.model}/bin/python','-m','suite.probe','--model',args.model,'--weights',args.weights,'--tasks',args.tasks,'--output',args.output,'--groups',str(args.groups),'--limit',str(args.limit)],check=True)
if __name__=='__main__':main()