CharlesZhang-USTC's picture
Add Control v2 incremental experiment results (evidence, visuals, 42 fixed samples)
a64600b verified
Raw History Blame Contribute Delete
17.3 kB
"""Container entry point; GPU models are imported only inside their environments."""
import argparse
import concurrent.futures
import hashlib
import importlib
import json
import os
import subprocess
import sys
import time
from pathlib import Path
from .design import ROOT,MODELS,read_tasks,plan
def selected(value):
result=MODELS if value=='all' else tuple(value.split(','))
if not result or not set(result)<=set(MODELS):raise ValueError('Unknown model selection')
return result
def validate_external_jobs(jobs):
"""Reject manifests that would inflate planned counts or overwrite outputs."""
from .design import job_id
if not jobs:raise ValueError('Empty external job manifest')
ids=[j['id'] for j in jobs]
if len(ids)!=len(set(ids)):raise ValueError('Duplicate external job IDs')
models=tuple(dict.fromkeys(j['model'] for j in jobs))
if not set(models)<=set(MODELS):raise ValueError('Unknown external model')
for j in jobs:
if j['id']!=job_id({k:v for k,v in j.items() if k!='id'}):raise ValueError('External job ID mismatch')
if j.get('planning_status')=='unreachable' or j.get('alpha') is None:
raise ValueError('Unreachable slots belong in expected_slots.json, not executable jobs.jsonl')
return models
def validate_expected_slots(jobs, metadata):
"""The sidecar denominator must describe exactly this executable plan."""
expected=metadata.get('expected_slots')
if not isinstance(expected,list):raise ValueError('Invalid expected_slots.json schema')
ids=[j['id'] for j in expected]
if len(ids)!=len(set(ids)):raise ValueError('Duplicate expected slot IDs')
executable={j['id']:j for j in expected if j.get('planning_status')!='unreachable'}
if executable!={j['id']:j for j in jobs}:
raise ValueError('Executable jobs differ from expected_slots.json; regenerate matching plan metadata')
def source_check():
lock=json.loads((ROOT/'packaging/sources.lock.json').read_text())
manifest=json.loads((ROOT/'packaging/sources.manifest.json').read_text())
for relative,expected in manifest.items():
path=ROOT/relative
if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest()!=expected:
raise ValueError(f'Vendored source checksum mismatch: {relative}')
return dict(verified_files=len(manifest),revisions=lock)
def import_check(model):
import torch
from .adapters import activate,sdpa_flash_fallback
if model=='sd35':
from diffusers import StableDiffusion3Pipeline
else:
activate(model)
if model=='meissonic':
from src.pipeline import Pipeline
from src.transformer import Transformer2DModel
elif model=='self_forcing':
# Upstream memory helper queries current_device at import time.
# Docker builds have no GPU; this only bypasses that device lookup.
from unittest.mock import patch
with patch('torch.cuda.current_device',return_value=0):
sdpa_flash_fallback()
from pipeline.causal_inference import CausalInferencePipeline
elif model=='momask':
from .momask_loading import loading_functions
loading_functions()
elif model=='showo':
from models import Showo,MAGVITv2
from training.prompting_utils import UniversalPrompting
return dict(model=model,imports='passed',torch=torch.__version__,cuda=torch.version.cuda)
def doctor(weights):
import torch
result=dict(python=sys.version,torch=torch.__version__,cuda_runtime=torch.version.cuda,
cuda_available=torch.cuda.is_available(),weights=str(weights),sources=source_check(),models={})
if not torch.cuda.is_available():raise RuntimeError('CUDA unavailable: use docker run --gpus all and a compatible host driver')
result['gpus']=[dict(name=torch.cuda.get_device_name(i),capability=torch.cuda.get_device_capability(i),memory_gib=torch.cuda.get_device_properties(i).total_memory/2**30) for i in range(torch.cuda.device_count())]
# Exercise a real CUDA kernel; availability alone does not prove Blackwell support.
x=torch.randn(256,256,device='cuda',dtype=torch.bfloat16);result['kernel_finite']=bool(torch.isfinite(x@x).all());torch.cuda.synchronize()
for model in MODELS:
p=subprocess.run([str(Path('/opt/envs')/model/'bin/python'),'-m','suite.cli','import-check','--model',model],capture_output=True,text=True)
result['models'][model]=dict(returncode=p.returncode,output=p.stdout[-2500:],error=p.stderr[-2500:])
print(json.dumps(result,indent=2,ensure_ascii=False))
if any(v['returncode'] for v in result['models'].values()):raise SystemExit(1)
def fetch(models,weights,offline=False):
from huggingface_hub import HfApi,snapshot_download,hf_hub_download
from .results import atomic_json
weights=Path(weights);weights.mkdir(parents=True,exist_ok=True)
path=weights/'weights.lock.json';lock=json.loads(path.read_text()) if path.exists() else {}
cfg=json.loads((ROOT/'suite/models.json').read_text())
for model in models:
if model=='momask':
destination=weights/'momask/checkpoints/t2m';destination.mkdir(parents=True,exist_ok=True)
if not (destination/cfg[model]['name']/'opt.txt').exists():
if offline:raise FileNotFoundError('Missing mounted MoMask checkpoints')
import gdown,zipfile
archive=weights/'momask/humanml3d_models.zip'
gdown.download(id='1vXS7SHJBgWPt59wupQ5UUzhFObrnGkQ0',output=str(archive),quiet=False)
with zipfile.ZipFile(archive) as z:
for member in z.infolist():
target=(destination/member.filename).resolve()
if not target.is_relative_to(destination.resolve()):raise ValueError('Unsafe checkpoint archive path')
z.extractall(destination)
lock['momask_archive_sha256']=hashlib.sha256(archive.read_bytes()).hexdigest();atomic_json(path,lock)
continue
repositories=[v for k,v in cfg[model].items() if k.endswith('repo') and k!='student_repo']
if model=='self_forcing':repositories.append(cfg[model]['student_repo'])
for repo in repositories:
if repo not in lock:
if offline:raise ValueError(f'No pinned weight revision for {repo}')
lock[repo]=HfApi().model_info(repo).sha;atomic_json(path,lock)
rev=lock[repo]
if model=='self_forcing':
if repo==cfg[model]['student_repo']:
downloaded=hf_hub_download(repo,filename=cfg[model]['student_file'],revision=rev,local_files_only=offline)
dest=weights/'self_forcing'/cfg[model]['student_file'];dest.parent.mkdir(parents=True,exist_ok=True)
if not dest.exists():dest.symlink_to(Path(downloaded).resolve())
else:
dest=weights/'self_forcing/wan_models/Wan2.1-T2V-1.3B'
snapshot_download(repo,revision=rev,local_dir=dest,local_files_only=offline)
else:snapshot_download(repo,revision=rev,local_files_only=offline)
print(json.dumps(dict(weights=str(weights),lock=lock),indent=2))
def run(args):
from .results import atomic_json,analyze
source_check()
tasks=read_tasks(args.tasks);models=selected(args.models)
jobs=plan(tasks,args.profile,models,args.extras) if not args.jobs else [json.loads(line) for line in Path(args.jobs).read_text().splitlines() if line.strip()]
if args.jobs:
models=validate_external_jobs(jobs)
out=Path(args.output).resolve();out.mkdir(parents=True,exist_ok=True)
# A run directory is immutable across code/config changes; refuse silent mixing.
code={str(p.relative_to(ROOT)):hashlib.sha256(p.read_bytes()).hexdigest() for p in (ROOT/'suite').glob('*.py')}
weightlock=Path(args.weights)/'weights.lock.json'
configuration_files={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in (ROOT/'suite').glob('*.json')}
external_metadata={}
if args.jobs:
for name in ('expected_slots.json','protocol.json'):
sidecar=Path(args.jobs).parent/name
if sidecar.exists():external_metadata[name]=dict(sha256=hashlib.sha256(sidecar.read_bytes()).hexdigest(),data=json.loads(sidecar.read_text(encoding='utf8')))
if 'expected_slots.json' in external_metadata:
validate_expected_slots(jobs,external_metadata['expected_slots.json']['data'])
freezes={m:hashlib.sha256(p.read_bytes()).hexdigest() for m in models if (p:=Path('/opt/envs')/m/'installed.freeze.txt').exists()}
identity=dict(profile=args.profile,models=models,jobs=jobs,code=code,configuration_files=configuration_files,environment_freezes=freezes,source_manifest_sha256=hashlib.sha256((ROOT/'packaging/sources.manifest.json').read_bytes()).hexdigest(),
weights=json.loads(weightlock.read_text()) if weightlock.exists() else {},render_motion=args.render_motion,
external_plan_metadata={k:v['sha256'] for k,v in external_metadata.items()})
digest=hashlib.sha256(json.dumps(identity,sort_keys=True).encode()).hexdigest()
manifest=out/'run.json'
if manifest.exists():
previous=json.loads(manifest.read_text())
if previous['digest']!=digest:raise ValueError('Run identity changed. Use a new --output directory.')
if not args.resume:raise ValueError('Existing run; pass --resume')
else:atomic_json(manifest,dict(digest=digest,configuration=identity,created_utc=time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())))
for name,metadata in external_metadata.items():atomic_json(out/name,metadata['data'])
(out/'plan.jsonl').write_text(''.join(json.dumps(j,ensure_ascii=False)+'\n' for j in jobs),encoding='utf-8')
counts={model:sum(j['model']==model for j in jobs) for model in models};print(json.dumps(dict(job_counts=counts,output=str(out)),indent=2),flush=True)
if args.dry_run:return
import torch
if not torch.cuda.is_available():raise RuntimeError('CUDA required. Use --dry-run for CPU planning.')
gpus=args.gpus.split(',') if args.gpus else [str(i) for i in range(torch.cuda.device_count())]
if len(gpus)!=len(set(gpus)):raise ValueError('GPU list must be unique')
# Jobs for one task remain together, minimizing duplicate reference generation.
shards=[]
for model in models:
bytask={}
for j in jobs:
if j['model']==model:bytask.setdefault(j['task']['id'],[]).append(j)
buckets=[[] for _ in range(min(len(gpus),len(bytask)))]
for group in sorted(bytask.values(),key=len,reverse=True):min(buckets,key=len).extend(group)
for i,bucket in enumerate(buckets):
p=out/'shards'/f'{model}_{i}.jsonl';p.parent.mkdir(exist_ok=True)
p.write_text(''.join(json.dumps(j)+'\n' for j in bucket));shards.append((model,p))
import queue
work=queue.Queue()
for item in sorted(shards,key=lambda x:x[1].stat().st_size,reverse=True):work.put(item)
def gpu_worker(gpu):
errors=[]
while not work.empty() and not (out/'STOP').exists():
try:model,shard=work.get_nowait()
except queue.Empty:break
env=dict(os.environ,CUDA_VISIBLE_DEVICES=gpu,PYTHONPATH=str(ROOT),HF_HOME=str(Path(args.weights)/'huggingface'),TORCH_HOME=str(Path(args.weights)/'torch'),XDG_CACHE_HOME=str(Path(args.weights)/'cache'))
cmd=[str(Path('/opt/envs')/model/'bin/python'),'-m','suite.worker','--shard',str(shard),'--run',str(out),'--weights',str(Path(args.weights).resolve())]
if args.render_motion:cmd.append('--render-motion')
logs=out/'logs';logs.mkdir(exist_ok=True)
with (logs/f'{shard.stem}.log').open('a') as f:
result=subprocess.run(cmd,env=env,stdout=f,stderr=subprocess.STDOUT)
if result.returncode:errors.append(dict(model=model,shard=str(shard),returncode=result.returncode))
return errors
with concurrent.futures.ThreadPoolExecutor(max_workers=len(gpus)) as pool:
failures=[error for batch in pool.map(gpu_worker,gpus) for error in batch]
summary=analyze(out);summary['worker_failures']=failures
summary['planned']=len(jobs);summary['incomplete']=len(jobs)-summary['completed'];atomic_json(out/'summary/status.json',summary)
print(json.dumps(summary,ensure_ascii=False,indent=2))
if failures or summary['incomplete']:raise SystemExit(1)
def main():
p=argparse.ArgumentParser(prog='snf-suite');sub=p.add_subparsers(dest='command',required=True)
sub.add_parser('source-check')
c=sub.add_parser('create-tasks');c.add_argument('--output',default='/data/tasks.curated.jsonl')
c=sub.add_parser('import-check');c.add_argument('--model',choices=MODELS,required=True)
c=sub.add_parser('doctor');c.add_argument('--weights',default='/weights')
c=sub.add_parser('fetch');c.add_argument('--models',default='all');c.add_argument('--weights',default='/weights');c.add_argument('--offline',action='store_true')
for name in ('run','plan'):
c=sub.add_parser(name);c.add_argument('--tasks',default=str(ROOT/'suite/tasks.example.jsonl'));c.add_argument('--models',default='all');c.add_argument('--profile',choices=('smoke','pilot','full'),default='smoke')
c.add_argument('--output',default='/outputs/smoke');c.add_argument('--weights',default='/weights');c.add_argument('--gpus');c.add_argument('--resume',action='store_true');c.add_argument('--dry-run',action='store_true')
c.add_argument('--extras',action=argparse.BooleanOptionalAction,default=None);c.add_argument('--render-motion',action='store_true')
c.add_argument('--jobs',help='Validated external jobs.jsonl, e.g. held-out calibration')
c=sub.add_parser('analyze');c.add_argument('--output',required=True)
c=sub.add_parser('calibrate');c.add_argument('--run',required=True);c.add_argument('--tasks',required=True);c.add_argument('--targets',required=True,help='Common negative-L1 targets, e.g. --targets=-0.3,-0.2');c.add_argument('--output',required=True)
c=sub.add_parser('analytic');c.add_argument('--output',default='/outputs/analytic');c.add_argument('--scripts',default='all')
c=sub.add_parser('recover');c.add_argument('--run',required=True);c.add_argument('--tasks',required=True);c.add_argument('--output',required=True);c.add_argument('--penalty',type=float,default=.5)
c=sub.add_parser('evaluate');c.add_argument('--run',required=True);c.add_argument('--weights',default='/weights');c.add_argument('--device',default='cuda');c.add_argument('--all-jobs',action='store_true')
c=sub.add_parser('probe');c.add_argument('--model',choices=('meissonic','momask','showo'),required=True);c.add_argument('--weights',default='/weights');c.add_argument('--tasks',required=True);c.add_argument('--output',required=True);c.add_argument('--groups',type=int,default=16);c.add_argument('--limit',type=int,default=8)
args=p.parse_args()
if args.command=='source-check':print(json.dumps(source_check(),indent=2))
elif args.command=='create-tasks':
from .create_tasks import create
path=Path(args.output);path.parent.mkdir(parents=True,exist_ok=True);path.write_text(''.join(json.dumps(t)+'\n' for t in create()),encoding='utf-8');print(path)
elif args.command=='import-check':print(json.dumps(import_check(args.model)))
elif args.command=='doctor':doctor(args.weights)
elif args.command=='fetch':fetch(selected(args.models),args.weights,args.offline)
elif args.command in ('run','plan'):
if args.command=='plan':args.dry_run=True
run(args)
elif args.command=='analyze':
from .results import analyze
print(json.dumps(analyze(args.output),indent=2))
elif args.command=='calibrate':
from .calibration import fit_and_plan
print(json.dumps(fit_and_plan(args.run,args.tasks,[float(x) for x in args.targets.split(',')],args.output),indent=2))
elif args.command=='analytic':
from .analytic import ORDER
chosen=ORDER if args.scripts=='all' else args.scripts.split(',');out=Path(args.output).resolve();out.mkdir(parents=True,exist_ok=True)
if not set(chosen)<=set(ORDER):raise ValueError('Unknown analytic script')
for script in chosen:
with (out/f'{script}.log').open('w') as f:subprocess.run([sys.executable,'-m','suite.analytic','--script',script,'--output',str(out)],stdout=f,stderr=subprocess.STDOUT,check=True)
elif args.command=='recover':
from .recovery import choose_and_plan
print(json.dumps(choose_and_plan(args.run,args.tasks,args.output,args.penalty),indent=2))
elif args.command=='evaluate':
from .evaluation import evaluate
print(json.dumps(evaluate(args.run,args.weights,args.device,args.all_jobs),indent=2))
elif args.command=='probe':
subprocess.run([f'/opt/envs/{args.model}/bin/python','-m','suite.probe','--model',args.model,'--weights',args.weights,'--tasks',args.tasks,'--output',args.output,'--groups',str(args.groups),'--limit',str(args.limit)],check=True)
if __name__=='__main__':main()