Add Control v2 incremental experiment results (evidence, visuals, 42 fixed samples)
a64600b verified Download method_v2/evidence/code/suite/cli.py from CharlesZhang-USTC/snf_experiments: direct link, hf CLI and curl.
- Browser
- Download file 17.3 kB
-
https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/method_v2/evidence/code/suite/cli.py
- Command line
-
hf download hf://CharlesZhang-USTC/snf_experiments/method_v2/evidence/code/suite/cli.py
-
curl -L -o cli.py https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/method_v2/evidence/code/suite/cli.py
17.3 kB
| """Container entry point; GPU models are imported only inside their environments.""" | |
| import argparse | |
| import concurrent.futures | |
| import hashlib | |
| import importlib | |
| import json | |
| import os | |
| import subprocess | |
| import sys | |
| import time | |
| from pathlib import Path | |
| from .design import ROOT,MODELS,read_tasks,plan | |
| def selected(value): | |
| result=MODELS if value=='all' else tuple(value.split(',')) | |
| if not result or not set(result)<=set(MODELS):raise ValueError('Unknown model selection') | |
| return result | |
| def validate_external_jobs(jobs): | |
| """Reject manifests that would inflate planned counts or overwrite outputs.""" | |
| from .design import job_id | |
| if not jobs:raise ValueError('Empty external job manifest') | |
| ids=[j['id'] for j in jobs] | |
| if len(ids)!=len(set(ids)):raise ValueError('Duplicate external job IDs') | |
| models=tuple(dict.fromkeys(j['model'] for j in jobs)) | |
| if not set(models)<=set(MODELS):raise ValueError('Unknown external model') | |
| for j in jobs: | |
| if j['id']!=job_id({k:v for k,v in j.items() if k!='id'}):raise ValueError('External job ID mismatch') | |
| if j.get('planning_status')=='unreachable' or j.get('alpha') is None: | |
| raise ValueError('Unreachable slots belong in expected_slots.json, not executable jobs.jsonl') | |
| return models | |
| def validate_expected_slots(jobs, metadata): | |
| """The sidecar denominator must describe exactly this executable plan.""" | |
| expected=metadata.get('expected_slots') | |
| if not isinstance(expected,list):raise ValueError('Invalid expected_slots.json schema') | |
| ids=[j['id'] for j in expected] | |
| if len(ids)!=len(set(ids)):raise ValueError('Duplicate expected slot IDs') | |
| executable={j['id']:j for j in expected if j.get('planning_status')!='unreachable'} | |
| if executable!={j['id']:j for j in jobs}: | |
| raise ValueError('Executable jobs differ from expected_slots.json; regenerate matching plan metadata') | |
| def source_check(): | |
| lock=json.loads((ROOT/'packaging/sources.lock.json').read_text()) | |
| manifest=json.loads((ROOT/'packaging/sources.manifest.json').read_text()) | |
| for relative,expected in manifest.items(): | |
| path=ROOT/relative | |
| if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest()!=expected: | |
| raise ValueError(f'Vendored source checksum mismatch: {relative}') | |
| return dict(verified_files=len(manifest),revisions=lock) | |
| def import_check(model): | |
| import torch | |
| from .adapters import activate,sdpa_flash_fallback | |
| if model=='sd35': | |
| from diffusers import StableDiffusion3Pipeline | |
| else: | |
| activate(model) | |
| if model=='meissonic': | |
| from src.pipeline import Pipeline | |
| from src.transformer import Transformer2DModel | |
| elif model=='self_forcing': | |
| # Upstream memory helper queries current_device at import time. | |
| # Docker builds have no GPU; this only bypasses that device lookup. | |
| from unittest.mock import patch | |
| with patch('torch.cuda.current_device',return_value=0): | |
| sdpa_flash_fallback() | |
| from pipeline.causal_inference import CausalInferencePipeline | |
| elif model=='momask': | |
| from .momask_loading import loading_functions | |
| loading_functions() | |
| elif model=='showo': | |
| from models import Showo,MAGVITv2 | |
| from training.prompting_utils import UniversalPrompting | |
| return dict(model=model,imports='passed',torch=torch.__version__,cuda=torch.version.cuda) | |
| def doctor(weights): | |
| import torch | |
| result=dict(python=sys.version,torch=torch.__version__,cuda_runtime=torch.version.cuda, | |
| cuda_available=torch.cuda.is_available(),weights=str(weights),sources=source_check(),models={}) | |
| if not torch.cuda.is_available():raise RuntimeError('CUDA unavailable: use docker run --gpus all and a compatible host driver') | |
| result['gpus']=[dict(name=torch.cuda.get_device_name(i),capability=torch.cuda.get_device_capability(i),memory_gib=torch.cuda.get_device_properties(i).total_memory/2**30) for i in range(torch.cuda.device_count())] | |
| # Exercise a real CUDA kernel; availability alone does not prove Blackwell support. | |
| x=torch.randn(256,256,device='cuda',dtype=torch.bfloat16);result['kernel_finite']=bool(torch.isfinite(x@x).all());torch.cuda.synchronize() | |
| for model in MODELS: | |
| p=subprocess.run([str(Path('/opt/envs')/model/'bin/python'),'-m','suite.cli','import-check','--model',model],capture_output=True,text=True) | |
| result['models'][model]=dict(returncode=p.returncode,output=p.stdout[-2500:],error=p.stderr[-2500:]) | |
| print(json.dumps(result,indent=2,ensure_ascii=False)) | |
| if any(v['returncode'] for v in result['models'].values()):raise SystemExit(1) | |
| def fetch(models,weights,offline=False): | |
| from huggingface_hub import HfApi,snapshot_download,hf_hub_download | |
| from .results import atomic_json | |
| weights=Path(weights);weights.mkdir(parents=True,exist_ok=True) | |
| path=weights/'weights.lock.json';lock=json.loads(path.read_text()) if path.exists() else {} | |
| cfg=json.loads((ROOT/'suite/models.json').read_text()) | |
| for model in models: | |
| if model=='momask': | |
| destination=weights/'momask/checkpoints/t2m';destination.mkdir(parents=True,exist_ok=True) | |
| if not (destination/cfg[model]['name']/'opt.txt').exists(): | |
| if offline:raise FileNotFoundError('Missing mounted MoMask checkpoints') | |
| import gdown,zipfile | |
| archive=weights/'momask/humanml3d_models.zip' | |
| gdown.download(id='1vXS7SHJBgWPt59wupQ5UUzhFObrnGkQ0',output=str(archive),quiet=False) | |
| with zipfile.ZipFile(archive) as z: | |
| for member in z.infolist(): | |
| target=(destination/member.filename).resolve() | |
| if not target.is_relative_to(destination.resolve()):raise ValueError('Unsafe checkpoint archive path') | |
| z.extractall(destination) | |
| lock['momask_archive_sha256']=hashlib.sha256(archive.read_bytes()).hexdigest();atomic_json(path,lock) | |
| continue | |
| repositories=[v for k,v in cfg[model].items() if k.endswith('repo') and k!='student_repo'] | |
| if model=='self_forcing':repositories.append(cfg[model]['student_repo']) | |
| for repo in repositories: | |
| if repo not in lock: | |
| if offline:raise ValueError(f'No pinned weight revision for {repo}') | |
| lock[repo]=HfApi().model_info(repo).sha;atomic_json(path,lock) | |
| rev=lock[repo] | |
| if model=='self_forcing': | |
| if repo==cfg[model]['student_repo']: | |
| downloaded=hf_hub_download(repo,filename=cfg[model]['student_file'],revision=rev,local_files_only=offline) | |
| dest=weights/'self_forcing'/cfg[model]['student_file'];dest.parent.mkdir(parents=True,exist_ok=True) | |
| if not dest.exists():dest.symlink_to(Path(downloaded).resolve()) | |
| else: | |
| dest=weights/'self_forcing/wan_models/Wan2.1-T2V-1.3B' | |
| snapshot_download(repo,revision=rev,local_dir=dest,local_files_only=offline) | |
| else:snapshot_download(repo,revision=rev,local_files_only=offline) | |
| print(json.dumps(dict(weights=str(weights),lock=lock),indent=2)) | |
| def run(args): | |
| from .results import atomic_json,analyze | |
| source_check() | |
| tasks=read_tasks(args.tasks);models=selected(args.models) | |
| jobs=plan(tasks,args.profile,models,args.extras) if not args.jobs else [json.loads(line) for line in Path(args.jobs).read_text().splitlines() if line.strip()] | |
| if args.jobs: | |
| models=validate_external_jobs(jobs) | |
| out=Path(args.output).resolve();out.mkdir(parents=True,exist_ok=True) | |
| # A run directory is immutable across code/config changes; refuse silent mixing. | |
| code={str(p.relative_to(ROOT)):hashlib.sha256(p.read_bytes()).hexdigest() for p in (ROOT/'suite').glob('*.py')} | |
| weightlock=Path(args.weights)/'weights.lock.json' | |
| configuration_files={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in (ROOT/'suite').glob('*.json')} | |
| external_metadata={} | |
| if args.jobs: | |
| for name in ('expected_slots.json','protocol.json'): | |
| sidecar=Path(args.jobs).parent/name | |
| if sidecar.exists():external_metadata[name]=dict(sha256=hashlib.sha256(sidecar.read_bytes()).hexdigest(),data=json.loads(sidecar.read_text(encoding='utf8'))) | |
| if 'expected_slots.json' in external_metadata: | |
| validate_expected_slots(jobs,external_metadata['expected_slots.json']['data']) | |
| freezes={m:hashlib.sha256(p.read_bytes()).hexdigest() for m in models if (p:=Path('/opt/envs')/m/'installed.freeze.txt').exists()} | |
| identity=dict(profile=args.profile,models=models,jobs=jobs,code=code,configuration_files=configuration_files,environment_freezes=freezes,source_manifest_sha256=hashlib.sha256((ROOT/'packaging/sources.manifest.json').read_bytes()).hexdigest(), | |
| weights=json.loads(weightlock.read_text()) if weightlock.exists() else {},render_motion=args.render_motion, | |
| external_plan_metadata={k:v['sha256'] for k,v in external_metadata.items()}) | |
| digest=hashlib.sha256(json.dumps(identity,sort_keys=True).encode()).hexdigest() | |
| manifest=out/'run.json' | |
| if manifest.exists(): | |
| previous=json.loads(manifest.read_text()) | |
| if previous['digest']!=digest:raise ValueError('Run identity changed. Use a new --output directory.') | |
| if not args.resume:raise ValueError('Existing run; pass --resume') | |
| else:atomic_json(manifest,dict(digest=digest,configuration=identity,created_utc=time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()))) | |
| for name,metadata in external_metadata.items():atomic_json(out/name,metadata['data']) | |
| (out/'plan.jsonl').write_text(''.join(json.dumps(j,ensure_ascii=False)+'\n' for j in jobs),encoding='utf-8') | |
| counts={model:sum(j['model']==model for j in jobs) for model in models};print(json.dumps(dict(job_counts=counts,output=str(out)),indent=2),flush=True) | |
| if args.dry_run:return | |
| import torch | |
| if not torch.cuda.is_available():raise RuntimeError('CUDA required. Use --dry-run for CPU planning.') | |
| gpus=args.gpus.split(',') if args.gpus else [str(i) for i in range(torch.cuda.device_count())] | |
| if len(gpus)!=len(set(gpus)):raise ValueError('GPU list must be unique') | |
| # Jobs for one task remain together, minimizing duplicate reference generation. | |
| shards=[] | |
| for model in models: | |
| bytask={} | |
| for j in jobs: | |
| if j['model']==model:bytask.setdefault(j['task']['id'],[]).append(j) | |
| buckets=[[] for _ in range(min(len(gpus),len(bytask)))] | |
| for group in sorted(bytask.values(),key=len,reverse=True):min(buckets,key=len).extend(group) | |
| for i,bucket in enumerate(buckets): | |
| p=out/'shards'/f'{model}_{i}.jsonl';p.parent.mkdir(exist_ok=True) | |
| p.write_text(''.join(json.dumps(j)+'\n' for j in bucket));shards.append((model,p)) | |
| import queue | |
| work=queue.Queue() | |
| for item in sorted(shards,key=lambda x:x[1].stat().st_size,reverse=True):work.put(item) | |
| def gpu_worker(gpu): | |
| errors=[] | |
| while not work.empty() and not (out/'STOP').exists(): | |
| try:model,shard=work.get_nowait() | |
| except queue.Empty:break | |
| env=dict(os.environ,CUDA_VISIBLE_DEVICES=gpu,PYTHONPATH=str(ROOT),HF_HOME=str(Path(args.weights)/'huggingface'),TORCH_HOME=str(Path(args.weights)/'torch'),XDG_CACHE_HOME=str(Path(args.weights)/'cache')) | |
| cmd=[str(Path('/opt/envs')/model/'bin/python'),'-m','suite.worker','--shard',str(shard),'--run',str(out),'--weights',str(Path(args.weights).resolve())] | |
| if args.render_motion:cmd.append('--render-motion') | |
| logs=out/'logs';logs.mkdir(exist_ok=True) | |
| with (logs/f'{shard.stem}.log').open('a') as f: | |
| result=subprocess.run(cmd,env=env,stdout=f,stderr=subprocess.STDOUT) | |
| if result.returncode:errors.append(dict(model=model,shard=str(shard),returncode=result.returncode)) | |
| return errors | |
| with concurrent.futures.ThreadPoolExecutor(max_workers=len(gpus)) as pool: | |
| failures=[error for batch in pool.map(gpu_worker,gpus) for error in batch] | |
| summary=analyze(out);summary['worker_failures']=failures | |
| summary['planned']=len(jobs);summary['incomplete']=len(jobs)-summary['completed'];atomic_json(out/'summary/status.json',summary) | |
| print(json.dumps(summary,ensure_ascii=False,indent=2)) | |
| if failures or summary['incomplete']:raise SystemExit(1) | |
| def main(): | |
| p=argparse.ArgumentParser(prog='snf-suite');sub=p.add_subparsers(dest='command',required=True) | |
| sub.add_parser('source-check') | |
| c=sub.add_parser('create-tasks');c.add_argument('--output',default='/data/tasks.curated.jsonl') | |
| c=sub.add_parser('import-check');c.add_argument('--model',choices=MODELS,required=True) | |
| c=sub.add_parser('doctor');c.add_argument('--weights',default='/weights') | |
| c=sub.add_parser('fetch');c.add_argument('--models',default='all');c.add_argument('--weights',default='/weights');c.add_argument('--offline',action='store_true') | |
| for name in ('run','plan'): | |
| c=sub.add_parser(name);c.add_argument('--tasks',default=str(ROOT/'suite/tasks.example.jsonl'));c.add_argument('--models',default='all');c.add_argument('--profile',choices=('smoke','pilot','full'),default='smoke') | |
| c.add_argument('--output',default='/outputs/smoke');c.add_argument('--weights',default='/weights');c.add_argument('--gpus');c.add_argument('--resume',action='store_true');c.add_argument('--dry-run',action='store_true') | |
| c.add_argument('--extras',action=argparse.BooleanOptionalAction,default=None);c.add_argument('--render-motion',action='store_true') | |
| c.add_argument('--jobs',help='Validated external jobs.jsonl, e.g. held-out calibration') | |
| c=sub.add_parser('analyze');c.add_argument('--output',required=True) | |
| c=sub.add_parser('calibrate');c.add_argument('--run',required=True);c.add_argument('--tasks',required=True);c.add_argument('--targets',required=True,help='Common negative-L1 targets, e.g. --targets=-0.3,-0.2');c.add_argument('--output',required=True) | |
| c=sub.add_parser('analytic');c.add_argument('--output',default='/outputs/analytic');c.add_argument('--scripts',default='all') | |
| c=sub.add_parser('recover');c.add_argument('--run',required=True);c.add_argument('--tasks',required=True);c.add_argument('--output',required=True);c.add_argument('--penalty',type=float,default=.5) | |
| c=sub.add_parser('evaluate');c.add_argument('--run',required=True);c.add_argument('--weights',default='/weights');c.add_argument('--device',default='cuda');c.add_argument('--all-jobs',action='store_true') | |
| c=sub.add_parser('probe');c.add_argument('--model',choices=('meissonic','momask','showo'),required=True);c.add_argument('--weights',default='/weights');c.add_argument('--tasks',required=True);c.add_argument('--output',required=True);c.add_argument('--groups',type=int,default=16);c.add_argument('--limit',type=int,default=8) | |
| args=p.parse_args() | |
| if args.command=='source-check':print(json.dumps(source_check(),indent=2)) | |
| elif args.command=='create-tasks': | |
| from .create_tasks import create | |
| path=Path(args.output);path.parent.mkdir(parents=True,exist_ok=True);path.write_text(''.join(json.dumps(t)+'\n' for t in create()),encoding='utf-8');print(path) | |
| elif args.command=='import-check':print(json.dumps(import_check(args.model))) | |
| elif args.command=='doctor':doctor(args.weights) | |
| elif args.command=='fetch':fetch(selected(args.models),args.weights,args.offline) | |
| elif args.command in ('run','plan'): | |
| if args.command=='plan':args.dry_run=True | |
| run(args) | |
| elif args.command=='analyze': | |
| from .results import analyze | |
| print(json.dumps(analyze(args.output),indent=2)) | |
| elif args.command=='calibrate': | |
| from .calibration import fit_and_plan | |
| print(json.dumps(fit_and_plan(args.run,args.tasks,[float(x) for x in args.targets.split(',')],args.output),indent=2)) | |
| elif args.command=='analytic': | |
| from .analytic import ORDER | |
| chosen=ORDER if args.scripts=='all' else args.scripts.split(',');out=Path(args.output).resolve();out.mkdir(parents=True,exist_ok=True) | |
| if not set(chosen)<=set(ORDER):raise ValueError('Unknown analytic script') | |
| for script in chosen: | |
| with (out/f'{script}.log').open('w') as f:subprocess.run([sys.executable,'-m','suite.analytic','--script',script,'--output',str(out)],stdout=f,stderr=subprocess.STDOUT,check=True) | |
| elif args.command=='recover': | |
| from .recovery import choose_and_plan | |
| print(json.dumps(choose_and_plan(args.run,args.tasks,args.output,args.penalty),indent=2)) | |
| elif args.command=='evaluate': | |
| from .evaluation import evaluate | |
| print(json.dumps(evaluate(args.run,args.weights,args.device,args.all_jobs),indent=2)) | |
| elif args.command=='probe': | |
| subprocess.run([f'/opt/envs/{args.model}/bin/python','-m','suite.probe','--model',args.model,'--weights',args.weights,'--tasks',args.tasks,'--output',args.output,'--groups',str(args.groups),'--limit',str(args.limit)],check=True) | |
| if __name__=='__main__':main() | |