"""Bounded qualitative previews using the unchanged original v4 inference/render protocol.""" import argparse import hashlib import importlib.util import json import os from pathlib import Path import subprocess import sys import tempfile import time ROOT = Path(__file__).resolve().parents[2] GROUP = ROOT / "scale/groups/sml_1m_224" sys.path.insert(0, str(ROOT)) import start_visualization as normal CASES = ["text_0997", "music_1912", "speech_1983", "video_0800"] PYTHON = ROOT.parent / "Psi0-yth-v26/.venv-psi/bin/python" MONITOR = ROOT.parent / "monitor/cmd.sh" def save(path, value): data = (json.dumps(value, ensure_ascii=False, indent=2) + "\n").encode() path.parent.mkdir(parents=True, exist_ok=True) with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as f: temporary = Path(f.name) f.write(data) f.flush() os.fsync(f.fileno()) if temporary.read_bytes() != data: raise OSError("Write verification failed") temporary.replace(path) def require_free(gpu): memory = subprocess.check_output(["nvidia-smi", f"--id={gpu}", "--query-gpu=memory.used", "--format=csv,noheader,nounits"], text=True) if int(memory.strip()) >= 500: raise RuntimeError(f"GPU {gpu} is occupied; no process will be stopped") def main(): global GROUP parser = argparse.ArgumentParser() parser.add_argument("--gpu", type=int, required=True) parser.add_argument("--name", required=True) parser.add_argument("--experiment-root", type=Path, default=GROUP) parser.add_argument("--step", type=int, help="Use the same completed checkpoint step for every model") args = parser.parse_args() GROUP = args.experiment_root.resolve() if not GROUP.is_relative_to((ROOT / "scale/groups").resolve()): parser.error("Preview group must be inside scale/groups") if Path(args.name).name != args.name or args.name in (".", ".."): parser.error("Invalid preview name") require_free(args.gpu) base = GROUP / "previews" / args.name base.mkdir(parents=True, exist_ok=True) # A static gallery page may be installed before the worker starts. with (base / ".preview_claim").open("x"): pass web = base / "web" web.mkdir(exist_ok=True) state = dict(status="preparing", qualitative_only=True, group=str(GROUP), gpu=args.gpu, models=[], cases=CASES) save(web / "status.json", state) spec = importlib.util.spec_from_file_location("original_watcher", normal.RESOURCE_PROJECT / "watch_training_cases.py") watcher = importlib.util.module_from_spec(spec) spec.loader.exec_module(watcher) watcher.PREVIEW = ROOT / "scale/protocol" original = normal.RESOURCE_PROJECT / "visualization/protocol" expected = (original / "infer_cases.py").read_text().replace("PROJECT = CODE / 'OmniHM-Zero_train_40d'", f"PROJECT = Path('{ROOT}')") assert (watcher.PREVIEW / "infer_cases.py").read_text() == expected, "Inference changed beyond project path" assert (watcher.PREVIEW / "render_cases.py").read_bytes() == (original / "render_cases.py").read_bytes(), "Renderer differs from original" normal.PROJECT = base jobs = [] used_gpu = False try: # Pin all three latest complete weight files before training retention can prune them. for size in ("S", "M", "L"): run = Path(json.loads((GROUP / "runs" / f"latest_{size}.json").read_text())["run_dir"]).resolve() if not run.is_relative_to((GROUP / "runs").resolve()): raise ValueError("Run escapes the active group") config = json.loads((run / "run_config.json").read_text()) candidates = watcher.completed_checkpoints(run, run / "training.log", config["world_size_expected"]) if args.step is not None: candidates = [candidate for candidate in candidates if candidate[0] == args.step] if not candidates: raise RuntimeError(f"{size}: no complete checkpoint") step, checkpoint, _ = candidates[0] selection = normal.prepare_selection(run, "main", "auto_cases", 35) directory = base / size / watcher.gallery_version(step) _, manifest = watcher.prepare_job(run, checkpoint, directory, selection) selected = [c for c in manifest["cases"] if c["id"] in CASES] assert len(selected) == len(CASES) entry = dict(size=size, step=step, run=run.name, status="queued", cases=[dict(id=c["id"], modality=c["modality"], frames=c["frames"], fps=c["fps"], prompt=json.loads((directory / c["directory"] / "selection_record.json").read_text()).get("caption", ""), ready=False) for c in selected]) state["models"].append(entry) jobs.append((directory, selected, entry)) state["protocol"] = watcher.PROTOCOL state["note"] = "原始 v4 推理;4 个固定样例完整时长;" + (f"统一 ckpt_{args.step};" if args.step is not None else "各模型 checkpoint 步数不同;") + "仅作预览,不是完整评测。左:预测,右:GT。" save(base / "provenance.json", dict(original_project=str(normal.RESOURCE_PROJECT), case_ids=CASES, inference_path_only_change=True, render_byte_identical=True, script_sha256={p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in watcher.PREVIEW.glob("*.py")})) env = dict(os.environ, CUDA_VISIBLE_DEVICES=str(args.gpu), PYTHONDONTWRITEBYTECODE="1", OMP_NUM_THREADS="4", OPENBLAS_NUM_THREADS="1", TOKENIZERS_PARALLELISM="false", HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1") for directory, selected, entry in jobs: require_free(args.gpu) state["status"] = entry["status"] = "inferring" state["updated_at"] = time.time() save(web / "status.json", state) used_gpu = True with (directory / "inferring.log").open("ab") as log: subprocess.run([str(PYTHON), "-B", "-u", str(directory / "protocol/infer_cases.py"), "--device", "cuda:0", "--threads", "4", "--output-dir", str(directory), "--case-ids", *CASES], cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, check=True) state["status"] = entry["status"] = "rendering" save(web / "status.json", state) for case, item in zip(selected, entry["cases"]): case_dir = directory / case["directory"] marker = json.loads((case_dir / "status.json").read_text()) if marker.get("status") != "inferred": raise RuntimeError(f"Prediction incomplete: {case['id']}") with (directory / "rendering.log").open("ab") as log: subprocess.run([watcher.RENDER_PYTHON, "-B", "-u", str(directory / "protocol/render_cases.py"), "--output-dir", str(directory), "--renderer", "software", "--case-ids", case["id"]], cwd=ROOT, env=dict(env, CUDA_VISIBLE_DEVICES=""), stdout=log, stderr=subprocess.STDOUT, check=True) rendered = json.loads((case_dir / "render_status.json").read_text()) if rendered.get("status") != "ready": raise RuntimeError(f"Render incomplete: {case['id']}") video = case_dir / "comparison.mp4" if not video.is_file() or not video.stat().st_size: raise RuntimeError("Missing rendered video") filename = f"{entry['size']}_{case['id']}.mp4" (web / filename).symlink_to(video) item.update(ready=True, video=filename) if case.get("input_media"): media = case_dir / case["input_media"] input_name = f"{entry['size']}_{case['id']}_input{media.suffix}" (web / input_name).symlink_to(media) item["input"] = input_name state["updated_at"] = time.time() save(web / "status.json", state) print(json.dumps(dict(event="video_ready", model=entry["size"], step=entry["step"], case=case["id"], file=str(video))), flush=True) entry["status"] = "ready" state["status"] = "ready" except BaseException as exc: state["status"] = "error" state["error"] = repr(exc) raise finally: state["updated_at"] = time.time() save(web / "status.json", state) if used_gpu: # User explicitly requested immediate occupancy restoration after GPU use. subprocess.run(["bash", "-c", 'source "$1"', "restore-monitor", str(MONITOR)], check=False) if __name__ == "__main__": main()