code-Movens-Zero_train / scale /script /preview_scale.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
13cc860 verified
Raw History Blame Contribute Delete
8.63 kB
"""Bounded qualitative previews using the unchanged original v4 inference/render protocol."""
import argparse
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import time
ROOT = Path(__file__).resolve().parents[2]
GROUP = ROOT / "scale/groups/sml_1m_224"
sys.path.insert(0, str(ROOT))
import start_visualization as normal
CASES = ["text_0997", "music_1912", "speech_1983", "video_0800"]
PYTHON = ROOT.parent / "Psi0-yth-v26/.venv-psi/bin/python"
MONITOR = ROOT.parent / "monitor/cmd.sh"
def save(path, value):
data = (json.dumps(value, ensure_ascii=False, indent=2) + "\n").encode()
path.parent.mkdir(parents=True, exist_ok=True)
with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as f:
temporary = Path(f.name)
f.write(data)
f.flush()
os.fsync(f.fileno())
if temporary.read_bytes() != data:
raise OSError("Write verification failed")
temporary.replace(path)
def require_free(gpu):
memory = subprocess.check_output(["nvidia-smi", f"--id={gpu}", "--query-gpu=memory.used", "--format=csv,noheader,nounits"], text=True)
if int(memory.strip()) >= 500:
raise RuntimeError(f"GPU {gpu} is occupied; no process will be stopped")
def main():
global GROUP
parser = argparse.ArgumentParser()
parser.add_argument("--gpu", type=int, required=True)
parser.add_argument("--name", required=True)
parser.add_argument("--experiment-root", type=Path, default=GROUP)
parser.add_argument("--step", type=int, help="Use the same completed checkpoint step for every model")
args = parser.parse_args()
GROUP = args.experiment_root.resolve()
if not GROUP.is_relative_to((ROOT / "scale/groups").resolve()):
parser.error("Preview group must be inside scale/groups")
if Path(args.name).name != args.name or args.name in (".", ".."):
parser.error("Invalid preview name")
require_free(args.gpu)
base = GROUP / "previews" / args.name
base.mkdir(parents=True, exist_ok=True)
# A static gallery page may be installed before the worker starts.
with (base / ".preview_claim").open("x"):
pass
web = base / "web"
web.mkdir(exist_ok=True)
state = dict(status="preparing", qualitative_only=True, group=str(GROUP), gpu=args.gpu, models=[], cases=CASES)
save(web / "status.json", state)
spec = importlib.util.spec_from_file_location("original_watcher", normal.RESOURCE_PROJECT / "watch_training_cases.py")
watcher = importlib.util.module_from_spec(spec)
spec.loader.exec_module(watcher)
watcher.PREVIEW = ROOT / "scale/protocol"
original = normal.RESOURCE_PROJECT / "visualization/protocol"
expected = (original / "infer_cases.py").read_text().replace("PROJECT = CODE / 'OmniHM-Zero_train_40d'", f"PROJECT = Path('{ROOT}')")
assert (watcher.PREVIEW / "infer_cases.py").read_text() == expected, "Inference changed beyond project path"
assert (watcher.PREVIEW / "render_cases.py").read_bytes() == (original / "render_cases.py").read_bytes(), "Renderer differs from original"
normal.PROJECT = base
jobs = []
used_gpu = False
try:
# Pin all three latest complete weight files before training retention can prune them.
for size in ("S", "M", "L"):
run = Path(json.loads((GROUP / "runs" / f"latest_{size}.json").read_text())["run_dir"]).resolve()
if not run.is_relative_to((GROUP / "runs").resolve()):
raise ValueError("Run escapes the active group")
config = json.loads((run / "run_config.json").read_text())
candidates = watcher.completed_checkpoints(run, run / "training.log", config["world_size_expected"])
if args.step is not None:
candidates = [candidate for candidate in candidates if candidate[0] == args.step]
if not candidates:
raise RuntimeError(f"{size}: no complete checkpoint")
step, checkpoint, _ = candidates[0]
selection = normal.prepare_selection(run, "main", "auto_cases", 35)
directory = base / size / watcher.gallery_version(step)
_, manifest = watcher.prepare_job(run, checkpoint, directory, selection)
selected = [c for c in manifest["cases"] if c["id"] in CASES]
assert len(selected) == len(CASES)
entry = dict(size=size, step=step, run=run.name, status="queued", cases=[dict(id=c["id"], modality=c["modality"], frames=c["frames"], fps=c["fps"], prompt=json.loads((directory / c["directory"] / "selection_record.json").read_text()).get("caption", ""), ready=False) for c in selected])
state["models"].append(entry)
jobs.append((directory, selected, entry))
state["protocol"] = watcher.PROTOCOL
state["note"] = "原始 v4 推理;4 个固定样例完整时长;" + (f"统一 ckpt_{args.step};" if args.step is not None else "各模型 checkpoint 步数不同;") + "仅作预览,不是完整评测。左:预测,右:GT。"
save(base / "provenance.json", dict(original_project=str(normal.RESOURCE_PROJECT), case_ids=CASES, inference_path_only_change=True, render_byte_identical=True, script_sha256={p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in watcher.PREVIEW.glob("*.py")}))
env = dict(os.environ, CUDA_VISIBLE_DEVICES=str(args.gpu), PYTHONDONTWRITEBYTECODE="1", OMP_NUM_THREADS="4", OPENBLAS_NUM_THREADS="1", TOKENIZERS_PARALLELISM="false", HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1")
for directory, selected, entry in jobs:
require_free(args.gpu)
state["status"] = entry["status"] = "inferring"
state["updated_at"] = time.time()
save(web / "status.json", state)
used_gpu = True
with (directory / "inferring.log").open("ab") as log:
subprocess.run([str(PYTHON), "-B", "-u", str(directory / "protocol/infer_cases.py"), "--device", "cuda:0", "--threads", "4", "--output-dir", str(directory), "--case-ids", *CASES], cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, check=True)
state["status"] = entry["status"] = "rendering"
save(web / "status.json", state)
for case, item in zip(selected, entry["cases"]):
case_dir = directory / case["directory"]
marker = json.loads((case_dir / "status.json").read_text())
if marker.get("status") != "inferred":
raise RuntimeError(f"Prediction incomplete: {case['id']}")
with (directory / "rendering.log").open("ab") as log:
subprocess.run([watcher.RENDER_PYTHON, "-B", "-u", str(directory / "protocol/render_cases.py"), "--output-dir", str(directory), "--renderer", "software", "--case-ids", case["id"]], cwd=ROOT, env=dict(env, CUDA_VISIBLE_DEVICES=""), stdout=log, stderr=subprocess.STDOUT, check=True)
rendered = json.loads((case_dir / "render_status.json").read_text())
if rendered.get("status") != "ready":
raise RuntimeError(f"Render incomplete: {case['id']}")
video = case_dir / "comparison.mp4"
if not video.is_file() or not video.stat().st_size:
raise RuntimeError("Missing rendered video")
filename = f"{entry['size']}_{case['id']}.mp4"
(web / filename).symlink_to(video)
item.update(ready=True, video=filename)
if case.get("input_media"):
media = case_dir / case["input_media"]
input_name = f"{entry['size']}_{case['id']}_input{media.suffix}"
(web / input_name).symlink_to(media)
item["input"] = input_name
state["updated_at"] = time.time()
save(web / "status.json", state)
print(json.dumps(dict(event="video_ready", model=entry["size"], step=entry["step"], case=case["id"], file=str(video))), flush=True)
entry["status"] = "ready"
state["status"] = "ready"
except BaseException as exc:
state["status"] = "error"
state["error"] = repr(exc)
raise
finally:
state["updated_at"] = time.time()
save(web / "status.json", state)
if used_gpu:
# User explicitly requested immediate occupancy restoration after GPU use.
subprocess.run(["bash", "-c", 'source "$1"', "restore-monitor", str(MONITOR)], check=False)
if __name__ == "__main__":
main()