"""Bounded, checkpoint-independent video jobs for the automatic infer watcher.""" from __future__ import annotations from collections import deque import json import os from pathlib import Path import signal import subprocess import time def swa_outputs_ready(path: Path) -> bool: """Legacy SWA .complete meant tensors/JSON only, not a rendered video.""" try: metrics = json.loads((path / "metrics.json").read_text()) if metrics.get('native_speech_version') == 1: for answer in metrics['answers']: audio = (path / answer['audio_path']).resolve() if (not audio.is_relative_to(path.resolve()) or not answer.get('native_speech') or audio.stat().st_size <= 44 or answer.get('samples', 0) <= 0): return False if metrics.get('overlay_fixture_file'): return bool(metrics.get('source_id') and metrics.get('events')) and ( Path(metrics['overlay_fixture_file']).is_file() and (path/'generated_abs.npy').stat().st_size>0) return bool(metrics.get("id") and metrics.get("events")) and all( (path / name).stat().st_size > 0 for name in ("generated_abs.npy", "body_pos.csv", "body_quat.csv", "joint_pos.csv") ) except (OSError, ValueError, TypeError, AttributeError, KeyError): return False def render_complete(path: Path) -> bool: if not (path / ".complete").is_file(): return False if (path / "metrics.json").is_file(): try: metrics = json.loads((path / 'metrics.json').read_text()) expected = 2 if metrics.get('native_speech_version') == 1 else 1 return ((path / "reference_motion_with_audio.mp4").stat().st_size > 0 and json.loads((path / "infer_meta.json").read_text()).get("swa_render_version") == expected) except (OSError, ValueError, AttributeError): return False return True def render_inputs_ready(path: Path) -> bool: required = ("infer_meta.json", "body_pos.csv", "body_quat.csv", "joint_pos.csv", "assembled_input_audio_16k.wav") try: return (all((path / name).stat().st_size > 0 for name in required) and int(json.loads((path / "infer_meta.json").read_text())["total_frames"]) > 0) except (OSError, ValueError, KeyError, TypeError): return False class AsyncRenderQueue: """Render workers do not consume inference GPU slots or checkpoint leases. .inferred + motion/audio files are the durable queue. On restart the watcher recovers unfinished results even if the original checkpoint was deleted. Backpressure bounds queued work (plus inference already in flight). """ def __init__(self, project_root: Path, run_dir: Path, *, workers: int = 4, capacity: int = 32): if workers < 1 or capacity < workers: raise ValueError("Render workers must be positive; pending limit must be >= workers") self.project_root, self.run_dir = project_root, run_dir self.workers, self.capacity = workers, capacity self.pending = deque() self.running = {} self.paths = set() self.errors = [] self.last_recovery = float("-inf") @property def full(self): return len(self.paths) >= self.capacity @property def busy(self): return bool(self.paths) def enqueue(self, path: Path): path = path.resolve() if not path.is_relative_to(self.run_dir.resolve()): raise ValueError(f"Render result outside watched run: {path}") if path in self.paths or render_complete(path) or any( (path / marker).exists() for marker in (".failed", ".render_failed")): return if not (render_inputs_ready(path) or swa_outputs_ready(path)): raise ValueError(f"Incomplete inference outputs: {path}") (path / ".inferred").touch() self.paths.add(path) self.pending.append(path) print(f"[render-queue] queued {path}; backlog={len(self.paths)}", flush=True) def recover(self, paths): if time.monotonic() - self.last_recovery < 30: return self.last_recovery = time.monotonic() for path in paths: if self.full: break if render_inputs_ready(path) or swa_outputs_ready(path): self.enqueue(path) def _failed(self, path, reason): (path / ".render_failed").write_text(reason + "\n", encoding="utf-8") self.errors.append((str(path), reason)) print(f"[render-queue:error] {path}: {reason}; model inference continues", flush=True) def poll(self): for process in list(self.running): if process.poll() is None: continue path, handle, started = self.running.pop(process) handle.close() self.paths.remove(path) if process.returncode != 0 or not render_complete(path): self._failed(path, f"render exit={process.returncode}; see render.log") else: print(f"[render-queue] done {path}; elapsed={time.monotonic()-started:.1f}s", flush=True) while self.pending and len(self.running) < self.workers: path = self.pending.popleft() environment = os.environ.copy() environment["HFD_RENDER_ONLY_DIR"] = str(path) if swa_outputs_ready(path): environment["HFD_RENDER_TASK"] = "swa" else: environment.pop("HFD_RENDER_TASK", None) # Software EGL otherwise creates a large thread pool for every # render. Explicit user settings remain respected. environment.setdefault("LP_NUM_THREADS", "2") environment.setdefault("OMP_NUM_THREADS", "1") handle = (path / "render.log").open("ab") try: command=["bash", str(self.project_root / "scripts/infer/run_audio_case.sh"), str(self.run_dir)] if (path/'metrics.json').is_file(): metrics=json.loads((path/'metrics.json').read_text()) if metrics.get('overlay_fixture_file'): command=[str(self.project_root/'.venv-psi/bin/python'), str(self.project_root/'scripts/infer/unified_overlay.py'),'render',str(path)] process = subprocess.Popen( command, cwd=self.project_root, env=environment, start_new_session=True, stdout=handle, stderr=subprocess.STDOUT, ) except OSError as error: handle.close() self.paths.remove(path) self._failed(path, str(error)) continue self.running[process] = (path, handle, time.monotonic()) print(f"[render-queue] start {path}; workers={len(self.running)}/{self.workers}", flush=True) def drain(self): while self.busy: self.poll() if self.busy: time.sleep(.25) def close(self): # Only our process groups; durable .inferred files permit resumption. for process in self.running: if process.poll() is None: try: os.killpg(process.pid, signal.SIGTERM) except ProcessLookupError: pass for process, (_, handle, _) in self.running.items(): try: process.wait(timeout=5) except subprocess.TimeoutExpired: try: os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass process.wait() handle.close() self.running.clear() def __enter__(self): return self def __exit__(self, *_): self.close()