code-HFD / src /psi /async_render_queue.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
8.02 kB
"""Bounded, checkpoint-independent video jobs for the automatic infer watcher."""
from __future__ import annotations
from collections import deque
import json
import os
from pathlib import Path
import signal
import subprocess
import time
def swa_outputs_ready(path: Path) -> bool:
"""Legacy SWA .complete meant tensors/JSON only, not a rendered video."""
try:
metrics = json.loads((path / "metrics.json").read_text())
if metrics.get('native_speech_version') == 1:
for answer in metrics['answers']:
audio = (path / answer['audio_path']).resolve()
if (not audio.is_relative_to(path.resolve()) or not answer.get('native_speech')
or audio.stat().st_size <= 44 or answer.get('samples', 0) <= 0):
return False
if metrics.get('overlay_fixture_file'):
return bool(metrics.get('source_id') and metrics.get('events')) and (
Path(metrics['overlay_fixture_file']).is_file() and
(path/'generated_abs.npy').stat().st_size>0)
return bool(metrics.get("id") and metrics.get("events")) and all(
(path / name).stat().st_size > 0
for name in ("generated_abs.npy", "body_pos.csv", "body_quat.csv", "joint_pos.csv")
)
except (OSError, ValueError, TypeError, AttributeError, KeyError):
return False
def render_complete(path: Path) -> bool:
if not (path / ".complete").is_file():
return False
if (path / "metrics.json").is_file():
try:
metrics = json.loads((path / 'metrics.json').read_text())
expected = 2 if metrics.get('native_speech_version') == 1 else 1
return ((path / "reference_motion_with_audio.mp4").stat().st_size > 0
and json.loads((path / "infer_meta.json").read_text()).get("swa_render_version") == expected)
except (OSError, ValueError, AttributeError):
return False
return True
def render_inputs_ready(path: Path) -> bool:
required = ("infer_meta.json", "body_pos.csv", "body_quat.csv", "joint_pos.csv",
"assembled_input_audio_16k.wav")
try:
return (all((path / name).stat().st_size > 0 for name in required)
and int(json.loads((path / "infer_meta.json").read_text())["total_frames"]) > 0)
except (OSError, ValueError, KeyError, TypeError):
return False
class AsyncRenderQueue:
"""Render workers do not consume inference GPU slots or checkpoint leases.
.inferred + motion/audio files are the durable queue. On restart the watcher
recovers unfinished results even if the original checkpoint was deleted.
Backpressure bounds queued work (plus inference already in flight).
"""
def __init__(self, project_root: Path, run_dir: Path, *, workers: int = 4,
capacity: int = 32):
if workers < 1 or capacity < workers:
raise ValueError("Render workers must be positive; pending limit must be >= workers")
self.project_root, self.run_dir = project_root, run_dir
self.workers, self.capacity = workers, capacity
self.pending = deque()
self.running = {}
self.paths = set()
self.errors = []
self.last_recovery = float("-inf")
@property
def full(self):
return len(self.paths) >= self.capacity
@property
def busy(self):
return bool(self.paths)
def enqueue(self, path: Path):
path = path.resolve()
if not path.is_relative_to(self.run_dir.resolve()):
raise ValueError(f"Render result outside watched run: {path}")
if path in self.paths or render_complete(path) or any(
(path / marker).exists() for marker in (".failed", ".render_failed")):
return
if not (render_inputs_ready(path) or swa_outputs_ready(path)):
raise ValueError(f"Incomplete inference outputs: {path}")
(path / ".inferred").touch()
self.paths.add(path)
self.pending.append(path)
print(f"[render-queue] queued {path}; backlog={len(self.paths)}", flush=True)
def recover(self, paths):
if time.monotonic() - self.last_recovery < 30:
return
self.last_recovery = time.monotonic()
for path in paths:
if self.full:
break
if render_inputs_ready(path) or swa_outputs_ready(path):
self.enqueue(path)
def _failed(self, path, reason):
(path / ".render_failed").write_text(reason + "\n", encoding="utf-8")
self.errors.append((str(path), reason))
print(f"[render-queue:error] {path}: {reason}; model inference continues", flush=True)
def poll(self):
for process in list(self.running):
if process.poll() is None:
continue
path, handle, started = self.running.pop(process)
handle.close()
self.paths.remove(path)
if process.returncode != 0 or not render_complete(path):
self._failed(path, f"render exit={process.returncode}; see render.log")
else:
print(f"[render-queue] done {path}; elapsed={time.monotonic()-started:.1f}s", flush=True)
while self.pending and len(self.running) < self.workers:
path = self.pending.popleft()
environment = os.environ.copy()
environment["HFD_RENDER_ONLY_DIR"] = str(path)
if swa_outputs_ready(path):
environment["HFD_RENDER_TASK"] = "swa"
else:
environment.pop("HFD_RENDER_TASK", None)
# Software EGL otherwise creates a large thread pool for every
# render. Explicit user settings remain respected.
environment.setdefault("LP_NUM_THREADS", "2")
environment.setdefault("OMP_NUM_THREADS", "1")
handle = (path / "render.log").open("ab")
try:
command=["bash", str(self.project_root / "scripts/infer/run_audio_case.sh"), str(self.run_dir)]
if (path/'metrics.json').is_file():
metrics=json.loads((path/'metrics.json').read_text())
if metrics.get('overlay_fixture_file'):
command=[str(self.project_root/'.venv-psi/bin/python'),
str(self.project_root/'scripts/infer/unified_overlay.py'),'render',str(path)]
process = subprocess.Popen(
command,
cwd=self.project_root, env=environment, start_new_session=True,
stdout=handle, stderr=subprocess.STDOUT,
)
except OSError as error:
handle.close()
self.paths.remove(path)
self._failed(path, str(error))
continue
self.running[process] = (path, handle, time.monotonic())
print(f"[render-queue] start {path}; workers={len(self.running)}/{self.workers}", flush=True)
def drain(self):
while self.busy:
self.poll()
if self.busy:
time.sleep(.25)
def close(self):
# Only our process groups; durable .inferred files permit resumption.
for process in self.running:
if process.poll() is None:
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
pass
for process, (_, handle, _) in self.running.items():
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
process.wait()
handle.close()
self.running.clear()
def __enter__(self):
return self
def __exit__(self, *_):
self.close()