File size: 8,015 Bytes
06f9561 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 | """Bounded, checkpoint-independent video jobs for the automatic infer watcher."""
from __future__ import annotations
from collections import deque
import json
import os
from pathlib import Path
import signal
import subprocess
import time
def swa_outputs_ready(path: Path) -> bool:
"""Legacy SWA .complete meant tensors/JSON only, not a rendered video."""
try:
metrics = json.loads((path / "metrics.json").read_text())
if metrics.get('native_speech_version') == 1:
for answer in metrics['answers']:
audio = (path / answer['audio_path']).resolve()
if (not audio.is_relative_to(path.resolve()) or not answer.get('native_speech')
or audio.stat().st_size <= 44 or answer.get('samples', 0) <= 0):
return False
if metrics.get('overlay_fixture_file'):
return bool(metrics.get('source_id') and metrics.get('events')) and (
Path(metrics['overlay_fixture_file']).is_file() and
(path/'generated_abs.npy').stat().st_size>0)
return bool(metrics.get("id") and metrics.get("events")) and all(
(path / name).stat().st_size > 0
for name in ("generated_abs.npy", "body_pos.csv", "body_quat.csv", "joint_pos.csv")
)
except (OSError, ValueError, TypeError, AttributeError, KeyError):
return False
def render_complete(path: Path) -> bool:
if not (path / ".complete").is_file():
return False
if (path / "metrics.json").is_file():
try:
metrics = json.loads((path / 'metrics.json').read_text())
expected = 2 if metrics.get('native_speech_version') == 1 else 1
return ((path / "reference_motion_with_audio.mp4").stat().st_size > 0
and json.loads((path / "infer_meta.json").read_text()).get("swa_render_version") == expected)
except (OSError, ValueError, AttributeError):
return False
return True
def render_inputs_ready(path: Path) -> bool:
required = ("infer_meta.json", "body_pos.csv", "body_quat.csv", "joint_pos.csv",
"assembled_input_audio_16k.wav")
try:
return (all((path / name).stat().st_size > 0 for name in required)
and int(json.loads((path / "infer_meta.json").read_text())["total_frames"]) > 0)
except (OSError, ValueError, KeyError, TypeError):
return False
class AsyncRenderQueue:
"""Render workers do not consume inference GPU slots or checkpoint leases.
.inferred + motion/audio files are the durable queue. On restart the watcher
recovers unfinished results even if the original checkpoint was deleted.
Backpressure bounds queued work (plus inference already in flight).
"""
def __init__(self, project_root: Path, run_dir: Path, *, workers: int = 4,
capacity: int = 32):
if workers < 1 or capacity < workers:
raise ValueError("Render workers must be positive; pending limit must be >= workers")
self.project_root, self.run_dir = project_root, run_dir
self.workers, self.capacity = workers, capacity
self.pending = deque()
self.running = {}
self.paths = set()
self.errors = []
self.last_recovery = float("-inf")
@property
def full(self):
return len(self.paths) >= self.capacity
@property
def busy(self):
return bool(self.paths)
def enqueue(self, path: Path):
path = path.resolve()
if not path.is_relative_to(self.run_dir.resolve()):
raise ValueError(f"Render result outside watched run: {path}")
if path in self.paths or render_complete(path) or any(
(path / marker).exists() for marker in (".failed", ".render_failed")):
return
if not (render_inputs_ready(path) or swa_outputs_ready(path)):
raise ValueError(f"Incomplete inference outputs: {path}")
(path / ".inferred").touch()
self.paths.add(path)
self.pending.append(path)
print(f"[render-queue] queued {path}; backlog={len(self.paths)}", flush=True)
def recover(self, paths):
if time.monotonic() - self.last_recovery < 30:
return
self.last_recovery = time.monotonic()
for path in paths:
if self.full:
break
if render_inputs_ready(path) or swa_outputs_ready(path):
self.enqueue(path)
def _failed(self, path, reason):
(path / ".render_failed").write_text(reason + "\n", encoding="utf-8")
self.errors.append((str(path), reason))
print(f"[render-queue:error] {path}: {reason}; model inference continues", flush=True)
def poll(self):
for process in list(self.running):
if process.poll() is None:
continue
path, handle, started = self.running.pop(process)
handle.close()
self.paths.remove(path)
if process.returncode != 0 or not render_complete(path):
self._failed(path, f"render exit={process.returncode}; see render.log")
else:
print(f"[render-queue] done {path}; elapsed={time.monotonic()-started:.1f}s", flush=True)
while self.pending and len(self.running) < self.workers:
path = self.pending.popleft()
environment = os.environ.copy()
environment["HFD_RENDER_ONLY_DIR"] = str(path)
if swa_outputs_ready(path):
environment["HFD_RENDER_TASK"] = "swa"
else:
environment.pop("HFD_RENDER_TASK", None)
# Software EGL otherwise creates a large thread pool for every
# render. Explicit user settings remain respected.
environment.setdefault("LP_NUM_THREADS", "2")
environment.setdefault("OMP_NUM_THREADS", "1")
handle = (path / "render.log").open("ab")
try:
command=["bash", str(self.project_root / "scripts/infer/run_audio_case.sh"), str(self.run_dir)]
if (path/'metrics.json').is_file():
metrics=json.loads((path/'metrics.json').read_text())
if metrics.get('overlay_fixture_file'):
command=[str(self.project_root/'.venv-psi/bin/python'),
str(self.project_root/'scripts/infer/unified_overlay.py'),'render',str(path)]
process = subprocess.Popen(
command,
cwd=self.project_root, env=environment, start_new_session=True,
stdout=handle, stderr=subprocess.STDOUT,
)
except OSError as error:
handle.close()
self.paths.remove(path)
self._failed(path, str(error))
continue
self.running[process] = (path, handle, time.monotonic())
print(f"[render-queue] start {path}; workers={len(self.running)}/{self.workers}", flush=True)
def drain(self):
while self.busy:
self.poll()
if self.busy:
time.sleep(.25)
def close(self):
# Only our process groups; durable .inferred files permit resumption.
for process in self.running:
if process.poll() is None:
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
pass
for process, (_, handle, _) in self.running.items():
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
process.wait()
handle.close()
self.running.clear()
def __enter__(self):
return self
def __exit__(self, *_):
self.close()
|