Download src/psi/async_render_queue.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 8.02 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/async_render_queue.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/async_render_queue.py
-
curl -L -o async_render_queue.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/async_render_queue.py
8.02 kB
| """Bounded, checkpoint-independent video jobs for the automatic infer watcher.""" | |
| from __future__ import annotations | |
| from collections import deque | |
| import json | |
| import os | |
| from pathlib import Path | |
| import signal | |
| import subprocess | |
| import time | |
| def swa_outputs_ready(path: Path) -> bool: | |
| """Legacy SWA .complete meant tensors/JSON only, not a rendered video.""" | |
| try: | |
| metrics = json.loads((path / "metrics.json").read_text()) | |
| if metrics.get('native_speech_version') == 1: | |
| for answer in metrics['answers']: | |
| audio = (path / answer['audio_path']).resolve() | |
| if (not audio.is_relative_to(path.resolve()) or not answer.get('native_speech') | |
| or audio.stat().st_size <= 44 or answer.get('samples', 0) <= 0): | |
| return False | |
| if metrics.get('overlay_fixture_file'): | |
| return bool(metrics.get('source_id') and metrics.get('events')) and ( | |
| Path(metrics['overlay_fixture_file']).is_file() and | |
| (path/'generated_abs.npy').stat().st_size>0) | |
| return bool(metrics.get("id") and metrics.get("events")) and all( | |
| (path / name).stat().st_size > 0 | |
| for name in ("generated_abs.npy", "body_pos.csv", "body_quat.csv", "joint_pos.csv") | |
| ) | |
| except (OSError, ValueError, TypeError, AttributeError, KeyError): | |
| return False | |
| def render_complete(path: Path) -> bool: | |
| if not (path / ".complete").is_file(): | |
| return False | |
| if (path / "metrics.json").is_file(): | |
| try: | |
| metrics = json.loads((path / 'metrics.json').read_text()) | |
| expected = 2 if metrics.get('native_speech_version') == 1 else 1 | |
| return ((path / "reference_motion_with_audio.mp4").stat().st_size > 0 | |
| and json.loads((path / "infer_meta.json").read_text()).get("swa_render_version") == expected) | |
| except (OSError, ValueError, AttributeError): | |
| return False | |
| return True | |
| def render_inputs_ready(path: Path) -> bool: | |
| required = ("infer_meta.json", "body_pos.csv", "body_quat.csv", "joint_pos.csv", | |
| "assembled_input_audio_16k.wav") | |
| try: | |
| return (all((path / name).stat().st_size > 0 for name in required) | |
| and int(json.loads((path / "infer_meta.json").read_text())["total_frames"]) > 0) | |
| except (OSError, ValueError, KeyError, TypeError): | |
| return False | |
| class AsyncRenderQueue: | |
| """Render workers do not consume inference GPU slots or checkpoint leases. | |
| .inferred + motion/audio files are the durable queue. On restart the watcher | |
| recovers unfinished results even if the original checkpoint was deleted. | |
| Backpressure bounds queued work (plus inference already in flight). | |
| """ | |
| def __init__(self, project_root: Path, run_dir: Path, *, workers: int = 4, | |
| capacity: int = 32): | |
| if workers < 1 or capacity < workers: | |
| raise ValueError("Render workers must be positive; pending limit must be >= workers") | |
| self.project_root, self.run_dir = project_root, run_dir | |
| self.workers, self.capacity = workers, capacity | |
| self.pending = deque() | |
| self.running = {} | |
| self.paths = set() | |
| self.errors = [] | |
| self.last_recovery = float("-inf") | |
| def full(self): | |
| return len(self.paths) >= self.capacity | |
| def busy(self): | |
| return bool(self.paths) | |
| def enqueue(self, path: Path): | |
| path = path.resolve() | |
| if not path.is_relative_to(self.run_dir.resolve()): | |
| raise ValueError(f"Render result outside watched run: {path}") | |
| if path in self.paths or render_complete(path) or any( | |
| (path / marker).exists() for marker in (".failed", ".render_failed")): | |
| return | |
| if not (render_inputs_ready(path) or swa_outputs_ready(path)): | |
| raise ValueError(f"Incomplete inference outputs: {path}") | |
| (path / ".inferred").touch() | |
| self.paths.add(path) | |
| self.pending.append(path) | |
| print(f"[render-queue] queued {path}; backlog={len(self.paths)}", flush=True) | |
| def recover(self, paths): | |
| if time.monotonic() - self.last_recovery < 30: | |
| return | |
| self.last_recovery = time.monotonic() | |
| for path in paths: | |
| if self.full: | |
| break | |
| if render_inputs_ready(path) or swa_outputs_ready(path): | |
| self.enqueue(path) | |
| def _failed(self, path, reason): | |
| (path / ".render_failed").write_text(reason + "\n", encoding="utf-8") | |
| self.errors.append((str(path), reason)) | |
| print(f"[render-queue:error] {path}: {reason}; model inference continues", flush=True) | |
| def poll(self): | |
| for process in list(self.running): | |
| if process.poll() is None: | |
| continue | |
| path, handle, started = self.running.pop(process) | |
| handle.close() | |
| self.paths.remove(path) | |
| if process.returncode != 0 or not render_complete(path): | |
| self._failed(path, f"render exit={process.returncode}; see render.log") | |
| else: | |
| print(f"[render-queue] done {path}; elapsed={time.monotonic()-started:.1f}s", flush=True) | |
| while self.pending and len(self.running) < self.workers: | |
| path = self.pending.popleft() | |
| environment = os.environ.copy() | |
| environment["HFD_RENDER_ONLY_DIR"] = str(path) | |
| if swa_outputs_ready(path): | |
| environment["HFD_RENDER_TASK"] = "swa" | |
| else: | |
| environment.pop("HFD_RENDER_TASK", None) | |
| # Software EGL otherwise creates a large thread pool for every | |
| # render. Explicit user settings remain respected. | |
| environment.setdefault("LP_NUM_THREADS", "2") | |
| environment.setdefault("OMP_NUM_THREADS", "1") | |
| handle = (path / "render.log").open("ab") | |
| try: | |
| command=["bash", str(self.project_root / "scripts/infer/run_audio_case.sh"), str(self.run_dir)] | |
| if (path/'metrics.json').is_file(): | |
| metrics=json.loads((path/'metrics.json').read_text()) | |
| if metrics.get('overlay_fixture_file'): | |
| command=[str(self.project_root/'.venv-psi/bin/python'), | |
| str(self.project_root/'scripts/infer/unified_overlay.py'),'render',str(path)] | |
| process = subprocess.Popen( | |
| command, | |
| cwd=self.project_root, env=environment, start_new_session=True, | |
| stdout=handle, stderr=subprocess.STDOUT, | |
| ) | |
| except OSError as error: | |
| handle.close() | |
| self.paths.remove(path) | |
| self._failed(path, str(error)) | |
| continue | |
| self.running[process] = (path, handle, time.monotonic()) | |
| print(f"[render-queue] start {path}; workers={len(self.running)}/{self.workers}", flush=True) | |
| def drain(self): | |
| while self.busy: | |
| self.poll() | |
| if self.busy: | |
| time.sleep(.25) | |
| def close(self): | |
| # Only our process groups; durable .inferred files permit resumption. | |
| for process in self.running: | |
| if process.poll() is None: | |
| try: | |
| os.killpg(process.pid, signal.SIGTERM) | |
| except ProcessLookupError: | |
| pass | |
| for process, (_, handle, _) in self.running.items(): | |
| try: | |
| process.wait(timeout=5) | |
| except subprocess.TimeoutExpired: | |
| try: | |
| os.killpg(process.pid, signal.SIGKILL) | |
| except ProcessLookupError: | |
| pass | |
| process.wait() | |
| handle.close() | |
| self.running.clear() | |
| def __enter__(self): | |
| return self | |
| def __exit__(self, *_): | |
| self.close() | |