File size: 8,015 Bytes
06f9561
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
"""Bounded, checkpoint-independent video jobs for the automatic infer watcher."""
from __future__ import annotations

from collections import deque
import json
import os
from pathlib import Path
import signal
import subprocess
import time


def swa_outputs_ready(path: Path) -> bool:
    """Legacy SWA .complete meant tensors/JSON only, not a rendered video."""
    try:
        metrics = json.loads((path / "metrics.json").read_text())
        if metrics.get('native_speech_version') == 1:
            for answer in metrics['answers']:
                audio = (path / answer['audio_path']).resolve()
                if (not audio.is_relative_to(path.resolve()) or not answer.get('native_speech')
                        or audio.stat().st_size <= 44 or answer.get('samples', 0) <= 0):
                    return False
        if metrics.get('overlay_fixture_file'):
            return bool(metrics.get('source_id') and metrics.get('events')) and (
                Path(metrics['overlay_fixture_file']).is_file() and
                (path/'generated_abs.npy').stat().st_size>0)
        return bool(metrics.get("id") and metrics.get("events")) and all(
            (path / name).stat().st_size > 0
            for name in ("generated_abs.npy", "body_pos.csv", "body_quat.csv", "joint_pos.csv")
        )
    except (OSError, ValueError, TypeError, AttributeError, KeyError):
        return False


def render_complete(path: Path) -> bool:
    if not (path / ".complete").is_file():
        return False
    if (path / "metrics.json").is_file():
        try:
            metrics = json.loads((path / 'metrics.json').read_text())
            expected = 2 if metrics.get('native_speech_version') == 1 else 1
            return ((path / "reference_motion_with_audio.mp4").stat().st_size > 0
                    and json.loads((path / "infer_meta.json").read_text()).get("swa_render_version") == expected)
        except (OSError, ValueError, AttributeError):
            return False
    return True


def render_inputs_ready(path: Path) -> bool:
    required = ("infer_meta.json", "body_pos.csv", "body_quat.csv", "joint_pos.csv",
                "assembled_input_audio_16k.wav")
    try:
        return (all((path / name).stat().st_size > 0 for name in required)
                and int(json.loads((path / "infer_meta.json").read_text())["total_frames"]) > 0)
    except (OSError, ValueError, KeyError, TypeError):
        return False


class AsyncRenderQueue:
    """Render workers do not consume inference GPU slots or checkpoint leases.

    .inferred + motion/audio files are the durable queue. On restart the watcher
    recovers unfinished results even if the original checkpoint was deleted.
    Backpressure bounds queued work (plus inference already in flight).
    """

    def __init__(self, project_root: Path, run_dir: Path, *, workers: int = 4,
                 capacity: int = 32):
        if workers < 1 or capacity < workers:
            raise ValueError("Render workers must be positive; pending limit must be >= workers")
        self.project_root, self.run_dir = project_root, run_dir
        self.workers, self.capacity = workers, capacity
        self.pending = deque()
        self.running = {}
        self.paths = set()
        self.errors = []
        self.last_recovery = float("-inf")

    @property
    def full(self):
        return len(self.paths) >= self.capacity

    @property
    def busy(self):
        return bool(self.paths)

    def enqueue(self, path: Path):
        path = path.resolve()
        if not path.is_relative_to(self.run_dir.resolve()):
            raise ValueError(f"Render result outside watched run: {path}")
        if path in self.paths or render_complete(path) or any(
                (path / marker).exists() for marker in (".failed", ".render_failed")):
            return
        if not (render_inputs_ready(path) or swa_outputs_ready(path)):
            raise ValueError(f"Incomplete inference outputs: {path}")
        (path / ".inferred").touch()
        self.paths.add(path)
        self.pending.append(path)
        print(f"[render-queue] queued {path}; backlog={len(self.paths)}", flush=True)

    def recover(self, paths):
        if time.monotonic() - self.last_recovery < 30:
            return
        self.last_recovery = time.monotonic()
        for path in paths:
            if self.full:
                break
            if render_inputs_ready(path) or swa_outputs_ready(path):
                self.enqueue(path)

    def _failed(self, path, reason):
        (path / ".render_failed").write_text(reason + "\n", encoding="utf-8")
        self.errors.append((str(path), reason))
        print(f"[render-queue:error] {path}: {reason}; model inference continues", flush=True)

    def poll(self):
        for process in list(self.running):
            if process.poll() is None:
                continue
            path, handle, started = self.running.pop(process)
            handle.close()
            self.paths.remove(path)
            if process.returncode != 0 or not render_complete(path):
                self._failed(path, f"render exit={process.returncode}; see render.log")
            else:
                print(f"[render-queue] done {path}; elapsed={time.monotonic()-started:.1f}s", flush=True)
        while self.pending and len(self.running) < self.workers:
            path = self.pending.popleft()
            environment = os.environ.copy()
            environment["HFD_RENDER_ONLY_DIR"] = str(path)
            if swa_outputs_ready(path):
                environment["HFD_RENDER_TASK"] = "swa"
            else:
                environment.pop("HFD_RENDER_TASK", None)
            # Software EGL otherwise creates a large thread pool for every
            # render. Explicit user settings remain respected.
            environment.setdefault("LP_NUM_THREADS", "2")
            environment.setdefault("OMP_NUM_THREADS", "1")
            handle = (path / "render.log").open("ab")
            try:
                command=["bash", str(self.project_root / "scripts/infer/run_audio_case.sh"), str(self.run_dir)]
                if (path/'metrics.json').is_file():
                    metrics=json.loads((path/'metrics.json').read_text())
                    if metrics.get('overlay_fixture_file'):
                        command=[str(self.project_root/'.venv-psi/bin/python'),
                                 str(self.project_root/'scripts/infer/unified_overlay.py'),'render',str(path)]
                process = subprocess.Popen(
                    command,
                    cwd=self.project_root, env=environment, start_new_session=True,
                    stdout=handle, stderr=subprocess.STDOUT,
                )
            except OSError as error:
                handle.close()
                self.paths.remove(path)
                self._failed(path, str(error))
                continue
            self.running[process] = (path, handle, time.monotonic())
            print(f"[render-queue] start {path}; workers={len(self.running)}/{self.workers}", flush=True)

    def drain(self):
        while self.busy:
            self.poll()
            if self.busy:
                time.sleep(.25)

    def close(self):
        # Only our process groups; durable .inferred files permit resumption.
        for process in self.running:
            if process.poll() is None:
                try:
                    os.killpg(process.pid, signal.SIGTERM)
                except ProcessLookupError:
                    pass
        for process, (_, handle, _) in self.running.items():
            try:
                process.wait(timeout=5)
            except subprocess.TimeoutExpired:
                try:
                    os.killpg(process.pid, signal.SIGKILL)
                except ProcessLookupError:
                    pass
                process.wait()
            handle.close()
        self.running.clear()

    def __enter__(self):
        return self

    def __exit__(self, *_):
        self.close()