Spaces:
Running on Zero
Running on Zero
Download zonos3/workers/pyhost.py from Zyphra/ZONOS3-Mini: direct link, hf CLI and curl.
- Browser
- Download file 19.9 kB
-
https://huggingface.co/spaces/Zyphra/ZONOS3-Mini/resolve/main/zonos3/workers/pyhost.py
- Command line
-
hf download hf://spaces/Zyphra/ZONOS3-Mini/zonos3/workers/pyhost.py
-
curl -L -o pyhost.py https://huggingface.co/spaces/Zyphra/ZONOS3-Mini/resolve/main/zonos3/workers/pyhost.py
19.9 kB
| #!/usr/bin/env python3 | |
| """The ggml server's Python host: speech editing (``zonos3.tts.edit_runtime.EditRuntime``) and the server plugins | |
| (``zonos3.server.plugins``, ``--plugins``) in a persistent process whose model is the server, reached over the same | |
| pipe. | |
| Runs with the workspace interpreter and ``PYTHONPATH=<model/src>`` (plugins installed in that interpreter). stdin and | |
| stdout carry one JSON object per line (UTF-8) in both directions, multiplexed: every request runs as its own asyncio | |
| task, so many are in flight and answer in any order. Stray output of the libraries goes to stderr (fd 1 points there; | |
| the protocol writes on a dup of the real stdout). The host exits when stdin closes, after closing its plugins and its | |
| ASR and FuseAlign workers. | |
| Requests ``{"id", "op", ...}`` are answered ``{"id", "status": "ok", "result"}`` or ``{"id", "status": "error", | |
| "code", "message"}``: the HTTP status and detail of ``zonos3.server.errors.call``; a body the schema | |
| (``zonos3.server.schemas``) rejects is 422 with FastAPI's detail list as ``message``; any other failure is 500 with | |
| ``str(exc)`` (traceback on stderr). | |
| Ops: | |
| init {edit: EditModel fields | null, host: HostFacts JSON | null, plugins: [name]} | |
| -> {plugins: [{name, keepalive_paths}]} | |
| edit_transcribe {request: EditTranscribeRequest} -> the /v2/edit/transcribe body | |
| edit_align {request: EditAlignRequest} -> the /v2/edit/align body | |
| edit {request: EditRequest} -> the /v2/edit body | |
| http {plugin, method, path, query, headers: [[k, v]], body: path | null, out: path} -> {status, headers: [[k, v]]} | |
| ``init`` comes first after every start; it builds the edit runtime over this process's own ``AsrWorker`` and | |
| ``FuseAlignWorker`` (ZONOS3_ASR_PYTHON, ZONOS3_ASR_DEVICE, ZONOS3_FUSEALIGN_DEVICE) and loads and mounts the plugins | |
| (``host``: ``zonos3.server.plugins.HostFacts.to_json``); a re-init closes the previous plugins. The ASR worker is | |
| optional: without its interpreter the runtime aligns and edits clips sent with their transcript and answers the | |
| requests that need speech recognition 503 (``zonos3.tts.edit_align.AsrUnavailable``). ``http`` runs one | |
| request of a plugin's routes: ``query`` is the encoded query string, ``body`` a file of the request body, and the | |
| response body is written to ``out``; error statuses are ``ok`` replies with that status. | |
| While a request runs, the model's work goes back to the server as engine calls ``{"call": N, "op", ...}`` (N unique per | |
| process), answered ``{"call": N, "status": "ok", ...}`` or ``{"call": N, "status": "error", "code", "message"}`` | |
| (code 400: ValueError, else RuntimeError, with the message): | |
| decode_audio {audio, out} -> {sample_rate}; float32 mono WAV at out | |
| embed {audio, max_seconds} -> {embedding: [float]} | |
| encode {audio, max_seconds} -> {codes: [[int] * K] * T} | |
| check {target_text, doses, condition_buckets, condition_values, | |
| steer_hold, steer_onset_seconds, steer_ramp_seconds} -> {raw: [id]} | |
| fill {every FillJob field} -> {rows: [[int] * K] * n, eos_frame (-1: none)} | |
| decode {codes, out, decoder} -> {}; float32 44.1 kHz WAV at out (decoder: id, null = default) | |
| laci_takes {takes: [{request: GenerateRequest body, target_frames, adjust_rate, | |
| scale: {edges, first, bucket, doses: {"<bucket>": dose}} | null, out}], session} | |
| -> {takes: [{audio: bool, record: LACI record}]}; WAV at out if audio | |
| Audio crosses as file paths (same host): ``audio`` is a file of the upload's bytes (``encode`` of a plugin's reference | |
| clip: a float WAV of the clip); the host creates and removes the temp dirs of its engine calls. | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| import dataclasses | |
| import json | |
| import os | |
| import sys | |
| import tempfile | |
| import threading | |
| import traceback | |
| from typing import TextIO | |
| import numpy as np | |
| import soundfile as sf | |
| BODY_CHUNK = 1 << 20 | |
| class InvalidBody(Exception): | |
| """A request body the schema rejects (HTTP 422); ``errors`` is FastAPI's detail list.""" | |
| def __init__(self, errors: list[dict]): | |
| super().__init__("invalid request body") | |
| self.errors = errors | |
| def validate(schema, body): | |
| """``body`` as a ``zonos3.server.schemas`` model (InvalidBody, never ValueError, when it does not fit).""" | |
| from pydantic import ValidationError | |
| try: | |
| return schema.model_validate(body) | |
| except ValidationError as exc: | |
| errors = json.loads(exc.json(include_url=False)) | |
| raise InvalidBody([{**e, "loc": ["body", *e["loc"]]} for e in errors]) from None | |
| class Pipe: | |
| """The protocol side of stdout (writes serialized) and the engine calls awaiting their answers.""" | |
| def __init__(self, out: TextIO): | |
| self._out = out | |
| self._lock = threading.Lock() | |
| self._calls: dict[int, asyncio.Future] = {} | |
| self._next_call = 0 | |
| self._closed = False | |
| def send(self, message: dict) -> None: | |
| line = json.dumps(message, ensure_ascii=False) + "\n" | |
| with self._lock: | |
| self._out.write(line) | |
| self._out.flush() | |
| async def call(self, op: str, **payload) -> dict: | |
| """One engine call; ValueError (code 400) or RuntimeError with the server's message when it fails.""" | |
| if self._closed: | |
| raise RuntimeError("the server closed the Python host's pipe") | |
| self._next_call += 1 | |
| call = self._next_call | |
| future = asyncio.get_running_loop().create_future() | |
| self._calls[call] = future | |
| try: | |
| self.send({"call": call, "op": op, **payload}) | |
| reply = await future | |
| finally: | |
| self._calls.pop(call, None) | |
| if reply.get("status") != "ok": | |
| message = reply.get("message") or f"engine call {op} failed" | |
| raise (ValueError if reply.get("code") == 400 else RuntimeError)(message) | |
| return reply | |
| def answer(self, reply: dict) -> None: | |
| future = self._calls.get(reply.get("call")) | |
| if future is None or future.done(): | |
| print(f"Python host: answer to no pending engine call: {reply.get('call')!r}", file=sys.stderr) | |
| return | |
| future.set_result(reply) | |
| def close(self) -> None: | |
| """Fail every pending engine call (stdin closed: no answer can come).""" | |
| self._closed = True | |
| for future in self._calls.values(): | |
| if not future.done(): | |
| future.set_exception(RuntimeError("the server closed the Python host's pipe")) | |
| def read_wav(path: str) -> np.ndarray: | |
| return sf.read(path, dtype="float32")[0] | |
| def write_upload(directory: str, audio: bytes) -> str: | |
| path = os.path.join(directory, "upload.bin") | |
| with open(path, "wb") as fh: | |
| fh.write(audio) | |
| return path | |
| class RpcEditEngine: | |
| """``zonos3.tts.edit_runtime.EditEngine`` over the server's engine calls.""" | |
| def __init__(self, pipe: Pipe, model): | |
| self.pipe = pipe | |
| self.model = model | |
| async def decode_audio(self, audio: bytes) -> tuple[np.ndarray, int]: | |
| with tempfile.TemporaryDirectory(prefix="zonos3-pyhost-") as tmp: | |
| out = os.path.join(tmp, "decoded.wav") | |
| reply = await self.pipe.call("decode_audio", audio=write_upload(tmp, audio), out=out) | |
| return read_wav(out), int(reply["sample_rate"]) | |
| async def embed(self, audio: bytes) -> list[float]: | |
| from zonos3.generation import EDIT_MAX_SECONDS | |
| with tempfile.TemporaryDirectory(prefix="zonos3-pyhost-") as tmp: | |
| reply = await self.pipe.call("embed", audio=write_upload(tmp, audio), max_seconds=EDIT_MAX_SECONDS) | |
| return [float(x) for x in reply["embedding"]] | |
| async def encode(self, audio: bytes, max_seconds: float) -> np.ndarray: | |
| with tempfile.TemporaryDirectory(prefix="zonos3-pyhost-") as tmp: | |
| reply = await self.pipe.call("encode", audio=write_upload(tmp, audio), max_seconds=max_seconds) | |
| return np.asarray(reply["codes"], dtype=np.int64).reshape(-1, self.model.n_codebooks) | |
| async def check(self, target_text: str, doses, condition_buckets, condition_values, steer) -> set[str]: | |
| reply = await self.pipe.call( | |
| "check", | |
| target_text=target_text, | |
| doses=doses, | |
| condition_buckets=condition_buckets, | |
| condition_values=condition_values, | |
| **steer, | |
| ) | |
| return set(reply["raw"]) | |
| async def fill(self, job) -> tuple[np.ndarray, int]: | |
| payload = {f.name: getattr(job, f.name) for f in dataclasses.fields(job)} | |
| payload["window_codes"] = np.asarray(job.window_codes).tolist() | |
| payload["fills"] = [np.asarray(fill).tolist() for fill in job.fills] | |
| reply = await self.pipe.call("fill", **payload) | |
| return np.asarray(reply["rows"], dtype=np.int64).reshape(-1, self.model.n_codebooks), int(reply["eos_frame"]) | |
| async def decode(self, codes: np.ndarray, decoder: str | None) -> np.ndarray: | |
| with tempfile.TemporaryDirectory(prefix="zonos3-pyhost-") as tmp: | |
| out = os.path.join(tmp, "decoded.wav") | |
| await self.pipe.call("decode", codes=np.asarray(codes).tolist(), out=out, decoder=decoder) | |
| return read_wav(out) | |
| class RpcHost: | |
| """``zonos3.server.plugins.Host`` over the server's engine calls.""" | |
| def __init__(self, pipe: Pipe, facts, asr, aligner): | |
| self.pipe = pipe | |
| self._facts = facts | |
| self.asr = asr | |
| self.aligner = aligner | |
| def facts(self): | |
| return self._facts | |
| async def synthesize_laci(self, takes, session_id: str | None) -> list: | |
| with tempfile.TemporaryDirectory(prefix="zonos3-pyhost-") as tmp: | |
| outs = [os.path.join(tmp, f"take{k}.wav") for k in range(len(takes))] | |
| reply = await self.pipe.call( | |
| "laci_takes", session=session_id, takes=[take.to_json(out) for take, out in zip(takes, outs)] | |
| ) | |
| return [ | |
| (read_wav(out) if take["audio"] else None, take["record"]) | |
| for take, out in zip(reply["takes"], outs, strict=True) | |
| ] | |
| async def encode_reference(self, audio: np.ndarray, sample_rate: int, max_seconds: float) -> list[list[int]]: | |
| with tempfile.TemporaryDirectory(prefix="zonos3-pyhost-") as tmp: | |
| path = os.path.join(tmp, "reference.wav") | |
| sf.write(path, audio, sample_rate, subtype="FLOAT") | |
| return (await self.pipe.call("encode", audio=path, max_seconds=max_seconds))["codes"] | |
| async def asgi_request( | |
| app, method: str, path: str, query: str, headers: list[list[str]], body_path: str | None, out_path: str | |
| ) -> tuple[int, list[list[str]]]: | |
| """Run one HTTP request through the ASGI ``app``: the body streamed from ``body_path`` in 1 MiB chunks, the | |
| response body written to ``out_path``. Returns the status and response headers.""" | |
| scope = { | |
| "type": "http", | |
| "asgi": {"version": "3.0", "spec_version": "2.3"}, | |
| "http_version": "1.1", | |
| "method": method.upper(), | |
| "scheme": "http", | |
| "path": path, | |
| "raw_path": path.encode("utf-8"), | |
| "query_string": (query or "").encode("latin-1"), | |
| "root_path": "", | |
| "headers": [(k.lower().encode("latin-1"), v.encode("latin-1")) for k, v in headers], | |
| "client": ("127.0.0.1", 0), | |
| "server": ("127.0.0.1", 80), | |
| } | |
| body = open(body_path, "rb") if body_path else None | |
| start: dict = {} | |
| done = asyncio.Event() | |
| received = False | |
| async def receive() -> dict: | |
| nonlocal received | |
| if not received: | |
| chunk = body.read(BODY_CHUNK) if body is not None else b"" | |
| received = len(chunk) < BODY_CHUNK | |
| return {"type": "http.request", "body": chunk, "more_body": not received} | |
| await done.wait() | |
| return {"type": "http.disconnect"} | |
| with open(out_path, "wb") as out: | |
| async def send(message: dict) -> None: | |
| if message["type"] == "http.response.start": | |
| start["status"] = int(message["status"]) | |
| start["headers"] = [[k.decode("latin-1"), v.decode("latin-1")] for k, v in message.get("headers", ())] | |
| elif message["type"] == "http.response.body": | |
| out.write(message.get("body", b"")) | |
| if not message.get("more_body", False): | |
| done.set() | |
| try: | |
| await app(scope, receive, send) | |
| except Exception: | |
| if "status" not in start: # nothing sent: the app's error is this request's 500 | |
| raise | |
| traceback.print_exc() | |
| finally: | |
| done.set() | |
| if body is not None and not body.closed: | |
| body.close() | |
| if "status" not in start: | |
| raise RuntimeError(f"{method} {path}: the app sent no response") | |
| return start["status"], start["headers"] | |
| class Worker: | |
| def __init__(self, pipe: Pipe): | |
| self.pipe = pipe | |
| self.asr = self.aligner = None | |
| self.edit = None | |
| self.plugins: list = [] | |
| self.app = None | |
| # ---- runtimes --------------------------------------------------------------------------------------------- | |
| async def init(self, edit: dict | None, host: dict | None, names: list[str] | None) -> dict: | |
| from fastapi import FastAPI | |
| from zonos3.server import plugins | |
| from zonos3.tts.edit_align import AsrWorker, FuseAlignWorker | |
| from zonos3.tts.edit_runtime import EditModel, EditRuntime | |
| if self.asr is None: | |
| self.asr, self.aligner = AsrWorker(), FuseAlignWorker() | |
| engine = None | |
| if edit is not None: | |
| engine = RpcEditEngine(self.pipe, EditModel(**{**edit, "codebook_sizes": tuple(edit["codebook_sizes"])})) | |
| self.edit = None if engine is None else EditRuntime(engine, self.asr, self.aligner) | |
| await asyncio.to_thread(self.close_plugins) | |
| names = list(names or ()) | |
| if names and host is None: | |
| raise ValueError("init: plugins need the host facts") | |
| loaded = plugins.load(names) | |
| app = FastAPI(openapi_url=None, docs_url=None, redoc_url=None) | |
| if loaded: | |
| rpc = RpcHost(self.pipe, plugins.HostFacts.from_json(host), self.asr, self.aligner) | |
| try: | |
| plugins.mount(app, loaded, rpc, []) | |
| except Exception: | |
| for plugin in loaded: | |
| plugin.close() | |
| raise | |
| self.plugins, self.app = loaded, app | |
| return {"plugins": [{"name": p.name, "keepalive_paths": list(p.keepalive_paths)} for p in loaded]} | |
| def edit_runtime(self): | |
| if self.edit is None: | |
| raise RuntimeError("speech editing is not initialized") | |
| return self.edit | |
| async def http(self, request: dict) -> dict: | |
| name = request.get("plugin") | |
| if not any(p.name == name for p in self.plugins): | |
| raise ValueError(f"plugin {name!r} is not loaded") | |
| status, headers = await asgi_request( | |
| self.app, | |
| request["method"], | |
| request["path"], | |
| request.get("query") or "", | |
| request.get("headers") or [], | |
| request.get("body"), | |
| request["out"], | |
| ) | |
| return {"status": status, "headers": headers} | |
| def close_plugins(self) -> None: | |
| plugins, self.plugins, self.app = self.plugins, [], None | |
| for plugin in plugins: | |
| try: | |
| plugin.close() | |
| except Exception: # noqa: BLE001 - close the rest | |
| traceback.print_exc() | |
| def close(self) -> None: | |
| self.close_plugins() | |
| for worker in (self.asr, self.aligner): | |
| if worker is not None: | |
| worker.close() | |
| # ---- requests --------------------------------------------------------------------------------------------- | |
| async def run(self, request: dict): | |
| from zonos3.server import schemas as S | |
| op = request.get("op") | |
| if op == "init": | |
| return await self.init(request.get("edit"), request.get("host"), request.get("plugins")) | |
| if op == "edit_transcribe": | |
| return await self.edit_runtime().transcribe(validate(S.EditTranscribeRequest, request.get("request"))) | |
| if op == "edit_align": | |
| return await self.edit_runtime().align(validate(S.EditAlignRequest, request.get("request"))) | |
| if op == "edit": | |
| return await self.edit_runtime().edit(validate(S.EditRequest, request.get("request"))) | |
| if op == "http": | |
| return await self.http(request) | |
| raise ValueError(f"unknown op {op!r}") | |
| async def respond(self, request: dict) -> None: | |
| """Run one request and send its answer (errors as ``zonos3.server.errors.call`` maps them).""" | |
| from fastapi import HTTPException | |
| from zonos3.server.errors import call | |
| rid = request.get("id") | |
| try: | |
| result = await call(self.run(request)) | |
| reply = {"id": rid, "status": "ok", "result": result} | |
| except HTTPException as exc: | |
| reply = {"id": rid, "status": "error", "code": exc.status_code, "message": exc.detail} | |
| except InvalidBody as exc: | |
| reply = {"id": rid, "status": "error", "code": 422, "message": exc.errors} | |
| except Exception as exc: # noqa: BLE001 - one failed request must not kill the worker | |
| traceback.print_exc() | |
| reply = {"id": rid, "status": "error", "code": 500, "message": str(exc)} | |
| try: | |
| self.pipe.send(reply) | |
| except (TypeError, ValueError) as exc: # an unserializable result fails this request, not the pipe | |
| traceback.print_exc() | |
| self.pipe.send({"id": rid, "status": "error", "code": 500, "message": str(exc)}) | |
| async def serve(lines: TextIO, out: TextIO) -> None: | |
| """Answer the requests of ``lines`` on ``out`` until ``lines`` ends; engine answers come on ``lines`` too.""" | |
| loop = asyncio.get_running_loop() | |
| inbox: asyncio.Queue[str | None] = asyncio.Queue() | |
| def read() -> None: # blocking reads off the event loop | |
| for line in lines: | |
| loop.call_soon_threadsafe(inbox.put_nowait, line) | |
| loop.call_soon_threadsafe(inbox.put_nowait, None) | |
| threading.Thread(target=read, name="pyhost-stdin", daemon=True).start() | |
| pipe = Pipe(out) | |
| worker = Worker(pipe) | |
| tasks: set[asyncio.Task] = set() | |
| while (line := await inbox.get()) is not None: | |
| if not line.strip(): | |
| continue | |
| try: | |
| message = json.loads(line) | |
| if not isinstance(message, dict): | |
| raise ValueError("a message must be a JSON object") | |
| except ValueError as exc: | |
| pipe.send({"id": None, "status": "error", "code": 400, "message": f"invalid message: {exc}"}) | |
| continue | |
| if "call" in message: | |
| pipe.answer(message) | |
| continue | |
| task = asyncio.create_task(worker.respond(message)) | |
| tasks.add(task) | |
| task.add_done_callback(tasks.discard) | |
| pipe.close() | |
| for task in tasks: | |
| task.cancel() | |
| await asyncio.gather(*tasks, return_exceptions=True) | |
| await asyncio.to_thread(worker.close) | |
| def main() -> None: | |
| # Keep stdout for the protocol only: library prints go to stderr. | |
| protocol = os.fdopen(os.dup(1), "w", buffering=1, encoding="utf-8") | |
| os.dup2(2, 1) | |
| sys.stdout = sys.stderr | |
| sys.stdin.reconfigure(encoding="utf-8") | |
| asyncio.run(serve(sys.stdin, protocol)) | |
| if __name__ == "__main__": | |
| main() | |