code/tt_diffrhythm/worker.py

#!/usr/bin/env python3
"""TT worker: loads a music model's DiT and its audio VAE decoder onto the P100a and answers one host pipeline.

Runs inside the TT image; the host (remote.py) owns the container for its whole life and talks to
this process over a Unix socket in a directory bind-mounted into both:

  python /work/tt_diffrhythm/worker.py --socket /ipc/worker.sock [--model diffrhythm] --checkpoint ... --vae ...

Messages are plain dicts (tensors via wire.py):
  -> {"op": "ready", ...}                                           once the weights are on the device
  <- {"op": "prepare", ...}                                         -> {"op": "prepared", ...}
       DiffRhythm: "text_embed": {drop_text: [T, 512]}, "cond": [T, 64]
  <- {"op": "forward", ...}                                         -> {"op": "velocity", "v": [T, 64], ...}
       DiffRhythm: "x": [T, 64], "drop_text": bool, "style": [512], "c": [512]
  <- {"op": "vae", "z": [64, L]}                                    -> {"op": "audio", "audio": [2, L * upsample], ...}
  <- {"op": "close"}
Any failure is sent back as {"op": "error", "error": "..."} and ends the worker.
"""
import argparse
from multiprocessing.connection import Listener
import os
import signal
import sys
import time
import traceback

import torch

from . import diffrhythm_dit, oobleck_vae, wire
from .shapes import VAE_WINDOWS


class DiffRhythm:
    def __init__(self, dev, checkpoint):
        from . import diffrhythm_host

        self.dit = diffrhythm_dit.DiffRhythmDiT(dev, diffrhythm_host.Checkpoint(checkpoint))

    def prepare(self, msg):
        text_embed = {bool(k): wire.decode(v) for k, v in msg["text_embed"].items()}
        song = self.dit.prepare(text_embed, wire.decode(msg["cond"]))
        return song, {"frames": song.geo.frames, "seq_pad": song.geo.seq_pad}

    def forward(self, msg, song):
        return self.dit.forward(wire.decode(msg["x"]), bool(msg["drop_text"]), wire.decode(msg["style"]),
                                wire.decode(msg["c"]), song)


MODELS = {"diffrhythm": (DiffRhythm, "/checkpoint/cfm_model.pt", "/vae/vae_model.pt")}


def serve(conn, model, vae):
    dit = model.dit
    song = None
    while True:
        msg = conn.recv()
        op = msg.get("op")
        started = time.monotonic()
        if op == "prepare":
            if song is not None:
                dit.release(song)
                song = None
            song, geometry = model.prepare(msg)
            conn.send({"op": "prepared", "seconds": time.monotonic() - started, "geometry": geometry})
        elif op == "forward":
            if song is None:
                raise RuntimeError("forward before prepare")
            v = model.forward(msg, song)
            conn.send({"op": "velocity", "v": wire.encode(v), "seconds": time.monotonic() - started})
        elif op == "vae":
            if song is not None:  # sampling is over; freeing the song's tensors first gives the VAE the same device
                dit.release(song)  # allocation state for every song length, so its compiled conv kernels are reused
                song = None
            audio = vae.decode(wire.decode(msg["z"]))
            conn.send({"op": "audio", "audio": wire.encode(audio), "seconds": time.monotonic() - started})
        elif op == "close":
            if song is not None:
                dit.release(song)
            return
        else:
            raise RuntimeError(f"unknown op {op!r}")


def main():
    # docker stop / a signal: leave through the finally below so the device is closed, not dropped
    signal.signal(signal.SIGTERM, lambda *_: sys.exit(143))
    parser = argparse.ArgumentParser()
    parser.add_argument("--socket", required=True)
    parser.add_argument("--model", choices=sorted(MODELS), default="diffrhythm")
    parser.add_argument("--checkpoint")
    parser.add_argument("--vae")
    args = parser.parse_args()
    model_class, checkpoint, vae_path = MODELS[args.model]
    authkey = os.environ.get("MUSIC_TT_AUTHKEY", "").encode() or None
    listener = Listener(args.socket, family="AF_UNIX", authkey=authkey)
    started = time.monotonic()
    dev = diffrhythm_dit.open_device()
    conn = None
    try:
        model = model_class(dev, args.checkpoint or checkpoint)
        vae = oobleck_vae.OobleckTT(dev, args.vae or vae_path)
        # Build the VAE programs now, in the same device state for every song: their conv kernels then compile
        # once (TT_METAL_CACHE) and later decodes reuse the in-process programs. Built after a song's DiT steps
        # instead, the device state (and so the kernels) depends on the song length and recompiles for 20-40 s.
        warm = time.monotonic()
        for window in VAE_WINDOWS:
            vae.decode(torch.zeros(64, window))
        ready = {"op": "ready", "model": args.model, "load_s": time.monotonic() - started,
                 "vae_warm_s": time.monotonic() - warm, "upsample": vae.upsample, "dram": diffrhythm_dit.dram_stats(dev)}
        conn = listener.accept()  # one pipeline per container
        conn.send(ready)
        serve(conn, model, vae)
    except BaseException as exc:
        traceback.print_exc()
        if conn is not None:
            try:
                conn.send({"op": "error", "error": f"{type(exc).__name__}: {exc}"})
            except OSError:
                pass
        raise
    finally:
        diffrhythm_dit.close_device(dev)
        listener.close()


if __name__ == "__main__":
    main()