Spaces:
Running on Zero
Running on Zero
Deploy ZONOS3 6b831d4
Browse files- app.py +7 -9
- zonos3/models/accent_format.py +45 -14
- zonos3/reference/__main__.py +1 -0
- zonos3/reference/accents.py +2 -2
- zonos3/reference/lm.py +12 -11
- zonos3/reference/synth.py +7 -0
app.py
CHANGED
|
@@ -100,18 +100,16 @@ STUDIO = ROOT / "studio"
|
|
| 100 |
DEVICE = torch.device("cuda:0")
|
| 101 |
CPU = torch.device("cpu")
|
| 102 |
|
| 103 |
-
# The edit
|
| 104 |
-
# sample (so they run exactly as on the release LM). Edit regions add the
|
| 105 |
-
# the CPU, then moved: ZeroGPU's emulated CUDA refuses mixing its tensors with
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
LM = Zonos3LM.load(RELEASE_LM, device="cpu", dtype=torch.float32)
|
| 109 |
-
LM.grow(_GROWN["rows"], span_end_id=_GROWN["span_end_id"], text_extra_tokens=_GROWN["text_extra_tokens"])
|
| 110 |
-
LM = LM.to(DEVICE)
|
| 111 |
CONFIG = LM.config
|
| 112 |
ASSETS = ReferenceAssets.load(steer_bundle=str(ROOT / "steering" / "studio_bundle_mect60k.json"))
|
| 113 |
CONTEXT = ReferenceContext(LM, ASSETS, DEVICE) # the request rules' view of the checkpoint (no GPU work)
|
| 114 |
-
EDIT_LORA = AccentState.for_model(LM,
|
| 115 |
ASSETS.decoder(CONFIG.audio_codec, DEVICE)
|
| 116 |
ASSETS.speaker_encoder(CONFIG.speaker_embedding_codec, CONFIG.speaker_embedding_dim, CPU)
|
| 117 |
# A clip in reference mode is encoded by the dual codec inside the GPU call; fetch its teacher folder now.
|
|
|
|
| 100 |
DEVICE = torch.device("cuda:0")
|
| 101 |
CPU = torch.device("cpu")
|
| 102 |
|
| 103 |
+
# The edit LoRA (zonos-accent-lora/3) brings the vocabulary it was trained with: its span-end audio row and edit-marker
|
| 104 |
+
# text rows, which plain takes never read or sample (so they run exactly as on the release LM). Edit regions add the
|
| 105 |
+
# LoRA per request (EDIT_LORA). Grown on the CPU, then moved: ZeroGPU's emulated CUDA refuses mixing its tensors with
|
| 106 |
+
# CPU ones in this process.
|
| 107 |
+
EDIT_ADAPTER = load_adapter(resolve_file(f"{RELEASE_LM}/edit_lora/z3m_omni3b_edit_lora.pt"))
|
| 108 |
+
LM = Zonos3LM.load(RELEASE_LM, device="cpu", dtype=torch.float32).grow(EDIT_ADAPTER.grown).to(DEVICE)
|
|
|
|
|
|
|
| 109 |
CONFIG = LM.config
|
| 110 |
ASSETS = ReferenceAssets.load(steer_bundle=str(ROOT / "steering" / "studio_bundle_mect60k.json"))
|
| 111 |
CONTEXT = ReferenceContext(LM, ASSETS, DEVICE) # the request rules' view of the checkpoint (no GPU work)
|
| 112 |
+
EDIT_LORA = AccentState.for_model(LM, EDIT_ADAPTER, 1.0)
|
| 113 |
ASSETS.decoder(CONFIG.audio_codec, DEVICE)
|
| 114 |
ASSETS.speaker_encoder(CONFIG.speaker_embedding_codec, CONFIG.speaker_embedding_dim, CPU)
|
| 115 |
# A clip in reference mode is encoded by the dual codec inside the GPU call; fetch its teacher folder now.
|
zonos3/models/accent_format.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""Accent adapter files (lingua ``apps.zonos.accent_adapter``, ``zonos-accent-lora/1`` and ``/
|
| 2 |
|
| 3 |
An adapter shifts linear weights by ``up @ down``, router balancing biases by a vector and, in ``/2``, every MoE
|
| 4 |
expert weight by its own ``up_e @ down_e``. A request served with adapter strength alpha runs ``base + alpha *
|
|
@@ -9,10 +9,17 @@ Module names are the checkpoint's: ``tools/convert_lingua_checkpoint.py`` keeps
|
|
| 9 |
lingua key. ``TARGETS`` mirrors lingua's ``apps.zonos.lora.DEFAULT_TARGETS``. Exact shifts: ``<router>.balancing_biases``
|
| 10 |
(float32 quantile thresholds); as in the converter, the legacy ZeroMean parametrization
|
| 11 |
``.parametrizations.balancing_biases.original`` maps to ``balancing_biases`` with its mean removed, and the training-only
|
| 12 |
-
router metrics (``ent_denom``, ``normalized_entropy``) are dropped. Any other exact shift is refused.
|
| 13 |
|
| 14 |
Expert factors (``/2``): ``layers.N.feed_forward.experts.{w13,w2}`` carry ``up`` [E, out, r] and ``down`` [E, r, in] in
|
| 15 |
lingua's SonicExperts layout (w13 rows interleave gate/up), ``shape`` (E, out, in) and ``experts`` E.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
"""
|
| 17 |
|
| 18 |
from __future__ import annotations
|
|
@@ -24,7 +31,8 @@ import torch
|
|
| 24 |
|
| 25 |
FORMAT = "zonos-accent-lora/1"
|
| 26 |
FORMAT_EXPERTS = "zonos-accent-lora/2"
|
| 27 |
-
|
|
|
|
| 28 |
# SonicExperts weights lingua's apps.zonos.lora adapts per expert.
|
| 29 |
EXPERT_WEIGHTS = ("w13", "w2")
|
| 30 |
|
|
@@ -81,6 +89,15 @@ def list_accent_adapters(directory: str | Path) -> list[str]:
|
|
| 81 |
return names
|
| 82 |
|
| 83 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
@dataclass(frozen=True)
|
| 85 |
class AccentAdapter:
|
| 86 |
"""One parsed adapter.
|
|
@@ -89,9 +106,9 @@ class AccentAdapter:
|
|
| 89 |
``(x @ down.T) @ up`` on the flat ``[out, in]`` product; ``shapes``: the weight shape each linear shift was trained
|
| 90 |
on. ``experts``: experts module (``layers.N.feed_forward.experts``) -> weight (``w13`` / ``w2``) -> (up [E, out, r],
|
| 91 |
down [E, r, in]) float32 in the SonicExperts layout. ``router_bias``: router module (``layers.N.feed_forward.router``)
|
| 92 |
-
-> float32 [experts] added to its balancing biases. ``
|
| 93 |
-
(``multi_embedder.embedders.<i>.weight``, ``multi_output.weight``) ->
|
| 94 |
-
|
| 95 |
|
| 96 |
name: str
|
| 97 |
format: str
|
|
@@ -100,6 +117,7 @@ class AccentAdapter:
|
|
| 100 |
experts: dict[str, dict[str, tuple[torch.Tensor, torch.Tensor]]] = field(default_factory=dict)
|
| 101 |
router_bias: dict[str, torch.Tensor] = field(default_factory=dict)
|
| 102 |
tables: dict[str, torch.Tensor] = field(default_factory=dict)
|
|
|
|
| 103 |
|
| 104 |
@property
|
| 105 |
def rank(self) -> int:
|
|
@@ -110,28 +128,31 @@ class AccentAdapter:
|
|
| 110 |
TABLES = ("multi_embedder.embedders.", "multi_output.weight")
|
| 111 |
|
| 112 |
|
| 113 |
-
def load_adapter(path: str | Path
|
| 114 |
-
"""Parse ``<accent>.pt``; raises ValueError for a file no server can apply.
|
| 115 |
-
shifts of the embedding tables and the output head (a reference-only path: the sgl, ggml and MLX servers refuse
|
| 116 |
-
them)."""
|
| 117 |
path = Path(path)
|
| 118 |
adapter = torch.load(path, map_location="cpu", weights_only=True)
|
| 119 |
fmt = adapter.get("format")
|
| 120 |
if fmt not in FORMATS:
|
| 121 |
raise ValueError(f"{path}: not a {' / '.join(FORMATS)} adapter")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
router_bias: dict[str, torch.Tensor] = {}
|
| 123 |
table_shifts: dict[str, torch.Tensor] = {}
|
| 124 |
for target, shift in adapter["exact"].items():
|
| 125 |
served, zero_mean = served_exact_name(target)
|
| 126 |
if served is None:
|
| 127 |
continue
|
| 128 |
-
if
|
| 129 |
table_shifts[served] = shift.float()
|
| 130 |
continue
|
| 131 |
if not served.endswith(".router" + BALANCING_BIASES):
|
| 132 |
raise ValueError(
|
| 133 |
f"{path}: shifts {target}; accent serving applies only linear "
|
| 134 |
-
"factors and router balancing biases"
|
| 135 |
)
|
| 136 |
shift = shift.float()
|
| 137 |
if zero_mean:
|
|
@@ -142,7 +163,7 @@ def load_adapter(path: str | Path, *, tables: bool = False) -> AccentAdapter:
|
|
| 142 |
experts: dict[str, dict[str, tuple[torch.Tensor, torch.Tensor]]] = {}
|
| 143 |
for target, pair in adapter["factors"].items():
|
| 144 |
if ".experts." in target or "experts" in pair:
|
| 145 |
-
if fmt
|
| 146 |
raise ValueError(f"{path}: {FORMAT} adapter shifts MoE expert weights ({target})")
|
| 147 |
module, _, weight = target.rpartition(".")
|
| 148 |
if not module.endswith("feed_forward.experts") or weight not in EXPERT_WEIGHTS:
|
|
@@ -173,4 +194,14 @@ def load_adapter(path: str | Path, *, tables: bool = False) -> AccentAdapter:
|
|
| 173 |
)
|
| 174 |
linear[module] = (pair["down"].float(), pair["up"].float().T)
|
| 175 |
shapes[module] = tuple(pair["shape"])
|
| 176 |
-
return AccentAdapter(path.stem, fmt, linear, shapes, experts, router_bias, table_shifts)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Accent adapter files (lingua ``apps.zonos.accent_adapter``, ``zonos-accent-lora/1``, ``/2`` and ``/3``).
|
| 2 |
|
| 3 |
An adapter shifts linear weights by ``up @ down``, router balancing biases by a vector and, in ``/2``, every MoE
|
| 4 |
expert weight by its own ``up_e @ down_e``. A request served with adapter strength alpha runs ``base + alpha *
|
|
|
|
| 9 |
lingua key. ``TARGETS`` mirrors lingua's ``apps.zonos.lora.DEFAULT_TARGETS``. Exact shifts: ``<router>.balancing_biases``
|
| 10 |
(float32 quantile thresholds); as in the converter, the legacy ZeroMean parametrization
|
| 11 |
``.parametrizations.balancing_biases.original`` maps to ``balancing_biases`` with its mean removed, and the training-only
|
| 12 |
+
router metrics (``ent_denom``, ``normalized_entropy``) are dropped. Any other exact shift is refused, except in ``/3``.
|
| 13 |
|
| 14 |
Expert factors (``/2``): ``layers.N.feed_forward.experts.{w13,w2}`` carry ``up`` [E, out, r] and ``down`` [E, r, in] in
|
| 15 |
lingua's SonicExperts layout (w13 rows interleave gate/up), ``shape`` (E, out, in) and ``experts`` E.
|
| 16 |
+
|
| 17 |
+
Grown vocabulary (``/3``, e.g. a speech-edit LoRA trained on a base grown with a span-end audio id and edit-marker text
|
| 18 |
+
rows): ``/2`` plus ``grown`` = ``{"span_end_id", "text_extra_tokens", "rows"}``, the rows such a base adds to the base it
|
| 19 |
+
came from (per audio embedder its span-end row [1, dim], for the text embedder its extra rows [extra, dim], for
|
| 20 |
+
``multi_output`` each codebook block's span-end row [n_codebooks, dim]). A server grows its model with them once (plain
|
| 21 |
+
requests never read or sample the new ids, so they run as before); the adapter's exact shifts may then also move the
|
| 22 |
+
embedding tables and the output head (grown shapes), applied per request like its other shifts.
|
| 23 |
"""
|
| 24 |
|
| 25 |
from __future__ import annotations
|
|
|
|
| 31 |
|
| 32 |
FORMAT = "zonos-accent-lora/1"
|
| 33 |
FORMAT_EXPERTS = "zonos-accent-lora/2"
|
| 34 |
+
FORMAT_GROWN = "zonos-accent-lora/3"
|
| 35 |
+
FORMATS = (FORMAT, FORMAT_EXPERTS, FORMAT_GROWN)
|
| 36 |
# SonicExperts weights lingua's apps.zonos.lora adapts per expert.
|
| 37 |
EXPERT_WEIGHTS = ("w13", "w2")
|
| 38 |
|
|
|
|
| 89 |
return names
|
| 90 |
|
| 91 |
|
| 92 |
+
@dataclass(frozen=True)
|
| 93 |
+
class GrownVocabulary:
|
| 94 |
+
"""The vocabulary a ``/3`` adapter needs: the span-end audio id, the extra text rows and the rows they add."""
|
| 95 |
+
|
| 96 |
+
span_end_id: int
|
| 97 |
+
text_extra_tokens: int
|
| 98 |
+
rows: dict[str, torch.Tensor] # tensor name -> float32 new rows (see the module docstring)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
@dataclass(frozen=True)
|
| 102 |
class AccentAdapter:
|
| 103 |
"""One parsed adapter.
|
|
|
|
| 106 |
``(x @ down.T) @ up`` on the flat ``[out, in]`` product; ``shapes``: the weight shape each linear shift was trained
|
| 107 |
on. ``experts``: experts module (``layers.N.feed_forward.experts``) -> weight (``w13`` / ``w2``) -> (up [E, out, r],
|
| 108 |
down [E, r, in]) float32 in the SonicExperts layout. ``router_bias``: router module (``layers.N.feed_forward.router``)
|
| 109 |
+
-> float32 [experts] added to its balancing biases. ``/3`` only: ``grown``, the vocabulary to grow the model to, and
|
| 110 |
+
``tables``, embedding / output-head weight (``multi_embedder.embedders.<i>.weight``, ``multi_output.weight``) ->
|
| 111 |
+
float32 shift of its rows in the grown shape."""
|
| 112 |
|
| 113 |
name: str
|
| 114 |
format: str
|
|
|
|
| 117 |
experts: dict[str, dict[str, tuple[torch.Tensor, torch.Tensor]]] = field(default_factory=dict)
|
| 118 |
router_bias: dict[str, torch.Tensor] = field(default_factory=dict)
|
| 119 |
tables: dict[str, torch.Tensor] = field(default_factory=dict)
|
| 120 |
+
grown: GrownVocabulary | None = None
|
| 121 |
|
| 122 |
@property
|
| 123 |
def rank(self) -> int:
|
|
|
|
| 128 |
TABLES = ("multi_embedder.embedders.", "multi_output.weight")
|
| 129 |
|
| 130 |
|
| 131 |
+
def load_adapter(path: str | Path) -> AccentAdapter:
|
| 132 |
+
"""Parse ``<accent>.pt``; raises ValueError for a file no server can apply."""
|
|
|
|
|
|
|
| 133 |
path = Path(path)
|
| 134 |
adapter = torch.load(path, map_location="cpu", weights_only=True)
|
| 135 |
fmt = adapter.get("format")
|
| 136 |
if fmt not in FORMATS:
|
| 137 |
raise ValueError(f"{path}: not a {' / '.join(FORMATS)} adapter")
|
| 138 |
+
grown = None
|
| 139 |
+
if fmt == FORMAT_GROWN:
|
| 140 |
+
g = adapter["grown"]
|
| 141 |
+
grown = GrownVocabulary(int(g["span_end_id"]), int(g["text_extra_tokens"]),
|
| 142 |
+
{name: rows.float() for name, rows in g["rows"].items()})
|
| 143 |
router_bias: dict[str, torch.Tensor] = {}
|
| 144 |
table_shifts: dict[str, torch.Tensor] = {}
|
| 145 |
for target, shift in adapter["exact"].items():
|
| 146 |
served, zero_mean = served_exact_name(target)
|
| 147 |
if served is None:
|
| 148 |
continue
|
| 149 |
+
if grown is not None and served.startswith(TABLES) and served.endswith(".weight"):
|
| 150 |
table_shifts[served] = shift.float()
|
| 151 |
continue
|
| 152 |
if not served.endswith(".router" + BALANCING_BIASES):
|
| 153 |
raise ValueError(
|
| 154 |
f"{path}: shifts {target}; accent serving applies only linear "
|
| 155 |
+
f"factors and router balancing biases (embedding and output-head shifts need {FORMAT_GROWN})"
|
| 156 |
)
|
| 157 |
shift = shift.float()
|
| 158 |
if zero_mean:
|
|
|
|
| 163 |
experts: dict[str, dict[str, tuple[torch.Tensor, torch.Tensor]]] = {}
|
| 164 |
for target, pair in adapter["factors"].items():
|
| 165 |
if ".experts." in target or "experts" in pair:
|
| 166 |
+
if fmt == FORMAT:
|
| 167 |
raise ValueError(f"{path}: {FORMAT} adapter shifts MoE expert weights ({target})")
|
| 168 |
module, _, weight = target.rpartition(".")
|
| 169 |
if not module.endswith("feed_forward.experts") or weight not in EXPERT_WEIGHTS:
|
|
|
|
| 194 |
)
|
| 195 |
linear[module] = (pair["down"].float(), pair["up"].float().T)
|
| 196 |
shapes[module] = tuple(pair["shape"])
|
| 197 |
+
return AccentAdapter(path.stem, fmt, linear, shapes, experts, router_bias, table_shifts, grown)
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def grow_adapter(adapter: dict, grown: dict) -> dict:
|
| 201 |
+
"""A ``/2`` adapter (as ``torch.load`` reads it) trained on a grown-vocabulary base, as a ``/3`` file's content:
|
| 202 |
+
``grown`` is ``{"span_end_id", "text_extra_tokens", "rows"}`` (the rows that base added; see the docstring)."""
|
| 203 |
+
if adapter.get("format") != FORMAT_EXPERTS:
|
| 204 |
+
raise ValueError(f"grow_adapter takes a {FORMAT_EXPERTS} adapter, got {adapter.get('format')!r}")
|
| 205 |
+
return {**adapter, "format": FORMAT_GROWN,
|
| 206 |
+
"grown": {"span_end_id": int(grown["span_end_id"]), "text_extra_tokens": int(grown["text_extra_tokens"]),
|
| 207 |
+
"rows": {name: rows.float().contiguous() for name, rows in grown["rows"].items()}}}
|
zonos3/reference/__main__.py
CHANGED
|
@@ -71,6 +71,7 @@ def main(argv=None) -> int:
|
|
| 71 |
lm = Zonos3LM.load(args.model, device=device, dtype=dtype)
|
| 72 |
assets = ReferenceAssets.load(steer_bundle=args.steer_bundle, accent_adapters=args.accent_adapters,
|
| 73 |
laci_heads=heads)
|
|
|
|
| 74 |
if args.dump_prompt:
|
| 75 |
plan = plan_prompt(lm, request, assets=assets, device=device)
|
| 76 |
path = Path(args.dump_prompt)
|
|
|
|
| 71 |
lm = Zonos3LM.load(args.model, device=device, dtype=dtype)
|
| 72 |
assets = ReferenceAssets.load(steer_bundle=args.steer_bundle, accent_adapters=args.accent_adapters,
|
| 73 |
laci_heads=heads)
|
| 74 |
+
assets.grow(lm)
|
| 75 |
if args.dump_prompt:
|
| 76 |
plan = plan_prompt(lm, request, assets=assets, device=device)
|
| 77 |
path = Path(args.dump_prompt)
|
zonos3/reference/accents.py
CHANGED
|
@@ -4,8 +4,8 @@ A target linear adds ``strength * ((x @ down.T) @ up)`` to its flat ``[out, in]`
|
|
| 4 |
``strength * shift`` to its balancing biases, and each routed (token, expert) pair adds
|
| 5 |
``strength * up[e] @ (down[e] @ row)`` to its gate/up pre-activation (w13, SonicExperts interleaved rows) and its
|
| 6 |
expert output (w2) -- zonos3_sgl.layers.accent ``accent_shift`` / ``accent_router_bias`` / ``expert_pair_shift``.
|
| 7 |
-
|
| 8 |
-
embedding rows a prompt reads and ``strength * normalized @ shift.T`` to the output head's logits.
|
| 9 |
"""
|
| 10 |
|
| 11 |
from __future__ import annotations
|
|
|
|
| 4 |
``strength * shift`` to its balancing biases, and each routed (token, expert) pair adds
|
| 5 |
``strength * up[e] @ (down[e] @ row)`` to its gate/up pre-activation (w13, SonicExperts interleaved rows) and its
|
| 6 |
expert output (w2) -- zonos3_sgl.layers.accent ``accent_shift`` / ``accent_router_bias`` / ``expert_pair_shift``.
|
| 7 |
+
A ``/3`` adapter (zonos3.models.accent_format; the model grown to its vocabulary first) also adds ``strength *
|
| 8 |
+
shift[id]`` to the embedding rows a prompt reads and ``strength * normalized @ shift.T`` to the output head's logits.
|
| 9 |
"""
|
| 10 |
|
| 11 |
from __future__ import annotations
|
zonos3/reference/lm.py
CHANGED
|
@@ -405,29 +405,30 @@ class Zonos3LM(nn.Module):
|
|
| 405 |
model.root = root
|
| 406 |
return model.cast(dtype)
|
| 407 |
|
| 408 |
-
def grow(self, grown
|
| 409 |
-
"""This model with
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
``multi_output.weight`` [n_codebooks, dim]). The existing rows keep their values, so a request that never reads
|
| 414 |
-
or samples the new ids runs exactly as before (the sampler masks every special id but the request's stop id)."""
|
| 415 |
c = self.config
|
|
|
|
|
|
|
| 416 |
if c.span_end_id is not None or c.text_extra_tokens:
|
| 417 |
-
raise ValueError("grow: the model already has
|
| 418 |
-
|
|
|
|
| 419 |
if new.audio_vocab != c.audio_vocab + 1 or new.span_end_id != c.codebook_size + 2:
|
| 420 |
raise ValueError("grow: the span-end id must be the one row after the audio pad id")
|
| 421 |
embedders = self.multi_embedder.embedders
|
| 422 |
for i in range(c.n_codebooks + 1):
|
| 423 |
key = f"multi_embedder.embedders.{i}.weight"
|
| 424 |
rows, width = (new.audio_vocab, c.audio_vocab) if i < c.n_codebooks else (new.text_rows, c.text_rows)
|
| 425 |
-
extra = grown[key].to(embedders[i].weight)
|
| 426 |
if tuple(extra.shape) != (rows - width, c.dim):
|
| 427 |
raise ValueError(f"grow: {key} new rows {tuple(extra.shape)} != {(rows - width, c.dim)}")
|
| 428 |
embedders[i].weight = nn.Parameter(torch.cat((embedders[i].weight, extra)), requires_grad=False)
|
| 429 |
head = self.multi_output.weight
|
| 430 |
-
extra = grown["multi_output.weight"].to(head)
|
| 431 |
if tuple(extra.shape) != (c.n_codebooks, c.dim):
|
| 432 |
raise ValueError(f"grow: multi_output.weight new rows {tuple(extra.shape)} != {(c.n_codebooks, c.dim)}")
|
| 433 |
blocks = head.view(c.n_codebooks, c.audio_vocab, c.dim)
|
|
|
|
| 405 |
model.root = root
|
| 406 |
return model.cast(dtype)
|
| 407 |
|
| 408 |
+
def grow(self, grown) -> "Zonos3LM":
|
| 409 |
+
"""This model with a grown vocabulary (``zonos3.models.accent_format.GrownVocabulary`` of a ``/3`` adapter: a
|
| 410 |
+
span-end audio id, extra text rows and the rows they add). The existing rows keep their values, so a request
|
| 411 |
+
that never reads or samples the new ids runs exactly as before (the sampler masks every special id but the
|
| 412 |
+
request's stop id). Growing again to the same vocabulary is a no-op."""
|
|
|
|
|
|
|
| 413 |
c = self.config
|
| 414 |
+
if (c.span_end_id, c.text_extra_tokens) == (grown.span_end_id, grown.text_extra_tokens):
|
| 415 |
+
return self
|
| 416 |
if c.span_end_id is not None or c.text_extra_tokens:
|
| 417 |
+
raise ValueError(f"grow: the model already has span_end_id {c.span_end_id} and {c.text_extra_tokens} extra "
|
| 418 |
+
f"text rows; the adapter wants {grown.span_end_id} and {grown.text_extra_tokens}")
|
| 419 |
+
new = replace(c, span_end_id=int(grown.span_end_id), text_extra_tokens=int(grown.text_extra_tokens))
|
| 420 |
if new.audio_vocab != c.audio_vocab + 1 or new.span_end_id != c.codebook_size + 2:
|
| 421 |
raise ValueError("grow: the span-end id must be the one row after the audio pad id")
|
| 422 |
embedders = self.multi_embedder.embedders
|
| 423 |
for i in range(c.n_codebooks + 1):
|
| 424 |
key = f"multi_embedder.embedders.{i}.weight"
|
| 425 |
rows, width = (new.audio_vocab, c.audio_vocab) if i < c.n_codebooks else (new.text_rows, c.text_rows)
|
| 426 |
+
extra = grown.rows[key].to(embedders[i].weight)
|
| 427 |
if tuple(extra.shape) != (rows - width, c.dim):
|
| 428 |
raise ValueError(f"grow: {key} new rows {tuple(extra.shape)} != {(rows - width, c.dim)}")
|
| 429 |
embedders[i].weight = nn.Parameter(torch.cat((embedders[i].weight, extra)), requires_grad=False)
|
| 430 |
head = self.multi_output.weight
|
| 431 |
+
extra = grown.rows["multi_output.weight"].to(head)
|
| 432 |
if tuple(extra.shape) != (c.n_codebooks, c.dim):
|
| 433 |
raise ValueError(f"grow: multi_output.weight new rows {tuple(extra.shape)} != {(c.n_codebooks, c.dim)}")
|
| 434 |
blocks = head.view(c.n_codebooks, c.audio_vocab, c.dim)
|
zonos3/reference/synth.py
CHANGED
|
@@ -70,6 +70,13 @@ class ReferenceAssets:
|
|
| 70 |
"""The Full AttnRes bank sources hidden deltas steer, ascending (sgl Engine._init_steering)."""
|
| 71 |
return () if self.steer_bundle is None else tuple(self.steer_bundle.hidden_sources)
|
| 72 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
def accent_names(self) -> list[str]:
|
| 74 |
return list(self.accent_adapters)
|
| 75 |
|
|
|
|
| 70 |
"""The Full AttnRes bank sources hidden deltas steer, ascending (sgl Engine._init_steering)."""
|
| 71 |
return () if self.steer_bundle is None else tuple(self.steer_bundle.hidden_sources)
|
| 72 |
|
| 73 |
+
def grow(self, lm: Zonos3LM) -> Zonos3LM:
|
| 74 |
+
"""``lm`` grown to the vocabulary its ``/3`` adapters need (zonos3.models.accent_format), before any request."""
|
| 75 |
+
for adapter in self.accent_adapters.values():
|
| 76 |
+
if adapter.grown is not None:
|
| 77 |
+
lm.grow(adapter.grown)
|
| 78 |
+
return lm
|
| 79 |
+
|
| 80 |
def accent_names(self) -> list[str]:
|
| 81 |
return list(self.accent_adapters)
|
| 82 |
|