Buckets:
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| from typing import Any | |
| import torch | |
| from .base import ModelBase, MmprojModel, gguf, logger | |
| from .qwen import Qwen3Model | |
| def _is_soprano_checkpoint(dir_model: Path) -> bool: | |
| if not (dir_model / "decoder.pth").is_file() or not (dir_model / "config.json").is_file(): | |
| return False | |
| with open(dir_model / "config.json", encoding="utf-8") as f: | |
| config = json.load(f) | |
| if config.get("model_type") != "qwen3" or config.get("bos_token_id") != 3 or config.get("eos_token_id") != 3: | |
| return False | |
| if not (dir_model / "tokenizer.json").is_file(): | |
| return False | |
| with open(dir_model / "tokenizer.json", encoding="utf-8") as f: | |
| added = {t["id"]: t["content"] for t in json.load(f).get("added_tokens", [])} | |
| return [added.get(i) for i in range(4)] == ["[UNK]", "[TEXT]", "[START]", "[STOP]"] | |
| def _load_soprano_hparams(dir_model: Path) -> dict[str, Any]: | |
| logger.info("gguf: detected Soprano checkpoint") | |
| hparams = ModelBase.load_hparams(dir_model, False, guess=False) | |
| # the text model stays Qwen3, the mmproj holds the decoder | |
| hparams["text_config"] = {"architectures": ["SopranoTextModel"]} | |
| hparams["architectures"] = ["SopranoModel"] | |
| return hparams | |
| class SopranoTextModel(Qwen3Model): | |
| model_arch = gguf.MODEL_ARCH.QWEN3 | |
| def get_vocab_base_pre(self, tokenizer) -> str: | |
| return "soprano" | |
| def set_vocab(self): | |
| super().set_vocab() | |
| self.gguf_writer.add_unk_token_id(0) | |
| class SopranoModel(MmprojModel): | |
| has_vision_encoder = False | |
| has_audio_encoder = False | |
| def _load_decoder(self) -> dict[str, torch.Tensor]: | |
| state = torch.load(self.dir_model / "decoder.pth", map_location="cpu", mmap=True, weights_only=True) | |
| return state.get("state_dict", state) | |
| def get_audio_config(self): | |
| state = self._load_decoder() | |
| n_embd, n_inp, _ = state["decoder.embed.weight"].shape | |
| n_ff = state["decoder.convnext.0.pwconv1.weight"].shape[0] | |
| n_layer = sum(1 for name in state if name.endswith(".pwconv1.weight")) | |
| if n_inp != self.n_embd_text: | |
| raise ValueError(f"Soprano decoder input dim {n_inp} does not match the backbone dim {self.n_embd_text}") | |
| return {"num_hidden_layers": n_layer, "hidden_size": n_embd, "intermediate_size": n_ff} | |
| def set_gguf_parameters(self): | |
| self.gguf_writer.add_file_type(self.ftype) | |
| self.gguf_writer.add_clip_has_gen_audio_encoder(True) | |
| self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.SOPRANO) | |
| self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text) | |
| self.gguf_writer.add_gen_audio_embedding_length(self.hparams["hidden_size"]) | |
| self.gguf_writer.add_gen_audio_feed_forward_length(self.hparams["intermediate_size"]) | |
| self.gguf_writer.add_gen_audio_block_count(self.hparams["num_hidden_layers"]) | |
| self.gguf_writer.add_gen_audio_head_count(1) | |
| self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-6) | |
| def get_tensors(self): | |
| yield from self._load_decoder().items() | |
| def tensor_force_quant(self, name, new_name, bid, n_dims): | |
| if "dwconv.weight" in new_name or (name == "decoder.embed.weight" and n_dims == 3): | |
| return gguf.GGMLQuantizationType.F16 | |
| return super().tensor_force_quant(name, new_name, bid, n_dims) | |
| def modify_tensors(self, data_torch, name, bid): | |
| if name == "head.istft.window": | |
| # the ISTFT in mtmd builds its own window | |
| if not torch.equal(data_torch.float(), torch.hann_window(data_torch.shape[0])): | |
| raise ValueError("Soprano requires a periodic Hann window") | |
| return | |
| if name == "decoder.embed.weight" and data_torch.shape[-1] == 1: | |
| data_torch = data_torch.squeeze(-1) | |
| T = gguf.MODEL_TENSOR | |
| if name.startswith("decoder.convnext."): | |
| _, _, blk, key, *suffix = name.split(".") | |
| block_map = { | |
| "dwconv": T.A_GEN_WAV_UP_DWCONV, | |
| "norm": T.A_GEN_WAV_UP_NORM, | |
| "pwconv1": T.A_GEN_WAV_UP_PW1, | |
| "pwconv2": T.A_GEN_WAV_UP_PW2, | |
| "gamma": T.A_GEN_WAV_UP_GAMMA, | |
| } | |
| yield self.format_tensor_name(block_map[key], int(blk), suffix="".join("." + s for s in suffix)), data_torch | |
| return | |
| tensor_map = { | |
| "decoder.embed": T.A_GEN_WAV_INPUT, | |
| "decoder.norm": T.A_GEN_WAV_NORM, | |
| "decoder.final_layer_norm": T.A_GEN_WAV_OUTPUT_NORM, | |
| "head.out": T.A_GEN_WAV_OUTPUT, | |
| } | |
| key, suffix = name.rsplit(".", 1) | |
| yield self.format_tensor_name(tensor_map[key], suffix="." + suffix), data_torch | |
Xet Storage Details
- Size:
- 5.06 kB
- Xet hash:
- f46f0ed8df49aaea25aa88cb0b393dbb33d5d2a6a872fba5add7215d575c8757
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.