Rumble-2-Flash-GGUF / llama-cpp-rumble.patch
javierbits's picture
Upload folder using huggingface_hub
aff6a88 verified
Raw History Blame Contribute Delete
4.06 kB
diff --git a/conversion/base.py b/conversion/base.py
index 7cfbff3..e7a850a 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -2471,6 +2471,10 @@ class TextModel(ModelBase):
# ref: https://huggingface.co/thinkingmachines/Inkling-2-Small
res = "inkling2"
+ if chkhsh == "9b33e5d83df24061dcdb1f30e51583be2f0879fb36c2e13d3b03c037742561f3":
+ # ref: https://huggingface.co/Reson8/Rumble-2-Flash (pruned Qwen3-ASR tokenizer)
+ res = "qwen2"
+
if res is None:
logger.warning("\n")
logger.warning("**************************************************************************************")
diff --git a/conversion/qwen3vl.py b/conversion/qwen3vl.py
index 9458366..2693b5b 100644
--- a/conversion/qwen3vl.py
+++ b/conversion/qwen3vl.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import json
+import os
from typing import Any, Callable, Iterable, TYPE_CHECKING
@@ -234,6 +235,23 @@ class Qwen3ASRMmprojModel(Qwen3OmniMmprojModel):
has_audio_encoder = True
has_vision_encoder = False
+ def get_audio_config(self) -> dict[str, Any] | None:
+ # The Transformers-5 checkpoint keeps audio_config at the top level (no thinker_config).
+ return self.global_config.get("audio_config") or self.global_config["thinker_config"].get("audio_config")
+
+ @classmethod
+ def filter_tensors(cls, item):
+ name, gen = item
+ if name.startswith("model."):
+ name = name.replace("model.", "", 1)
+ if name.startswith("multi_modal_projector.linear_"):
+ # Transformers-5 layout: projector is multi_modal_projector.linear_1/2; the GGUF map
+ # (and llama.cpp's qwen3a graph) expects audio_tower.proj1/2.
+ name = "audio_tower.proj" + name.rsplit("_", 1)[1]
+ if "audio_tower." not in name:
+ return None
+ return MmprojModel.filter_tensors((name, gen))
+
@ModelBase.register("Glm4vForConditionalGeneration", "Glm4vMoeForConditionalGeneration", "GlmOcrForConditionalGeneration")
@ModelBase.example("zai-org/GLM-4.1V-9B-Thinking", "zai-org/GLM-4.5V")
@@ -390,11 +408,24 @@ class Qwen3ASRTextModel(Qwen3VLTextModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
self.gguf_writer.add_num_deepstack_layers(0)
+ # Rumble's config drops mrope_section; Qwen3-ASR-0.6B uses [24, 20, 20, 0] and the
+ # audio/text positions are identical in every section, so this is plain RoPE.
+ if "mrope_section" not in self.rope_parameters:
+ self.gguf_writer.add_rope_dimension_sections([24, 20, 20, 0])
def set_vocab(self):
super().set_vocab()
- # fix chat template, use correct chatml format
- self.gguf_writer.add_chat_template("{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}")
+ # llama-server runs the model with a media marker inside the user text, and the vendor
+ # template is written for typed content plus {% generation %}. RUMBLE_CHAT_TEMPLATE points
+ # at a llama.cpp-compatible adaptation (see the model repo); fall back to plain chatml.
+ template = None
+ template_path = os.environ.get("RUMBLE_CHAT_TEMPLATE")
+ if template_path and os.path.isfile(template_path):
+ with open(template_path, "r", encoding="utf-8") as f:
+ template = f.read()
+ if template is None:
+ template = "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}"
+ self.gguf_writer.add_chat_template(template)
# correct BOS/EOS tokens
with open(self.dir_model / "tokenizer_config.json", "r", encoding="utf-8") as f:
tokenizer_config = json.load(f)