diff --git a/common/arg.cpp b/common/arg.cpp index e346863..525c857 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -9,6 +9,7 @@ #include "llama.h" #include "log.h" #include "sampling.h" +#include "jais-language.h" #include "speculative.h" #include "preset.h" @@ -817,6 +818,14 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context const std::string arg_prefix = "--"; std::string arg = argv[i]; + const bool inline_language = arg.compare(0, 11, "--language=") == 0 || + arg.compare(0, 16, "--jais-language=") == 0; + std::string language_value; + if (inline_language) { + const auto equals = arg.find('='); + language_value = arg.substr(equals + 1); + arg.resize(equals); + } if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) { std::replace(arg.begin(), arg.end(), '_', '-'); } @@ -847,8 +856,13 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context } // arg with single value - check_arg(i); - std::string val = argv[++i]; + std::string val; + if (inline_language) { + val = language_value; + } else { + check_arg(i); + val = argv[++i]; + } if (opt.handler_int) { opt.handler_int(params, std::stoi(val)); continue; @@ -2071,6 +2085,15 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.sampling.typ_p = std::stof(value); } ).set_sampling()); + add_opt(common_arg( + {"--language", "--jais-language"}, "TAG", + "Jais-ASR language pin (case-insensitive: ar, EG, en, or comma-separated tags; empty disables)", + [](common_params & params, const std::string & value) { + common_jais_language_pin pin; + pin.configure(value); + params.sampling.jais_language = value; + } + ).set_sampling()); add_opt(common_arg( {"--repeat-last-n"}, "N", string_format("last n tokens to consider for penalize (default: %d, 0 = disabled)", params.sampling.penalty_last_n), diff --git a/common/chat-diff-analyzer.cpp b/common/chat-diff-analyzer.cpp index a7e3705..0470524 100644 --- a/common/chat-diff-analyzer.cpp +++ b/common/chat-diff-analyzer.cpp @@ -49,6 +49,19 @@ static std::vector ... + [](const common_chat_template & tmpl, autoparser & analysis) -> void { + if (tmpl.src.find("") != std::string::npos && + tmpl.src.find("") != std::string::npos && + analysis.reasoning.mode == reasoning_mode::NONE) { + analysis.reasoning.mode = reasoning_mode::TAG_BASED; + analysis.reasoning.start = ""; + analysis.reasoning.end = ""; + analysis.preserved_tokens.push_back(""); + analysis.preserved_tokens.push_back(""); + LOG_DBG(ANSI_ORANGE "[Patch: IFM K2-Horizon thinking template]\n" ANSI_RESET); + } + }, // Granite 3.3, with separate reasoning and content markers [](const common_chat_template & tmpl, autoparser & analysis) -> void { if (tmpl.src.find("Write your thoughts between and write your response between " diff --git a/common/chat.cpp b/common/chat.cpp index 743ecde..7c4c892 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -724,6 +724,17 @@ static bool is_lfm2_template(const std::string & src) { src.find("<|tool_list_end|>") != std::string::npos; } +// Jais-ASR: the user turn holds only the audio, and the language is chosen with +// language (pin) or language_prefill (template kwarg), never with prompt text +bool common_chat_templates_is_jais_asr(const struct common_chat_templates * tmpls) { + if (!tmpls || !tmpls->template_default) { + return false; + } + const auto & src = tmpls->template_default->source(); + return src.find("<|ifm|audio|>") != std::string::npos && + src.find("language_prefill") != std::string::npos; +} + common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates) { common_chat_prompt_preset asr_preset; asr_preset.system = ""; @@ -734,6 +745,11 @@ common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates asr_preset.user = ""; } + if (common_chat_templates_is_jais_asr(chat_templates)) { + asr_preset.system = ""; + asr_preset.user = ""; + } + return asr_preset; } @@ -3816,6 +3832,15 @@ static common_chat_params common_chat_templates_apply_legacy(const struct common common_chat_params common_chat_templates_apply(const struct common_chat_templates * tmpls, const struct common_chat_templates_inputs & inputs) { GGML_ASSERT(tmpls != nullptr); + if (common_chat_templates_is_jais_asr(tmpls) && + (inputs.enable_thinking || inputs.chat_template_kwargs.count("enable_thinking"))) { + // Jais-ASR is trained with an empty, closed thinking block and never closes an open + // one (the language pin would then never act), so thinking stays off for it + auto jais_inputs = inputs; + jais_inputs.enable_thinking = false; + jais_inputs.chat_template_kwargs.erase("enable_thinking"); + return common_chat_templates_apply(tmpls, jais_inputs); + } return inputs.use_jinja ? common_chat_templates_apply_jinja(tmpls, inputs) : common_chat_templates_apply_legacy(tmpls, inputs); } diff --git a/common/chat.h b/common/chat.h index cb39e34..5ad3254 100644 --- a/common/chat.h +++ b/common/chat.h @@ -385,4 +385,6 @@ struct common_chat_prompt_preset { common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates); +bool common_chat_templates_is_jais_asr(const struct common_chat_templates * tmpls); + common_chat_msg_delimiters common_chat_msg_delimiters_parse(const common_json & delimiters); diff --git a/common/common.cpp b/common/common.cpp index 347e8e9..e04310c 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1287,8 +1287,50 @@ struct common_init_result::impl { std::vector samplers_seq_config; }; +void common_params_apply_model_defaults(common_params & params) { + if (params.model.path.empty()) { + return; + } + struct gguf_init_params meta_gguf_params = { + /* .no_alloc = */ true, + /* .ctx = */ nullptr, + }; + struct gguf_context * ctx_gguf = gguf_init_from_file(params.model.path.c_str(), meta_gguf_params); + if (!ctx_gguf) { + return; + } + bool jais_asr = false; + const int64_t key = gguf_find_key(ctx_gguf, "tokenizer.chat_template"); + if (key >= 0 && gguf_get_kv_type(ctx_gguf, key) == GGUF_TYPE_STRING) { + const std::string tmpl = gguf_get_val_str(ctx_gguf, key); + jais_asr = tmpl.find("<|ifm|audio|>") != std::string::npos && tmpl.find("language_prefill") != std::string::npos; + } + gguf_free(ctx_gguf); + if (!jais_asr) { + return; + } + if (params.n_ctx == 0) { + params.n_ctx = 4096; // the evaluated context, instead of all the free GPU memory + } + if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO) { + params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED; // flash attention gave invalid outputs on AMD GPUs + } + // the model's generation config: greedy decoding, repetition penalty 1.05 over the whole sequence + auto & sparams = params.sampling; + if (!(sparams.user_sampling_config & common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_TEMP)) { + sparams.temp = 0.0f; + } + if (!(sparams.user_sampling_config & common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_REPEAT)) { + sparams.penalty_repeat = 1.05f; + } + if (!(sparams.user_sampling_config & common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_LAST_N)) { + sparams.penalty_last_n = params.n_ctx; + } +} + common_init_result::common_init_result(common_params & params, bool model_only) : pimpl(new impl{}) { + common_params_apply_model_defaults(params); auto mparams = common_model_params_to_llama(params); auto cparams = common_context_params_to_llama(params); diff --git a/common/common.h b/common/common.h index a333f70..601cab0 100644 --- a/common/common.h +++ b/common/common.h @@ -222,6 +222,7 @@ inline bool common_grammar_needs_prefill(const common_grammar & g) { // sampling parameters struct common_params_sampling { + std::string jais_language; // empty: automatic detection; otherwise tag or comma-separated tags uint32_t seed = LLAMA_DEFAULT_SEED; // the seed used to initialize llama_sampler int32_t n_prev = 64; // number of previous tokens to remember @@ -946,6 +947,10 @@ using common_init_result_ptr = std::unique_ptr; common_init_result_ptr common_init_from_params(common_params & params, bool model_only = false); +// Jais-ASR (detected from the chat template in the GGUF file): fill the settings the user left unset with +// the model's generation settings and the llama.cpp settings it was evaluated with +void common_params_apply_model_defaults(common_params & params); + struct llama_model_params common_model_params_to_llama ( common_params & params); struct llama_context_params common_context_params_to_llama(const common_params & params); diff --git a/common/jais-language.h b/common/jais-language.h new file mode 100644 index 0000000..78c2c3c --- /dev/null +++ b/common/jais-language.h @@ -0,0 +1,165 @@ +#pragma once + +#include "llama.h" + +#include +#include +#include +#include +#include +#include +#include + +// Token inventory from Jais-ASR postprocess.py. Validate against the loaded vocabulary. +struct common_jais_tag { llama_token id; const char * name; }; +static const std::array common_jais_tags = {{{ + 226606, "msa"}, {272, "en"}, {2075, "fr"}, {273, "es"}, {286, "it"}, + {7646, "EG"}, {8048, "SA"}, {6101, "MA"}, {29354, "IQ"}, {23129, "JO"}, + {20056, "SY"}, {27526, "QA"}, {64489, "DZ"}, {23852, "LB"}, {5594, "OM"}, + {28472, "YE"}, {43852, "TN"}, {17604, "MR"}, {25893, "KW"}, {9373, "SD"}, + {12933, "LY"}, {5402, "AE"}, {4295, "PS"}, {41905, "BH"} +}}; + +// Same messages as the Python LanguagePin and the vLLM plugin. +static const char * const common_jais_pin_prefill_conflict = + "Use either language (pinning) or language_prefill, not both"; + +static inline bool common_jais_is_tag(llama_token id) { + for (const auto & tag : common_jais_tags) { + if (tag.id == id) { return true; } + } + return false; +} + +static inline const char * common_jais_tag_name(llama_token id) { + for (const auto & tag : common_jais_tags) { + if (tag.id == id) { return tag.name; } + } + return nullptr; +} + +// postprocess.has_prefilled_tag: the prompt ends with a complete "[tag]". +static inline bool common_jais_ends_with_tag(llama_token a, llama_token b, llama_token c) { + return a == 60 && common_jais_is_tag(b) && c == 62; +} + +struct common_jais_language_pin { + static constexpr llama_token open = 60, close = 62, refusal = 5126; + static constexpr llama_token think_open = 250029, think_close = 250030; + std::vector allowed; + std::array prompt_tail = {{LLAMA_TOKEN_NULL, LLAMA_TOKEN_NULL, LLAMA_TOKEN_NULL}}; + bool thinking = false; + bool started = false; + int position = 0; + llama_token decision = LLAMA_TOKEN_NULL; + + bool enabled() const { return !allowed.empty(); } + bool contains(llama_token id) const { + return std::find(allowed.begin(), allowed.end(), id) != allowed.end(); + } + static bool is_tag(llama_token id) { return common_jais_is_tag(id); } + void reset() { + prompt_tail = {{LLAMA_TOKEN_NULL, LLAMA_TOKEN_NULL, LLAMA_TOKEN_NULL}}; + thinking = false; + started = false; + position = 0; + decision = LLAMA_TOKEN_NULL; + } + void configure(const std::string & language) { + allowed.clear(); + reset(); + if (language.empty()) { return; } + size_t begin = 0; + do { + const auto end = language.find(',', begin); + auto s = language.substr(begin, end == std::string::npos ? end : end - begin); + const auto first = s.find_first_not_of(" \t\r\n"); + const auto last = s.find_last_not_of(" \t\r\n"); + s = first == std::string::npos ? "" : s.substr(first, last - first + 1); + auto iso = s; + for (auto & c : iso) { c = static_cast(std::tolower(static_cast(c))); } + bool found = false; + for (size_t i = 0; i < common_jais_tags.size(); ++i) { + const auto & tag = common_jais_tags[i]; + std::string canonical = tag.name; + for (auto & c : canonical) { c = static_cast(std::tolower(static_cast(c))); } + if ((iso == "ar" && (i == 0 || i >= 5)) || iso == canonical) { + if (!contains(tag.id)) { allowed.push_back(tag.id); } + found = true; + } + } + if (!found) { throw std::invalid_argument("unknown Jais-ASR language: " + s); } + if (end == std::string::npos) { break; } + begin = end + 1; + } while (true); + } + bool prefilled() const { + return common_jais_ends_with_tag(prompt_tail[0], prompt_tail[1], prompt_tail[2]); + } + void accept(llama_token id, bool generated) { + if (!enabled()) { return; } + if (!generated) { + if (id == think_open) { thinking = true; } + if (id == think_close) { thinking = false; } + prompt_tail = {{prompt_tail[1], prompt_tail[2], id}}; + return; + } + if (!started && prefilled()) { position = 3; } + started = true; + if (thinking) { + if (id == think_close) { thinking = false; } + return; + } + if (position == 0 && id != open) { position = 3; return; } + if (position == 1) { decision = id; } + if (position < 3) { ++position; } + } + template + llama_token select(Score score) const { + if (!enabled() || thinking || position >= 3) { return LLAMA_TOKEN_NULL; } + if (!started && prefilled()) { return LLAMA_TOKEN_NULL; } + llama_token selected = LLAMA_TOKEN_NULL; + if (position == 0) { + selected = open; + } else if (position == 2) { + if (contains(decision)) { selected = close; } + } else { + auto best = common_jais_tags.front().id; + for (const auto & tag : common_jais_tags) { + if (score(tag.id) > score(best)) { best = tag.id; } + } + // Refusal competes against ALL tags before the caller's restriction. + if (score(refusal) > score(best)) { + selected = refusal; + } else { + selected = allowed.front(); + for (const auto id : allowed) { + if (score(id) > score(selected)) { selected = id; } + } + } + } + if (selected != LLAMA_TOKEN_NULL && !std::isfinite(score(selected))) { + throw std::runtime_error("Jais-ASR pin selected a token without a finite score"); + } + return selected; + } +}; + +// language_prefill takes exactly one tag: "en" or "EG" work, a family such as "ar" does not. +static inline std::string common_jais_prefill_tag(const std::string & language) { + common_jais_language_pin pin; + pin.configure(language); + if (pin.allowed.size() != 1) { + throw std::invalid_argument("language_prefill must resolve to one tag; use language for a family"); + } + return common_jais_tag_name(pin.allowed.front()); +} + +// Two specs name the same tags in the same order ("ar" and the 20 Arabic tags do). +static inline bool common_jais_same_language(const std::string & a, const std::string & b) { + common_jais_language_pin pa; + common_jais_language_pin pb; + pa.configure(a); + pb.configure(b); + return pa.allowed == pb.allowed; +} diff --git a/common/sampling.cpp b/common/sampling.cpp index 06dea1e..a2eb6da 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -1,4 +1,5 @@ #include "sampling.h" +#include "jais-language.h" #include "common.h" #include "fit.h" @@ -120,9 +121,11 @@ struct common_sampler { std::vector cur; llama_token_data_array cur_p; + common_jais_language_pin jais_pin; void reset() { prev.clear(); + jais_pin.reset(); llama_sampler_reset(chain); } @@ -199,6 +202,25 @@ struct common_sampler * common_sampler_init( throw std::invalid_argument("penalty_present must be finite"); } const llama_vocab * vocab = llama_model_get_vocab(model); + common_jais_language_pin jais_pin; + jais_pin.configure(params.jais_language); + if (jais_pin.enabled()) { + if (!common_grammar_value(params.grammar).empty() || !params.logit_bias.empty()) { + throw std::invalid_argument("Jais-ASR pinning cannot be combined with grammar or logit bias"); + } + auto verify = [&](const std::string & text, const llama_tokens & expected) { + if (common_tokenize(vocab, text, false, true) != expected) { + throw std::invalid_argument("Jais-ASR tokenizer mismatch for " + text); + } + }; + for (const auto & tag : common_jais_tags) { + verify("[" + std::string(tag.name) + "]", {60, tag.id, 62}); + } + verify("[IFM-NoTranscript]", {60, 5126, 46, 131126, 195862, 62}); + verify("", {250029}); + verify("", {250030}); + params.backend_sampling = false; + } llama_sampler_chain_params lparams = llama_sampler_chain_default_params(); lparams.no_perf = params.no_perf; @@ -432,6 +454,7 @@ struct common_sampler * common_sampler_init( /* .prev = */ ring_buffer(std::max(32, params.n_prev)), /* .cur = */ {}, /* .cur_p = */ {}, + /* .jais_pin = */ jais_pin, }; return result; @@ -471,6 +494,8 @@ void common_sampler_accept(struct common_sampler * gsmpl, llama_token token, boo const auto tm = gsmpl->tm(); + gsmpl->jais_pin.accept(token, is_generated); + // grammar_should_apply() checks the reasoning budget state, so calculate this before we accept const auto accept_grammar = is_generated && grammar_should_apply(gsmpl); @@ -515,6 +540,7 @@ struct common_sampler * common_sampler_clone(common_sampler * gsmpl) { /* .prev = */ gsmpl->prev, /* .cur = */ gsmpl->cur, /* .cur_p = */ gsmpl->cur_p, + /* .jais_pin = */ gsmpl->jais_pin, }; } @@ -530,6 +556,7 @@ void common_sampler_copy(const common_sampler * src, common_sampler * dst) { llama_sampler_copy(src->rbudget, dst->rbudget); llama_sampler_copy(src->chain, dst->chain); + dst->jais_pin = src->jais_pin; dst->params = src->params; dst->prev = src->prev; dst->cur = src->cur; @@ -606,6 +633,24 @@ llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_co gsmpl->set_logits(ctx, idx); + if (gsmpl->jais_pin.enabled()) { + const float * raw = llama_get_logits_ith(ctx, idx); + if (!raw || llama_get_sampled_token_ith(ctx, idx) != LLAMA_TOKEN_NULL) { + throw std::runtime_error("Jais-ASR pinning requires CPU sampling and full logits"); + } + const auto forced = gsmpl->jais_pin.select([&](llama_token token) { return raw[token]; }); + if (forced != LLAMA_TOKEN_NULL) { + for (size_t i = 0; i < cur_p.size; ++i) { + const bool keep = cur_p.data[i].id == forced; + cur_p.data[i].p = keep ? 1.0f : 0.0f; + if (keep) { cur_p.selected = i; } + else { cur_p.data[i].logit = -INFINITY; } + } + if (cur_p.selected < 0) { throw std::runtime_error("missing Jais-ASR pin candidate"); } + return forced; + } + } + // Check if a backend sampler has already sampled a token in which case we // return that token id directly. { diff --git a/conversion/__init__.py b/conversion/__init__.py index 648dd73..1355717 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -224,6 +224,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "Qwen2VLModel": "qwenvl", "Qwen2_5OmniModel": "qwenvl", "Qwen2_5_VLForConditionalGeneration": "qwenvl", + "JaisASRForConditionalGeneration": "jais_asr", "Qwen3ASRForConditionalGeneration": "qwen3vl", "Qwen3ForCausalLM": "qwen", "Qwen3Model": "qwen", @@ -329,6 +330,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "Qwen2VLModel": "qwenvl", "Qwen2_5OmniModel": "qwenvl", "Qwen2_5_VLForConditionalGeneration": "qwenvl", + "JaisASRForConditionalGeneration": "jais_asr", "Qwen3ASRForConditionalGeneration": "qwen3vl", "Qwen3OmniMoeForConditionalGeneration": "qwen3vl", "PocketTTSModel": "pockettts", diff --git a/conversion/jais_asr.py b/conversion/jais_asr.py new file mode 100644 index 0000000..69b1836 --- /dev/null +++ b/conversion/jais_asr.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from typing import Any, Callable, Iterable, TYPE_CHECKING + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, TextModel, gguf + +from .k2_horizon import K2HorizonModel +from .qwenvl import Qwen25AudioModel + + +# Jais-ASR = Qwen3-Omni audio tower + extra MLP projector + K2-Horizon text decoder. +# Checkpoint prefixes: model.audio_tower.*, model.embed_audio.*, model.language_model.*, lm_head.* + + +@ModelBase.register("JaisASRForConditionalGeneration") +@ModelBase.example("MBZUAI-Paris/Jais-ASR") +class JaisASRTextModel(K2HorizonModel): + model_arch = gguf.MODEL_ARCH.K2HORIZON + + # "model.language_model." is reduced to "model." by ModelBase.filter_tensors, and the + # audio tensors are dropped by TextModel.filter_tensors, so only set_vocab differs. + + # generation_config.json stops on this in addition to the tokenizer's eos token + _EXTRA_STOP_TOKEN = "<|ifm|endoftext|>" + + def set_vocab(self): + # skip K2HorizonModel.set_vocab: it replaces the chat template with the + # K2-Horizon one, but Jais-ASR ships its own audio chat template + TextModel.set_vocab(self) + + # llama.cpp derives its end-of-generation set from the single eos token, so the + # second stop token would never terminate generation. Put it in the EOT slot. + tokens, _, _ = self.get_vocab_base() + self.gguf_writer.add_eot_token_id(tokens.index(self._EXTRA_STOP_TOKEN)) + + +@ModelBase.register("JaisASRForConditionalGeneration") +@ModelBase.example("MBZUAI-Paris/Jais-ASR") +class JaisASRMmprojModel(Qwen25AudioModel): + has_audio_encoder = True + has_vision_encoder = False + + # model.embed_audio.* -> the extra projector stage read by PROJECTOR_TYPE_JAISASR + _EMBED_AUDIO_MAP = { + "norm": gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_MM_NORM_PRE], + "linear_1": gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_MMPROJ].format(bid=3), + "linear_2": gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_MMPROJ].format(bid=4), + } + + def get_vision_config(self) -> dict[str, Any] | None: + return None + + def set_gguf_parameters(self): + super().set_gguf_parameters() + self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.JAISASR) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + + if name.startswith("model.audio_tower."): + # rename to the prefix the qwen3omni tensor mappings expect + return name.replace("model.audio_tower.", "audio_tower.", 1), gen + + if name.startswith("model.embed_audio."): + return item + + return None + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if name.startswith("model.embed_audio."): + module, suffix = name[len("model.embed_audio."):].rsplit(".", 1) + yield (f"{self._EMBED_AUDIO_MAP[module]}.{suffix}", data_torch) + return + + if "conv2d" in name and name.endswith(".bias"): + # transform conv2d bias [n_embd] --> [1, 1, n_embd] + data_torch = data_torch.unsqueeze(-1).unsqueeze(-1) + + yield from Qwen25AudioModel.modify_tensors(self, data_torch, name, bid) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 13d5567..7c4ddae 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -5673,6 +5673,7 @@ class VisionProjectorType: INTERNVL = "internvl" QWEN2A = "qwen2a" # audio QWEN3A = "qwen3a" # audio + JAISASR = "jais-asr" # audio (Qwen3-Omni audio tower + Jais-ASR MLP projector) GLMA = "glma" # audio QWEN25O = "qwen2.5o" # omni VOXTRAL = "voxtral" diff --git a/tests/test-arg-parser.cpp b/tests/test-arg-parser.cpp index e090763..823a4e6 100644 --- a/tests/test-arg-parser.cpp +++ b/tests/test-arg-parser.cpp @@ -172,6 +172,22 @@ static void test(void) { std::vector argv; + for (const auto & example : {LLAMA_EXAMPLE_MTMD, LLAMA_EXAMPLE_SERVER}) { + for (const auto & option : {"--language=ar", "--language=eg", "--language=EG", "--language=eG", "--jais-language=ar", "--language="}) { + common_params language_params; + argv = {"binary_name", "-m", "jais-test.gguf", option}; + assert(common_params_parse(argv.size(), list_str_to_char(argv).data(), language_params, example)); + assert(language_params.sampling.jais_language == std::string(option).substr(std::string(option).find('=') + 1)); + } + } + + for (const auto & option : {"--language", "--jais-language"}) { + common_params language_params; + argv = {"binary_name", "-m", "jais-test.gguf", option, "eG"}; + assert(common_params_parse(argv.size(), list_str_to_char(argv).data(), language_params, LLAMA_EXAMPLE_MTMD)); + assert(language_params.sampling.jais_language == "eG"); + } + printf("test-arg-parser: test invalid usage\n\n"); // missing value diff --git a/tests/test-sampling.cpp b/tests/test-sampling.cpp index d727ab6..a53f479 100644 --- a/tests/test-sampling.cpp +++ b/tests/test-sampling.cpp @@ -1,11 +1,13 @@ #include "ggml.h" #include "llama.h" +#include "jais-language.h" #ifdef NDEBUG #undef NDEBUG #endif #include +#include #include #include #include @@ -334,7 +336,90 @@ static void test_perf() { BENCH(llama_sampler_init_xtc (1.0f, 0.1f, 1, 1), data, 32); } +static void test_jais_language_pin() { + common_jais_language_pin pin; + for (const auto & tag : common_jais_tags) { + std::string lower = tag.name; + for (auto & c : lower) { c = static_cast(std::tolower(static_cast(c))); } + pin.configure(lower); + assert(pin.allowed == std::vector{tag.id}); + } + for (const auto & spelling : {"eg", "EG", "eG", "Eg"}) { + pin.configure(spelling); + assert(pin.allowed == std::vector{7646}); + } + pin.configure("AR"); + assert(pin.allowed.size() == 20); + pin.configure("eg,EG,eG,mA"); + assert((pin.allowed == std::vector{7646, 6101})); + pin.configure("MSA"); + assert(pin.allowed == std::vector{226606}); + pin.configure("EG"); + std::vector logits(250031, -20.0f); + auto score = [&](llama_token id) { return logits[id]; }; + assert(pin.select(score) == 60); + pin.accept(60, true); + logits[272] = 9; logits[7646] = 6; logits[5126] = 7; + assert(pin.select(score) == 7646); // An unallowed tag still wins over refusal. + logits[5126] = 9; + assert(pin.select(score) == 7646); // Ties transcribe. + logits[5126] = 10; + assert(pin.select(score) == 5126); + auto refusal = pin; + refusal.accept(5126, true); + assert(refusal.select(score) == LLAMA_TOKEN_NULL); + pin.accept(7646, true); + assert(pin.select(score) == 62); + pin.accept(62, true); + assert(pin.select(score) == LLAMA_TOKEN_NULL); + pin.configure("ar"); + assert(pin.allowed.size() == 20); + pin.accept(60, true); + logits[5126] = 7; logits[8048] = 8; + assert(pin.select(score) == 8048); + pin.configure("EG,MA,EG"); + assert(pin.allowed.size() == 2); + pin.accept(250029, false); + assert(pin.select(score) == LLAMA_TOKEN_NULL); + pin.accept(123, true); + pin.accept(250030, true); + assert(pin.select(score) == 60); + pin.reset(); + assert(pin.select(score) == 60); + pin.accept(60, false); pin.accept(7646, false); pin.accept(62, false); + assert(pin.select(score) == LLAMA_TOKEN_NULL); + pin.accept(123, true); + assert(pin.select(score) == LLAMA_TOKEN_NULL); + bool threw = false; + pin.configure(""); + assert(pin.select(score) == LLAMA_TOKEN_NULL); + threw = false; + try { pin.configure("unknown"); } catch (const std::invalid_argument &) { threw = true; } + assert(threw); + pin.configure("en"); + logits[60] = -INFINITY; + threw = false; + try { pin.select(score); } catch (const std::runtime_error &) { threw = true; } + assert(threw); + + // prefill takes one tag; ISO codes are canonicalized like resolve_language + assert(common_jais_prefill_tag("EG") == "EG"); + assert(common_jais_prefill_tag(" EN ") == "en"); + assert(common_jais_prefill_tag("eg") == "EG"); + assert(common_jais_prefill_tag("eG") == "EG"); + for (const char * family : {"ar", "EG,MA"}) { + threw = false; + try { common_jais_prefill_tag(family); } catch (const std::invalid_argument &) { threw = true; } + assert(threw); + } + assert(common_jais_same_language("ar", "msa,EG,SA,MA,IQ,JO,SY,QA,DZ,LB,OM,YE,TN,MR,KW,SD,LY,AE,PS,BH")); + assert(!common_jais_same_language("EG,MA", "MA,EG")); + assert(common_jais_ends_with_tag(60, 7646, 62)); + assert(!common_jais_ends_with_tag(60, 5126, 62)); +} + int main(void) { + test_jais_language_pin(); ggml_time_init(); test_dist_singleton_rng(); diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index f604509..5e6ae15 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -464,6 +464,7 @@ enum projector_type { PROJECTOR_TYPE_LLAMA4, PROJECTOR_TYPE_QWEN2A, PROJECTOR_TYPE_QWEN3A, + PROJECTOR_TYPE_JAISASR, PROJECTOR_TYPE_GLMA, PROJECTOR_TYPE_QWEN25O, // will be replaced by QWEN2A or QWEN25VL depending on clip_ctx PROJECTOR_TYPE_VOXTRAL, @@ -528,6 +529,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_LLAMA4, "llama4"}, { PROJECTOR_TYPE_QWEN2A, "qwen2a"}, { PROJECTOR_TYPE_QWEN3A, "qwen3a"}, + { PROJECTOR_TYPE_JAISASR, "jais-asr"}, { PROJECTOR_TYPE_GLMA, "glma"}, { PROJECTOR_TYPE_QWEN25O, "qwen2.5o"}, { PROJECTOR_TYPE_VOXTRAL, "voxtral"}, diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 90de195..e83826c 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1082,6 +1082,7 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const builder = std::make_unique(ctx, img); } break; case PROJECTOR_TYPE_QWEN3A: + case PROJECTOR_TYPE_JAISASR: { builder = std::make_unique(ctx, img); } break; @@ -1736,6 +1737,7 @@ struct clip_model_loader { case PROJECTOR_TYPE_ULTRAVOX: case PROJECTOR_TYPE_QWEN2A: case PROJECTOR_TYPE_QWEN3A: + case PROJECTOR_TYPE_JAISASR: case PROJECTOR_TYPE_GLMA: case PROJECTOR_TYPE_VOXTRAL: case PROJECTOR_TYPE_MERALION: @@ -2809,6 +2811,7 @@ struct clip_model_loader { model.mm_fc_b = get_tensor(string_format(TN_MM_AUDIO_FC, "bias")); } break; case PROJECTOR_TYPE_QWEN3A: + case PROJECTOR_TYPE_JAISASR: { model.conv2d_1_w = get_tensor(string_format(TN_CONV2D, 1, "weight")); model.conv2d_1_b = get_tensor(string_format(TN_CONV2D, 1, "bias")); @@ -2821,6 +2824,16 @@ struct clip_model_loader { model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias")); model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight")); model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "bias")); + if (model.proj_type == PROJECTOR_TYPE_JAISASR) { + // Jais-ASR adds LayerNorm -> Linear -> GELU -> Linear on top of the + // Qwen3-Omni audio tower, mapping audio output_dim -> LLM hidden size + model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight")); + model.mm_norm_pre_b = get_tensor(string_format(TN_MM_NORM_PRE, "bias")); + model.mm_3_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "weight")); + model.mm_3_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "bias")); + model.mm_4_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 4, "weight")); + model.mm_4_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 4, "bias")); + } } break; case PROJECTOR_TYPE_MIMO_AUDIO: { @@ -4201,6 +4214,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { } } break; case PROJECTOR_TYPE_QWEN3A: + case PROJECTOR_TYPE_JAISASR: { // chunk_size=100 frames --> 3x stride-2 conv2d --> 13 tokens per chunk const int chunk_size = 100; @@ -5152,6 +5166,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { case PROJECTOR_TYPE_NEMOTRON_V2_VL: case PROJECTOR_TYPE_QWEN2A: case PROJECTOR_TYPE_QWEN3A: + case PROJECTOR_TYPE_JAISASR: case PROJECTOR_TYPE_GLMA: case PROJECTOR_TYPE_ULTRAVOX: case PROJECTOR_TYPE_LFM2: @@ -5860,6 +5875,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { return ctx->model.mm_fc_w->ne[1]; case PROJECTOR_TYPE_QWEN3A: return ctx->model.mm_2_w->ne[1]; + case PROJECTOR_TYPE_JAISASR: + return ctx->model.mm_4_w->ne[1]; // extra projector maps audio dim -> LLM hidden case PROJECTOR_TYPE_GLMA: case PROJECTOR_TYPE_LFM2: case PROJECTOR_TYPE_KIMIVL: diff --git a/tools/mtmd/models/qwen3a.cpp b/tools/mtmd/models/qwen3a.cpp index 4de9695..a898e9f 100644 --- a/tools/mtmd/models/qwen3a.cpp +++ b/tools/mtmd/models/qwen3a.cpp @@ -83,6 +83,18 @@ ggml_cgraph * clip_graph_qwen3a::build() { FFN_GELU_ERF, -1); cb(cur, "projected", -1); + // Jais-ASR: extra projector on top of the Qwen3-Omni tower + // (LayerNorm -> Linear -> GELU -> Linear), audio output_dim -> LLM hidden size + if (proj_type == PROJECTOR_TYPE_JAISASR) { + cur = build_norm(cur, model.mm_norm_pre_w, model.mm_norm_pre_b, NORM_TYPE_NORMAL, eps, -1); + cur = build_ffn(cur, + model.mm_3_w, model.mm_3_b, + nullptr, nullptr, + model.mm_4_w, model.mm_4_b, + FFN_GELU_ERF, -1); + cb(cur, "projected_jais", -1); + } + ggml_build_forward_expand(gf, cur); return gf; } diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp index 97678c6..ddad415 100644 --- a/tools/mtmd/mtmd-cli.cpp +++ b/tools/mtmd/mtmd-cli.cpp @@ -3,6 +3,7 @@ #include "log.h" #include "common.h" #include "sampling.h" +#include "jais-language.h" #include "llama.h" #include "ggml.h" #include "console.h" @@ -96,6 +97,8 @@ struct mtmd_cli_context { common_chat_templates_ptr tmpls; std::vector chat_history; bool use_jinja = false; + bool jais_pin_enabled = false; + bool jais_asr = false; // TODO: support for --system-prompt with /clear command // support for legacy templates (models not having EOT token) @@ -111,6 +114,7 @@ struct mtmd_cli_context { lctx = llama_init->context(); vocab = llama_model_get_vocab(model); smpl = common_sampler_init(model, params.sampling); + jais_pin_enabled = !params.sampling.jais_language.empty(); n_threads = params.cpuparams.n_threads; batch = llama_batch_init(1, 0, 1); // batch for next token generation n_batch = params.n_batch; @@ -136,9 +140,10 @@ struct mtmd_cli_context { } tmpls = common_chat_templates_init(model, params.chat_template); - use_jinja = params.use_jinja; + jais_asr = common_chat_templates_is_jais_asr(tmpls.get()); + use_jinja = params.use_jinja || jais_asr; // the Jais-ASR template needs the jinja engine chat_history.clear(); - LOG_INF("%s: chat template example:\n%s\n", __func__, common_chat_format_example(tmpls.get(), params.use_jinja, params.default_template_kwargs).c_str()); + LOG_INF("%s: chat template example:\n%s\n", __func__, common_chat_format_example(tmpls.get(), use_jinja, params.default_template_kwargs).c_str()); // load antiprompt tokens for legacy templates if (params.chat_template == "vicuna") { @@ -213,7 +218,14 @@ static int generate_response(mtmd_cli_context & ctx, int n_predict) { break; } - llama_token token_id = common_sampler_sample(ctx.smpl, ctx.lctx, -1); + llama_token token_id; + try { + token_id = common_sampler_sample(ctx.smpl, ctx.lctx, -1); + } catch (const std::exception & e) { + // e.g. the Jais-ASR language pin found no finite score: report it like llama-server does + LOG_ERR("\n%s: %s\n", __func__, e.what()); + return 1; + } generated_tokens.push_back(token_id); common_sampler_accept(ctx.smpl, token_id, true); @@ -288,6 +300,30 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) { ctx.bitmaps.entries.clear(); ctx.videos.clear(); + if (ctx.jais_pin_enabled || ctx.jais_asr) { + // like llama-server: the prompt text goes through the sampler, so the repeat penalty + // sees the same history and the pin sees the prompt + common_sampler_reset(ctx.smpl); + llama_token tail[3] = {LLAMA_TOKEN_NULL, LLAMA_TOKEN_NULL, LLAMA_TOKEN_NULL}; + for (size_t i = 0; i < mtmd_input_chunks_size(chunks.ptr.get()); ++i) { + const auto * chunk = mtmd_input_chunks_get(chunks.ptr.get(), i); + if (mtmd_input_chunk_get_type(chunk) == MTMD_INPUT_CHUNK_TYPE_TEXT) { + size_t count = 0; + const auto * ids = mtmd_input_chunk_get_tokens_text(chunk, &count); + for (size_t j = 0; j < count; ++j) { + common_sampler_accept(ctx.smpl, ids[j], false); + tail[0] = tail[1]; + tail[1] = tail[2]; + tail[2] = ids[j]; + } + } + } + if (ctx.jais_pin_enabled && common_jais_ends_with_tag(tail[0], tail[1], tail[2])) { + LOG_ERR("%s\n", common_jais_pin_prefill_conflict); + return 1; + } + } + // batch encode all media chunks, then decode each size_t n_chunks = mtmd_input_chunks_size(chunks.ptr.get()); for (size_t i = 0; i < n_chunks; i++) { diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 5b30618..9f7e305 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -935,6 +935,13 @@ struct mtmd_context { aud_end = "<|audio_end|>"; audio_preproc = std::make_unique(ctx_a); } break; + case PROJECTOR_TYPE_JAISASR: + { + // Jais-ASR wraps the audio span with + aud_beg = ""; + audio_preproc = std::make_unique(ctx_a); + } break; case PROJECTOR_TYPE_VOXTRAL: { // [BEGIN_AUDIO] ... (embeddings) ... diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp index a6fe3c6..ca402a5 100644 --- a/tools/server/server-chat.cpp +++ b/tools/server/server-chat.cpp @@ -656,20 +656,30 @@ json convert_transcriptions_to_chatcmpl( if (response_format != "json") { throw std::invalid_argument("Only 'json' response_format is supported for transcription"); } - const common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls); - if (prompt.empty()) { - prompt = preset.user; - } - if (!language.empty()) { - prompt += string_format(" (language: %s)", language.c_str()); - } - prompt += get_media_marker(); - json messages = json::array(); - if (!preset.system.empty()) { - messages.push_back({{"role", "system"}, {"content", preset.system}}); + if (common_chat_templates_is_jais_asr(tmpls)) { + // Jais-ASR, as in its vLLM plugin: the user turn holds only the audio and `prompt` + // is the system turn. `language` and `language_prefill` stay in the body as pin + // and prefill controls, handled by oaicompat_chat_params_parse. + if (!prompt.empty()) { + messages.push_back({{"role", "system"}, {"content", prompt}}); + } + messages.push_back({{"role", "user"}, {"content", get_media_marker()}}); + } else { + const common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls); + if (prompt.empty()) { + prompt = preset.user; + } + if (!language.empty()) { + prompt += string_format(" (language: %s)", language.c_str()); + } + prompt += get_media_marker(); + + if (!preset.system.empty()) { + messages.push_back({{"role", "system"}, {"content", preset.system}}); + } + messages.push_back({{"role", "user"}, {"content", prompt}}); } - messages.push_back({{"role", "user"}, {"content", prompt}}); json chatcmpl_body = inp_body; // copy all fields chatcmpl_body["messages"] = messages; diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index c30955e..4edcd0a 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -5,6 +5,7 @@ #include "mtmd.h" #include "mtmd-helper.h" #include "chat.h" +#include "jais-language.h" #include "base64.hpp" #include "server-common.h" @@ -1210,6 +1211,25 @@ json oaicompat_chat_params_parse( throw std::invalid_argument("Expected 'content' to be a string or an array"); } + // Jais-ASR: the user turn is the audio alone. llama-cli adds its -p text after the + // audio, and "<__media__>" there is only a placeholder for the attached audio. + if (common_chat_templates_is_jais_asr(opt.tmpls.get())) { + bool has_audio = false; + for (const auto & p : content) { + has_audio = has_audio || json_value(p, "type", std::string()) == "input_audio"; + } + if (has_audio) { + json kept = json::array(); + for (const auto & p : content) { + if (json_value(p, "type", std::string()) != "text" || + json_value(p, "text", std::string()) != mtmd_default_marker()) { + kept.push_back(p); + } + } + content = kept; + } + } + for (auto & p : content) { std::string type = json_value(p, "type", std::string()); if (type == "image_url") { @@ -1331,6 +1351,82 @@ json oaicompat_chat_params_parse( } } + // Jais-ASR language controls, with the same contract as its vLLM plugin: + // language, jais_language or the `language` template kwarg -> generation-time pin + // language_prefill, top level or template kwarg -> "[tag]" written into the prompt + // Setting either control in the request replaces the server defaults for both. + if (common_chat_templates_is_jais_asr(opt.tmpls.get())) { + auto to_spec = [](const json & value) { + if (value.is_string()) { + return value.get(); + } + std::string spec; + if (value.is_array() && !value.empty()) { + for (const auto & item : value) { + if (!item.is_string()) { + throw std::invalid_argument("language must be a string or a list of strings"); + } + spec += (spec.empty() ? "" : ",") + item.get(); + } + return spec; + } + throw std::invalid_argument("language must be a string or a non-empty list of strings"); + }; + // the first value given wins; any other must name the same tags + auto choose = [&](const char * name, const std::vector & values, std::string & chosen) { + bool found = false; + for (const auto & value : values) { + if (value.is_null()) { + continue; + } + const std::string spec = to_spec(value); + if (!found) { + chosen = spec; + found = true; + } else if (!common_jais_same_language(spec, chosen)) { + throw std::invalid_argument(string_format("Conflicting %s values in the request", name)); + } + } + return found; + }; + auto request_kwarg = [&](const char * key) { + return json_value(chat_template_kwargs_object, key, json()); + }; + auto default_kwarg = [&](const char * key) { + const auto it = opt.chat_template_kwargs.find(key); + return it == opt.chat_template_kwargs.end() ? json() : json::parse(it->second); + }; + + std::string pin; + std::string prefill; + bool has_pin = choose("language", + {json_value(body, "jais_language", json()), json_value(body, "language", json()), request_kwarg("language")}, pin); + bool has_prefill = choose("language_prefill", + {json_value(body, "language_prefill", json()), request_kwarg("language_prefill")}, prefill); + const bool explicit_control = has_pin || has_prefill; + if (has_pin && has_prefill) { + throw std::invalid_argument(common_jais_pin_prefill_conflict); + } + if (!explicit_control) { + has_pin = choose("language", {default_kwarg("language")}, pin); + has_prefill = choose("language_prefill", {default_kwarg("language_prefill")}, prefill); + } + + // the template rejects `language`: a pin cannot be written into a prompt + inputs.chat_template_kwargs.erase("language"); + inputs.chat_template_kwargs.erase("language_prefill"); + if (has_prefill) { + inputs.chat_template_kwargs["language_prefill"] = json(common_jais_prefill_tag(prefill)).dump(); + } + if (has_pin) { + common_jais_language_pin check; + check.configure(pin); + llama_params["jais_language"] = pin; + } else if (explicit_control) { + llama_params["jais_language"] = ""; // a requested prefill replaces --jais-language + } + } + inputs.force_pure_content = opt.force_pure_content; // Apply chat template to the list of messages diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index f547735..f2c1dbe 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -12,6 +12,7 @@ #include "fit.h" #include "llama.h" #include "log.h" +#include "jais-language.h" #include "sampling.h" #include "speculative.h" #include "mtmd.h" @@ -468,7 +469,7 @@ struct server_slot { } bool can_speculate() const { - return !!spec; + return !!spec && (!task || task->params.sampling.jais_language.empty()); } void add_token(const completion_token_output & token) { @@ -1011,6 +1012,7 @@ private: const bool is_resume = sleeping; params_base = params; + common_params_apply_model_defaults(params_base); // Jais-ASR defaults, before anything reads them const auto output_limits = server_output_limits(params_base); params_base.n_outputs_max = output_limits.total; params_base.n_outputs_max_per_seq = output_limits.per_seq; @@ -4291,6 +4293,14 @@ std::unique_ptr server_routes::handle_completions_impl( meta->logit_bias_eog, data); + // Jais-ASR: a language pin and a prompt that already ends with "[tag]" are exclusive + if (!task.params.sampling.jais_language.empty() && task.tokens.size() >= 3) { + const size_t n = task.tokens.size(); + if (common_jais_ends_with_tag(task.tokens[n - 3], task.tokens[n - 2], task.tokens[n - 1])) { + throw std::invalid_argument(common_jais_pin_prefill_conflict); + } + } + task.params.message_spans = task.tokens.find_message_spans(delimiters); task.id_slot = json_value(data, "id_slot", -1); diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp index 64b9251..90756d2 100644 --- a/tools/server/server-schema.cpp +++ b/tools/server/server-schema.cpp @@ -1,4 +1,5 @@ #include "server-schema.h" +#include "jais-language.h" #include "json-schema-to-grammar.h" @@ -127,6 +128,15 @@ std::vector> make_llama_cmpl_schema(const common_params & ->set_hard_limits(0, INT32_MAX) ->set_desc("Last n tokens to consider for penalizing repetition (0 = disabled)")); + add((new field_str("jais_language")) + ->set_desc("Jais-ASR language pin: ar, a concrete tag, or comma-separated tags; empty disables (chat and transcription requests also accept `language`)") + ->set_handler([&](field_eval_context & ctx, const json & data) { + const auto language = data.at("jais_language").get(); + common_jais_language_pin pin; + pin.configure(language); + ctx.params.sampling.jais_language = language; + })); + add((new field_num("repeat_penalty", params.sampling.penalty_repeat)) ->set_desc("Control the repetition of token sequences in the generated text (1.0 = disabled)")); diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 0d3beb3..fac93f6 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -54,6 +54,7 @@ json task_params::to_json(bool only_metrics) const { {"typical_p", sampling.typ_p}, {"repeat_last_n", sampling.penalty_last_n}, {"repeat_penalty", sampling.penalty_repeat}, + {"jais_language", sampling.jais_language}, {"presence_penalty", sampling.penalty_present}, {"frequency_penalty", sampling.penalty_freq}, {"dry_multiplier", sampling.dry_multiplier}, @@ -106,6 +107,7 @@ json task_params::to_json(bool only_metrics) const { {"typical_p", sampling.typ_p}, {"repeat_last_n", sampling.penalty_last_n}, {"repeat_penalty", sampling.penalty_repeat}, + {"jais_language", sampling.jais_language}, {"presence_penalty", sampling.penalty_present}, {"frequency_penalty", sampling.penalty_freq}, {"dry_multiplier", sampling.dry_multiplier},