"""Chat-template rendering and prefix/suffix token split, shared by the MLX engine and tests.""" from __future__ import annotations from pathlib import Path from typing import Any from .prompt import DEFAULT_STYLE, LETTERS, SLOT_LABELS, PromptStyle, RenderedQuestion, build_messages PREFIX_MARK = "QEV_PREFIX_MARK" def load_tokenizer(model_dir: str | Path): from transformers import AutoTokenizer tok = AutoTokenizer.from_pretrained(str(model_dir), local_files_only=Path(model_dir).is_dir()) jinja = Path(model_dir) / "chat_template.jinja" if tok.chat_template is None and jinja.is_file(): tok.chat_template = jinja.read_text() return tok def template_kwargs_for(model_type: str | None) -> dict[str, Any]: # Flide sets this for both Qwen and Gemma 4. The Gemma 4 jinja default is already false, # but passing it keeps the path explicit. mt = (model_type or "").lower() if mt.startswith(("qwen", "gemma4")): return {"enable_thinking": False} return {} def render_prompt(tokenizer, state: Any, suffix: str, style: PromptStyle = DEFAULT_STYLE, template_kwargs: dict[str, Any] | None = None) -> str: kw = {"enable_thinking": False} if template_kwargs is None else template_kwargs return tokenizer.apply_chat_template( build_messages(state, suffix, style), tokenize=False, add_generation_prompt=True, **kw, ) def encode(tokenizer, text: str) -> list[int]: return tokenizer.encode(text, add_special_tokens=False) def prefix_and_suffix_ids( tokenizer, state: Any, rendered: list[RenderedQuestion], style: PromptStyle = DEFAULT_STYLE, template_kwargs: dict[str, Any] | None = None, ) -> tuple[list[int], list[list[int]], list[str]]: """Longest common token prefix that ends at or before the end of the state block. Same rule as OneJev DecisionEngine._prefix_and_suffix_ids / Flide's block-aware variant. """ prompts = [render_prompt(tokenizer, state, r.suffix, style, template_kwargs) for r in rendered] full = [encode(tokenizer, p) for p in prompts] prefix_text = render_prompt(tokenizer, state, PREFIX_MARK, style, template_kwargs) cut = prefix_text.index(PREFIX_MARK) prefix_ids = encode(tokenizer, prefix_text[:cut]) n = len(prefix_ids) while n > 0 and not all(ids[:n] == prefix_ids[:n] for ids in full): n -= 1 if n == 0: raise ValueError("branches share no token prefix") prefix_ids = prefix_ids[:n] suffixes = [ids[n:] for ids in full] if any(len(s) == 0 for s in suffixes): raise ValueError("a branch has an empty suffix; state block boundary mismatch") return prefix_ids, suffixes, prompts def verify_slot_ids(tokenizer) -> list[int]: """Single round-trip token ids for A..Z, plus two-letter labels if the tokenizer holds them all.""" def one(label: str): enc = tokenizer.encode(label, add_special_tokens=False) return enc[0] if len(enc) == 1 and tokenizer.decode(enc) == label else None ids = [one(l) for l in LETTERS] if any(i is None for i in ids): bad = [l for l, i in zip(LETTERS, ids) if i is None] raise ValueError(f"answer slots {bad} are not single round-trip tokens") extra = [one(l) for l in SLOT_LABELS[len(LETTERS) :]] if all(i is not None for i in extra): ids += extra if len(set(ids)) != len(ids): raise ValueError("answer slot tokens collide") return ids