File size: 3,497 Bytes
9eefb05
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
"""Chat-template rendering and prefix/suffix token split, shared by the MLX engine and tests."""
from __future__ import annotations

from pathlib import Path
from typing import Any

from .prompt import DEFAULT_STYLE, LETTERS, SLOT_LABELS, PromptStyle, RenderedQuestion, build_messages

PREFIX_MARK = "QEV_PREFIX_MARK"


def load_tokenizer(model_dir: str | Path):
    from transformers import AutoTokenizer

    tok = AutoTokenizer.from_pretrained(str(model_dir), local_files_only=Path(model_dir).is_dir())
    jinja = Path(model_dir) / "chat_template.jinja"
    if tok.chat_template is None and jinja.is_file():
        tok.chat_template = jinja.read_text()
    return tok


def template_kwargs_for(model_type: str | None) -> dict[str, Any]:
    # Flide sets this for both Qwen and Gemma 4. The Gemma 4 jinja default is already false,
    # but passing it keeps the path explicit.
    mt = (model_type or "").lower()
    if mt.startswith(("qwen", "gemma4")):
        return {"enable_thinking": False}
    return {}


def render_prompt(tokenizer, state: Any, suffix: str, style: PromptStyle = DEFAULT_STYLE,
                  template_kwargs: dict[str, Any] | None = None) -> str:
    kw = {"enable_thinking": False} if template_kwargs is None else template_kwargs
    return tokenizer.apply_chat_template(
        build_messages(state, suffix, style),
        tokenize=False,
        add_generation_prompt=True,
        **kw,
    )


def encode(tokenizer, text: str) -> list[int]:
    return tokenizer.encode(text, add_special_tokens=False)


def prefix_and_suffix_ids(
    tokenizer,
    state: Any,
    rendered: list[RenderedQuestion],
    style: PromptStyle = DEFAULT_STYLE,
    template_kwargs: dict[str, Any] | None = None,
) -> tuple[list[int], list[list[int]], list[str]]:
    """Longest common token prefix that ends at or before the end of the state block.

    Same rule as OneJev DecisionEngine._prefix_and_suffix_ids / Flide's block-aware variant.
    """
    prompts = [render_prompt(tokenizer, state, r.suffix, style, template_kwargs) for r in rendered]
    full = [encode(tokenizer, p) for p in prompts]
    prefix_text = render_prompt(tokenizer, state, PREFIX_MARK, style, template_kwargs)
    cut = prefix_text.index(PREFIX_MARK)
    prefix_ids = encode(tokenizer, prefix_text[:cut])
    n = len(prefix_ids)
    while n > 0 and not all(ids[:n] == prefix_ids[:n] for ids in full):
        n -= 1
    if n == 0:
        raise ValueError("branches share no token prefix")
    prefix_ids = prefix_ids[:n]
    suffixes = [ids[n:] for ids in full]
    if any(len(s) == 0 for s in suffixes):
        raise ValueError("a branch has an empty suffix; state block boundary mismatch")
    return prefix_ids, suffixes, prompts


def verify_slot_ids(tokenizer) -> list[int]:
    """Single round-trip token ids for A..Z, plus two-letter labels if the tokenizer holds them all."""

    def one(label: str):
        enc = tokenizer.encode(label, add_special_tokens=False)
        return enc[0] if len(enc) == 1 and tokenizer.decode(enc) == label else None

    ids = [one(l) for l in LETTERS]
    if any(i is None for i in ids):
        bad = [l for l, i in zip(LETTERS, ids) if i is None]
        raise ValueError(f"answer slots {bad} are not single round-trip tokens")
    extra = [one(l) for l in SLOT_LABELS[len(LETTERS) :]]
    if all(i is not None for i in extra):
        ids += extra
    if len(set(ids)) != len(ids):
        raise ValueError("answer slot tokens collide")
    return ids