File size: 2,342 Bytes
e29f88c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | """Validate the actual tokenizer in the exact pointer prompt/answer contexts."""
from typing import Any
import re
from scripts.dataset.pointer import SYMBOLS
MODEL_ID = "Qwen/Qwen2.5-0.5B-Instruct"
MODEL_REVISION = "7ae557604adf67be50417f59c2c2f167def9a775"
def validate_prompt_tokens(tokenizer: Any, prompt: str, token_ids: dict[str, int]) -> int:
"""Check every space-prefixed symbol span; return the answer readout index.
Prompts are raw text, with no chat template or added special tokens. The
answer position is the final ':' token's index (a next-token readout).
"""
if not prompt.endswith("\nAnswer:"):
raise ValueError("Prompt must end with '\\nAnswer:' without a trailing space")
encoded = tokenizer(prompt, add_special_tokens=False, return_offsets_mapping=True)
spans = dict(zip(map(tuple, encoded["offset_mapping"]), encoded["input_ids"]))
for match in re.finditer(r" [A-Z](?=[,)\n])", prompt):
if spans.get(match.span()) != token_ids[match.group()[1:]]:
raise ValueError(f"Symbol does not retain its token at {match.span()}: {match.group()!r}")
return len(encoded["input_ids"]) - 1
def validate_symbols(tokenizer: Any) -> dict[str, int]:
"""Require one unique token per ' '+symbol in rules, starts, and answers."""
token_ids = {}
for symbol in SYMBOLS:
ids = tokenizer.encode(" " + symbol, add_special_tokens=False)
if len(ids) != 1 or tokenizer.decode(ids) != " " + symbol:
raise ValueError(f"Symbol {symbol!r} is not one reversible space-prefixed token")
token_ids[symbol] = ids[0]
if len(set(token_ids.values())) != len(SYMBOLS):
raise ValueError("Answer tokens must be unique")
# Exhaustive pair contexts, and all 26 possible answer continuations.
for source in SYMBOLS:
prompt = "Rules: " + " ".join(f"( {source}, {target})" for target in SYMBOLS)
prompt += f"\nStart: {source}\nSteps: 1\nAnswer:"
validate_prompt_tokens(tokenizer, prompt, token_ids)
prefix = tokenizer.encode(prompt, add_special_tokens=False)
for target in SYMBOLS:
if tokenizer.encode(prompt + " " + target, add_special_tokens=False) != [*prefix, token_ids[target]]:
raise ValueError(f"Answer boundary merges for {target!r}")
return token_ids
|