decision-4b-preview / protocol.py
EpochEcho's picture
Upload folder using huggingface_hub
883a747 verified
Raw History Blame Contribute Delete
2.72 kB
"""Deterministic text rendering for a single decision question."""
import json
from typing import Any
SYSTEM_PROMPT = (
"Read the complete state and schema. Decide every field jointly. Each answer "
"must be exactly one of that field's allowed options."
)
SUFFIX = (
"\n<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
"JOINT SCHEMA DECISIONS:"
)
def render(value: Any) -> str:
if isinstance(value, str):
return value
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
def encode(tokenizer, state: Any, question_id: str, question: dict, codes: list) -> dict:
kind = question.get("type")
if kind == "choice":
criteria = question.get("criteria")
if not isinstance(criteria, dict) or not 1 <= len(criteria) <= 255:
raise ValueError("choice requires 1..255 named criteria")
options = sorted((str(key), value) for key, value in criteria.items())
elif kind == "noul":
descriptions = {
"true": "The proposition is true or the answer is yes.",
"false": "The proposition is false or the answer is no.",
}
descriptions.update(question.get("criteria") or {})
options = [(key, descriptions[key]) for key in ("true", "false")]
else:
raise ValueError("Supported question types: choice, noul")
def tokens(text: str) -> list[int]:
return tokenizer(text, add_special_tokens=False).input_ids
instruction = question.get("instructions")
if instruction is None or instruction == "":
instruction = str(question_id)
ids = tokens(
f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n"
"<|im_start|>user\nSTATE:\n"
)
ids.extend(tokens(render(state)))
ids.extend(tokens("\n\nSCHEMA FIELDS:\n"))
ids.extend(
tokens(f"\nFIELD 1\nID: {question_id}\nTYPE: {kind}\nINSTRUCTION: ")
)
ids.extend(tokens(render(instruction)))
ids.extend(tokens("\nALLOWED OPTIONS:\n"))
for index, (key, description) in enumerate(options):
ids.extend(tokens(f"OPTION {codes[index]}: "))
semantics = {"option_id": key}
if description is not None:
semantics["description"] = description
ids.extend(tokens(render(semantics)))
ids.extend(tokens("\n"))
ids.extend(tokens("END FIELD\n"))
position = len(ids)
ids.append(tokenizer.convert_tokens_to_ids("<|fim_suffix|>"))
ids.extend(tokens(SUFFIX))
if len(ids) > 32768:
raise ValueError("complete input exceeds limit; never truncate")
return {
"input_ids": ids,
"position": position,
"option_ids": [key for key, _ in options],
}