"""Deterministic text rendering for a single decision question.""" import json from typing import Any SYSTEM_PROMPT = ( "Read the complete state and schema. Decide every field jointly. Each answer " "must be exactly one of that field's allowed options." ) SUFFIX = ( "\n<|im_end|>\n<|im_start|>assistant\n\n\n\n\n" "JOINT SCHEMA DECISIONS:" ) def render(value: Any) -> str: if isinstance(value, str): return value return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True) def encode(tokenizer, state: Any, question_id: str, question: dict, codes: list) -> dict: kind = question.get("type") if kind == "choice": criteria = question.get("criteria") if not isinstance(criteria, dict) or not 1 <= len(criteria) <= 255: raise ValueError("choice requires 1..255 named criteria") options = sorted((str(key), value) for key, value in criteria.items()) elif kind == "noul": descriptions = { "true": "The proposition is true or the answer is yes.", "false": "The proposition is false or the answer is no.", } descriptions.update(question.get("criteria") or {}) options = [(key, descriptions[key]) for key in ("true", "false")] else: raise ValueError("Supported question types: choice, noul") def tokens(text: str) -> list[int]: return tokenizer(text, add_special_tokens=False).input_ids instruction = question.get("instructions") if instruction is None or instruction == "": instruction = str(question_id) ids = tokens( f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n" "<|im_start|>user\nSTATE:\n" ) ids.extend(tokens(render(state))) ids.extend(tokens("\n\nSCHEMA FIELDS:\n")) ids.extend( tokens(f"\nFIELD 1\nID: {question_id}\nTYPE: {kind}\nINSTRUCTION: ") ) ids.extend(tokens(render(instruction))) ids.extend(tokens("\nALLOWED OPTIONS:\n")) for index, (key, description) in enumerate(options): ids.extend(tokens(f"OPTION {codes[index]}: ")) semantics = {"option_id": key} if description is not None: semantics["description"] = description ids.extend(tokens(render(semantics))) ids.extend(tokens("\n")) ids.extend(tokens("END FIELD\n")) position = len(ids) ids.append(tokenizer.convert_tokens_to_ids("<|fim_suffix|>")) ids.extend(tokens(SUFFIX)) if len(ids) > 32768: raise ValueError("complete input exceeds limit; never truncate") return { "input_ids": ids, "position": position, "option_ids": [key for key, _ in options], }