File size: 2,720 Bytes
883a747
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
"""Deterministic text rendering for a single decision question."""

import json
from typing import Any


SYSTEM_PROMPT = (
    "Read the complete state and schema. Decide every field jointly. Each answer "
    "must be exactly one of that field's allowed options."
)
SUFFIX = (
    "\n<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
    "JOINT SCHEMA DECISIONS:"
)


def render(value: Any) -> str:
    if isinstance(value, str):
        return value
    return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True)


def encode(tokenizer, state: Any, question_id: str, question: dict, codes: list) -> dict:
    kind = question.get("type")
    if kind == "choice":
        criteria = question.get("criteria")
        if not isinstance(criteria, dict) or not 1 <= len(criteria) <= 255:
            raise ValueError("choice requires 1..255 named criteria")
        options = sorted((str(key), value) for key, value in criteria.items())
    elif kind == "noul":
        descriptions = {
            "true": "The proposition is true or the answer is yes.",
            "false": "The proposition is false or the answer is no.",
        }
        descriptions.update(question.get("criteria") or {})
        options = [(key, descriptions[key]) for key in ("true", "false")]
    else:
        raise ValueError("Supported question types: choice, noul")

    def tokens(text: str) -> list[int]:
        return tokenizer(text, add_special_tokens=False).input_ids

    instruction = question.get("instructions")
    if instruction is None or instruction == "":
        instruction = str(question_id)

    ids = tokens(
        f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n"
        "<|im_start|>user\nSTATE:\n"
    )
    ids.extend(tokens(render(state)))
    ids.extend(tokens("\n\nSCHEMA FIELDS:\n"))
    ids.extend(
        tokens(f"\nFIELD 1\nID: {question_id}\nTYPE: {kind}\nINSTRUCTION: ")
    )
    ids.extend(tokens(render(instruction)))
    ids.extend(tokens("\nALLOWED OPTIONS:\n"))
    for index, (key, description) in enumerate(options):
        ids.extend(tokens(f"OPTION {codes[index]}: "))
        semantics = {"option_id": key}
        if description is not None:
            semantics["description"] = description
        ids.extend(tokens(render(semantics)))
        ids.extend(tokens("\n"))
    ids.extend(tokens("END FIELD\n"))
    position = len(ids)
    ids.append(tokenizer.convert_tokens_to_ids("<|fim_suffix|>"))
    ids.extend(tokens(SUFFIX))

    if len(ids) > 32768:
        raise ValueError("complete input exceeds limit; never truncate")
    return {
        "input_ids": ids,
        "position": position,
        "option_ids": [key for key, _ in options],
    }