"""The GameScaling action format: parse model text into one 200 ms keyboard/mouse chunk.
Every turn the model emits exactly one chunk:
<|action_start|>X Y Z ; g1 ; g2 ; g3 ; g4 ; g5 ; g6<|action_end|>
- ``X Y`` is the mouse displacement over the whole 200 ms, in per-mille of the
screen (``X = sum(dx) / width * 1000``, clamped to +-1000; X>0 right, Y>0 down).
- ``Z`` is the scroll amount in wheel clicks (clamped to +-5; Z>0 up).
- ``g1..g6`` are six 33 ms substeps. Each group lists the keys/buttons that are
held during that substep (at most 4). A key present in consecutive groups is
held continuously; an empty group releases everything.
Reasoning models may prefix the chunk with ``...``. The action is
always read from the text *after* the last ````: thinking models quote
the format back at themselves ("the format is <|action_start|>X Y Z ; ..."), and
reading the whole reply would execute the quote.
"""
from __future__ import annotations
import re
from dataclasses import dataclass, field
ACTION_START = "<|action_start|>"
ACTION_END = "<|action_end|>"
THINK_OPEN = ""
THINK_CLOSE = ""
CHUNK_MS = 200.0
CHUNK_STEPS = 6
STEP_MS = CHUNK_MS / CHUNK_STEPS
MAX_KEYS_PER_STEP = 4
MOUSE_LIMIT = 1000.0
SCROLL_LIMIT = 5.0
NOOP_ACTION_TEXT = f"{ACTION_START}0 0 0 ; ; ; ; ; ; {ACTION_END}"
# The 74-token vocabulary the model was trained on. Case matters:
# `one`..`nine` are number-row keys, `One`..`Twelve` are F1..F12.
LETTER_KEYS = tuple("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
DIGIT_KEYS = ("zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine")
FUNCTION_KEYS = ("One", "Two", "Three", "Four", "Five", "Six", "Seven", "Eight",
"Nine", "Ten", "Eleven", "Twelve")
MOUSE_BUTTON_KEYS = ("LB", "RB", "MB")
ARROW_KEYS = ("Up", "Down", "Left", "Right")
SPECIAL_KEYS = ("Shift", "Ctrl", "Alt", "Tab", "Caps", "Esc", "Space", "Enter", "Back",
"Delete", "Insert", "Home", "End", "Pause")
PUNCTUATION_KEYS = ("Equal", "Minus", "Period", "Slash", "Quote")
KEY_VOCAB: frozenset[str] = frozenset(
LETTER_KEYS + DIGIT_KEYS + FUNCTION_KEYS + MOUSE_BUTTON_KEYS + ARROW_KEYS
+ SPECIAL_KEYS + PUNCTUATION_KEYS
)
# `}` instead of `>` is a closing-bracket typo some models make.
_TAG_RE = re.compile(r"<\|action_start\|[>}](.*?)<\|action_end\|[>}]", re.DOTALL)
_BARE_RE = re.compile(r"^\s*-?\d")
# Lenient fallbacks (see `parse_action(lenient=True)`).
_WRAPPED_INT_RE = re.compile(
r"<\|action(?:_start)?\|>\s*(-?\d+)\s+(-?\d+)\s+(-?\d+)(.*?)<\|action(?:_end)?\|>",
re.DOTALL,
)
_SKELETON_RE = re.compile(r"[^\n;<]*(?:;[^\n;<]*){5,}")
_BARE_INT_RE = re.compile(r"(-?\d+)\s+(-?\d+)\s+(-?\d+)\s*(;[^\n<]*)", re.DOTALL)
_INT_RE = re.compile(r"^-?\d+$")
def split_reasoning(text: str) -> tuple[str, str, bool]:
"""``text`` -> (reasoning, text after the reasoning, reasoning was closed).
An unclosed ```` means generation ended before any action existed
(the token budget ran out, or a stop string matched inside the reasoning),
so the tail is empty.
"""
text = text or ""
if THINK_CLOSE in text:
head, _, tail = text.rpartition(THINK_CLOSE)
return head.split(THINK_OPEN)[-1].strip(), tail, True
if THINK_OPEN in text:
return text.split(THINK_OPEN)[-1].strip(), "", False
return "", text, True
@dataclass
class Action:
"""One parsed 200 ms chunk. Unparseable text is a no-op with ``parsed=False``."""
mouse: tuple[float, float, float] = (0.0, 0.0, 0.0)
steps: list[list[str]] = field(default_factory=lambda: [[] for _ in range(CHUNK_STEPS)])
parsed: bool = False
truncated: bool = False
body: str = ""
dropped: list[str] = field(default_factory=list)
@property
def history_text(self) -> str:
"""What this turn looks like as an assistant message in the next prompt.
Always a well-formed chunk: an unparseable turn is stored as an explicit
no-op, because putting the model's own failure into context teaches it
to repeat the failure.
"""
if not self.parsed:
return NOOP_ACTION_TEXT
return f"{ACTION_START}{self.body}{ACTION_END}"
@property
def is_noop(self) -> bool:
return not any(self.steps) and self.mouse == (0.0, 0.0, 0.0)
@property
def pressed(self) -> set[str]:
return {k for group in self.steps for k in group}
def to_text(self) -> str:
x, y, z = (int(round(v)) for v in self.mouse)
groups = " ; ".join(" ".join(g) for g in self.steps)
return f"{ACTION_START}{x} {y} {z} ; {groups}{ACTION_END}"
def _build(nums: list[float], groups: list[str], body: str) -> Action:
x = max(-MOUSE_LIMIT, min(MOUSE_LIMIT, nums[0]))
y = max(-MOUSE_LIMIT, min(MOUSE_LIMIT, nums[1]))
z = max(-SCROLL_LIMIT, min(SCROLL_LIMIT, nums[2]))
steps: list[list[str]] = []
dropped: list[str] = []
for group in groups[:CHUNK_STEPS]:
keys = []
for tok in group.split():
(keys if tok in KEY_VOCAB else dropped).append(tok)
steps.append(keys[:MAX_KEYS_PER_STEP])
while len(steps) < CHUNK_STEPS:
steps.append([])
return Action(mouse=(x, y, z), steps=steps, parsed=True, body=body, dropped=dropped)
def _parse_strict(tail: str) -> Action:
m = _TAG_RE.search(tail)
if m:
body = m.group(1)
elif _BARE_RE.match(tail) and ";" in tail:
# Tags are special tokens; a server that strips them returns a bare body.
body = tail.strip()
else:
return Action()
parts = [seg.strip() for seg in body.split(";")]
if not parts or not parts[0]:
return Action()
nums: list[float] = []
for tok in parts[0].split()[:3]:
try:
nums.append(float(tok))
except ValueError:
return Action()
if len(nums) < 3:
return Action()
return _build(nums, parts[1:], body)
def _parse_lenient(text: str) -> Action:
"""Recovery for malformed replies (see `parse_action`)."""
text = text.replace(THINK_OPEN, " ").replace(THINK_CLOSE, " ")
m = _WRAPPED_INT_RE.search(text)
if m:
nums = [float(m.group(i)) for i in (1, 2, 3)]
key_part = m.group(4).strip()
body = text[m.start():m.end()]
body = body[body.find(">") + 1:body.rfind("<")]
else:
skel = None
for sm in _SKELETON_RE.finditer(text):
skel = sm.group(0)
if skel is None:
bm = _BARE_INT_RE.search(text)
if not bm:
return Action()
nums = [float(bm.group(i)) for i in (1, 2, 3)]
key_part = bm.group(4).strip()
body = bm.group(0)
else:
head = skel.partition(";")[0]
head_tokens = head.split()
if head_tokens and len(head_tokens) <= 3 and all(_INT_RE.match(t) for t in head_tokens):
nums = [float(t) for t in head_tokens] + [0.0, 0.0, 0.0]
key_part = skel[len(head):]
else:
nums = [0.0, 0.0, 0.0]
key_part = skel
body = skel
segments = key_part.split(";") if key_part else []
if segments and segments[0].strip() == "":
segments = segments[1:]
return _build(nums[:3], segments, body.strip())
def parse_action(text: str, *, lenient: bool = False) -> Action:
"""Raw model reply -> :class:`Action`. Never raises; failures are no-ops.
``lenient=False`` (default): only a chunk after the
last ```` counts, either tagged or as a bare ``X Y Z ; ...`` body.
``lenient=True``: additionally recover shapes a
well-trained checkpoint rarely emits -- an action inside an unclosed
````, a 2-number mouse prefix, a keys-only skeleton -- by locating the
``;``-separated six-group skeleton anywhere in the reply.
"""
_, tail, closed = split_reasoning(text)
if closed:
action = _parse_strict(tail)
if action.parsed or not lenient:
return action
elif not lenient:
return Action(truncated=True)
action = _parse_lenient(text or "")
if not closed and not action.parsed:
action.truncated = True
return action