| """Offline contracts, not a verified adapter for the raw AgentNet schema. |
| |
| Callers must establish screenshot timing and source coordinate semantics first. |
| No source code, screenshot, or desktop action is executed here. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import ast |
| import copy |
| import hashlib |
| import json |
| import math |
| import random |
| from collections.abc import Mapping |
|
|
| from .actions import SPECIAL_KEYS, ActionCall |
|
|
| |
| SOURCE_KEYS = tuple( |
| sorted( |
| SPECIAL_KEYS |
| | set( |
| "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" |
| " !\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" |
| ) |
| | { |
| "cmd", |
| "command", |
| "win", |
| "winleft", |
| "winright", |
| "esc", |
| "caps_lock", |
| "capslock", |
| "num_lock", |
| "numlock", |
| "scroll_lock", |
| "scrolllock", |
| "print_screen", |
| "printscreen", |
| "menu", |
| "pause", |
| "media_volume_up", |
| "media_volume_down", |
| "media_volume_mute", |
| "ctrl_l", |
| "ctrl_r", |
| "alt_l", |
| "alt_r", |
| "shift_l", |
| "shift_r", |
| "cmd_l", |
| "cmd_r", |
| "altleft", |
| "altright", |
| "ctrlleft", |
| "ctrlright", |
| "shiftleft", |
| "shiftright", |
| "ż", |
| "ś", |
| "ł", |
| *(f"f{number}" for number in range(1, 25)), |
| } |
| ) |
| ) |
|
|
| ACTIONS = ( |
| "click", |
| "double_click", |
| "scroll", |
| "press_key", |
| "hotkey", |
| "type_text", |
| "drag", |
| "wait", |
| "done", |
| "end_step", |
| "move_to", |
| "drag_to", |
| "wheel", |
| "horizontal_wheel", |
| "triple_click", |
| "key_down", |
| "key_up", |
| ) |
|
|
|
|
| def validate_action(payload): |
| if payload == {"done": True} and type(payload.get("done")) is bool: |
| return {"done": True} |
| if not isinstance(payload, dict) or set(payload) != {"skill", "args"}: |
| raise ValueError("Expected one exact action object") |
| skill, args = payload["skill"], payload["args"] |
| if skill not in ACTIONS or skill == "done": |
| raise ValueError("Unsupported action") |
| if skill == "end_step": |
| if args != {}: |
| raise ValueError("end_step takes no arguments") |
| return {"skill": skill, "args": {}} |
| if skill in {"move_to", "drag_to", "triple_click"}: |
| required = {"x", "y"} | ({"button"} if skill != "move_to" else set()) |
| if not isinstance(args, dict) or set(args) != required: |
| raise ValueError("Unexpected pointer arguments") |
| point = ActionCall("click", args).args |
| if skill == "move_to": |
| point.pop("button") |
| return {"skill": skill, "args": point} |
| if skill in {"wheel", "horizontal_wheel"}: |
| if not isinstance(args, dict) or set(args) != {"wheel_units"}: |
| raise ValueError("Wheel requires an amount") |
| units = args["wheel_units"] |
| if type(units) is not int or units == 0: |
| raise ValueError("Invalid wheel units") |
| return {"skill": skill, "args": {"wheel_units": units}} |
| if skill == "type_text": |
| if not isinstance(args, dict) or set(args) != {"text"} or not isinstance(args["text"], str): |
| raise ValueError("Text action requires a string") |
| |
| |
| return {"skill": skill, "args": dict(args)} |
| if skill in {"press_key", "hotkey", "key_down", "key_up"}: |
| expected = "keys" if skill == "hotkey" else "key" |
| if not isinstance(args, dict) or set(args) != {expected}: |
| raise ValueError("Unexpected keyboard arguments") |
| keys = args["keys"] if skill == "hotkey" else [args["key"]] |
| if not isinstance(keys, list) or not keys or any(key not in SOURCE_KEYS for key in keys): |
| raise ValueError("Unsupported source keyboard key") |
| if skill == "hotkey" and (not 2 <= len(keys) <= 4 or len(set(keys)) != len(keys)): |
| raise ValueError("Unsupported hotkey arity or duplicate keys") |
| return {"skill": skill, "args": copy.deepcopy(args)} |
| if skill == "scroll": |
| if not isinstance(args, dict) or set(args) != {"x", "y", "wheel_units"}: |
| raise ValueError("Scroll requires location and explicit wheel units") |
| position = ActionCall("click", {k: args[k] for k in ("x", "y")}).args |
| units = args["wheel_units"] |
| if type(units) is not int or units == 0 or not -100 <= units <= 100: |
| raise ValueError("Invalid wheel units") |
| return { |
| "skill": skill, |
| "args": {"x": position["x"], "y": position["y"], "wheel_units": units}, |
| } |
| return ActionCall.from_payload(payload).to_payload() |
|
|
|
|
| def validate_step(payload): |
| """A source step is one or more ordered actions under ONE observation.""" |
| if isinstance(payload, dict) and set(payload) == {"actions"}: |
| actions = payload["actions"] |
| if not isinstance(actions, list) or not actions: |
| raise ValueError("A step requires a nonempty action sequence") |
| clean = [validate_action(action) for action in actions] |
| if any(action.get("skill") == "end_step" for action in clean): |
| raise ValueError("end_step is a decoder boundary, not a source action") |
| if any(action.get("done") for action in clean[:-1]): |
| raise ValueError("No action may follow termination") |
| return {"actions": clean} |
| return validate_action(payload) |
|
|
|
|
| def step_actions(payload): |
| value = validate_step(payload) |
| return value.get("actions", [value]) |
|
|
|
|
| def validate_history_action(payload): |
| if isinstance(payload, dict) and set(payload) == {"source_code"}: |
| if not isinstance(payload["source_code"], str) or not payload["source_code"]: |
| raise ValueError("Missing recorded history code") |
| return dict(payload) |
| return validate_step(payload) |
|
|
|
|
| def encode_coordinate(value, grid=32): |
| if type(value) not in (int, float) or not math.isfinite(value) or not 0 <= value <= 1: |
| raise ValueError("Expected a finite normalized coordinate") |
| if type(grid) is not int or grid < 1: |
| raise ValueError("Invalid grid") |
| cell = min(grid - 1, math.floor(grid * value)) |
| return cell, grid * value - cell |
|
|
|
|
| def pixel_coordinate(value, dimension): |
| encode_coordinate(value) |
| if type(dimension) is not int or dimension < 1: |
| raise ValueError("Invalid image dimension") |
| return min(dimension - 1, math.floor(value * dimension)) |
|
|
|
|
| def parse_literal_action(code): |
| """Parse our compact interchange language, NOT arbitrary pyautogui/AgentNet code. |
| |
| Keyword-only calls make units and fields explicit. Raw source adaptation must |
| separately verify aliases, omitted defaults, timing and coordinate conventions. |
| """ |
| if not isinstance(code, str) or len(code) > 10000: |
| raise ValueError("Invalid action expression") |
| try: |
| node = ast.parse(code, mode="eval").body |
| if ( |
| not isinstance(node, ast.Call) |
| or not isinstance(node.func, ast.Name) |
| or node.func.id not in ACTIONS |
| or node.args |
| ): |
| raise ValueError("Only a single keyword-only whitelisted call is allowed") |
| args = {} |
| for keyword in node.keywords: |
| if keyword.arg is None or keyword.arg in args: |
| raise ValueError("Expanded or duplicate arguments are forbidden") |
| args[keyword.arg] = ast.literal_eval(keyword.value) |
| if node.func.id == "done": |
| if args: |
| raise ValueError("done takes no arguments") |
| return {"done": True} |
| return validate_action({"skill": node.func.id, "args": args}) |
| except (SyntaxError, TypeError, RecursionError) as exc: |
| raise ValueError("Invalid literal action") from exc |
|
|
|
|
| def serialize_action(payload, arm): |
| action = validate_action(payload) |
| if arm == "G-json": |
| return json.dumps(action, ensure_ascii=False, separators=(",", ":"), allow_nan=False) |
| if arm == "G-compact": |
| if action.get("done"): |
| return "done()" |
| return ( |
| action["skill"] + "(" + ",".join(f"{k}={v!r}" for k, v in action["args"].items()) + ")" |
| ) |
| raise ValueError("Unknown text baseline") |
|
|
|
|
| def build_record( |
| *, |
| record_id, |
| trajectory_id, |
| duplicate_group, |
| step_index, |
| task, |
| screenshot, |
| history, |
| target, |
| audit_only, |
| synthetic=False, |
| ): |
| """Build from explicit reviewed fields, never merge untrusted source dictionaries.""" |
| for value in (record_id, trajectory_id, duplicate_group, task): |
| if not isinstance(value, str) or not value.strip(): |
| raise ValueError("Identity, duplicate grouping and original task are required") |
| if type(step_index) is not int or step_index < 0 or type(synthetic) is not bool: |
| raise ValueError("Invalid step/synthetic marker") |
| fields = {"asset_id", "sha256", "width", "height", "timing"} |
| if not isinstance(screenshot, dict) or set(screenshot) != fields: |
| raise ValueError("Unexpected screenshot fields") |
| if screenshot["timing"] != "before_target_action": |
| raise ValueError("Unverified screenshot timing") |
| if not isinstance(screenshot["asset_id"], str) or not screenshot["asset_id"]: |
| raise ValueError("Missing image identity") |
| for key in ("width", "height"): |
| pixel_coordinate(0, screenshot[key]) |
| digest = screenshot["sha256"] |
| if not synthetic and ( |
| not isinstance(digest, str) |
| or len(digest) != 64 |
| or any(c not in "0123456789abcdef" for c in digest) |
| ): |
| raise ValueError("Real images require verified SHA256") |
| clean_history = [] |
| previous = -1 |
| for item in history: |
| if set(item) != {"step_index", "action", "executed"}: |
| raise ValueError("Unexpected history fields") |
| index = item["step_index"] |
| if ( |
| type(index) is not int |
| or not previous < index < step_index |
| or item["executed"] is not True |
| ): |
| raise ValueError("History must contain ordered previously executed actions") |
| action = validate_history_action(item["action"]) |
| if "source_code" not in action and any(a.get("done") for a in step_actions(action)): |
| raise ValueError("History cannot continue after done") |
| clean_history.append({"step_index": index, "action": action, "executed": True}) |
| previous = index |
| target = validate_step(target) |
| if any(a.get("done") for a in step_actions(target)) and not ( |
| audit_only.get("terminal_state_observed") is True |
| or audit_only.get("terminal_label_basis") |
| == "source_demonstration_stop_not_verified_success" |
| ): |
| raise ValueError("done supervision requires an observed terminal state") |
| return { |
| "record_id": record_id, |
| "trajectory_id": trajectory_id, |
| "duplicate_group": duplicate_group, |
| "step_index": step_index, |
| "synthetic": synthetic, |
| "input": { |
| "task": task, |
| "screenshot": copy.deepcopy(screenshot), |
| "history": clean_history[-4:], |
| }, |
| "target": target, |
| "audit_only": copy.deepcopy(audit_only), |
| } |
|
|
|
|
| def policy_input(record): |
| """Only this allowlisted object may be passed to a future backbone processor.""" |
| value = record["input"] |
| if set(value) != {"task", "screenshot", "history"}: |
| raise ValueError("Unexpected policy input fields") |
| return copy.deepcopy(value) |
|
|
|
|
| def grouped_splits(records, salt="visual-nanojev-v1"): |
| """Connected trajectory/template/exact-image groups; no near-duplicate detector. |
| |
| duplicate_group must be supplied by a separate reviewed grouping procedure. |
| Four deterministic partitions are development/train/calibration/internal_test. |
| """ |
| parent = list(range(len(records))) |
|
|
| def root(i): |
| while parent[i] != i: |
| parent[i] = parent[parent[i]] |
| i = parent[i] |
| return i |
|
|
| seen = {} |
| ids = set() |
| for i, record in enumerate(records): |
| if record["record_id"] in ids: |
| raise ValueError("Duplicate record identity") |
| ids.add(record["record_id"]) |
| keys = [("trajectory", record["trajectory_id"]), ("template", record["duplicate_group"])] |
| digest = record["input"]["screenshot"]["sha256"] |
| if digest: |
| keys.append(("image", digest)) |
| for key in keys: |
| if key in seen: |
| parent[root(i)] = root(seen[key]) |
| seen[key] = i |
| groups = {} |
| for i, record in enumerate(records): |
| groups.setdefault(root(i), []).append(record["record_id"]) |
| assignments = {} |
| for members in groups.values(): |
| identity = json.dumps([salt, sorted(members)], separators=(",", ":")) |
| bucket = int(hashlib.sha256(identity.encode()).hexdigest(), 16) % 100 |
| split = ( |
| "train" |
| if bucket < 80 |
| else "development" |
| if bucket < 90 |
| else "calibration" |
| if bucket < 95 |
| else "internal_test" |
| ) |
| assignments.update(dict.fromkeys(members, split)) |
| return assignments |
|
|
|
|
| def trajectory_splits(records, seed=42, validation_fraction=0.05): |
| """Assign complete trajectories to deterministic train/validation splits. |
| |
| Trajectory identities are sorted before fixed-seed sampling, so assignments do |
| not depend on record input order. The validation trajectory count is |
| ``ceil(n * validation_fraction)`` and is clipped to leave at least one |
| trajectory in each split when ``n >= 2``. Zero or one trajectory is assigned |
| entirely to train because a disjoint validation split is then impossible. |
| |
| This intentionally performs no template, image, or similarity grouping. |
| """ |
| if type(seed) is not int: |
| raise ValueError("Seed must be an integer") |
| if ( |
| type(validation_fraction) not in (int, float) |
| or not math.isfinite(validation_fraction) |
| or not 0 < validation_fraction < 1 |
| ): |
| raise ValueError("Validation fraction must be finite and strictly between zero and one") |
|
|
| record_trajectories = {} |
| trajectory_ids = set() |
| for record in records: |
| if not isinstance(record, Mapping): |
| raise TypeError("Each record must be a mapping") |
| record_id = record.get("record_id") |
| trajectory_id = record.get("trajectory_id") |
| if not isinstance(record_id, str) or not record_id.strip(): |
| raise ValueError("Each record requires a non-empty string record_id") |
| if not isinstance(trajectory_id, str) or not trajectory_id.strip(): |
| raise ValueError("Each record requires a non-empty string trajectory_id") |
| if record_id in record_trajectories: |
| if record_trajectories[record_id] != trajectory_id: |
| raise ValueError("Conflicting duplicate record identity") |
| raise ValueError("Duplicate record identity") |
| record_trajectories[record_id] = trajectory_id |
| trajectory_ids.add(trajectory_id) |
|
|
| ordered_trajectories = sorted(trajectory_ids) |
| validation_count = 0 |
| if len(ordered_trajectories) >= 2: |
| validation_count = min( |
| len(ordered_trajectories) - 1, |
| max(1, math.ceil(len(ordered_trajectories) * validation_fraction)), |
| ) |
| validation = set(random.Random(seed).sample(ordered_trajectories, validation_count)) |
| return { |
| record_id: "validation" if trajectory_id in validation else "train" |
| for record_id, trajectory_id in record_trajectories.items() |
| } |
|
|
