"""Expected backend checks for generated tasks. A check is built from the control's probed backend signature (never by an LLM), so it is exact: the request the site receives when the option is applied. {"kind": "request", "method": "GET", "path": "/sites/e-commerce/", "params": {"category": "Sports & Outdoors"}, # what the step must set "keep": {"sort": "price_asc"}, # state that must survive (mid-chain / preset) "final": true} # the LAST request to that path still has it `evaluate` reads a session request log (/_admin/log entries). It passes when an accepted request carries params+keep AND (final) the last accepted request to the same endpoint still does — so "filter applied, then cleared" fails. The same check is also emitted as an `src/miniweb/evaluation/verifiers.py` spec (request_made), so generated tasks can be graded by the evaluation machinery and `verify_task` is the reused gate. A `state` check is for steps that change data (toggles, reactions, deletes, forms). It is recorded by the privileged dry run — the session's effective data changes after the step (/_admin/changes: rows inserted / updated / deleted vs the base tables) — and passes when an attempt's changes contain each expected change AND nothing else changed in those collections (so a like-then-unlike, a different item, or extra deletes fail): {"kind": "state", "site": "forums", "expect": [{"collection": "posts", "op": "update", "item_id": "127582", "fields": {"score": "41322"}}], "scope": ["posts"], # collections where any other change fails the step "request": {"method": "POST", "path": "/sites/forums/api/posts/127582/vote"}} # for verify_task An `answer` check is for steps that end by replying to the human (report_information, count_entries, search_by_playback, …). The expected answer is computed by the privileged sampler, never by an LLM; the reply is graded by the evaluation harness's answer chain, and every `evidence` string (the facts the answer rests on) must have been readable in the viewport at one of the recorded screenshots — so the trajectory shows where the answer came from. `requires` optionally adds a request/state check that must also hold (a 2FA reveal): {"kind": "answer", "value": "12", "alternatives": ["twelve"], "evidence": ["Oak Street", "$1,200"], "requires": null} """ from __future__ import annotations import re from urllib.parse import urlencode def _norm(v): return re.sub(r"\s+", " ", str(v if v is not None else "")).strip().casefold() def _path(p): return (p or "").rstrip("/") or "/" def build_check(signature, value, keep=None, final=True): return {"kind": "request", "method": (signature.get("method") or "GET").upper(), "path": signature["path"], "params": {signature["param"]: value}, "keep": dict(keep or {}), "final": bool(final)} def _fields(entry): f = {} f.update(entry.get("query") or {}) if isinstance(entry.get("body"), dict): f.update({k: v for k, v in entry["body"].items() if not isinstance(v, dict)}) return f def _same(got, want): """A logged field matches the wanted value; a repeated field (list) matches a wanted value it contains, or a wanted list with the same members; {"value": [lo, hi], "mode": "between"} is a numeric range (a drop position anywhere inside a region).""" if isinstance(want, dict) and want.get("mode") == "between": try: return float(want["value"][0]) <= float(got) <= float(want["value"][1]) except (TypeError, ValueError, IndexError): return False if isinstance(want, (list, tuple)): return isinstance(got, list) and sorted(map(_norm, got)) == sorted(map(_norm, want)) if isinstance(got, list): return _norm(want) in map(_norm, got) return _norm(got) == _norm(want) def _carries(entry, want): f = _fields(entry) return all(k in f and _same(f[k], v) for k, v in want.items()) def _answered(resp, want): """A JSON response holds `want` (strings compared per line without trailing blanks).""" if not isinstance(resp, dict): return False norm = lambda v: "\n".join(l.rstrip() for l in str(v).rstrip().split("\n")) if isinstance(v, str) else v return all(k in resp and norm(resp[k]) == norm(v) for k, v in want.items()) _LOGIN_PATH = re.compile(r"/(login|signin|sign-in|sign_in|auth)\b", re.I) def _bounced_to_login(e, entries): """A request answered with a redirect straight to a login page was refused (a follow while signed out).""" if str(e.get("status", ""))[:1] != "3" or _LOGIN_PATH.search(e.get("path") or ""): return False try: i = entries.index(e) except ValueError: return False nxt = next((x for x in entries[i + 1:i + 3] if (x.get("method") or "GET").upper() == "GET"), None) return bool(nxt and _LOGIN_PATH.search(nxt.get("path") or "")) def _status_class_ok(e, cls): """A request check may pin the status class its dry run succeeded with ("3xx": a form that redirects on success re-renders with 200 on failure, e.g. "Invalid password", "Username already taken").""" return not cls or str(e.get("status", ""))[:1] == str(cls)[:1] def _accepted(e): return isinstance(e.get("status"), int) and 200 <= e["status"] < 400 # fields that change as a side effect of any write: ignored when comparing states INCIDENTAL = re.compile(r"(^|_)(date|time|timestamp|created|updated|modified|last_seen|last_active|last_login|" r"seen|viewed|views|view_count|visits|updated_at|created_at|modified_at|ts|at|" r"read|is_read|unread|unread_count|read_at)$", re.I) # opening a chat marks it read def _canon(v): from miniweb.app.db import _canon as canon return canon(v) ANY = "*" # an expected field value: present and non-empty (drawn images, generated media) def _essential(change): """The fields that identify a change: for an insert its content (not its generated id or timestamps), for an update the changed fields (not bookkeeping ones). Drawn images and other data: URIs are execution-specific pixels: only their presence counts.""" if change["op"] == "delete": return {} fields = {k: v[1] for k, v in change["changed"].items() if not INCIDENTAL.search(k)} if change["op"] == "insert": fields.pop("id", None) return {k: (ANY if isinstance(v, str) and v.startswith("data:image/") else _canon(_images(v))) for k, v in fields.items()} def _images(v): """Nested drawn images (a signature inside a stored form) compare as present, not as pixels.""" if isinstance(v, str) and v.startswith("data:image/"): return ANY if isinstance(v, list): return [_images(x) for x in v] if isinstance(v, dict): # nested records: their timestamps are incidental too return {k: _images(x) for k, x in v.items() if not INCIDENTAL.search(str(k))} return v def _field_ok(got, want): want = _images(want) # checks recorded before images became ANY return bool(got) and got not in ("", "0") if want == ANY else got == want _DATE_IN_VALUE = re.compile(r"(?