Spaces:
Running
Running
Download src/miniwebgen/kinds/forms.py from sparklabuofu/MiniWeb-demo: direct link, hf CLI and curl.
- Browser
- Download file 19 kB
-
https://huggingface.co/spaces/sparklabuofu/MiniWeb-demo/resolve/main/src/miniwebgen/kinds/forms.py
- Command line
-
hf download hf://spaces/sparklabuofu/MiniWeb-demo/src/miniwebgen/kinds/forms.py
-
curl -L -o forms.py https://huggingface.co/spaces/sparklabuofu/MiniWeb-demo/resolve/main/src/miniwebgen/kinds/forms.py
19 kB
| """Forms: fill fields and submit (create, edit, configure, share, pay, book, checkout, cancel, | |
| sign in, message, review, typed signature, translate, directions, join by code β¦). | |
| A control is one form (a <form>, or a scripted panel's fields sharing an Apply/Submit button); | |
| its role β which macro it serves β comes from the submit label, the action path and the page | |
| heading. Arguments are value sets (src/miniwebgen/formvalues.py): an LLM proposes them once per form, | |
| sign-in forms use the site's real accounts. The check is recorded by the dry run: the data the | |
| submission changes (checks.state_check), else β sign-in, a translation β the request it sent | |
| with the entered values. | |
| """ | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import re | |
| from miniwebgen import browser as B | |
| from miniwebgen import formvalues as FV | |
| from miniwebgen.kinds import register | |
| from miniwebgen.kinds.base import Kind | |
| # (macro, pattern over "submit text | action path | heading | page title"), first match wins | |
| ROLES = [ | |
| ("authenticate_by_form", r"\b(log ?in|sign ?in)\b"), | |
| ("checkout_by_form", r"\b(checkout|check out|place (your )?order|complete (purchase|order)|buy now|promo|coupon|" | |
| r"redeem|discount code)\b"), | |
| ("pay_by_form", r"\b(pay|payment|transfer|send money|pay bill)\b"), | |
| ("share_by_form", r"\b(share|invite)\b"), | |
| ("book_by_form", r"\b(book|reserve|reservation|appointment)\b"), | |
| ("cancel_by_form", r"\bcancel\b"), | |
| ("translate_by_query", r"\btranslat"), | |
| ("get_nav_route", r"\b(directions|route|navigate)\b"), | |
| ("join_meeting", r"\bjoin\b"), | |
| ("sign_by_text", r"\b(sign|signature|adopt)\b"), | |
| ("feedback_by_text", r"\b(review|feedback)\b"), | |
| ("message_from_free_text", r"\b(send|reply|message|comment|chat)\b"), | |
| ("configure_by_form", r"\b(settings|preferences|configure|notification)\b"), | |
| ("edit_by_form", r"\b(update|save changes|edit)\b"), | |
| ("create_by_form", r"\b(create|add|new|post|publish|submit|save|register|apply|compose|schedule|upload)\b"), | |
| ] | |
| FORM_MACROS = [m for m, _ in ROLES] | |
| TEXTY = {"text", "email", "number", "tel", "url", "password", "search", "textarea", "time", "month", "week"} | |
| SKIP_FORM = re.compile(r"(log ?out|sign ?out|delete (my )?account|search)", re.I) | |
| MAX_FIELDS = 12 | |
| def role_of(blob, site_macros): | |
| """The form's macro. `blob` is "submit | action | heading | title": the most specific part that | |
| names a role decides (a Translation site's export form is an export, not a translation).""" | |
| for part in str(blob).split(" | "): | |
| hits = [m for m, pat in ROLES if re.search(pat, part, re.I)] | |
| if hits: | |
| listed = [m for m in hits if m in site_macros] | |
| return (listed or hits)[0] | |
| return None | |
| def _fields(inputs, selects): | |
| """Form fields from the scan: radio groups become one field with options.""" | |
| out, radios = [], {} | |
| for i in inputs: | |
| t = i["type"] | |
| if t in ("hidden", "submit", "button", "image", "reset", "file"): | |
| continue | |
| if t == "radio": | |
| radios.setdefault(i["name"] or i["css"], []).append(i) | |
| continue | |
| f = {"css": i["css"], "type": "date" if t == "date" else ("checkbox" if t == "checkbox" else ("range" if t == "range" else | |
| ("textarea" if t == "textarea" else ("text" if t not in TEXTY else t)))), | |
| "name": i["name"], "label": i["label"] or i["option_text"] or i["placeholder"], "placeholder": i["placeholder"], | |
| "value": i["value"], "checked": i["checked"], "min": i["min"], "max": i["max"], "label_css": i["label_css"], | |
| "visible": i["visible"]} | |
| out.append(f) | |
| for name, group in radios.items(): | |
| out.append({"css": group[0]["css"], "type": "radio", "name": group[0]["name"], | |
| "label": group[0]["group_label"] or name, "value": next((g["value"] for g in group if g["checked"]), ""), | |
| "options": [{"index": n, "value": g["value"], "text": g["option_text"] or g["value"], "css": g["css"], | |
| "label_css": g["label_css"], "visible": g["visible"]} for n, g in enumerate(group)]}) | |
| for s in selects: | |
| out.append({"css": s["css"], "type": "select", "name": s["name"], "label": s["label"] or s["placeholder"], | |
| "value": next((o["text"] for o in s["options"] if o["index"] == s["selected_index"]), ""), | |
| "options": [{"index": o["index"], "value": o["value"], "text": o["text"]} for o in s["options"] | |
| if not o.get("disabled")]}) | |
| return out | |
| class FormKind(Kind): | |
| name = "form" | |
| macros = {m: m for m in FORM_MACROS} | |
| oracle = True | |
| example = '''# example: fill each field as given, then submit the form | |
| for f in task["fields"]: | |
| el = dom.one(css=f["css"]) if f["visible"] else dom.one(css=f["label_css"]) | |
| if not el.in_viewport: | |
| el = act.scroll_into_view(el) | |
| if f["type"] == "select": | |
| act.click(el) # opens the native option list | |
| act.type(f["option_text"]) # type-ahead to the option | |
| act.press("Enter") | |
| elif f["type"] in ("radio", "checkbox"): | |
| act.click(el) | |
| elif f["type"] == "date": | |
| act.click(el, at=(0.12, 0.5)) # month segment of a native date input | |
| act.type(f["keys"]) # MMDDYYYY | |
| else: | |
| act.click(el) | |
| if el.value: | |
| act.press("Control+a") # replace what is there | |
| act.type(f["value"]) | |
| btn = dom.one(css=task["submit_css"]) | |
| if not btn.in_viewport: | |
| btn = act.scroll_into_view(btn) | |
| act.click(btn) | |
| expect.backend()''' | |
| def discover(self, scan, page, site): | |
| from miniweb.annotation.macro_locations import MACRO_LOCATIONS | |
| site_macros = MACRO_LOCATIONS.get(site) or {} | |
| groups = {} | |
| for i in scan.get("inputs") or []: | |
| host = (i.get("form") or {}).get("css") or ((i.get("apply_hint") or {}).get("css") and "hint:" + i["apply_hint"]["css"]) | |
| if host: | |
| groups.setdefault(host, {"inputs": [], "selects": [], "form": i.get("form"), "hint": i.get("apply_hint")})["inputs"].append(i) | |
| for s in scan.get("selects") or []: | |
| host = (s.get("form") or {}).get("css") or ((s.get("apply_hint") or {}).get("css") and "hint:" + s["apply_hint"]["css"]) | |
| if host: | |
| groups.setdefault(host, {"inputs": [], "selects": [], "form": s.get("form"), "hint": s.get("apply_hint")})["selects"].append(s) | |
| out = [] | |
| for host, g in groups.items(): | |
| form = g["form"] | |
| submit = (form or {}).get("submit") or g["hint"] | |
| if not submit or any(i["type"] == "file" for i in g["inputs"]): | |
| continue # no submit, or an upload (upload_file's own kind) | |
| fields = _fields(g["inputs"], g["selects"]) | |
| if not fields or len(fields) > MAX_FIELDS: | |
| continue | |
| method = (form or {}).get("method", "post") | |
| blob = " | ".join([submit.get("text") or "", (form or {}).get("action", ""), scan.get("h1") or "", scan["title"]]) | |
| if SKIP_FORM.search(submit.get("text") or ""): | |
| continue | |
| # a GET form's role must come from its own submit label or action, never the page title | |
| role = role_of(blob if method != "get" else " | ".join(blob.split(" | ")[:2]), site_macros) | |
| if role is None: | |
| continue | |
| if method == "get" and role not in ("translate_by_query", "get_nav_route", "join_meeting", "authenticate_by_form"): | |
| continue # GET forms narrow listings: the filter kinds' job | |
| if role == "authenticate_by_form" and not any(f["type"] == "password" for f in fields): | |
| continue | |
| ctrl = {"kind": "form", "role": role, "macro": role, "css": fields[0]["css"], "name": host, "id": "", | |
| "label": scan.get("h1") or scan["title"], "heading": scan.get("h1") or "", "fields": fields, | |
| "submit": {"css": submit["css"], "text": submit.get("text") or ""}, "form": form, | |
| "context": (scan.get("items") or [])[:25], "site_id": site, "page": page, "page_title": scan["title"]} | |
| out.append((("form", role, host, _sm()._pattern(page)), ctrl)) | |
| return out | |
| # ββ arguments: value sets ββββββββββββββββββββββββββββββββββββββββββββββ | |
| def arguments(self, ctrl): | |
| site = ctrl.get("site_id") or ctrl["page"].split("/")[2] | |
| if ctrl["role"] == "authenticate_by_form": | |
| return [self._login_arg(ctrl, a, n) for n, a in enumerate(FV.accounts(site)[:8])] | |
| sets = ctrl.get("value_sets") | |
| want = ctrl.get("_k") or FV.K | |
| if sets is None or (len(sets) < want and ctrl.get("_topped_up") != want): | |
| ctrl["_topped_up"] = None # (a saved map may carry an older size) | |
| sets = FV.value_sets(ctrl, site, k=want) # cached; tops up older, smaller caches | |
| ctrl["value_sets"], ctrl["_topped_up"] = sets, want | |
| out = [] | |
| for n, s in enumerate(sets): | |
| vals = _changed(ctrl, s["values"]) | |
| if vals: | |
| out.append(self._arg(ctrl, vals, s.get("summary", ""), n)) | |
| return out | |
| def has_arguments(self, ctrl): | |
| if ctrl["role"] == "authenticate_by_form": | |
| return bool(FV.accounts(ctrl.get("site_id") or ctrl["page"].split("/")[2])) | |
| return ctrl.get("value_sets") != [] # unknown until generated: assume yes | |
| def _login_arg(self, ctrl, acct, n): | |
| vals = {} | |
| for f in ctrl["fields"]: | |
| if f["type"] == "password": | |
| vals[FV.field_key(f)] = acct["password"] | |
| elif f["type"] in ("text", "email") and not vals.get("_login"): | |
| # "Email or Username" fields take the username: several sites look sign-ins up by it only | |
| mail = f["type"] == "email" or ("mail" in (f["name"] or "").lower() and "user" not in (f["name"] or "").lower()) | |
| use = acct["email"] if mail and acct["email"] else acct["login"] | |
| vals[FV.field_key(f)] = use | |
| vals["_login"] = use | |
| vals.pop("_login", None) | |
| return self._arg(ctrl, vals, f"sign in as {acct['login']}", n) | |
| def _arg(self, ctrl, values, summary, n): | |
| h = hashlib.sha1(json.dumps(values, sort_keys=True).encode()).hexdigest()[:10] | |
| return {"index": n, "value": h, "text": summary or ", ".join(values.values())[:80], "values": values} | |
| def probe_argument(self, ctrl): | |
| args = self.arguments(ctrl) | |
| return args[0] if args else None | |
| def element_key(self, ctrl): | |
| action = (ctrl.get("form") or {}).get("action") or ctrl["page"] | |
| return f"FORM {ctrl['role']} {_sm()._pattern(re.sub(r'^https?://[^/]+', '', action))}" | |
| def dedup_value(self, arg): | |
| return arg["value"] | |
| def view(self, ctrl): | |
| v = super().view(ctrl) | |
| v.update({k: ctrl.get(k) for k in ("fields", "submit", "heading", "macro")}) | |
| v.pop("options", None) | |
| return v | |
| def setup(self, ctx, base, ctrl, arg=None): | |
| from miniwebgen.kinds.base import prepare | |
| prepare(ctx, base, ctrl) | |
| if ctrl["role"] == "authenticate_by_form": # 2FA codes are their own macro (reveal_by_2fa) | |
| B.session_flags(ctx, base, logout=True, disable_2fa=True) | |
| elif ctrl["role"] in ("pay_by_form", "checkout_by_form", "book_by_form"): | |
| B.session_flags(ctx, base, disable_2fa=True) | |
| # ββ privileged fill + submit βββββββββββββββββββββββββββββββββββββββββββ | |
| def apply(self, page, ctrl, arg, base, ctx, log_before): | |
| page.once("dialog", lambda d: d.accept()) | |
| for f in ctrl["fields"]: | |
| v = arg["values"].get(FV.field_key(f)) | |
| if v is None: | |
| continue | |
| if f["type"] == "select": | |
| o = FV.option_for(f, v) | |
| if o: | |
| page.select_option(f["css"], value=o["value"], timeout=5000) | |
| elif f["type"] == "radio": | |
| o = FV.option_for(f, v) | |
| if o: | |
| loc = page.locator(o["css"]).first | |
| loc.scroll_into_view_if_needed(timeout=3000) | |
| loc.check(timeout=5000, force=True) | |
| elif f["type"] == "checkbox": | |
| want = str(v).strip().lower() in ("1", "true", "yes", "on", "checked") | |
| loc = page.locator(f["css"]).first | |
| loc.scroll_into_view_if_needed(timeout=3000) | |
| loc.set_checked(want, timeout=5000, force=True) | |
| elif f["type"] == "range": | |
| page.locator(f["css"]).first.evaluate("(el, v) => { el.value = v; el.dispatchEvent(new Event('input', {bubbles: true})); el.dispatchEvent(new Event('change', {bubbles: true})); }", v) | |
| else: | |
| page.fill(f["css"], v, timeout=5000) | |
| page.locator(ctrl["submit"]["css"]).first.click(timeout=5000) | |
| B.settle(page) | |
| new = B.session_log(ctx, base)[log_before:] | |
| if ctrl["role"] == "authenticate_by_form" and _still_signed_out(page, ctrl): | |
| return None, None, None, new # rejected credentials re-render the form (often with a 200) | |
| acc = [e for e in new if isinstance(e.get("status"), int) and 200 <= e["status"] < 400] | |
| muts = [e for e in acc if (e.get("method") or "GET").upper() != "GET"] | |
| gets = [e for e in acc if (e.get("method") or "GET").upper() == "GET" and any( | |
| _sm().find_carrier([e], v)[0] for v in arg["values"].values())] | |
| e = (muts or gets or [None])[-1] | |
| return (e, None, "submit", new) if e else (None, None, None, new) | |
| def check(self, ctrl, arg, keep=None): | |
| return {"kind": "state", "pending": True} | |
| def request_check(self, entry): | |
| """No data change (sign-in, translation): the request, with the entered values it carries.""" | |
| fields = {} | |
| fields.update(entry.get("query") or {}) | |
| if isinstance(entry.get("body"), dict): | |
| fields.update({k: v for k, v in entry["body"].items() if isinstance(v, (str, int, float))}) | |
| secret = re.compile(r"(csrf|token|nonce|_ts$|timestamp)", re.I) | |
| return {"kind": "request", "method": entry["method"].upper(), "path": entry["path"], | |
| "params": {k: v for k, v in fields.items() if not secret.search(k) and str(v).strip()}, "keep": {}, | |
| "final": False} | |
| # ββ wording ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def describe(self, ctrl): | |
| return {"type": f"a form ({ctrl.get('heading') or ctrl.get('page_title')})", "label": ctrl["submit"].get("text"), | |
| "fields": [f.get("label") or f.get("name") for f in ctrl["fields"]], "choices": [], | |
| "note": "state every value to enter exactly as given (quote free text); do not add values"} | |
| def say(self, ctrl, arg): | |
| lab = {FV.field_key(f): (f.get("label") or f.get("name") or "field") for f in ctrl["fields"]} | |
| return {lab.get(k, k): _human(v) for k, v in arg["values"].items()} | |
| def mentions(self, arg): | |
| out = [] | |
| for v in arg["values"].values(): | |
| v = _human(str(v)) | |
| words = v.split() | |
| if not words: | |
| continue | |
| out.append(v if len(words) <= 6 else " ".join(words[:6])) | |
| return out | |
| def _field_task(self, f, v): | |
| t = {"css": f["css"], "type": f["type"], "value": str(v), "visible": bool(f.get("visible", True)), | |
| "label_css": f.get("label_css"), "label": f.get("label")} | |
| if f["type"] in ("select", "radio"): | |
| o = FV.option_for(f, v) or {} | |
| t.update(option_text=o.get("text", str(v)), option_index=o.get("index")) | |
| if f["type"] == "radio" and o.get("css"): | |
| t.update(css=o["css"], label_css=o.get("label_css"), visible=bool(o.get("visible", True))) | |
| if f["type"] == "date" and re.fullmatch(r"\d{4}-\d{2}-\d{2}", str(v)): | |
| t["keys"] = f"{v[5:7]}{v[8:10]}{v[0:4]}" | |
| return t | |
| def step_spec(self, step): | |
| c, o = step["control"], step["option"] | |
| lines = [f"TARGET: the form {c.get('heading') or c.get('page_title')!r}, submit button css={c['submit']['css']!r} " | |
| f"text={c['submit'].get('text')!r}", "FIELDS TO FILL (leave all others as they are):"] | |
| for f in c["fields"]: | |
| v = o["values"].get(FV.field_key(f)) | |
| if v is not None: | |
| lines.append(f" - {f.get('label') or f.get('name')!r} ({f['type']}, css={f['css']!r}) = {v!r}") | |
| return lines | |
| def script_task(self, step): | |
| c, o = step["control"], step["option"] | |
| fields = [self._field_task(f, o["values"][FV.field_key(f)]) for f in c["fields"] | |
| if FV.field_key(f) in o["values"]] | |
| return {"subtask": step["subtask"], "fields": fields, "submit_css": c["submit"]["css"], | |
| "submit_text": c["submit"].get("text")} | |
| def locate(self, driver, step): | |
| return driver.query(css=step["control"]["submit"]["css"], limit=1) | |
| def _changed(ctrl, values): | |
| """Only what the step changes: an edit/settings form's value set repeats the fields it leaves | |
| as they are; those are neither entered nor worded (the check records what really changed).""" | |
| cur = {FV.field_key(f): str(f.get("value") or "").strip() for f in ctrl["fields"]} | |
| return {k: v for k, v in values.items() if str(v).strip() != cur.get(k, None)} | |
| def _still_signed_out(page, ctrl): | |
| """After a sign-in submit: the same page still asks for a password, or shows an error.""" | |
| try: | |
| if page.locator("input[type=password]:visible").count() and _sm()._pattern(page.url.split("?")[0].replace( | |
| re.match(r"^https?://[^/]+", page.url).group(0), "")) == _sm()._pattern(ctrl["page"]): | |
| return True | |
| return bool(page.get_by_text(re.compile(r"(invalid|incorrect|wrong) (username|password|credentials|email)", re.I)).count()) | |
| except Exception: | |
| return False | |
| def _human(v): | |
| """ISO dates in words (the executor still types the digits).""" | |
| import calendar | |
| m = re.fullmatch(r"(\d{4})-(\d{2})-(\d{2})", str(v)) | |
| return f"{calendar.month_name[int(m[2])]} {int(m[3])}, {m[1]}" if m and 1 <= int(m[2]) <= 12 else str(v) | |
| def _sm(): | |
| from miniwebgen import sitemap | |
| return sitemap | |