"""The Computer use tab: real runs of blink driving apps from screenshots, then how one step works. Everything here is HTML built from space/cua/gallery.json (see build_gallery.py) and space/cua/pipeline.json (build_pipeline.py: one recorded step re-rendered from its raw screen and checked against the run), plus one script that plays it: the stage (a strip of runs, or one run in the player), the scenario gallery, how a step goes from the raw screen to blink's decision, and the tiles beyond clicks. Nothing here calls a model; Screen click, which does, sits between that explainer and the tiles (app.py). Every word the tab shows is in COPY. """ from __future__ import annotations import functools import html import json import math import os import urllib.parse import screens HERE = os.path.dirname(os.path.abspath(__file__)) GALLERY = os.path.join(HERE, "cua", "gallery.json") SCHEMA = "blink-cua-gallery/1" PIPELINE = os.path.join(HERE, "cua", "pipeline.json") PIPELINE_SCHEMA = "blink-cua-pipeline/1" TAB = "computer-use" PARAMS = ("scenario", "model", "run", "step") # Every string the tab shows, in one place, so the words can change without touching the code. # Placeholders in braces are filled in by the page; keep them. COPY = { "title": "Watch blink use ten apps", "pitch": "Each app's own page supplies the boxes; no detector model finds them. blink sees them numbered " "on a screenshot and answers three questions.", "how_link": "How a step works", "mock": "Test data, not real blink runs.", "stat_apps": "original apps", "stat_runs": "of {n} runs completed", "stat_step": "median per step", "stat_gate": "risky clicks paused on", "runs": "{ok}/{n} runs", "open": "Watch", "gallery": "Pick an app", "phone": "phone", "canvas": "canvas app", "turns": "turn-based", # the player "close": "Back to apps", "video": "Video", "steps": "Steps", "play": "Play steps", "pause": "Pause", "prev": "Previous step", "next": "Next step", "step_of": "Step {n}/{total}", "finished": "Task done", "failed": "Task not done", "steps_taken": "{n} steps", "steps_taken_one": "1 step", "asked": "{n} times asked first", "per_step": "{ms} ms per step", "interval": "95% interval {lo} to {hi}%", "matched": "{pct}% picked the right box", "show_failed": "Watch a miss", "show_best": "Watch a finish", "show_other": "Watch another", "seed": "seed {seed}", "expected": "Right box: {n}", "click": "Click {n}", "lane": "Lane {n}", "ask": "Ask before clicking {n}", "type": "Type {value} in {n}", "select": "Pick {value} in {n}", "done": "Looks done", "none": "Nothing to click", "done_q": "Done?", "risky_q": "Risky?", "ms": "{ms} ms", "tokens": "{n} image tokens", "models": "Same tasks, other models", "task": "Task", "no_video": "No video here. The steps still work.", "no_steps": "Steps unavailable. The video is still here.", # what blink was sent at a step, under its answers "in_title": "What blink got", "in_shot": "Screenshot", "in_dom": "{n} numbered boxes, listed by the page's DOM", "in_canvas": "{n} numbered boxes, from the app's own hit regions", "in_prev": "Earlier actions", "in_none": "None yet", "in_more": "+{n} earlier", "in_asked": "Questions", "in_second": "Then, for box {n}", "in_boxes": "boxes 1\u2013{n}", "in_yesno": "yes or no", "in_options": "{n} options", "in_picked": "picked", "in_tokens": "{n} input tokens over every pass, 0 generated", "in_names": "Names beside the numbers come from the page, for you to read. blink is not sent them: it gets " "the numbered screenshot, and a name only inside its own earlier actions.", "in_json": "Request JSON", "in_json_second": "Second request, after the box", "in_image": "data:image/png;base64,\u2026 (the numbered screenshot)", # how it reads a screenshot "how_eyebrow": "From screenshot to click", "how_line": "One pass per question. A probability for every answer, no text generated.", "how_nodes": ( ("Marked screenshot", "Boxes show what can be clicked."), ("Vision encoder", "{base}, unchanged by blink training."), ("Image tokens", "One per {side} \u00d7 {side} px patch."), ("Three questions", "Next box, done yet, risky click."), ("Probabilities", "One for every offered answer."), ("Next step", "Click, pause, stop, or hand over."), ), "how_facts": ( "{width} \u00d7 {height} px", "{blocks} blocks \u00b7 hidden {hidden}", "{tokens}", "3 questions", "one forward pass each", "{outcomes}", ), "how_outcomes": "click \u00b7 pause \u00b7 done \u00b7 hand over", "how_questions": ("Which box?", "Done?", "Risky?"), # how one step works: a recorded step from its raw screen to blink's decision (pipeline.json) "pipe_title": "How one step works", "pipe_line": "Step {step} of a recorded {title} run, from the raw screen to blink's decision. Everything " "before blink is ordinary browser automation, so blink is the only model in the loop.", "pipe_before": "Before blink: no model", "pipe_inside": "Inside blink: one forward pass per question", "pipe_nodes": ( ("Raw screen", "Playwright opens the app and screenshots it."), ("What can be clicked", ("One query to the page's DOM lists its links, buttons, fields, menus and other " "controls, with positions. Hidden, disabled and covered ones drop out.")), ("Numbered boxes", ("Each gets a numbered box, drawn into the pixels. This picture is all blink sees of " "the page.")), ("The request", ("The picture, the task, blink's earlier actions and three questions. No page code, " "element list or answer key.")), ("Image tokens", ("Each model's base vision encoder, unchanged by blink training, makes one token per " "{side} \u00d7 {side} px patch.")), ("Three questions", ("Each is one forward pass. The options get letters, and blink scores the letters " "instead of writing.")), ("Probabilities", "One for every offered answer."), ("Decision", ("Done at {done_at} stops; risky at {risky_at} asks first; otherwise the top box is " "clicked. A field gets one more question for its value.")), ), "pipe_facts": ( "{width} \u00d7 {height} px \u00b7 {shot_ms}", "{kept} elements \u00b7 {dom_ms}", "boxes 1\u2013{kept} \u00b7 drawn in {draw_ms}", "{prev} \u00b7 3 questions", "{tokens} image tokens", "{input_tokens} input tokens in 3 passes \u00b7 0 generated", "{agree}", "{act} \u00b7 {lead} {ms}", ), "pipe_prev": ("no earlier actions", "1 earlier action", "{n} earlier actions"), "pipe_agree": "all {n} models pick box {pick}", "pipe_split": "{lead} picks box {pick}", "pipe_raw_alt": "The app as Playwright captured it, before any marks", "pipe_marked_alt": "The same screen with numbered boxes, as blink got it", "pipe_src_title": "Where the boxes come from", "pipe_src": ( (("{dom} app with HTML controls", "{dom} apps with HTML controls"), "The page's DOM, read in the browser by one query."), (("{canvas} app drawn on a canvas", "{canvas} apps drawn on a canvas"), "Each app reports where it drew its targets: {names}."), (("A screenshot of your own", "A screenshot of your own"), "You draw the boxes, or a numbered grid stands in. Try it below."), ), "pipe_none": "No detector, OCR or second vision model in any of them.", "pipe_time": "Times before blink are medians from a local re-run of every app's first screen ({apps} apps, " "{reps} passes each); the recorded runs do not time those stages. Drawing the boxes is quick: the " "whole marking step, mostly saving the picture as a PNG, takes about {mark}. blink's time is the " "recorded call for all three questions.", "pipe_open": "Step through this run", "pipe_request": "The exact request for this step", "pipe_request_note": "One request carries all three questions. The screenshot field holds the numbered " "picture as a PNG; nothing else is sent.", "pipe_list": "What the page listed", "pipe_list_note": "The harness's DOM query found these {kept} elements, in reading order. Their roles and " "names are shown here for you; blink is not sent them.", "pipe_cols": ("Box", "Role", "Name", "Pixels (x1, y1, x2, y2)"), "pipe_check": "Re-rendered from the app, seed and task of the recorded run: the page listed the same " "{kept} boxes, and marking this raw screen reproduces the screenshot every model was sent, " "pixel for pixel.", # try it "try_title": "Try your screenshot", # beyond clicks "more_eyebrow": "Beyond clicks", "more_line": "Voice, audio, screen checks, and document pages.", "sound": "Play with sound", } def esc(s) -> str: return html.escape(str(s), quote=True) # --- the gallery ----------------------------------------------------------------------- def path() -> str: """BLINK_GALLERY points the page at another build (a mock, a local check); the Space ships cua/gallery.json.""" return os.environ.get("BLINK_GALLERY") or GALLERY @functools.lru_cache(maxsize=4) def _load(where: str, stamp: float): with open(where, encoding="utf-8") as fh: g = json.load(fh) if g.get("schema") != SCHEMA or not g.get("scenarios"): return None return g def load() -> dict | None: """The gallery, or None when there is none to show (the tab then leads with the explainer and Screen click).""" where = path() try: stamp = os.path.getmtime(where) except OSError: return None try: return _load(where, stamp) except (OSError, ValueError): return None def available() -> bool: return load() is not None def media(g: dict, rel: str | None) -> str: return g["media_base"] + urllib.parse.quote(rel) if rel else "" def scenario(g: dict, sid: str) -> dict | None: return next((s for s in g["scenarios"] if s["id"] == sid), None) def runs_line(st: dict) -> str: return COPY["runs"].format(ok=st["successes"], n=st["episodes"]) # --- deep links --------------------------------------------------------------------- def parse_link(raw, g: dict | None = None) -> dict: """?tab=computer-use&scenario=shop&model=blink-4b&run=failed&step=3 -> what the player opens. Unknown scenarios, models and steps fall back rather than fail: a stale link still opens the tab.""" g = g if g is not None else load() out = {"scenario": None, "model": None, "run": "showcase", "step": None} if not isinstance(raw, str) or g is None: return out params = urllib.parse.parse_qs(raw.partition("#")[0].lstrip("?")) asked = (params.get("scenario") or [""])[0].strip().lower() sc = scenario(g, asked) if sc is None: return out out["scenario"] = sc["id"] models = [m for m in g["models"] if m in sc["models"]] want = (params.get("model") or [""])[0].strip() lead = g.get("lead_model") or g["vision_model"] out["model"] = want if want in models else (lead if lead in models else models[0]) item = sc["models"][out["model"]] if (params.get("run") or [""])[0].strip().lower() in ("failed", "failure") and item.get("failure"): out["run"] = "failure" ep = g["episodes"][item["showcase" if out["run"] == "showcase" else "failure"]] try: step = int((params.get("step") or [""])[0]) except ValueError: step = None if step is not None and ep["steps"]: out["step"] = min(max(step, 1), len(ep["steps"])) return out # --- the stage: a strip of runs -------------------------------------------------------- def _video(g: dict, ep: dict, *, auto: bool, cls: str = "") -> str: poster = media(g, ep.get("poster")) src = media(g, ep.get("preview") or ep.get("video")) if not src: return f'' if poster else "" return (f'') def hero_order(g: dict) -> list[str]: """The strip's runs: a phone between two screens where there is one, then the rest, in registry order.""" ids = [e for e in g.get("hero", []) if e in g["episodes"]] phones = [e for e in ids if g["episodes"][e]["frame"][1] > g["episodes"][e]["frame"][0]] wide = [e for e in ids if e not in phones] if phones and len(wide) >= 2: return [wide[0], phones[0], wide[1], *[e for e in ids if e not in (wide[0], phones[0], wide[1])]] return ids def _link(sid: str) -> str: return "?" + urllib.parse.urlencode({"tab": TAB, "scenario": sid}) def hero(g: dict) -> str: reels = [] for i, eid in enumerate(hero_order(g)): ep = g["episodes"][eid] sc = scenario(g, ep["scenario"]) st = sc["models"][ep["model"]]["stats"] w, h = ep["frame"] reels.append( f' w else ""}" href="{esc(_link(sc["id"]))}" data-sid="{esc(sc["id"])}"' f' style="--ar:{w} / {h};--i:{i}">{_video(g, ep, auto=True)}' f'{esc(sc["title"])}{esc(runs_line(st))}' ) ids = hero_order(g)[:3] wide = all(g["episodes"][e]["frame"][0] > g["episodes"][e]["frame"][1] for e in ids) return (f'
{"".join(reels)}
' if reels else "") def _pct(v: float | None) -> str: return "\u2013" if v is None else f"{round(v * 100):d}%" def stat_row(g: dict) -> str: lead = g.get("lead_model") or g["vision_model"] total = g["totals"].get(lead) or next(iter(g["totals"].values()), None) if not total: return "" who = lead.rsplit("/", 1)[-1] if lead in g["totals"] else "" items = [(str(total["scenarios"]), COPY["stat_apps"]), (_pct(total["rate"]), COPY["stat_runs"].format(n=total["episodes"]) + (f" \u00b7 {who}" if who else ""))] if total.get("p50_ms"): items.append((f"{total['p50_ms'] / 1000:.1f} s", COPY["stat_step"])) if total.get("gate_recall") is not None: items.append((_pct(total["gate_recall"]), COPY["stat_gate"])) return ('
' + "".join(f'{esc(v)}{esc(k)}' for i, (v, k) in enumerate(items)) + "
") def _dots(outcomes: list[dict]) -> str: return ('") def cards(g: dict) -> str: out = [] for i, sc in enumerate(g["scenarios"]): lead = g.get("lead_model") or g["vision_model"] model = lead if lead in sc["models"] else next(iter(sc["models"])) item = sc["models"][model] ep = g["episodes"].get(item["showcase"] or "") if ep is None: continue w, h = ep["frame"] # the game waits for each decision (the harness lockstep clock), so its card says so tag = (COPY["phone"] if sc["device"] == "phone" else COPY["turns"] if sc["kind"] == "game" else COPY["canvas"] if sc["kind"] != "dom" else "") out.append( f'' f' w else ""}">{_video(g, ep, auto=False)}' + (f'{esc(tag)}' if tag else "") + f'{esc(sc["title"])}' f'{_dots(item["outcomes"])}{esc(runs_line(item["stats"]))}' f'{esc(model)}' ) return (f'

{esc(COPY["gallery"])}

' f'') # --- how it reads a screenshot ------------------------------------------------------------- def explainer(facts: dict, thumb: str = "", answers: dict | None = None, outcome: str = "") -> str: """Six nodes, told in order when they come into view: the numbered screenshot, the vision encoder, the image tokens laid over the screenshot as the patches they are, the typed questions, their probabilities and the action. `facts` are ui.vision_facts(); `answers` a saved run's, when there is one.""" tokens = facts.get("tokens") or 0 side = facts.get("side") or 32 width, height = facts.get("width") or 1440, facts.get("height") or 864 cols, rows = max(1, math.ceil(width / side)), max(1, math.ceil(height / side)) values = dict(facts, tokens=f"{tokens:,} image tokens" if tokens else "image tokens", side=side, width=width, height=height, outcome=outcome or COPY["click"].format(n="N"), outcomes=COPY["how_outcomes"]) probs = [] if answers: element = answers.get("element", {}) top = sorted(element.get("probabilities", {}).items(), key=lambda kv: -kv[1])[:3] probs = [(f"box {k}", v, k == element.get("choice")) for k, v in top] probs += [(COPY["done_q"], answers.get("done", {}).get("noul"), False), (COPY["risky_q"], answers.get("risky", {}).get("noul"), False)] win_class = ' class="win"' bars = "".join( f'{esc(k)}' f'{esc(_pct(p))}' for i, (k, p, win) in enumerate(probs)) art = [ f'' if thumb else '', '' + "".join(f'' for k in range(int(facts.get("blocks") or 27))) + "", (f'' + (f'' if thumb else "") + '' + f'{tokens:,}'), '' + "".join(f'{esc(q)}' for k, q in enumerate(COPY["how_questions"])) + "", f'', f'{esc(values["outcome"])}', ] nodes = [] for i, ((label, caption), fact, figure) in enumerate(zip(COPY["how_nodes"], COPY["how_facts"], art, strict=True)): nodes.append( f'
  • {figure}' f'{i + 1}{esc(label)}' f'{esc(caption.format(**values))}' f'{esc(fact.format(**values))}
  • ' ) return (f'

    {esc(COPY["how_eyebrow"])}

    ' f'

    {esc(COPY["how_line"])}

      {"".join(nodes)}
    ') # --- how one step works: a recorded step from its raw screen to blink's decision ---------------------- def pipeline_path() -> str: """BLINK_PIPELINE points the page at another record; the Space ships cua/pipeline.json.""" return os.environ.get("BLINK_PIPELINE") or PIPELINE @functools.lru_cache(maxsize=4) def _load_pipeline(where: str, stamp: float): with open(where, encoding="utf-8") as fh: p = json.load(fh) return p if p.get("schema") == PIPELINE_SCHEMA else None def load_pipeline() -> dict | None: """The re-rendered step build_pipeline.py checked against its run, or None.""" where = pipeline_path() try: return _load_pipeline(where, os.path.getmtime(where)) except (OSError, ValueError): return None def pipeline_fits(g: dict | None, p: dict | None) -> bool: """Whether the record describes a scenario this gallery shows, with answers to draw.""" if not g or not p: return False try: return (scenario(g, p["scenario"]) is not None and bool(p["answers"]) and bool(p["candidates"]) and all(a.get("p") and len(a["p"]) == len(p["candidates"]) for a in p["answers"].values())) except (KeyError, TypeError): return False def _ms(v) -> str: v = float(v) return f"{v / 1000:.1f} s" if v >= 1000 else f"{max(1, round(v)):d} ms" def _box_style(b, frame) -> str: w, h = frame return (f"--x:{b[0] / w * 100:.3f}%;--y:{b[1] / h * 100:.3f}%;" f"--w:{(b[2] - b[0]) / w * 100:.3f}%;--h:{(b[3] - b[1]) / h * 100:.3f}%") def pipeline(g: dict, p: dict, try_here: bool = True) -> str: """How one step works: the recorded step in pipeline.json, from its raw screen to blink's decision. The first row is the browser's work (no model): the raw screen, the DOM's list of what can be clicked, the numbered boxes and the request. The second is blink's: image tokens, one forward pass per question, the probabilities and the decision. Then where boxes come from in every app, the exact request, and the page's list. Every number comes from pipeline.json (checked against the run when it was built) or the gallery; the nodes are told in order when they come into view, like explainer()'s. `try_here` says Screen click follows on the page, so a screenshot of one's own gets its line.""" sc = scenario(g, p["scenario"]) models = [m for m in g["models"] if m in p["answers"]] or list(p["answers"]) lead = g.get("lead_model") if g.get("lead_model") in p["answers"] else models[0] a = p["answers"][lead] frame = p["frame"] width, height = frame kept = len(p["candidates"]) tokens = int(a.get("visual_tokens") or 0) side = math.isqrt(int(a["image_pixels"]) // tokens) if tokens and a.get("image_pixels") else 32 t = p["timing"] state = p["request"]["state"] questions = p["request"]["questions"] prev = len(state.get("previous_actions") or []) pick = a["pick"] picks = {p["answers"][m]["pick"] for m in models} act = (COPY["done"] if a.get("act") == "done" else COPY["click"].format(n=pick) if pick else COPY["none"]) values = { "step": p["step"], "title": sc["title"], "width": width, "height": height, "side": side, "shot_ms": _ms(t["screenshot_ms"]["p50"]), "dom_ms": _ms(t["dom_ms"]["p50"]), "draw_ms": _ms(t["draw_ms"]["p50"]), "kept": kept, "prev": COPY["pipe_prev"][min(prev, 2)].format(n=prev), "tokens": f"{tokens:,}", "input_tokens": f"{int(a.get('input_tokens') or 0):,}", "agree": (COPY["pipe_agree"].format(n=len(models), pick=pick) if len(picks) == 1 and len(models) > 1 else COPY["pipe_split"].format(lead=lead, pick=pick)), "act": act, "lead": lead, "ms": _ms(a["ms"]), "done_at": _pct(screens.DONE_AT), "risky_at": _pct(screens.RISKY_AT), } raw, marked = media(p, p["raw"]), media(p, p["marked"]) # 1-4: the browser's work, drawn from the real screen outlines = "".join(f'' for k, c in enumerate(p["candidates"])) lines = [("task", f"\u201c{state.get('task', '')}\u201d"), ("previous_actions", json.dumps(state.get("previous_actions") or [], ensure_ascii=False)), ("screenshot_note", f"\u201c{state.get('screenshot_note', '')}\u201d")] asked = " \u00b7 ".join(COPY["how_questions"]) request_art = ('' + "".join(f"{esc(k)} {esc(v)}" for k, v in lines) + 'screenshot [image 1]' + f'questions {esc(asked)}') before = [ (f'' f'{esc(COPY['), (f'' f'{outlines}{kept}'), (f'' f'{esc(COPY['), request_art, ] # 5-8: blink's work, as explainer() draws it cols, rows = max(1, math.ceil(width / side)), max(1, math.ceil(height / side)) ranked = sorted(range(kept), key=lambda i: -a["p"][i])[:3] probs = [(f"box {i + 1}", a["p"][i], i + 1 == pick) for i in ranked] probs += [(COPY["done_q"], a.get("done"), False), (COPY["risky_q"], a.get("risky"), False)] win_class = ' class="win"' bars = "".join( f'{esc(k)}' f'{esc(_pct(v))}' for i, (k, v, win) in enumerate(probs)) inside = [ (f'' f'{tokens:,}'), '' + "".join(f'{esc(q)}' for k, q in enumerate(COPY["how_questions"])) + "", f'', f'{esc(act)}', ] def flow(arts, first: int, cls: str) -> str: items = [] for i, art in enumerate(arts, first): label, caption = COPY["pipe_nodes"][i] items.append( f'
  • {art}' f'{i + 1}{esc(label)}' f'{esc(caption.format(**values))}' f'{esc(COPY["pipe_facts"][i].format(**values))}
  • ') return f'
      {"".join(items)}
    ' # where the boxes come from, in every app the gallery shows and in Screen click kinds = [s.get("kind") or "dom" for s in g["scenarios"]] drawn = [s["title"] for s in g["scenarios"] if (s.get("kind") or "dom") != "dom"] src_values = {"dom": kinds.count("dom"), "canvas": len(drawn), "names": ", ".join(drawn)} counts = (src_values["dom"], len(drawn), 2 if try_here else 0) sources = "".join( f'
  • {esc(heads[n != 1].format(**src_values))}{esc(text.format(**src_values))}
  • ' for (heads, text), n in zip(COPY["pipe_src"], counts) if n) # the exact request, and the page's own list of what it found body = {"model": "blink", "state": {**state, "screenshot": COPY["in_image"]}, "questions": questions} request_json = json.dumps(body, indent=2, ensure_ascii=False) table = "".join(f'{c["n"]}{esc(c["role"])}{esc(c["name"])}' f'{esc(", ".join(str(v) for v in c["box"]))}' for c in p["candidates"]) head_cells = "".join(f"{esc(h)}" for h in COPY["pipe_cols"]) item = sc["models"].get(lead) or {} opener = (f'' if item.get("showcase") == p["episodes"].get(lead) else "") timing = COPY["pipe_time"].format(apps=t["apps"], reps=t["reps"], mark=_ms(t["mark_ms"]["p50"])) return ( f'
    ' f'

    {esc(COPY["pipe_title"])}

    {opener}
    ' f'

    {esc(COPY["pipe_line"].format(**values))}

    ' f'

    {esc(COPY["pipe_before"])}

    {flow(before, 0, "blk-cxp-big")}' f'

    {esc(COPY["pipe_inside"])}

    {flow(inside, 4, "blk-cxp-model")}' f'

    {esc(COPY["pipe_src_title"])}

    ' f'
      {sources}

    {esc(COPY["pipe_none"])}

    ' f'

    {esc(timing)}

    ' f'
    ' f'
    {esc(COPY["pipe_request"])}' f'

    {esc(COPY["pipe_request_note"])}

    {esc(request_json)}
    ' f'
    {esc(COPY["pipe_list"])}' f'

    {esc(COPY["pipe_list_note"].format(kept=kept))}

    ' f'
    {head_cells}' f'{table}
    ' f'

    {esc(COPY["pipe_check"].format(kept=kept))}

    ' ) # --- beyond clicks ------------------------------------------------------------------- def tiles(g: dict | None) -> str: items = (g or {}).get("more") or [] if not items: return "" out = [] for i, t in enumerate(items): still = media(g, t.get("poster") or t.get("chart") or (t.get("images") or [None])[0]) metric = t.get("metric") or {} out.append( f'" ) data = esc(json.dumps({"more": items, "media_base": g["media_base"], "copy": {"sound": COPY["sound"]}}, separators=(",", ":"), ensure_ascii=False)) return (f'
    ' f'

    {esc(COPY["more_eyebrow"])}

    {esc(COPY["more_line"])}

    ' f'
    {"".join(out)}
    ') # --- assembly ----------------------------------------------------------------------- def client_data(g: dict) -> dict: """What the page carries: every run's summary and step rail. Each run's steps are fetched from beside its media when the player first needs them (build_gallery.py stages them), so the page stays light.""" keep = ("media_base", "mock", "models", "vision_model", "lead_model", "scenarios") out = {k: g[k] for k in keep if k in g} out["episodes"] = {eid: {k: v for k, v in ep.items() if k != "steps"} for eid, ep in g["episodes"].items()} out["rule_cut"] = {"done_at": screens.DONE_AT, "risky_at": screens.RISKY_AT} return out def pad_for(width: int) -> int: """How far a page mark reaches past its box: screens.style's stroke and halo.""" stroke = max(2, round(width / 360)) return stroke + max(1, stroke // 2) def top(g: dict | None, explain: str = "") -> str: """The tab above Screen click: the stage, the gallery, then how one step works.""" how = (f' ' if g is not None and 'id="cua-how"' in explain else "") head = (f'

    {esc(COPY["title"])}

    ' f'

    {esc(COPY["pitch"])}{how}

    ') if g is None: return f'
    {head}{explain}
    ' data = client_data(g) for ep in data["episodes"].values(): ep.setdefault("pad", pad_for(ep["frame"][0])) banner = f'
    {esc(COPY["mock"])}
    ' if g.get("mock") else "" blob = esc(json.dumps(data, separators=(",", ":"), ensure_ascii=False)) copy = esc(json.dumps(COPY, separators=(",", ":"), ensure_ascii=False)) return (f'
    {banner}{head}' f'
    {hero(g)}
    {stat_row(g)}{cards(g)}{explain}
    ') def try_head(note: str = "") -> str: """Screen click's heading, with the line under it that says what it does.""" return (f'

    {esc(COPY["try_title"])}

    ' f'{note}
    ') def home_figure(g: dict) -> str: """Home's figure: the first run of the strip, looping muted, with whose it is and how often it finishes.""" ids = hero_order(g) if not ids: return "" ep = g["episodes"][ids[0]] sc = scenario(g, ep["scenario"]) st = sc["models"][ep["model"]]["stats"] src = media(g, ep.get("preview") or ep.get("video")) caption = " \u00b7 ".join((sc["title"], runs_line(st), ep["model"])) return (f'
    {esc(caption)}
    ') def more(g: dict | None) -> str: return tiles(g) # The player: plays the runs gallery.json holds, steps through them and keeps the address in step. PLAYER_JS = r"""(() => { // The Computer use tab's player. Reads the gallery the page was built with (data-g), plays runs, // steps through them and keeps the address in step. Talks to nothing but the page it is on. const CHECK = ''; const BACK = ''; const PREV = ''; const NEXT = ''; const PLAY = ''; const PAUSE = ''; const esc = (s) => String(s == null ? '' : s).replace(/[&<>"']/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c])); const fmt = (s, o) => String(s || '').replace(/\{(\w+)\}/g, (m, k) => (k in o ? o[k] : m)); const pct = (p) => (p == null ? '\u2013' : `${Math.round(p * 100)}%`); const pct1 = (p) => (p == null ? '\u2013' : `${(p * 100).toFixed(1)}%`); const reduce = !!(window.matchMedia && window.matchMedia('(prefers-reduced-motion: reduce)').matches); const boot = (tries) => { const root = element.querySelector('.blk-cuax'); if (!root) { if (tries < 50) setTimeout(() => boot(tries + 1), 100); return; } if (root.__blink) return; root.__blink = true; start(root); }; const start = (root) => { let G = null; let C = {}; try { G = JSON.parse(root.dataset.g || 'null'); C = JSON.parse(root.dataset.copy || '{}'); } catch (err) { G = null; } // how it reads a screenshot: told once, in order, when it comes into view const how = root.querySelector('.blk-cx'); if (how) { if (!reduce) how.classList.add('arm'); const count = how.querySelector('.blk-cx-count'); const go = () => { if (how.classList.contains('on')) return; how.classList.add('on'); if (!count || reduce) return; const to = Number(count.dataset.to || 0); if (!to) return; const t0 = performance.now() + Number(count.dataset.delay || 900); const tick = (t) => { const k = Math.min(1, Math.max(0, (t - t0) / 1300)); count.textContent = Math.round(to * (1 - Math.pow(1 - k, 3))).toLocaleString('en-US'); if (k < 1) requestAnimationFrame(tick); }; count.textContent = '0'; requestAnimationFrame(tick); }; if ('IntersectionObserver' in window) { const seen = new IntersectionObserver((es) => { if (es.some((e) => e.isIntersecting)) { go(); seen.disconnect(); } }, { threshold: 0.3 }); // the first row of nodes, not the whole section: a tall section on a phone may never be 30% in view seen.observe(how.querySelector('.blk-cx-flow') || how); } else { go(); } } // the address the page opened with, read once: a run to open, or Screen click to bring into view const memo = window.__blinkCua || (window.__blinkCua = { open: null, first: true }); const raw = memo.first && typeof window.__blinkFirstUrl === 'string' ? window.__blinkFirstUrl : window.location.search; const firstLoad = memo.first; memo.first = false; const q = new URLSearchParams(raw.split('#')[0].replace(/^\?/, '')); const toTry = firstLoad && !q.get('scenario') && (q.get('shot') || /^screen$/i.test((q.get('case') || '').trim())); if (toTry) { const bring = (n) => { const at = document.getElementById('cua-try'); if (at && at.offsetParent !== null && document.querySelector('#screen-out .blk-scr')) { at.scrollIntoView({ behavior: 'auto', block: 'start' }); } else if (n < 60) { setTimeout(() => bring(n + 1), 150); } }; setTimeout(() => bring(0), 300); } if (!G) return; const base = G.media_base || ''; const media = (rel) => (rel ? base + encodeURI(rel) : ''); const byId = {}; G.scenarios.forEach((s) => { byId[s.id] = s; }); const stage = root.querySelector('.blk-cuax-stage'); if (!stage) return; const hasHow = !!root.querySelector('#cua-how'); const heroHTML = stage.innerHTML; const doneAt = (G.rule_cut || {}).done_at || 0.6; const riskyAt = (G.rule_cut || {}).risky_at || 0.5; let cur = null; let timer = null; // muted loops play while they are on screen, and stop when they are not const io = 'IntersectionObserver' in window ? new IntersectionObserver((es) => { es.forEach((en) => { const v = en.target; if (en.isIntersecting && en.intersectionRatio >= 0.35) { if (!reduce) { v.muted = true; const p = v.play(); if (p && p.catch) p.catch(() => {}); } } else if (!v.paused) { v.pause(); } }); }, { threshold: [0, 0.35, 0.7] }) : null; const loops = (scope) => scope.querySelectorAll('video[data-auto="1"]').forEach((v) => { v.muted = true; v.preload = 'metadata'; if (io) io.observe(v); }); loops(stage); root.querySelectorAll('.blk-cuax-card').forEach((card) => { const v = card.querySelector('video'); if (!v) return; card.addEventListener('pointerenter', (e) => { if (e.pointerType !== 'mouse' || reduce) return; v.muted = true; v.preload = 'auto'; const p = v.play(); if (p && p.catch) p.catch(() => {}); }); card.addEventListener('pointerleave', () => v.pause()); }); const itemOf = (st) => byId[st.sid].models[st.model]; const epOf = (st) => G.episodes[itemOf(st)[st.role]]; const count = (ep) => (ep.rail || []).length; // a run's steps sit beside its media; fetched once, when the player first needs them const fetched = memo.steps || (memo.steps = {}); const stepsOf = (ep) => { const at = ep.steps_url; if (!at) return Promise.resolve([]); if (!fetched[at]) { fetched[at] = fetch(media(at), { credentials: 'omit' }) .then((r) => (r.ok ? r.json() : Promise.reject(new Error(String(r.status))))) .catch((err) => { delete fetched[at]; throw err; }); } return fetched[at]; }; const ready = memo.ready || (memo.ready = {}); // the address says what is open, so it can be shared; an embedding page hears it too const sync = () => { const url = new URL(window.location.href); ['scenario', 'model', 'run', 'step'].forEach((k) => url.searchParams.delete(k)); if (cur) { url.searchParams.set('tab', 'computer-use'); ['case', 'shot'].forEach((k) => url.searchParams.delete(k)); url.searchParams.set('scenario', cur.sid); if (cur.model !== (G.lead_model || G.vision_model)) url.searchParams.set('model', cur.model); if (cur.role === 'failure') url.searchParams.set('run', 'failed'); if (cur.view === 'steps') url.searchParams.set('step', String(cur.step + 1)); } const next = url.pathname + url.search; if (next !== window.location.pathname + window.location.search) { window.history.replaceState(null, '', next); } try { window.parent.postMessage({ queryString: url.search.replace(/^\?/, ''), hash: '' }, 'https://huggingface.co'); } catch (err) { /* not embedded */ } }; const meter = (label, p, on, kind) => (p == null ? '' : `
    ${pct(p)}${esc(label)}
    `); const verdict = (s, lane) => { const n = s.pick; if (s.act === 'done') return { head: C.done, mark: CHECK, act: 'done' }; if (!n) return { head: C.none, mark: '', act: 'unsure' }; const value = s.value == null ? '' : `\u201c${s.value}\u201d`; if (s.gate) return { head: fmt(C.ask, { n }), mark: esc(n), act: 'ask' }; if (s.act === 'type') return { head: fmt(C.type, { n, value }), mark: esc(n), act: 'click' }; if (s.act === 'select') return { head: fmt(C.select, { n, value }), mark: esc(n), act: 'click' }; return { head: fmt(lane ? C.lane : C.click, { n }), mark: esc(n), act: 'click' }; }; const figure = (ep, s, narrow) => { const [W, H] = ep.frame; const pad = ep.pad || 6; const share = (v, t) => `${(v / t * 100).toFixed(3)}%`; const act = verdict(s).act; const marks = s.boxes.map((b, i) => { const n = i + 1; const p = s.p[i]; const win = s.pick === n && s.act !== 'done' ? ' win' : ''; const want = s.oracle === n && s.pick !== n && s.act !== 'done' ? ' expect' : ''; const x1 = Math.max(0, b[0] - pad); const y1 = Math.max(0, b[1] - pad); const x2 = Math.min(W, b[2] + pad); const y2 = Math.min(H, b[3] + pad); const t = s.tags[i]; const shown = p != null && (!narrow || !!win); const chars = String(n).length + (shown ? pct(p).length : 0); return `` + `` + `${n}${shown ? `${pct(p)}` : ''}`; }).join(''); return `
    ` + `
    ${marks}
    `; }; const stepSide = (ep, steps, i) => { const s = steps[i]; const v = verdict(s, (byId[ep.scenario] || {}).kind === 'game'); const label = s.pick ? (s.labels[s.pick - 1] || '') : ''; const top = s.p.map((p, k) => [k + 1, p]).filter((x) => x[1] != null) .sort((a, b) => b[1] - a[1]).slice(0, 5); const bars = top.map(([k, p], j) => { const name = s.labels[k - 1] || ''; return `
  • ` + `${k}${name ? ` \u00b7 ${esc(name)}` : ''}` + `` + `${pct1(p)}
  • `; }).join(''); const miss = s.oracle && s.pick && s.oracle !== s.pick && s.act !== 'done' ? `

    ${esc(fmt(C.expected, { n: s.oracle }))}

    ` : ''; return `

    ${esc(fmt(C.step_of, { n: i + 1, total: steps.length }))}` + `${esc(fmt(C.ms, { ms: Math.round(s.ms).toLocaleString('en-US') }))}

    ` + `
    ` + `

    ${esc(v.head)}

    ${label ? `

    ${esc(label)}

    ` : ''}
    ` + miss + (bars ? `` : '') + `
    ${meter(C.done_q, s.done, s.done != null && s.done >= doneAt, 'done')}` + `${meter(C.risky_q, s.risky, s.risky != null && s.risky >= riskyAt, 'risky')}
    ` + got(ep, s); }; // what blink was sent at this step, under what it answered: the numbered screenshot, the task above, its own // earlier actions and the questions, word for word from the run (build_gallery.step_view). Older galleries // carry no questions, and then nothing is said rather than something guessed. const optionsOf = (q) => { const said = Object.values((q && q.criteria) || {}); if (q && q.type === 'noul') return C.in_yesno; if (said.length && said.every((d, j) => d === `box ${j + 1}`)) return fmt(C.in_boxes, { n: said.length }); return fmt(C.in_options, { n: said.length }); }; const got = (ep, s) => { const qs = s.questions || {}; const keys = Object.keys(qs); if (!keys.length) return ''; const kind = (byId[ep.scenario] || {}).kind; const prev = Array.isArray(s.previous_actions) ? s.previous_actions : []; const recent = prev.slice(-3); const main = keys.filter((k) => k !== 'value'); const ask = (k) => `
  • ${esc(qs[k].instructions || '')} ${esc(optionsOf(qs[k]))}
  • `; // a field's own question: its options are inputs too, so they are listed, the one blink picked marked const follow = () => { const said = Object.values(qs.value.criteria || {}); const ps = Array.isArray(s.value_p) ? s.value_p : []; const at = said.indexOf(s.value); const mark = (d, j) => `${esc(d)}${ps[j] != null ? ` ${esc(pct1(ps[j]))}` : ''}`; const opts = said.length <= 8 ? said.map((d, j) => (j === at ? mark(d, j) : esc(d))).join(' \u00b7 ') : esc(fmt(C.in_options, { n: said.length })) + (at >= 0 ? ` \u00b7 ${esc(C.in_picked)} ${mark(said[at], at)}` : ''); return `
  • ${esc(qs.value.instructions || '')}${opts}
  • `; }; const state = { task: ep.task, previous_actions: prev, screenshot_note: s.screenshot_note || '', screenshot: C.in_image }; const only = (list) => Object.fromEntries(list.map((k) => [k, qs[k]])); const body = (list) => `
    ${esc(JSON.stringify({ model: 'blink', state, questions: only(list) }, null, 2))}
    `; const earlier = prev.length ? `
      ${prev.length > recent.length ? `
    1. ${esc(fmt(C.in_more, { n: prev.length - recent.length }))}
    2. ` : ''}` + `${recent.map((x) => `
    3. ${esc(x)}
    4. `).join('')}
    ` : esc(C.in_none); return `
    ${esc(C.in_title)}` + `
    ${esc(C.in_shot)}
    ${esc(fmt(kind === 'dom' ? C.in_dom : C.in_canvas, { n: s.boxes.length }))}
    ` + `
    ${esc(C.in_prev)}
    ${earlier}
    ` + (main.length ? `
    ${esc(C.in_asked)}
      ${main.map(ask).join('')}
    ` : '') + (qs.value ? `
    ${esc(fmt(C.in_second, { n: s.pick }))}
      ${follow()}
    ` : '') + '
    ' + (s.input_tokens ? `

    ${esc(fmt(C.in_tokens, { n: Number(s.input_tokens).toLocaleString('en-US') }))}

    ` : '') + `

    ${esc(C.in_names)}

    ` + `
    ${esc(C.in_json)}${body(main)}` + (qs.value ? `

    ${esc(C.in_json_second)}

    ${body(['value'])}` : '') + '
    ' + (hasHow ? `` : '') + '
    '; }; const dots = (item) => ``; const summary = (sc, item, ep) => { const st = item.stats; const models = G.models.filter((m) => sc.models[m]); const facts = [ep.steps_taken === 1 ? C.steps_taken_one : fmt(C.steps_taken, { n: ep.steps_taken })]; if (ep.gates) facts.push(fmt(C.asked, { n: ep.gates })); if (ep.p50_ms) facts.push(fmt(C.per_step, { ms: Math.round(ep.p50_ms).toLocaleString('en-US') })); const lo = st.wilson95 ? Math.round(st.wilson95[0] * 100) : null; const hi = st.wilson95 ? Math.round(st.wilson95[1] * 100) : null; // the other run on offer, said as what it is: finished, failed, or simply another const alt = G.episodes[item[cur.role === 'failure' ? 'showcase' : 'failure']] || {}; const other = cur.role === 'failure' ? (alt.success ? C.show_best : C.show_other) : (ep.success ? C.show_failed : C.show_other); const compare = models.length > 1 ? `

    ${esc(C.models)}

    ` : ''; return `

    ${esc(ep.success ? C.finished : C.failed)}` + `${!ep.success && ep.detail ? `${esc(ep.detail)}` : ''}

    ` + `

    ${facts.map((f) => `${esc(f)}`).join('')}

    ` + `
    ${dots(item)}${esc(fmt(C.runs, { ok: st.successes, n: st.episodes }))}` + (lo != null ? `${esc(fmt(C.interval, { lo, hi }))}` : '') + '
    ' + (st.step_accuracy != null ? `

    ${esc(fmt(C.matched, { pct: Math.round(st.step_accuracy * 100) }))}

    ` : '') + (item.failure ? `` : '') + compare + (hasHow ? `` : ''); }; const rail = (ep) => (ep.rail || []).map((s, i) => { const cls = [s.a === 'done' ? 'done' : '', s.g ? 'gate' : '', s.m ? 'miss' : '', cur.view === 'steps' && i === cur.step ? 'on' : ''].filter(Boolean).join(' '); return `
  • `; }).join(''); const paint = () => { const player = stage.querySelector('.blk-cuax-player'); if (!player || !cur) return; const sc = byId[cur.sid]; const item = itemOf(cur); const ep = epOf(cur); player.dataset.view = cur.view; const screen = player.querySelector('.blk-cuax-screen'); const side = player.querySelector('.blk-cuax-side'); if (cur.view === 'video' && ep.video) { const [W, H] = ep.frame; if (!screen.querySelector('video')) { screen.innerHTML = `
    '; const v = screen.querySelector('video'); v.muted = true; if (!reduce) { const p = v.play(); if (p && p.catch) p.catch(() => {}); } } side.innerHTML = summary(sc, item, ep); } else if (count(ep)) { const steps = ready[ep.steps_url]; if (steps) { screen.innerHTML = figure(ep, steps[cur.step], screen.clientWidth < 520); side.innerHTML = stepSide(ep, steps, cur.step); [cur.step - 1, cur.step + 1].forEach((k) => { const s = steps[k]; if (s && s.img) (new Image()).src = media(s.img); }); } else { const [W, H] = ep.frame; screen.innerHTML = `
    ` + (ep.poster ? `` : '') + '
    '; side.innerHTML = '

    '; const want = cur; stepsOf(ep).then((got) => { ready[ep.steps_url] = got; if (cur === want && got.length) paint(); }).catch(() => { if (cur === want) side.innerHTML = `

    ${esc(C.no_steps)}

    `; }); } } player.querySelectorAll('.blk-cuax-rail button').forEach((b) => { b.classList.toggle('on', cur.view === 'steps' && Number(b.dataset.i) === cur.step); }); player.querySelectorAll('[data-do="view"]').forEach((b) => { b.classList.toggle('on', b.dataset.view === cur.view); b.setAttribute('aria-pressed', String(b.dataset.view === cur.view)); }); const at = player.querySelector('.blk-cuax-rail .on'); if (at && at.scrollIntoView) at.scrollIntoView({ block: 'nearest', inline: 'nearest' }); }; const render = () => { const sc = byId[cur.sid]; const ep = epOf(cur); const [W, H] = ep.frame; const tall = H > W; stage.innerHTML = `
    ` + '
    ' + `` + `

    ${esc(sc.title)}

    ` + `${esc(cur.model)}${cur.role === 'failure' ? ` \u00b7 ${esc(fmt(C.seed, { seed: ep.seed }))}` : ''}` + '
    ' + `

    ${esc(C.task)}${esc(ep.task)}

    ` + '
    ' + '
    ' + `
    ${ep.video ? `` : ''}` + `
    ` + '
    ' + `` + `` + `
      ${rail(ep)}
    ` + `` + '
    '; paint(); }; const mark = () => root.querySelectorAll('.blk-cuax-card').forEach((c) => { c.classList.toggle('on', !!cur && c.dataset.sid === cur.sid); }); const open = (o, scroll) => { const sc = byId[o.sid]; if (!sc) return; halt(); const models = G.models.filter((m) => sc.models[m]); const lead = G.lead_model || G.vision_model; const model = models.includes(o.model) ? o.model : (models.includes(lead) ? lead : models[0]); const item = sc.models[model]; const role = o.role === 'failure' && item.failure ? 'failure' : 'showcase'; const ep = G.episodes[item[role]]; const steps = count(ep); const view = ep.video ? (o.view || 'video') : 'steps'; cur = { sid: sc.id, model, role, view, step: Math.min(Math.max(o.step || 0, 0), Math.max(steps - 1, 0)) }; memo.open = cur; render(); mark(); sync(); if (scroll) { const top = stage.getBoundingClientRect().top; if (top < 0 || top > window.innerHeight * 0.4) { stage.scrollIntoView({ behavior: reduce ? 'auto' : 'smooth', block: 'start' }); } } }; // the steps can play on their own, one decision at a time; anything else a visitor does stops them const halt = () => { if (timer) clearInterval(timer); timer = null; const b = stage.querySelector('.blk-cuax-autoplay'); if (b) { b.innerHTML = PLAY; b.setAttribute('aria-pressed', 'false'); b.setAttribute('aria-label', C.play); } }; const play = () => { if (timer) { halt(); return; } const ep = epOf(cur); if (!count(ep)) return; go(cur.view === 'steps' && cur.step < count(ep) - 1 ? cur.step : 0); const b = stage.querySelector('.blk-cuax-autoplay'); if (b) { b.innerHTML = PAUSE; b.setAttribute('aria-pressed', 'true'); b.setAttribute('aria-label', C.pause); } timer = setInterval(() => { if (!cur || cur.step >= count(epOf(cur)) - 1) halt(); else go(cur.step + 1); }, 1700); }; const close = () => { halt(); cur = null; memo.open = null; stage.innerHTML = heroHTML; loops(stage); mark(); sync(); }; const go = (i) => { const ep = epOf(cur); if (!count(ep)) return; cur.step = Math.min(Math.max(i, 0), count(ep) - 1); cur.view = 'steps'; paint(); sync(); }; root.addEventListener('click', (e) => { const card = e.target.closest('[data-sid]'); if (card && root.contains(card)) { if (e.metaKey || e.ctrlKey || e.shiftKey || e.button) return; e.preventDefault(); open({ sid: card.dataset.sid }, true); return; } const btn = e.target.closest('[data-do], [data-open]'); if (btn && btn.dataset.do === 'how') { // how one step works, below the gallery; the player keeps its place e.preventDefault(); halt(); const at = root.querySelector('#cua-how'); if (at) at.scrollIntoView({ behavior: reduce ? 'auto' : 'smooth', block: 'start' }); return; } if (btn && btn.dataset.open) { // the explainer's own step, opened in the player at that step e.preventDefault(); open({ sid: btn.dataset.open, model: btn.dataset.model || '', view: 'steps', step: Math.max(Number(btn.dataset.step || 1) - 1, 0) }, true); return; } if (!btn || !cur) return; const act = btn.dataset.do; if (act === 'play') { play(); return; } halt(); if (act === 'close') close(); else if (act === 'step') go(Number(btn.dataset.i)); else if (act === 'prev') go((cur.view === 'steps' ? cur.step : 0) - 1); else if (act === 'next') go(cur.view === 'steps' ? cur.step + 1 : 0); else if (act === 'view') { cur.view = btn.dataset.view; const screen = stage.querySelector('.blk-cuax-screen'); if (screen) screen.innerHTML = ''; paint(); sync(); } else if (act === 'model') open({ sid: cur.sid, model: btn.dataset.model, view: cur.view }, false); else if (act === 'run') open({ sid: cur.sid, model: cur.model, role: cur.role === 'failure' ? 'showcase' : 'failure', view: cur.view }, false); }); // What blink got stays open or shut, and its JSON too, as the visitor leaves them from step to step root.addEventListener('toggle', (e) => { const d = e.target; if (!d || !d.classList) return; if (d.classList.contains('blk-cuax-got')) memo.got = d.open; else if (d.classList.contains('blk-cuax-json')) memo.json = d.open; }, true); root.addEventListener('keydown', (e) => { if (!cur || !e.target.closest('.blk-cuax-player')) return; if (['ArrowRight', 'ArrowLeft', 'Escape'].includes(e.key)) halt(); if (e.key === 'ArrowRight') { e.preventDefault(); go(cur.view === 'steps' ? cur.step + 1 : 0); } else if (e.key === 'ArrowLeft') { e.preventDefault(); go(cur.view === 'steps' ? cur.step - 1 : 0); } else if (e.key === 'Escape') close(); }); // a link opens its run once, on the page's first load; coming back to the tab keeps what was open const sid = (q.get('scenario') || '').trim().toLowerCase(); if (sid && byId[sid] && firstLoad) { const step = parseInt(q.get('step') || '', 10); open({ sid, model: q.get('model') || '', role: /^fail/i.test(q.get('run') || '') ? 'failure' : 'showcase', view: Number.isFinite(step) ? 'steps' : '', step: Number.isFinite(step) ? step - 1 : 0 }, true); } else if (memo.open && byId[memo.open.sid]) { open(memo.open, false); } }; boot(0); })(); """ # Beyond clicks: one tile open at a time. TILES_JS = r"""(() => { // Beyond clicks: one tile open at a time, its video or chart under the row. A video with sound // never starts on its own; one without plays muted. const esc = (s) => String(s == null ? '' : s).replace(/[&<>"']/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c])); const boot = (tries) => { const box = element.querySelector('.blk-cuax-more'); if (!box) { if (tries < 50) setTimeout(() => boot(tries + 1), 100); return; } if (box.__blink) return; box.__blink = true; let T = {}; try { T = JSON.parse(box.dataset.t || '{}'); } catch (err) { return; } const items = T.more || []; const C = T.copy || {}; const media = (rel) => (rel ? (T.media_base || '') + encodeURI(rel) : ''); const pane = box.querySelector('.blk-cuax-open'); let at = null; const show = (id) => { const t = items.find((x) => x.id === id); const again = at === id; box.querySelectorAll('.blk-cuax-tile').forEach((b) => { const on = b.dataset.tile === id && !again; b.classList.toggle('on', on); b.setAttribute('aria-expanded', String(on)); }); if (!t || again) { at = null; pane.hidden = true; pane.innerHTML = ''; return; } at = id; let fig = ''; if (t.video) { const track = t.captions ? `` : ''; const cors = t.captions ? ' crossorigin="anonymous"' : ''; fig += t.sound ? `` + `

    ${esc(C.sound)}

    ` : ``; } if (t.chart) fig += `${esc(t.title)}`; if (t.images && t.images.length) { fig += `
    ${t.images.map((i) => ``).join('')}
    `; } pane.innerHTML = `

    ${esc(t.title)}

    ${esc(t.line)}

    ${fig}
    `; pane.hidden = false; const v = pane.querySelector('video[muted]'); if (v) { v.muted = true; const p = v.play(); if (p && p.catch) p.catch(() => {}); } }; box.addEventListener('click', (e) => { const b = e.target.closest('.blk-cuax-tile'); if (b) show(b.dataset.tile); }); }; boot(0); })(); """