| """The Computer use tab: real runs of blink driving apps from screenshots, then how one step works. |
| |
| Everything here is HTML built from space/cua/gallery.json (see build_gallery.py) and space/cua/pipeline.json |
| (build_pipeline.py: one recorded step re-rendered from its raw screen and checked against the run), plus one |
| script that plays it: the stage (a strip of runs, or one run in the player), the scenario gallery, how a step |
| goes from the raw screen to blink's decision, and the tiles beyond clicks. Nothing here calls a model; Screen |
| click, which does, sits between that explainer and the tiles (app.py). Every word the tab shows is in COPY. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import functools |
| import html |
| import json |
| import math |
| import os |
| import urllib.parse |
|
|
| import screens |
|
|
| HERE = os.path.dirname(os.path.abspath(__file__)) |
| GALLERY = os.path.join(HERE, "cua", "gallery.json") |
| SCHEMA = "blink-cua-gallery/1" |
| PIPELINE = os.path.join(HERE, "cua", "pipeline.json") |
| PIPELINE_SCHEMA = "blink-cua-pipeline/1" |
| TAB = "computer-use" |
| PARAMS = ("scenario", "model", "run", "step") |
|
|
| |
| |
| COPY = { |
| "title": "Watch blink use ten apps", |
| "pitch": "Each app's own page supplies the boxes; no detector model finds them. blink sees them numbered " |
| "on a screenshot and answers three questions.", |
| "how_link": "How a step works", |
| "mock": "Test data, not real blink runs.", |
| "stat_apps": "original apps", |
| "stat_runs": "of {n} runs completed", |
| "stat_step": "median per step", |
| "stat_gate": "risky clicks paused on", |
| "runs": "{ok}/{n} runs", |
| "open": "Watch", |
| "gallery": "Pick an app", |
| "phone": "phone", |
| "canvas": "canvas app", |
| "turns": "turn-based", |
| |
| "close": "Back to apps", |
| "video": "Video", |
| "steps": "Steps", |
| "play": "Play steps", |
| "pause": "Pause", |
| "prev": "Previous step", |
| "next": "Next step", |
| "step_of": "Step {n}/{total}", |
| "finished": "Task done", |
| "failed": "Task not done", |
| "steps_taken": "{n} steps", |
| "steps_taken_one": "1 step", |
| "asked": "{n} times asked first", |
| "per_step": "{ms} ms per step", |
| "interval": "95% interval {lo} to {hi}%", |
| "matched": "{pct}% picked the right box", |
| "show_failed": "Watch a miss", |
| "show_best": "Watch a finish", |
| "show_other": "Watch another", |
| "seed": "seed {seed}", |
| "expected": "Right box: {n}", |
| "click": "Click {n}", |
| "lane": "Lane {n}", |
| "ask": "Ask before clicking {n}", |
| "type": "Type {value} in {n}", |
| "select": "Pick {value} in {n}", |
| "done": "Looks done", |
| "none": "Nothing to click", |
| "done_q": "Done?", |
| "risky_q": "Risky?", |
| "ms": "{ms} ms", |
| "tokens": "{n} image tokens", |
| "models": "Same tasks, other models", |
| "task": "Task", |
| "no_video": "No video here. The steps still work.", |
| "no_steps": "Steps unavailable. The video is still here.", |
| |
| "in_title": "What blink got", |
| "in_shot": "Screenshot", |
| "in_dom": "{n} numbered boxes, listed by the page's DOM", |
| "in_canvas": "{n} numbered boxes, from the app's own hit regions", |
| "in_prev": "Earlier actions", |
| "in_none": "None yet", |
| "in_more": "+{n} earlier", |
| "in_asked": "Questions", |
| "in_second": "Then, for box {n}", |
| "in_boxes": "boxes 1\u2013{n}", |
| "in_yesno": "yes or no", |
| "in_options": "{n} options", |
| "in_picked": "picked", |
| "in_tokens": "{n} input tokens over every pass, 0 generated", |
| "in_names": "Names beside the numbers come from the page, for you to read. blink is not sent them: it gets " |
| "the numbered screenshot, and a name only inside its own earlier actions.", |
| "in_json": "Request JSON", |
| "in_json_second": "Second request, after the box", |
| "in_image": "data:image/png;base64,\u2026 (the numbered screenshot)", |
| |
| "how_eyebrow": "From screenshot to click", |
| "how_line": "One pass per question. A probability for every answer, no text generated.", |
| "how_nodes": ( |
| ("Marked screenshot", "Boxes show what can be clicked."), |
| ("Vision encoder", "{base}, unchanged by blink training."), |
| ("Image tokens", "One per {side} \u00d7 {side} px patch."), |
| ("Three questions", "Next box, done yet, risky click."), |
| ("Probabilities", "One for every offered answer."), |
| ("Next step", "Click, pause, stop, or hand over."), |
| ), |
| "how_facts": ( |
| "{width} \u00d7 {height} px", |
| "{blocks} blocks \u00b7 hidden {hidden}", |
| "{tokens}", |
| "3 questions", |
| "one forward pass each", |
| "{outcomes}", |
| ), |
| "how_outcomes": "click \u00b7 pause \u00b7 done \u00b7 hand over", |
| "how_questions": ("Which box?", "Done?", "Risky?"), |
| |
| "pipe_title": "How one step works", |
| "pipe_line": "Step {step} of a recorded {title} run, from the raw screen to blink's decision. Everything " |
| "before blink is ordinary browser automation, so blink is the only model in the loop.", |
| "pipe_before": "Before blink: no model", |
| "pipe_inside": "Inside blink: one forward pass per question", |
| "pipe_nodes": ( |
| ("Raw screen", "Playwright opens the app and screenshots it."), |
| ("What can be clicked", ("One query to the page's DOM lists its links, buttons, fields, menus and other " |
| "controls, with positions. Hidden, disabled and covered ones drop out.")), |
| ("Numbered boxes", ("Each gets a numbered box, drawn into the pixels. This picture is all blink sees of " |
| "the page.")), |
| ("The request", ("The picture, the task, blink's earlier actions and three questions. No page code, " |
| "element list or answer key.")), |
| ("Image tokens", ("Each model's base vision encoder, unchanged by blink training, makes one token per " |
| "{side} \u00d7 {side} px patch.")), |
| ("Three questions", ("Each is one forward pass. The options get letters, and blink scores the letters " |
| "instead of writing.")), |
| ("Probabilities", "One for every offered answer."), |
| ("Decision", ("Done at {done_at} stops; risky at {risky_at} asks first; otherwise the top box is " |
| "clicked. A field gets one more question for its value.")), |
| ), |
| "pipe_facts": ( |
| "{width} \u00d7 {height} px \u00b7 {shot_ms}", |
| "{kept} elements \u00b7 {dom_ms}", |
| "boxes 1\u2013{kept} \u00b7 drawn in {draw_ms}", |
| "{prev} \u00b7 3 questions", |
| "{tokens} image tokens", |
| "{input_tokens} input tokens in 3 passes \u00b7 0 generated", |
| "{agree}", |
| "{act} \u00b7 {lead} {ms}", |
| ), |
| "pipe_prev": ("no earlier actions", "1 earlier action", "{n} earlier actions"), |
| "pipe_agree": "all {n} models pick box {pick}", |
| "pipe_split": "{lead} picks box {pick}", |
| "pipe_raw_alt": "The app as Playwright captured it, before any marks", |
| "pipe_marked_alt": "The same screen with numbered boxes, as blink got it", |
| "pipe_src_title": "Where the boxes come from", |
| "pipe_src": ( |
| (("{dom} app with HTML controls", "{dom} apps with HTML controls"), |
| "The page's DOM, read in the browser by one query."), |
| (("{canvas} app drawn on a canvas", "{canvas} apps drawn on a canvas"), |
| "Each app reports where it drew its targets: {names}."), |
| (("A screenshot of your own", "A screenshot of your own"), |
| "You draw the boxes, or a numbered grid stands in. Try it below."), |
| ), |
| "pipe_none": "No detector, OCR or second vision model in any of them.", |
| "pipe_time": "Times before blink are medians from a local re-run of every app's first screen ({apps} apps, " |
| "{reps} passes each); the recorded runs do not time those stages. Drawing the boxes is quick: the " |
| "whole marking step, mostly saving the picture as a PNG, takes about {mark}. blink's time is the " |
| "recorded call for all three questions.", |
| "pipe_open": "Step through this run", |
| "pipe_request": "The exact request for this step", |
| "pipe_request_note": "One request carries all three questions. The screenshot field holds the numbered " |
| "picture as a PNG; nothing else is sent.", |
| "pipe_list": "What the page listed", |
| "pipe_list_note": "The harness's DOM query found these {kept} elements, in reading order. Their roles and " |
| "names are shown here for you; blink is not sent them.", |
| "pipe_cols": ("Box", "Role", "Name", "Pixels (x1, y1, x2, y2)"), |
| "pipe_check": "Re-rendered from the app, seed and task of the recorded run: the page listed the same " |
| "{kept} boxes, and marking this raw screen reproduces the screenshot every model was sent, " |
| "pixel for pixel.", |
| |
| "try_title": "Try your screenshot", |
| |
| "more_eyebrow": "Beyond clicks", |
| "more_line": "Voice, audio, screen checks, and document pages.", |
| "sound": "Play with sound", |
| } |
|
|
|
|
| def esc(s) -> str: |
| return html.escape(str(s), quote=True) |
|
|
|
|
| |
|
|
|
|
| def path() -> str: |
| """BLINK_GALLERY points the page at another build (a mock, a local check); the Space ships cua/gallery.json.""" |
| return os.environ.get("BLINK_GALLERY") or GALLERY |
|
|
|
|
| @functools.lru_cache(maxsize=4) |
| def _load(where: str, stamp: float): |
| with open(where, encoding="utf-8") as fh: |
| g = json.load(fh) |
| if g.get("schema") != SCHEMA or not g.get("scenarios"): |
| return None |
| return g |
|
|
|
|
| def load() -> dict | None: |
| """The gallery, or None when there is none to show (the tab then leads with the explainer and Screen click).""" |
| where = path() |
| try: |
| stamp = os.path.getmtime(where) |
| except OSError: |
| return None |
| try: |
| return _load(where, stamp) |
| except (OSError, ValueError): |
| return None |
|
|
|
|
| def available() -> bool: |
| return load() is not None |
|
|
|
|
| def media(g: dict, rel: str | None) -> str: |
| return g["media_base"] + urllib.parse.quote(rel) if rel else "" |
|
|
|
|
| def scenario(g: dict, sid: str) -> dict | None: |
| return next((s for s in g["scenarios"] if s["id"] == sid), None) |
|
|
|
|
| def runs_line(st: dict) -> str: |
| return COPY["runs"].format(ok=st["successes"], n=st["episodes"]) |
|
|
|
|
| |
|
|
|
|
| def parse_link(raw, g: dict | None = None) -> dict: |
| """?tab=computer-use&scenario=shop&model=blink-4b&run=failed&step=3 -> what the player opens. |
| |
| Unknown scenarios, models and steps fall back rather than fail: a stale link still opens the tab.""" |
| g = g if g is not None else load() |
| out = {"scenario": None, "model": None, "run": "showcase", "step": None} |
| if not isinstance(raw, str) or g is None: |
| return out |
| params = urllib.parse.parse_qs(raw.partition("#")[0].lstrip("?")) |
| asked = (params.get("scenario") or [""])[0].strip().lower() |
| sc = scenario(g, asked) |
| if sc is None: |
| return out |
| out["scenario"] = sc["id"] |
| models = [m for m in g["models"] if m in sc["models"]] |
| want = (params.get("model") or [""])[0].strip() |
| lead = g.get("lead_model") or g["vision_model"] |
| out["model"] = want if want in models else (lead if lead in models else models[0]) |
| item = sc["models"][out["model"]] |
| if (params.get("run") or [""])[0].strip().lower() in ("failed", "failure") and item.get("failure"): |
| out["run"] = "failure" |
| ep = g["episodes"][item["showcase" if out["run"] == "showcase" else "failure"]] |
| try: |
| step = int((params.get("step") or [""])[0]) |
| except ValueError: |
| step = None |
| if step is not None and ep["steps"]: |
| out["step"] = min(max(step, 1), len(ep["steps"])) |
| return out |
|
|
|
|
| |
|
|
|
|
| def _video(g: dict, ep: dict, *, auto: bool, cls: str = "") -> str: |
| poster = media(g, ep.get("poster")) |
| src = media(g, ep.get("preview") or ep.get("video")) |
| if not src: |
| return f'<img class="{esc(cls)}" src="{esc(poster)}" alt="" loading="lazy">' if poster else "" |
| return (f'<video class="{esc(cls)}" data-auto="{1 if auto else 0}" muted loop playsinline preload="none"' |
| f' poster="{esc(poster)}" aria-hidden="true"><source src="{esc(src)}" type="video/mp4"></video>') |
|
|
|
|
| def hero_order(g: dict) -> list[str]: |
| """The strip's runs: a phone between two screens where there is one, then the rest, in registry order.""" |
| ids = [e for e in g.get("hero", []) if e in g["episodes"]] |
| phones = [e for e in ids if g["episodes"][e]["frame"][1] > g["episodes"][e]["frame"][0]] |
| wide = [e for e in ids if e not in phones] |
| if phones and len(wide) >= 2: |
| return [wide[0], phones[0], wide[1], *[e for e in ids if e not in (wide[0], phones[0], wide[1])]] |
| return ids |
|
|
|
|
| def _link(sid: str) -> str: |
| return "?" + urllib.parse.urlencode({"tab": TAB, "scenario": sid}) |
|
|
|
|
| def hero(g: dict) -> str: |
| reels = [] |
| for i, eid in enumerate(hero_order(g)): |
| ep = g["episodes"][eid] |
| sc = scenario(g, ep["scenario"]) |
| st = sc["models"][ep["model"]]["stats"] |
| w, h = ep["frame"] |
| reels.append( |
| f'<a class="blk-cuax-reel{" tall" if h > w else ""}" href="{esc(_link(sc["id"]))}" data-sid="{esc(sc["id"])}"' |
| f' style="--ar:{w} / {h};--i:{i}">{_video(g, ep, auto=True)}' |
| f'<span class="blk-cuax-cap"><b>{esc(sc["title"])}</b><em>{esc(runs_line(st))}</em></span></a>' |
| ) |
| ids = hero_order(g)[:3] |
| wide = all(g["episodes"][e]["frame"][0] > g["episodes"][e]["frame"][1] for e in ids) |
| return (f'<div class="blk-cuax-strip{" wide" if wide else ""}">{"".join(reels)}</div>' |
| if reels else "") |
|
|
|
|
| def _pct(v: float | None) -> str: |
| return "\u2013" if v is None else f"{round(v * 100):d}%" |
|
|
|
|
| def stat_row(g: dict) -> str: |
| lead = g.get("lead_model") or g["vision_model"] |
| total = g["totals"].get(lead) or next(iter(g["totals"].values()), None) |
| if not total: |
| return "" |
| who = lead.rsplit("/", 1)[-1] if lead in g["totals"] else "" |
| items = [(str(total["scenarios"]), COPY["stat_apps"]), |
| (_pct(total["rate"]), COPY["stat_runs"].format(n=total["episodes"]) + (f" \u00b7 {who}" if who else ""))] |
| if total.get("p50_ms"): |
| items.append((f"{total['p50_ms'] / 1000:.1f} s", COPY["stat_step"])) |
| if total.get("gate_recall") is not None: |
| items.append((_pct(total["gate_recall"]), COPY["stat_gate"])) |
| return ('<div class="blk-cuax-stats">' |
| + "".join(f'<span style="--i:{i}"><b>{esc(v)}</b>{esc(k)}</span>' for i, (v, k) in enumerate(items)) |
| + "</div>") |
|
|
|
|
| def _dots(outcomes: list[dict]) -> str: |
| return ('<span class="blk-cuax-dots" aria-hidden="true">' |
| + "".join(f'<i class="{"ok" if o["ok"] else "no"}"></i>' for o in outcomes) + "</span>") |
|
|
|
|
| def cards(g: dict) -> str: |
| out = [] |
| for i, sc in enumerate(g["scenarios"]): |
| lead = g.get("lead_model") or g["vision_model"] |
| model = lead if lead in sc["models"] else next(iter(sc["models"])) |
| item = sc["models"][model] |
| ep = g["episodes"].get(item["showcase"] or "") |
| if ep is None: |
| continue |
| w, h = ep["frame"] |
| |
| tag = (COPY["phone"] if sc["device"] == "phone" else COPY["turns"] if sc["kind"] == "game" |
| else COPY["canvas"] if sc["kind"] != "dom" else "") |
| out.append( |
| f'<a class="blk-cuax-card" href="{esc(_link(sc["id"]))}" data-sid="{esc(sc["id"])}" style="--i:{i}">' |
| f'<span class="blk-cuax-thumb{" tall" if h > w else ""}">{_video(g, ep, auto=False)}' |
| + (f'<em class="blk-cuax-kind">{esc(tag)}</em>' if tag else "") |
| + f'</span><b>{esc(sc["title"])}</b>' |
| f'<span class="blk-cuax-rate">{_dots(item["outcomes"])}<span>{esc(runs_line(item["stats"]))}</span></span>' |
| f'<span class="blk-cuax-by">{esc(model)}</span></a>' |
| ) |
| return (f'<h3 class="blk-section blk-cuax-gl">{esc(COPY["gallery"])}</h3>' |
| f'<nav class="blk-cuax-cards">{"".join(out)}</nav>') |
|
|
|
|
| |
|
|
|
|
| def explainer(facts: dict, thumb: str = "", answers: dict | None = None, outcome: str = "") -> str: |
| """Six nodes, told in order when they come into view: the numbered screenshot, the vision encoder, |
| the image tokens laid over the screenshot as the patches they are, the typed questions, their |
| probabilities and the action. `facts` are ui.vision_facts(); `answers` a saved run's, when there is one.""" |
| tokens = facts.get("tokens") or 0 |
| side = facts.get("side") or 32 |
| width, height = facts.get("width") or 1440, facts.get("height") or 864 |
| cols, rows = max(1, math.ceil(width / side)), max(1, math.ceil(height / side)) |
| values = dict(facts, tokens=f"{tokens:,} image tokens" if tokens else "image tokens", |
| side=side, width=width, height=height, outcome=outcome or COPY["click"].format(n="N"), |
| outcomes=COPY["how_outcomes"]) |
| probs = [] |
| if answers: |
| element = answers.get("element", {}) |
| top = sorted(element.get("probabilities", {}).items(), key=lambda kv: -kv[1])[:3] |
| probs = [(f"box {k}", v, k == element.get("choice")) for k, v in top] |
| probs += [(COPY["done_q"], answers.get("done", {}).get("noul"), False), |
| (COPY["risky_q"], answers.get("risky", {}).get("noul"), False)] |
| win_class = ' class="win"' |
| bars = "".join( |
| f'<li{win_class if win else ""} style="--p:{(p or 0):.3f};--i:{i}"><span>{esc(k)}</span>' |
| f'<i></i><b>{esc(_pct(p))}</b></li>' for i, (k, p, win) in enumerate(probs)) |
| art = [ |
| f'<span class="blk-cx-shot"><img src="{esc(thumb)}" alt=""></span>' if thumb else '<span class="blk-cx-shot"></span>', |
| '<span class="blk-cx-enc">' + "".join(f'<i style="--k:{k}"></i>' for k in range(int(facts.get("blocks") or 27))) |
| + "</span>", |
| (f'<span class="blk-cx-grid" style="--cols:{cols};--rows:{rows}">' |
| + (f'<img src="{esc(thumb)}" alt="">' if thumb else "") + '<span class="blk-cx-cells"></span>' |
| + f'<b class="blk-cx-count" data-to="{tokens}">{tokens:,}</b></span>'), |
| '<span class="blk-cx-qs">' + "".join(f'<em style="--k:{k}">{esc(q)}</em>' |
| for k, q in enumerate(COPY["how_questions"])) + "</span>", |
| f'<ul class="blk-cx-bars">{bars}</ul>', |
| f'<span class="blk-cx-act"><b>{esc(values["outcome"])}</b></span>', |
| ] |
| nodes = [] |
| for i, ((label, caption), fact, figure) in enumerate(zip(COPY["how_nodes"], COPY["how_facts"], art, strict=True)): |
| nodes.append( |
| f'<li style="--i:{i}"><span class="blk-cx-art">{figure}</span>' |
| f'<span class="blk-cx-n">{i + 1}</span><b>{esc(label)}</b>' |
| f'<span class="blk-cx-cap">{esc(caption.format(**values))}</span>' |
| f'<code>{esc(fact.format(**values))}</code></li>' |
| ) |
| return (f'<section class="blk-cx" id="cua-how"><h3 class="blk-section">{esc(COPY["how_eyebrow"])}</h3>' |
| f'<p class="blk-note">{esc(COPY["how_line"])}</p><ol class="blk-cx-flow">{"".join(nodes)}</ol></section>') |
|
|
|
|
| |
|
|
|
|
| def pipeline_path() -> str: |
| """BLINK_PIPELINE points the page at another record; the Space ships cua/pipeline.json.""" |
| return os.environ.get("BLINK_PIPELINE") or PIPELINE |
|
|
|
|
| @functools.lru_cache(maxsize=4) |
| def _load_pipeline(where: str, stamp: float): |
| with open(where, encoding="utf-8") as fh: |
| p = json.load(fh) |
| return p if p.get("schema") == PIPELINE_SCHEMA else None |
|
|
|
|
| def load_pipeline() -> dict | None: |
| """The re-rendered step build_pipeline.py checked against its run, or None.""" |
| where = pipeline_path() |
| try: |
| return _load_pipeline(where, os.path.getmtime(where)) |
| except (OSError, ValueError): |
| return None |
|
|
|
|
| def pipeline_fits(g: dict | None, p: dict | None) -> bool: |
| """Whether the record describes a scenario this gallery shows, with answers to draw.""" |
| if not g or not p: |
| return False |
| try: |
| return (scenario(g, p["scenario"]) is not None and bool(p["answers"]) and bool(p["candidates"]) |
| and all(a.get("p") and len(a["p"]) == len(p["candidates"]) for a in p["answers"].values())) |
| except (KeyError, TypeError): |
| return False |
|
|
|
|
| def _ms(v) -> str: |
| v = float(v) |
| return f"{v / 1000:.1f} s" if v >= 1000 else f"{max(1, round(v)):d} ms" |
|
|
|
|
| def _box_style(b, frame) -> str: |
| w, h = frame |
| return (f"--x:{b[0] / w * 100:.3f}%;--y:{b[1] / h * 100:.3f}%;" |
| f"--w:{(b[2] - b[0]) / w * 100:.3f}%;--h:{(b[3] - b[1]) / h * 100:.3f}%") |
|
|
|
|
| def pipeline(g: dict, p: dict, try_here: bool = True) -> str: |
| """How one step works: the recorded step in pipeline.json, from its raw screen to blink's decision. |
| |
| The first row is the browser's work (no model): the raw screen, the DOM's list of what can be clicked, |
| the numbered boxes and the request. The second is blink's: image tokens, one forward pass per question, |
| the probabilities and the decision. Then where boxes come from in every app, the exact request, and the |
| page's list. Every number comes from pipeline.json (checked against the run when it was built) or the |
| gallery; the nodes are told in order when they come into view, like explainer()'s. `try_here` says Screen |
| click follows on the page, so a screenshot of one's own gets its line.""" |
| sc = scenario(g, p["scenario"]) |
| models = [m for m in g["models"] if m in p["answers"]] or list(p["answers"]) |
| lead = g.get("lead_model") if g.get("lead_model") in p["answers"] else models[0] |
| a = p["answers"][lead] |
| frame = p["frame"] |
| width, height = frame |
| kept = len(p["candidates"]) |
| tokens = int(a.get("visual_tokens") or 0) |
| side = math.isqrt(int(a["image_pixels"]) // tokens) if tokens and a.get("image_pixels") else 32 |
| t = p["timing"] |
| state = p["request"]["state"] |
| questions = p["request"]["questions"] |
| prev = len(state.get("previous_actions") or []) |
| pick = a["pick"] |
| picks = {p["answers"][m]["pick"] for m in models} |
| act = (COPY["done"] if a.get("act") == "done" else COPY["click"].format(n=pick) if pick else COPY["none"]) |
| values = { |
| "step": p["step"], "title": sc["title"], "width": width, "height": height, "side": side, |
| "shot_ms": _ms(t["screenshot_ms"]["p50"]), "dom_ms": _ms(t["dom_ms"]["p50"]), |
| "draw_ms": _ms(t["draw_ms"]["p50"]), "kept": kept, |
| "prev": COPY["pipe_prev"][min(prev, 2)].format(n=prev), |
| "tokens": f"{tokens:,}", "input_tokens": f"{int(a.get('input_tokens') or 0):,}", |
| "agree": (COPY["pipe_agree"].format(n=len(models), pick=pick) if len(picks) == 1 and len(models) > 1 |
| else COPY["pipe_split"].format(lead=lead, pick=pick)), |
| "act": act, "lead": lead, "ms": _ms(a["ms"]), |
| "done_at": _pct(screens.DONE_AT), "risky_at": _pct(screens.RISKY_AT), |
| } |
| raw, marked = media(p, p["raw"]), media(p, p["marked"]) |
|
|
| |
| outlines = "".join(f'<i style="{_box_style(c["box"], frame)};--k:{k}"></i>' |
| for k, c in enumerate(p["candidates"])) |
| lines = [("task", f"\u201c{state.get('task', '')}\u201d"), |
| ("previous_actions", json.dumps(state.get("previous_actions") or [], ensure_ascii=False)), |
| ("screenshot_note", f"\u201c{state.get('screenshot_note', '')}\u201d")] |
| asked = " \u00b7 ".join(COPY["how_questions"]) |
| request_art = ('<span class="blk-cxp-req">' |
| + "".join(f"<span><i>{esc(k)}</i> {esc(v)}</span>" for k, v in lines) |
| + '<span><i>screenshot</i> <em>[image 1]</em></span>' |
| + f'<span><i>questions</i> {esc(asked)}</span></span>') |
| before = [ |
| (f'<a class="blk-cxp-shot" href="{esc(raw)}" target="_blank" rel="noopener">' |
| f'<img src="{esc(raw)}" alt="{esc(COPY["pipe_raw_alt"])}" loading="lazy"></a>'), |
| (f'<span class="blk-cxp-shot blk-cxp-found"><img src="{esc(raw)}" alt="" loading="lazy">' |
| f'<span class="blk-cxp-boxes">{outlines}</span><b class="blk-cxp-tally">{kept}</b></span>'), |
| (f'<a class="blk-cxp-shot" href="{esc(marked)}" target="_blank" rel="noopener">' |
| f'<img src="{esc(marked)}" alt="{esc(COPY["pipe_marked_alt"])}" loading="lazy"></a>'), |
| request_art, |
| ] |
|
|
| |
| cols, rows = max(1, math.ceil(width / side)), max(1, math.ceil(height / side)) |
| ranked = sorted(range(kept), key=lambda i: -a["p"][i])[:3] |
| probs = [(f"box {i + 1}", a["p"][i], i + 1 == pick) for i in ranked] |
| probs += [(COPY["done_q"], a.get("done"), False), (COPY["risky_q"], a.get("risky"), False)] |
| win_class = ' class="win"' |
| bars = "".join( |
| f'<li{win_class if win else ""} style="--p:{(v or 0):.3f};--i:{i}"><span>{esc(k)}</span>' |
| f'<i></i><b>{esc(_pct(v))}</b></li>' for i, (k, v, win) in enumerate(probs)) |
| inside = [ |
| (f'<span class="blk-cx-grid" style="--cols:{cols};--rows:{rows}"><img src="{esc(marked)}" alt=""' |
| f' loading="lazy"><span class="blk-cx-cells"></span>' |
| f'<b class="blk-cx-count" data-to="{tokens}" data-delay="2000">{tokens:,}</b></span>'), |
| '<span class="blk-cx-qs">' + "".join(f'<em style="--k:{k}">{esc(q)}</em>' |
| for k, q in enumerate(COPY["how_questions"])) + "</span>", |
| f'<ul class="blk-cx-bars">{bars}</ul>', |
| f'<span class="blk-cx-act"><b>{esc(act)}</b></span>', |
| ] |
|
|
| def flow(arts, first: int, cls: str) -> str: |
| items = [] |
| for i, art in enumerate(arts, first): |
| label, caption = COPY["pipe_nodes"][i] |
| items.append( |
| f'<li style="--i:{i}"><span class="blk-cx-art">{art}</span>' |
| f'<span class="blk-cx-n">{i + 1}</span><b>{esc(label)}</b>' |
| f'<span class="blk-cx-cap">{esc(caption.format(**values))}</span>' |
| f'<code>{esc(COPY["pipe_facts"][i].format(**values))}</code></li>') |
| return f'<ol class="blk-cx-flow blk-cx-four {cls}">{"".join(items)}</ol>' |
|
|
| |
| kinds = [s.get("kind") or "dom" for s in g["scenarios"]] |
| drawn = [s["title"] for s in g["scenarios"] if (s.get("kind") or "dom") != "dom"] |
| src_values = {"dom": kinds.count("dom"), "canvas": len(drawn), "names": ", ".join(drawn)} |
| counts = (src_values["dom"], len(drawn), 2 if try_here else 0) |
| sources = "".join( |
| f'<li><b>{esc(heads[n != 1].format(**src_values))}</b><span>{esc(text.format(**src_values))}</span></li>' |
| for (heads, text), n in zip(COPY["pipe_src"], counts) if n) |
|
|
| |
| body = {"model": "blink", "state": {**state, "screenshot": COPY["in_image"]}, "questions": questions} |
| request_json = json.dumps(body, indent=2, ensure_ascii=False) |
| table = "".join(f'<tr><td>{c["n"]}</td><td>{esc(c["role"])}</td><td>{esc(c["name"])}</td>' |
| f'<td>{esc(", ".join(str(v) for v in c["box"]))}</td></tr>' for c in p["candidates"]) |
| head_cells = "".join(f"<th>{esc(h)}</th>" for h in COPY["pipe_cols"]) |
| item = sc["models"].get(lead) or {} |
| opener = (f'<button type="button" class="blk-cxp-open" data-open="{esc(sc["id"])}" data-model="{esc(lead)}"' |
| f' data-step="{p["step"]}">{esc(COPY["pipe_open"])}</button>' |
| if item.get("showcase") == p["episodes"].get(lead) else "") |
| timing = COPY["pipe_time"].format(apps=t["apps"], reps=t["reps"], mark=_ms(t["mark_ms"]["p50"])) |
| return ( |
| f'<section class="blk-cx blk-cxp" id="cua-how"><div class="blk-cxp-head">' |
| f'<h3 class="blk-section">{esc(COPY["pipe_title"])}</h3>{opener}</div>' |
| f'<p class="blk-note">{esc(COPY["pipe_line"].format(**values))}</p>' |
| f'<p class="blk-eyebrow blk-cxp-row">{esc(COPY["pipe_before"])}</p>{flow(before, 0, "blk-cxp-big")}' |
| f'<p class="blk-eyebrow blk-cxp-row">{esc(COPY["pipe_inside"])}</p>{flow(inside, 4, "blk-cxp-model")}' |
| f'<div class="blk-cxp-src"><p class="blk-eyebrow">{esc(COPY["pipe_src_title"])}</p>' |
| f'<ul>{sources}</ul><p class="blk-cxp-none">{esc(COPY["pipe_none"])}</p></div>' |
| f'<p class="blk-cxp-time">{esc(timing)}</p>' |
| f'<div class="blk-cxp-folds">' |
| f'<details class="blk-d"><summary>{esc(COPY["pipe_request"])}</summary>' |
| f'<p>{esc(COPY["pipe_request_note"])}</p><div class="blk-pre">{esc(request_json)}</div></details>' |
| f'<details class="blk-d"><summary>{esc(COPY["pipe_list"])}</summary>' |
| f'<p>{esc(COPY["pipe_list_note"].format(kept=kept))}</p>' |
| f'<div class="blk-tablewrap"><table class="blk-table blk-cxp-table"><thead><tr>{head_cells}</tr></thead>' |
| f'<tbody>{table}</tbody></table></div>' |
| f'<p>{esc(COPY["pipe_check"].format(kept=kept))}</p></details></div></section>' |
| ) |
|
|
|
|
| |
|
|
|
|
| def tiles(g: dict | None) -> str: |
| items = (g or {}).get("more") or [] |
| if not items: |
| return "" |
| out = [] |
| for i, t in enumerate(items): |
| still = media(g, t.get("poster") or t.get("chart") or (t.get("images") or [None])[0]) |
| metric = t.get("metric") or {} |
| out.append( |
| f'<button type="button" class="blk-cuax-tile" data-tile="{esc(t["id"])}" style="--i:{i}">' |
| + (f'<span class="blk-cuax-still"><img src="{esc(still)}" alt="" loading="lazy">' |
| + ('<span class="blk-cuax-play" aria-hidden="true"></span>' if t.get("video") else "") |
| + "</span>" if still else "") |
| + f'<b>{esc(t["title"])}</b>' |
| + (f'<span class="blk-cuax-metric"><strong>{esc(metric.get("value", ""))}</strong>' |
| f'{esc(metric.get("label", ""))}</span>' if metric else "") |
| + "</button>" |
| ) |
| data = esc(json.dumps({"more": items, "media_base": g["media_base"], "copy": {"sound": COPY["sound"]}}, |
| separators=(",", ":"), ensure_ascii=False)) |
| return (f'<section class="blk-cuax-more" id="cua-more" data-t="{data}">' |
| f'<h3 class="blk-section">{esc(COPY["more_eyebrow"])}</h3><p class="blk-note">{esc(COPY["more_line"])}</p>' |
| f'<div class="blk-cuax-tiles">{"".join(out)}</div><div class="blk-cuax-open" hidden></div></section>') |
|
|
|
|
| |
|
|
|
|
| def client_data(g: dict) -> dict: |
| """What the page carries: every run's summary and step rail. Each run's steps are fetched from beside its |
| media when the player first needs them (build_gallery.py stages them), so the page stays light.""" |
| keep = ("media_base", "mock", "models", "vision_model", "lead_model", "scenarios") |
| out = {k: g[k] for k in keep if k in g} |
| out["episodes"] = {eid: {k: v for k, v in ep.items() if k != "steps"} for eid, ep in g["episodes"].items()} |
| out["rule_cut"] = {"done_at": screens.DONE_AT, "risky_at": screens.RISKY_AT} |
| return out |
|
|
|
|
| def pad_for(width: int) -> int: |
| """How far a page mark reaches past its box: screens.style's stroke and halo.""" |
| stroke = max(2, round(width / 360)) |
| return stroke + max(1, stroke // 2) |
|
|
|
|
| def top(g: dict | None, explain: str = "") -> str: |
| """The tab above Screen click: the stage, the gallery, then how one step works.""" |
| how = (f' <button type="button" class="blk-cuax-howbtn" data-do="how">{esc(COPY["how_link"])}</button>' |
| if g is not None and 'id="cua-how"' in explain else "") |
| head = (f'<h2 class="blk-cuax-title">{esc(COPY["title"])}</h2>' |
| f'<p class="blk-note blk-cuax-pitch">{esc(COPY["pitch"])}{how}</p>') |
| if g is None: |
| return f'<div class="blk-cuax" data-g="null">{head}{explain}</div>' |
| data = client_data(g) |
| for ep in data["episodes"].values(): |
| ep.setdefault("pad", pad_for(ep["frame"][0])) |
| banner = f'<div class="blk-warn blk-mock">{esc(COPY["mock"])}</div>' if g.get("mock") else "" |
| blob = esc(json.dumps(data, separators=(",", ":"), ensure_ascii=False)) |
| copy = esc(json.dumps(COPY, separators=(",", ":"), ensure_ascii=False)) |
| return (f'<div class="blk-cuax" data-g="{blob}" data-copy="{copy}">{banner}{head}' |
| f'<div class="blk-cuax-stage" aria-live="polite">{hero(g)}</div>{stat_row(g)}{cards(g)}{explain}</div>') |
|
|
|
|
| def try_head(note: str = "") -> str: |
| """Screen click's heading, with the line under it that says what it does.""" |
| return (f'<div class="blk-cuax-try" id="cua-try"><h3 class="blk-section">{esc(COPY["try_title"])}</h3>' |
| f'{note}</div>') |
|
|
|
|
| def home_figure(g: dict) -> str: |
| """Home's figure: the first run of the strip, looping muted, with whose it is and how often it finishes.""" |
| ids = hero_order(g) |
| if not ids: |
| return "" |
| ep = g["episodes"][ids[0]] |
| sc = scenario(g, ep["scenario"]) |
| st = sc["models"][ep["model"]]["stats"] |
| src = media(g, ep.get("preview") or ep.get("video")) |
| caption = " \u00b7 ".join((sc["title"], runs_line(st), ep["model"])) |
| return (f'<figure class="blk-cua-shot"><video autoplay muted loop playsinline preload="metadata"' |
| f' poster="{esc(media(g, ep.get("poster")))}" aria-label="{esc(caption)}">' |
| f'<source src="{esc(src)}" type="video/mp4"></video><figcaption>{esc(caption)}</figcaption></figure>') |
|
|
|
|
| def more(g: dict | None) -> str: |
| return tiles(g) |
|
|
|
|
|
|
| |
| PLAYER_JS = r"""(() => { |
| // The Computer use tab's player. Reads the gallery the page was built with (data-g), plays runs, |
| // steps through them and keeps the address in step. Talks to nothing but the page it is on. |
| const CHECK = '<svg viewBox="0 0 24 24" aria-hidden="true"><path d="M5 12.5 10 17.5 19 7"/></svg>'; |
| const BACK = '<svg viewBox="0 0 24 24" aria-hidden="true"><path d="M15 5 8 12l7 7"/></svg>'; |
| const PREV = '<svg viewBox="0 0 24 24" aria-hidden="true"><path d="M15 5 8 12l7 7"/></svg>'; |
| const NEXT = '<svg viewBox="0 0 24 24" aria-hidden="true"><path d="m9 5 7 7-7 7"/></svg>'; |
| const PLAY = '<svg viewBox="0 0 24 24" aria-hidden="true"><path d="M8 5.5v13l10.5-6.5z"/></svg>'; |
| const PAUSE = '<svg viewBox="0 0 24 24" aria-hidden="true"><path d="M8 5v14M16 5v14"/></svg>'; |
| const esc = (s) => String(s == null ? '' : s).replace(/[&<>"']/g, |
| (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c])); |
| const fmt = (s, o) => String(s || '').replace(/\{(\w+)\}/g, (m, k) => (k in o ? o[k] : m)); |
| const pct = (p) => (p == null ? '\u2013' : `${Math.round(p * 100)}%`); |
| const pct1 = (p) => (p == null ? '\u2013' : `${(p * 100).toFixed(1)}%`); |
| const reduce = !!(window.matchMedia && window.matchMedia('(prefers-reduced-motion: reduce)').matches); |
| |
| const boot = (tries) => { |
| const root = element.querySelector('.blk-cuax'); |
| if (!root) { |
| if (tries < 50) setTimeout(() => boot(tries + 1), 100); |
| return; |
| } |
| if (root.__blink) return; |
| root.__blink = true; |
| start(root); |
| }; |
| |
| const start = (root) => { |
| let G = null; |
| let C = {}; |
| try { |
| G = JSON.parse(root.dataset.g || 'null'); |
| C = JSON.parse(root.dataset.copy || '{}'); |
| } catch (err) { |
| G = null; |
| } |
| |
| // how it reads a screenshot: told once, in order, when it comes into view |
| const how = root.querySelector('.blk-cx'); |
| if (how) { |
| if (!reduce) how.classList.add('arm'); |
| const count = how.querySelector('.blk-cx-count'); |
| const go = () => { |
| if (how.classList.contains('on')) return; |
| how.classList.add('on'); |
| if (!count || reduce) return; |
| const to = Number(count.dataset.to || 0); |
| if (!to) return; |
| const t0 = performance.now() + Number(count.dataset.delay || 900); |
| const tick = (t) => { |
| const k = Math.min(1, Math.max(0, (t - t0) / 1300)); |
| count.textContent = Math.round(to * (1 - Math.pow(1 - k, 3))).toLocaleString('en-US'); |
| if (k < 1) requestAnimationFrame(tick); |
| }; |
| count.textContent = '0'; |
| requestAnimationFrame(tick); |
| }; |
| if ('IntersectionObserver' in window) { |
| const seen = new IntersectionObserver((es) => { |
| if (es.some((e) => e.isIntersecting)) { |
| go(); |
| seen.disconnect(); |
| } |
| }, { threshold: 0.3 }); |
| // the first row of nodes, not the whole section: a tall section on a phone may never be 30% in view |
| seen.observe(how.querySelector('.blk-cx-flow') || how); |
| } else { |
| go(); |
| } |
| } |
| // the address the page opened with, read once: a run to open, or Screen click to bring into view |
| const memo = window.__blinkCua || (window.__blinkCua = { open: null, first: true }); |
| const raw = memo.first && typeof window.__blinkFirstUrl === 'string' ? window.__blinkFirstUrl : window.location.search; |
| const firstLoad = memo.first; |
| memo.first = false; |
| const q = new URLSearchParams(raw.split('#')[0].replace(/^\?/, '')); |
| const toTry = firstLoad && !q.get('scenario') && (q.get('shot') || /^screen$/i.test((q.get('case') || '').trim())); |
| if (toTry) { |
| const bring = (n) => { |
| const at = document.getElementById('cua-try'); |
| if (at && at.offsetParent !== null && document.querySelector('#screen-out .blk-scr')) { |
| at.scrollIntoView({ behavior: 'auto', block: 'start' }); |
| } else if (n < 60) { |
| setTimeout(() => bring(n + 1), 150); |
| } |
| }; |
| setTimeout(() => bring(0), 300); |
| } |
| if (!G) return; |
| |
| const base = G.media_base || ''; |
| const media = (rel) => (rel ? base + encodeURI(rel) : ''); |
| const byId = {}; |
| G.scenarios.forEach((s) => { byId[s.id] = s; }); |
| const stage = root.querySelector('.blk-cuax-stage'); |
| if (!stage) return; |
| const hasHow = !!root.querySelector('#cua-how'); |
| const heroHTML = stage.innerHTML; |
| const doneAt = (G.rule_cut || {}).done_at || 0.6; |
| const riskyAt = (G.rule_cut || {}).risky_at || 0.5; |
| let cur = null; |
| let timer = null; |
| |
| // muted loops play while they are on screen, and stop when they are not |
| const io = 'IntersectionObserver' in window ? new IntersectionObserver((es) => { |
| es.forEach((en) => { |
| const v = en.target; |
| if (en.isIntersecting && en.intersectionRatio >= 0.35) { |
| if (!reduce) { |
| v.muted = true; |
| const p = v.play(); |
| if (p && p.catch) p.catch(() => {}); |
| } |
| } else if (!v.paused) { |
| v.pause(); |
| } |
| }); |
| }, { threshold: [0, 0.35, 0.7] }) : null; |
| const loops = (scope) => scope.querySelectorAll('video[data-auto="1"]').forEach((v) => { |
| v.muted = true; |
| v.preload = 'metadata'; |
| if (io) io.observe(v); |
| }); |
| loops(stage); |
| root.querySelectorAll('.blk-cuax-card').forEach((card) => { |
| const v = card.querySelector('video'); |
| if (!v) return; |
| card.addEventListener('pointerenter', (e) => { |
| if (e.pointerType !== 'mouse' || reduce) return; |
| v.muted = true; |
| v.preload = 'auto'; |
| const p = v.play(); |
| if (p && p.catch) p.catch(() => {}); |
| }); |
| card.addEventListener('pointerleave', () => v.pause()); |
| }); |
| |
| const itemOf = (st) => byId[st.sid].models[st.model]; |
| const epOf = (st) => G.episodes[itemOf(st)[st.role]]; |
| const count = (ep) => (ep.rail || []).length; |
| |
| // a run's steps sit beside its media; fetched once, when the player first needs them |
| const fetched = memo.steps || (memo.steps = {}); |
| const stepsOf = (ep) => { |
| const at = ep.steps_url; |
| if (!at) return Promise.resolve([]); |
| if (!fetched[at]) { |
| fetched[at] = fetch(media(at), { credentials: 'omit' }) |
| .then((r) => (r.ok ? r.json() : Promise.reject(new Error(String(r.status))))) |
| .catch((err) => { delete fetched[at]; throw err; }); |
| } |
| return fetched[at]; |
| }; |
| const ready = memo.ready || (memo.ready = {}); |
| |
| // the address says what is open, so it can be shared; an embedding page hears it too |
| const sync = () => { |
| const url = new URL(window.location.href); |
| ['scenario', 'model', 'run', 'step'].forEach((k) => url.searchParams.delete(k)); |
| if (cur) { |
| url.searchParams.set('tab', 'computer-use'); |
| ['case', 'shot'].forEach((k) => url.searchParams.delete(k)); |
| url.searchParams.set('scenario', cur.sid); |
| if (cur.model !== (G.lead_model || G.vision_model)) url.searchParams.set('model', cur.model); |
| if (cur.role === 'failure') url.searchParams.set('run', 'failed'); |
| if (cur.view === 'steps') url.searchParams.set('step', String(cur.step + 1)); |
| } |
| const next = url.pathname + url.search; |
| if (next !== window.location.pathname + window.location.search) { |
| window.history.replaceState(null, '', next); |
| } |
| try { |
| window.parent.postMessage({ queryString: url.search.replace(/^\?/, ''), hash: '' }, 'https://huggingface.co'); |
| } catch (err) { /* not embedded */ } |
| }; |
| |
| const meter = (label, p, on, kind) => (p == null ? '' : |
| `<div class="blk-meter${on ? ' on' : ''}" data-kind="${kind}"><span class="blk-ring" style="--sp:${p.toFixed(4)}"` |
| + ` role="img" aria-label="${esc(label)} ${pct(p)}"><b>${pct(p)}</b></span><span class="k">${esc(label)}</span></div>`); |
| |
| const verdict = (s, lane) => { |
| const n = s.pick; |
| if (s.act === 'done') return { head: C.done, mark: CHECK, act: 'done' }; |
| if (!n) return { head: C.none, mark: '', act: 'unsure' }; |
| const value = s.value == null ? '' : `\u201c${s.value}\u201d`; |
| if (s.gate) return { head: fmt(C.ask, { n }), mark: esc(n), act: 'ask' }; |
| if (s.act === 'type') return { head: fmt(C.type, { n, value }), mark: esc(n), act: 'click' }; |
| if (s.act === 'select') return { head: fmt(C.select, { n, value }), mark: esc(n), act: 'click' }; |
| return { head: fmt(lane ? C.lane : C.click, { n }), mark: esc(n), act: 'click' }; |
| }; |
| |
| const figure = (ep, s, narrow) => { |
| const [W, H] = ep.frame; |
| const pad = ep.pad || 6; |
| const share = (v, t) => `${(v / t * 100).toFixed(3)}%`; |
| const act = verdict(s).act; |
| const marks = s.boxes.map((b, i) => { |
| const n = i + 1; |
| const p = s.p[i]; |
| const win = s.pick === n && s.act !== 'done' ? ' win' : ''; |
| const want = s.oracle === n && s.pick !== n && s.act !== 'done' ? ' expect' : ''; |
| const x1 = Math.max(0, b[0] - pad); |
| const y1 = Math.max(0, b[1] - pad); |
| const x2 = Math.min(W, b[2] + pad); |
| const y2 = Math.min(H, b[3] + pad); |
| const t = s.tags[i]; |
| const shown = p != null && (!narrow || !!win); |
| const chars = String(n).length + (shown ? pct(p).length : 0); |
| return `<span class="blk-som${win}${want}" data-n="${n}" style="--sx:${share(x1, W)};--sy:${share(y1, H)};` |
| + `--sw:${share(x2 - x1, W)};--sh:${share(y2 - y1, H)};--sp:${(p || 0).toFixed(4)}"` |
| + ` title="${esc(n + (s.labels[i] ? ` \u00b7 ${s.labels[i]}` : ''))}"></span>` |
| + `<span class="blk-somtag ${esc(t[4])}${win}" data-n="${n}" style="--tx:${share(t[0], W)};` |
| + `--ty:${share(t[1], H)};--tw:${share(t[2] - t[0], W)};--th:${share(t[3] - t[1], H)};` |
| + `--tr:${((t[3] - t[1]) / W).toFixed(5)};--tc:${chars}${shown ? ';--tg:4px' : ''}">` |
| + `<b>${n}</b>${shown ? `<i>${pct(p)}</i>` : ''}</span>`; |
| }).join(''); |
| return `<figure class="blk-shot answered" data-act="${act}"><div class="blk-shot-frame"` |
| + ` style="aspect-ratio:${W} / ${H}"><img src="${esc(media(s.img))}" alt="" draggable="false">` |
| + `<div class="blk-marks">${marks}</div></div></figure>`; |
| }; |
| |
| const stepSide = (ep, steps, i) => { |
| const s = steps[i]; |
| const v = verdict(s, (byId[ep.scenario] || {}).kind === 'game'); |
| const label = s.pick ? (s.labels[s.pick - 1] || '') : ''; |
| const top = s.p.map((p, k) => [k + 1, p]).filter((x) => x[1] != null) |
| .sort((a, b) => b[1] - a[1]).slice(0, 5); |
| const bars = top.map(([k, p], j) => { |
| const name = s.labels[k - 1] || ''; |
| return `<li class="blk-opt${k === s.pick && s.act !== 'done' ? ' win' : ''}" style="--i:${j}">` |
| + `<span class="k" title="${esc(name)}">${k}${name ? ` \u00b7 ${esc(name)}` : ''}</span>` |
| + `<span class="track"><i style="width:${Math.max(p * 100, 0.6).toFixed(1)}%"></i></span>` |
| + `<span class="v">${pct1(p)}</span></li>`; |
| }).join(''); |
| const miss = s.oracle && s.pick && s.oracle !== s.pick && s.act !== 'done' |
| ? `<p class="blk-cuax-miss">${esc(fmt(C.expected, { n: s.oracle }))}</p>` : ''; |
| return `<p class="blk-cuax-stepn"><b>${esc(fmt(C.step_of, { n: i + 1, total: steps.length }))}</b>` |
| + `<span>${esc(fmt(C.ms, { ms: Math.round(s.ms).toLocaleString('en-US') }))}</span></p>` |
| + `<div class="blk-sv" data-act="${v.act}"><span class="blk-sv-mark" aria-hidden="true">${v.mark}</span>` |
| + `<div class="blk-sv-text"><h3>${esc(v.head)}</h3>${label ? `<p>${esc(label)}</p>` : ''}</div></div>` |
| + miss + (bars ? `<ul class="blk-opts">${bars}</ul>` : '') |
| + `<div class="blk-meters">${meter(C.done_q, s.done, s.done != null && s.done >= doneAt, 'done')}` |
| + `${meter(C.risky_q, s.risky, s.risky != null && s.risky >= riskyAt, 'risky')}</div>` |
| + got(ep, s); |
| }; |
| |
| // what blink was sent at this step, under what it answered: the numbered screenshot, the task above, its own |
| // earlier actions and the questions, word for word from the run (build_gallery.step_view). Older galleries |
| // carry no questions, and then nothing is said rather than something guessed. |
| const optionsOf = (q) => { |
| const said = Object.values((q && q.criteria) || {}); |
| if (q && q.type === 'noul') return C.in_yesno; |
| if (said.length && said.every((d, j) => d === `box ${j + 1}`)) return fmt(C.in_boxes, { n: said.length }); |
| return fmt(C.in_options, { n: said.length }); |
| }; |
| const got = (ep, s) => { |
| const qs = s.questions || {}; |
| const keys = Object.keys(qs); |
| if (!keys.length) return ''; |
| const kind = (byId[ep.scenario] || {}).kind; |
| const prev = Array.isArray(s.previous_actions) ? s.previous_actions : []; |
| const recent = prev.slice(-3); |
| const main = keys.filter((k) => k !== 'value'); |
| const ask = (k) => `<li>${esc(qs[k].instructions || '')} <em>${esc(optionsOf(qs[k]))}</em></li>`; |
| // a field's own question: its options are inputs too, so they are listed, the one blink picked marked |
| const follow = () => { |
| const said = Object.values(qs.value.criteria || {}); |
| const ps = Array.isArray(s.value_p) ? s.value_p : []; |
| const at = said.indexOf(s.value); |
| const mark = (d, j) => `<b>${esc(d)}${ps[j] != null ? ` ${esc(pct1(ps[j]))}` : ''}</b>`; |
| const opts = said.length <= 8 ? said.map((d, j) => (j === at ? mark(d, j) : esc(d))).join(' \u00b7 ') |
| : esc(fmt(C.in_options, { n: said.length })) + (at >= 0 ? ` \u00b7 ${esc(C.in_picked)} ${mark(said[at], at)}` : ''); |
| return `<li>${esc(qs.value.instructions || '')}<span class="opts">${opts}</span></li>`; |
| }; |
| const state = { task: ep.task, previous_actions: prev, screenshot_note: s.screenshot_note || '', screenshot: C.in_image }; |
| const only = (list) => Object.fromEntries(list.map((k) => [k, qs[k]])); |
| const body = (list) => `<pre class="blk-pre">${esc(JSON.stringify({ model: 'blink', state, questions: only(list) }, null, 2))}</pre>`; |
| const earlier = prev.length |
| ? `<ol>${prev.length > recent.length ? `<li class="more">${esc(fmt(C.in_more, { n: prev.length - recent.length }))}</li>` : ''}` |
| + `${recent.map((x) => `<li>${esc(x)}</li>`).join('')}</ol>` |
| : esc(C.in_none); |
| return `<details class="blk-cuax-got"${memo.got === false ? '' : ' open'}><summary>${esc(C.in_title)}</summary>` |
| + `<dl><dt>${esc(C.in_shot)}</dt><dd>${esc(fmt(kind === 'dom' ? C.in_dom : C.in_canvas, { n: s.boxes.length }))}</dd>` |
| + `<dt>${esc(C.in_prev)}</dt><dd>${earlier}</dd>` |
| + (main.length ? `<dt>${esc(C.in_asked)}</dt><dd><ol>${main.map(ask).join('')}</ol></dd>` : '') |
| + (qs.value ? `<dt>${esc(fmt(C.in_second, { n: s.pick }))}</dt><dd><ol>${follow()}</ol></dd>` : '') |
| + '</dl>' |
| + (s.input_tokens ? `<p class="blk-cuax-gotn">${esc(fmt(C.in_tokens, { n: Number(s.input_tokens).toLocaleString('en-US') }))}</p>` : '') |
| + `<p class="blk-cuax-gotn">${esc(C.in_names)}</p>` |
| + `<details class="blk-cuax-json"${memo.json ? ' open' : ''}><summary>${esc(C.in_json)}</summary>${body(main)}` |
| + (qs.value ? `<p class="blk-cuax-gotn">${esc(C.in_json_second)}</p>${body(['value'])}` : '') + '</details>' |
| + (hasHow ? `<button type="button" class="blk-cuax-howbtn" data-do="how">${esc(C.how_link)}</button>` : '') |
| + '</details>'; |
| }; |
| |
| const dots = (item) => `<span class="blk-cuax-dots" aria-hidden="true">${item.outcomes |
| .map((o) => `<i class="${o.ok ? 'ok' : 'no'}"></i>`).join('')}</span>`; |
| |
| const summary = (sc, item, ep) => { |
| const st = item.stats; |
| const models = G.models.filter((m) => sc.models[m]); |
| const facts = [ep.steps_taken === 1 ? C.steps_taken_one : fmt(C.steps_taken, { n: ep.steps_taken })]; |
| if (ep.gates) facts.push(fmt(C.asked, { n: ep.gates })); |
| if (ep.p50_ms) facts.push(fmt(C.per_step, { ms: Math.round(ep.p50_ms).toLocaleString('en-US') })); |
| const lo = st.wilson95 ? Math.round(st.wilson95[0] * 100) : null; |
| const hi = st.wilson95 ? Math.round(st.wilson95[1] * 100) : null; |
| // the other run on offer, said as what it is: finished, failed, or simply another |
| const alt = G.episodes[item[cur.role === 'failure' ? 'showcase' : 'failure']] || {}; |
| const other = cur.role === 'failure' ? (alt.success ? C.show_best : C.show_other) |
| : (ep.success ? C.show_failed : C.show_other); |
| const compare = models.length > 1 ? `<p class="blk-eyebrow">${esc(C.models)}</p><ul class="blk-cuax-cmp">${models |
| .map((m) => { |
| const ms = sc.models[m].stats; |
| return `<li class="${m === cur.model ? 'on' : ''}"><button type="button" data-do="model" data-model="${esc(m)}">` |
| + `<span>${esc(m)}</span><i style="--p:${(ms.rate || 0).toFixed(3)}"></i>` |
| + `<b>${esc(fmt(C.runs, { ok: ms.successes, n: ms.episodes }))}</b></button></li>`; |
| }).join('')}</ul>` : ''; |
| return `<p class="blk-cuax-outcome ${ep.success ? 'ok' : 'no'}"><span>${esc(ep.success ? C.finished : C.failed)}</span>` |
| + `${!ep.success && ep.detail ? `<em>${esc(ep.detail)}</em>` : ''}</p>` |
| + `<p class="blk-cuax-facts">${facts.map((f) => `<span>${esc(f)}</span>`).join('')}</p>` |
| + `<div class="blk-cuax-runs">${dots(item)}<b>${esc(fmt(C.runs, { ok: st.successes, n: st.episodes }))}</b>` |
| + (lo != null ? `<span>${esc(fmt(C.interval, { lo, hi }))}</span>` : '') + '</div>' |
| + (st.step_accuracy != null ? `<p class="blk-cuax-acc">${esc(fmt(C.matched, { pct: Math.round(st.step_accuracy * 100) }))}</p>` : '') |
| + (item.failure ? `<button type="button" class="blk-cuax-alt" data-do="run">${esc(other)}</button>` : '') |
| + compare |
| + (hasHow ? `<button type="button" class="blk-cuax-howbtn" data-do="how">${esc(C.how_link)}</button>` : ''); |
| }; |
| |
| const rail = (ep) => (ep.rail || []).map((s, i) => { |
| const cls = [s.a === 'done' ? 'done' : '', s.g ? 'gate' : '', s.m ? 'miss' : '', |
| cur.view === 'steps' && i === cur.step ? 'on' : ''].filter(Boolean).join(' '); |
| return `<li><button type="button" data-do="step" data-i="${i}" class="${cls}"` |
| + ` aria-label="${esc(fmt(C.step_of, { n: i + 1, total: count(ep) }))}">${i + 1}</button></li>`; |
| }).join(''); |
| |
| const paint = () => { |
| const player = stage.querySelector('.blk-cuax-player'); |
| if (!player || !cur) return; |
| const sc = byId[cur.sid]; |
| const item = itemOf(cur); |
| const ep = epOf(cur); |
| player.dataset.view = cur.view; |
| const screen = player.querySelector('.blk-cuax-screen'); |
| const side = player.querySelector('.blk-cuax-side'); |
| if (cur.view === 'video' && ep.video) { |
| const [W, H] = ep.frame; |
| if (!screen.querySelector('video')) { |
| screen.innerHTML = `<div class="blk-cuax-vid" style="aspect-ratio:${W} / ${H}"><video controls playsinline muted` |
| + ` preload="auto" poster="${esc(media(ep.poster))}"><source src="${esc(media(ep.video))}" type="video/mp4">` |
| + '</video></div>'; |
| const v = screen.querySelector('video'); |
| v.muted = true; |
| if (!reduce) { |
| const p = v.play(); |
| if (p && p.catch) p.catch(() => {}); |
| } |
| } |
| side.innerHTML = summary(sc, item, ep); |
| } else if (count(ep)) { |
| const steps = ready[ep.steps_url]; |
| if (steps) { |
| screen.innerHTML = figure(ep, steps[cur.step], screen.clientWidth < 520); |
| side.innerHTML = stepSide(ep, steps, cur.step); |
| [cur.step - 1, cur.step + 1].forEach((k) => { |
| const s = steps[k]; |
| if (s && s.img) (new Image()).src = media(s.img); |
| }); |
| } else { |
| const [W, H] = ep.frame; |
| screen.innerHTML = `<div class="blk-cuax-vid wait" style="aspect-ratio:${W} / ${H}">` |
| + (ep.poster ? `<img src="${esc(media(ep.poster))}" alt="">` : '') + '</div>'; |
| side.innerHTML = '<p class="blk-cuax-load" aria-busy="true"><i></i><i></i><i></i></p>'; |
| const want = cur; |
| stepsOf(ep).then((got) => { |
| ready[ep.steps_url] = got; |
| if (cur === want && got.length) paint(); |
| }).catch(() => { |
| if (cur === want) side.innerHTML = `<p class="blk-note">${esc(C.no_steps)}</p>`; |
| }); |
| } |
| } |
| player.querySelectorAll('.blk-cuax-rail button').forEach((b) => { |
| b.classList.toggle('on', cur.view === 'steps' && Number(b.dataset.i) === cur.step); |
| }); |
| player.querySelectorAll('[data-do="view"]').forEach((b) => { |
| b.classList.toggle('on', b.dataset.view === cur.view); |
| b.setAttribute('aria-pressed', String(b.dataset.view === cur.view)); |
| }); |
| const at = player.querySelector('.blk-cuax-rail .on'); |
| if (at && at.scrollIntoView) at.scrollIntoView({ block: 'nearest', inline: 'nearest' }); |
| }; |
| |
| const render = () => { |
| const sc = byId[cur.sid]; |
| const ep = epOf(cur); |
| const [W, H] = ep.frame; |
| const tall = H > W; |
| stage.innerHTML = `<div class="blk-cuax-player${tall ? ' tall' : ''}" data-view="${cur.view}" tabindex="-1">` |
| + '<div class="blk-cuax-bar">' |
| + `<button type="button" class="blk-cuax-x" data-do="close">${BACK}<span>${esc(C.close)}</span></button>` |
| + `<h3 class="blk-cuax-name">${esc(sc.title)}</h3>` |
| + `<span class="blk-cuax-who">${esc(cur.model)}${cur.role === 'failure' ? ` \u00b7 ${esc(fmt(C.seed, { seed: ep.seed }))}` : ''}</span>` |
| + '</div>' |
| + `<p class="blk-cuax-task"><span>${esc(C.task)}</span>${esc(ep.task)}</p>` |
| + '<div class="blk-cuax-body"><div class="blk-cuax-screen"></div><aside class="blk-cuax-side"></aside>' |
| + '<div class="blk-cuax-nav">' |
| + `<div class="blk-cuax-views" role="group">${ep.video ? `<button type="button" data-do="view" data-view="video">${esc(C.video)}</button>` : ''}` |
| + `<button type="button" data-do="view" data-view="steps">${esc(C.steps)}</button></div>` |
| + '<div class="blk-cuax-walk">' |
| + `<button type="button" class="blk-cuax-step blk-cuax-autoplay" data-do="play" aria-pressed="false"` |
| + ` aria-label="${esc(C.play)}">${PLAY}</button>` |
| + `<button type="button" class="blk-cuax-step" data-do="prev" aria-label="${esc(C.prev)}">${PREV}</button>` |
| + `<ol class="blk-cuax-rail">${rail(ep)}</ol>` |
| + `<button type="button" class="blk-cuax-step" data-do="next" aria-label="${esc(C.next)}">${NEXT}</button>` |
| + '</div></div></div></div>'; |
| paint(); |
| }; |
| |
| const mark = () => root.querySelectorAll('.blk-cuax-card').forEach((c) => { |
| c.classList.toggle('on', !!cur && c.dataset.sid === cur.sid); |
| }); |
| |
| const open = (o, scroll) => { |
| const sc = byId[o.sid]; |
| if (!sc) return; |
| halt(); |
| const models = G.models.filter((m) => sc.models[m]); |
| const lead = G.lead_model || G.vision_model; |
| const model = models.includes(o.model) ? o.model : (models.includes(lead) ? lead : models[0]); |
| const item = sc.models[model]; |
| const role = o.role === 'failure' && item.failure ? 'failure' : 'showcase'; |
| const ep = G.episodes[item[role]]; |
| const steps = count(ep); |
| const view = ep.video ? (o.view || 'video') : 'steps'; |
| cur = { sid: sc.id, model, role, view, step: Math.min(Math.max(o.step || 0, 0), Math.max(steps - 1, 0)) }; |
| memo.open = cur; |
| render(); |
| mark(); |
| sync(); |
| if (scroll) { |
| const top = stage.getBoundingClientRect().top; |
| if (top < 0 || top > window.innerHeight * 0.4) { |
| stage.scrollIntoView({ behavior: reduce ? 'auto' : 'smooth', block: 'start' }); |
| } |
| } |
| }; |
| |
| // the steps can play on their own, one decision at a time; anything else a visitor does stops them |
| const halt = () => { |
| if (timer) clearInterval(timer); |
| timer = null; |
| const b = stage.querySelector('.blk-cuax-autoplay'); |
| if (b) { |
| b.innerHTML = PLAY; |
| b.setAttribute('aria-pressed', 'false'); |
| b.setAttribute('aria-label', C.play); |
| } |
| }; |
| const play = () => { |
| if (timer) { |
| halt(); |
| return; |
| } |
| const ep = epOf(cur); |
| if (!count(ep)) return; |
| go(cur.view === 'steps' && cur.step < count(ep) - 1 ? cur.step : 0); |
| const b = stage.querySelector('.blk-cuax-autoplay'); |
| if (b) { |
| b.innerHTML = PAUSE; |
| b.setAttribute('aria-pressed', 'true'); |
| b.setAttribute('aria-label', C.pause); |
| } |
| timer = setInterval(() => { |
| if (!cur || cur.step >= count(epOf(cur)) - 1) halt(); |
| else go(cur.step + 1); |
| }, 1700); |
| }; |
| |
| const close = () => { |
| halt(); |
| cur = null; |
| memo.open = null; |
| stage.innerHTML = heroHTML; |
| loops(stage); |
| mark(); |
| sync(); |
| }; |
| |
| const go = (i) => { |
| const ep = epOf(cur); |
| if (!count(ep)) return; |
| cur.step = Math.min(Math.max(i, 0), count(ep) - 1); |
| cur.view = 'steps'; |
| paint(); |
| sync(); |
| }; |
| |
| root.addEventListener('click', (e) => { |
| const card = e.target.closest('[data-sid]'); |
| if (card && root.contains(card)) { |
| if (e.metaKey || e.ctrlKey || e.shiftKey || e.button) return; |
| e.preventDefault(); |
| open({ sid: card.dataset.sid }, true); |
| return; |
| } |
| const btn = e.target.closest('[data-do], [data-open]'); |
| if (btn && btn.dataset.do === 'how') { |
| // how one step works, below the gallery; the player keeps its place |
| e.preventDefault(); |
| halt(); |
| const at = root.querySelector('#cua-how'); |
| if (at) at.scrollIntoView({ behavior: reduce ? 'auto' : 'smooth', block: 'start' }); |
| return; |
| } |
| if (btn && btn.dataset.open) { |
| // the explainer's own step, opened in the player at that step |
| e.preventDefault(); |
| open({ sid: btn.dataset.open, model: btn.dataset.model || '', view: 'steps', |
| step: Math.max(Number(btn.dataset.step || 1) - 1, 0) }, true); |
| return; |
| } |
| if (!btn || !cur) return; |
| const act = btn.dataset.do; |
| if (act === 'play') { |
| play(); |
| return; |
| } |
| halt(); |
| if (act === 'close') close(); |
| else if (act === 'step') go(Number(btn.dataset.i)); |
| else if (act === 'prev') go((cur.view === 'steps' ? cur.step : 0) - 1); |
| else if (act === 'next') go(cur.view === 'steps' ? cur.step + 1 : 0); |
| else if (act === 'view') { |
| cur.view = btn.dataset.view; |
| const screen = stage.querySelector('.blk-cuax-screen'); |
| if (screen) screen.innerHTML = ''; |
| paint(); |
| sync(); |
| } else if (act === 'model') open({ sid: cur.sid, model: btn.dataset.model, view: cur.view }, false); |
| else if (act === 'run') open({ sid: cur.sid, model: cur.model, role: cur.role === 'failure' ? 'showcase' : 'failure', view: cur.view }, false); |
| }); |
| |
| // What blink got stays open or shut, and its JSON too, as the visitor leaves them from step to step |
| root.addEventListener('toggle', (e) => { |
| const d = e.target; |
| if (!d || !d.classList) return; |
| if (d.classList.contains('blk-cuax-got')) memo.got = d.open; |
| else if (d.classList.contains('blk-cuax-json')) memo.json = d.open; |
| }, true); |
| |
| root.addEventListener('keydown', (e) => { |
| if (!cur || !e.target.closest('.blk-cuax-player')) return; |
| if (['ArrowRight', 'ArrowLeft', 'Escape'].includes(e.key)) halt(); |
| if (e.key === 'ArrowRight') { e.preventDefault(); go(cur.view === 'steps' ? cur.step + 1 : 0); } |
| else if (e.key === 'ArrowLeft') { e.preventDefault(); go(cur.view === 'steps' ? cur.step - 1 : 0); } |
| else if (e.key === 'Escape') close(); |
| }); |
| |
| // a link opens its run once, on the page's first load; coming back to the tab keeps what was open |
| const sid = (q.get('scenario') || '').trim().toLowerCase(); |
| if (sid && byId[sid] && firstLoad) { |
| const step = parseInt(q.get('step') || '', 10); |
| open({ sid, model: q.get('model') || '', role: /^fail/i.test(q.get('run') || '') ? 'failure' : 'showcase', |
| view: Number.isFinite(step) ? 'steps' : '', step: Number.isFinite(step) ? step - 1 : 0 }, true); |
| } else if (memo.open && byId[memo.open.sid]) { |
| open(memo.open, false); |
| } |
| }; |
| |
| boot(0); |
| })(); |
| """ |
|
|
| |
| TILES_JS = r"""(() => { |
| // Beyond clicks: one tile open at a time, its video or chart under the row. A video with sound |
| // never starts on its own; one without plays muted. |
| const esc = (s) => String(s == null ? '' : s).replace(/[&<>"']/g, |
| (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c])); |
| const boot = (tries) => { |
| const box = element.querySelector('.blk-cuax-more'); |
| if (!box) { |
| if (tries < 50) setTimeout(() => boot(tries + 1), 100); |
| return; |
| } |
| if (box.__blink) return; |
| box.__blink = true; |
| let T = {}; |
| try { |
| T = JSON.parse(box.dataset.t || '{}'); |
| } catch (err) { |
| return; |
| } |
| const items = T.more || []; |
| const C = T.copy || {}; |
| const media = (rel) => (rel ? (T.media_base || '') + encodeURI(rel) : ''); |
| const pane = box.querySelector('.blk-cuax-open'); |
| let at = null; |
| const show = (id) => { |
| const t = items.find((x) => x.id === id); |
| const again = at === id; |
| box.querySelectorAll('.blk-cuax-tile').forEach((b) => { |
| const on = b.dataset.tile === id && !again; |
| b.classList.toggle('on', on); |
| b.setAttribute('aria-expanded', String(on)); |
| }); |
| if (!t || again) { |
| at = null; |
| pane.hidden = true; |
| pane.innerHTML = ''; |
| return; |
| } |
| at = id; |
| let fig = ''; |
| if (t.video) { |
| const track = t.captions ? `<track kind="captions" src="${esc(media(t.captions))}" srclang="en" default>` : ''; |
| const cors = t.captions ? ' crossorigin="anonymous"' : ''; |
| fig += t.sound |
| ? `<video controls playsinline preload="metadata"${cors} poster="${esc(media(t.poster))}">` |
| + `<source src="${esc(media(t.video))}" type="video/mp4">${track}</video>` |
| + `<p class="blk-cuax-sound">${esc(C.sound)}</p>` |
| : `<video controls playsinline muted loop preload="metadata"${cors} poster="${esc(media(t.poster))}">` |
| + `<source src="${esc(media(t.video))}" type="video/mp4">${track}</video>`; |
| } |
| if (t.chart) fig += `<img class="blk-cuax-chart" src="${esc(media(t.chart))}" alt="${esc(t.title)}">`; |
| if (t.images && t.images.length) { |
| fig += `<div class="blk-cuax-imgs">${t.images.map((i) => `<img src="${esc(media(i))}" alt="" loading="lazy">`).join('')}</div>`; |
| } |
| pane.innerHTML = `<div class="blk-cuax-pane"><h4>${esc(t.title)}</h4><p class="blk-note">${esc(t.line)}</p>${fig}</div>`; |
| pane.hidden = false; |
| const v = pane.querySelector('video[muted]'); |
| if (v) { |
| v.muted = true; |
| const p = v.play(); |
| if (p && p.catch) p.catch(() => {}); |
| } |
| }; |
| box.addEventListener('click', (e) => { |
| const b = e.target.closest('.blk-cuax-tile'); |
| if (b) show(b.dataset.tile); |
| }); |
| }; |
| boot(0); |
| })(); |
| """ |
|
|
