| """Text 'eyes' without any model: what a rendered page shows, read from the browser's layout (DOM) and, for |
| drawings, from the geometry of the SVG elements. Used as feedback for the model, which cannot see its result. |
| |
| outline(page) -> lines like "first screen: navbar 'Golden Crust | Home About Shop'", "y 820-1100 (below the first |
| screen): form with inputs Email, Password; checkbox 'Remember me'; button 'Sign in'". |
| svg_outline(page) -> the largest shapes and all texts with their positions and colours, back to front. |
| """ |
|
|
| OUTLINE_JS = r""" |
| () => { |
| const H = innerHeight, out = [], seen = new Set(); |
| const vis = (el) => { const r = el.getBoundingClientRect(), s = getComputedStyle(el); |
| return r.width > 3 && r.height > 3 && s.visibility !== 'hidden' && s.display !== 'none' && +s.opacity > 0.05; }; |
| const txt = (el, n = 60) => (el.innerText || el.value || el.placeholder || el.getAttribute('aria-label') || '') |
| .replace(/\s+/g, ' ').trim().slice(0, n); |
| const where = (r) => (r.top < H - 40 ? 'first screen' : `y ${Math.round(r.top + scrollY)} (below the first screen)`); |
| const colour = (el) => getComputedStyle(el).backgroundColor; |
| const sel = 'nav,header,footer,aside,h1,h2,h3,form,table,button,a[class*=btn],input,select,textarea,img,canvas,svg,' + |
| '[class*=card],[class*=modal],[class*=hero],[class*=sidebar],details,blockquote,pre,ul,ol'; |
| for (const el of document.querySelectorAll(sel)) { |
| if (!vis(el)) continue; |
| if ([...seen].some((p) => p.contains(el) && p.matches('form,table,ul,ol,nav,[class*=card],details,pre,blockquote'))) continue; |
| const r = el.getBoundingClientRect(), tag = el.tagName.toLowerCase(), cls = (el.className && el.className.baseVal === undefined) ? String(el.className) : ''; |
| let kind = tag, detail = txt(el); |
| if (tag === 'form') { |
| const parts = [...el.querySelectorAll('input,select,textarea,button')].filter(vis).map((x) => { |
| const t = x.type || x.tagName.toLowerCase(), lab = x.labels && x.labels[0] ? x.labels[0].innerText.trim() : (x.placeholder || x.name || x.value || x.innerText || '').trim(); |
| return `${t === 'submit' || x.tagName === 'BUTTON' ? 'button' : t} '${lab.slice(0, 30)}'`; }); |
| detail = parts.join('; '); |
| } else if (tag === 'table') { |
| const rows = el.rows.length, head = [...(el.rows[0] || { cells: [] }).cells].map((c) => c.innerText.trim()).join(' | '); |
| detail = `${rows} rows, header: ${head.slice(0, 80)}`; |
| } else if (tag === 'ul' || tag === 'ol') { |
| detail = `${el.children.length} items: ` + [...el.children].slice(0, 4).map((li) => li.innerText.trim().slice(0, 25)).join(' / '); |
| } else if (/card/.test(cls)) { kind = 'card'; } |
| else if (/modal/.test(cls)) { kind = 'modal (open, covers the page)'; } |
| else if (/hero/.test(cls)) { kind = 'hero section'; detail = txt(el, 80); } |
| else if (tag === 'img' || tag === 'canvas' || tag === 'svg') { detail = `${Math.round(r.width)}x${Math.round(r.height)} px`; } |
| else if (tag === 'input') { detail = `${el.type} '${txt(el, 30)}'`; } |
| seen.add(el); |
| out.push(`${where(r)}: ${kind}${detail ? " '" + detail + "'" : ''} [${Math.round(r.left)},${Math.round(r.top + scrollY)} ${Math.round(r.width)}x${Math.round(r.height)}, bg ${colour(el)}]`); |
| if (out.length >= 40) break; |
| } |
| const h = document.documentElement.scrollHeight; |
| out.unshift(`page height ${h} px (the first screen is 1024x768); body background ${getComputedStyle(document.body).backgroundColor}`); |
| return out; |
| } |
| """ |
|
|
| SVG_JS = r""" |
| () => { |
| const svg = document.querySelector('svg'); if (!svg) return ['no <svg> element']; |
| const vb = svg.viewBox && svg.viewBox.baseVal && svg.viewBox.baseVal.width ? svg.viewBox.baseVal : { x: 0, y: 0, width: svg.clientWidth, height: svg.clientHeight }; |
| const out = [`viewBox ${vb.width}x${vb.height}`], items = []; |
| svg.querySelectorAll('rect,circle,ellipse,path,polygon,polyline,line,text').forEach((el, i) => { |
| let b; try { b = el.getBBox(); } catch (e) { return; } |
| if (!b || (b.width < 0.5 && b.height < 0.5)) return; |
| const s = getComputedStyle(el), fill = s.fill, stroke = s.stroke; |
| const area = b.width * b.height, outside = b.x + b.width < vb.x || b.y + b.height < vb.y || b.x > vb.x + vb.width || b.y > vb.y + vb.height; |
| items.push({ i, tag: el.tagName, x: Math.round(b.x), y: Math.round(b.y), w: Math.round(b.width), h: Math.round(b.height), |
| fill, stroke, text: el.tagName === 'text' ? el.textContent.trim().slice(0, 30) : '', area, outside }); |
| }); |
| const big = items.filter((o) => o.tag !== 'text').sort((a, b) => b.area - a.area).slice(0, 18).sort((a, b) => a.i - b.i); |
| for (const o of big) out.push(`${o.tag} at x ${o.x}-${o.x + o.w}, y ${o.y}-${o.y + o.h}, fill ${o.fill}${o.stroke !== 'none' ? ', stroke ' + o.stroke : ''}${o.outside ? ' (OUTSIDE the picture)' : ''}`); |
| for (const o of items.filter((o) => o.tag === 'text')) out.push(`text '${o.text}' at x ${o.x}-${o.x + o.w}, y ${o.y}-${o.y + o.h}`); |
| out.push(`${items.length} shapes in total (back to front in the order listed; y grows downwards)`); |
| return out; |
| } |
| """ |
|
|
|
|
| def outline(page): |
| return page.evaluate(OUTLINE_JS) |
|
|
|
|
| def svg_outline(page): |
| return page.evaluate(SVG_JS) |
|
|
|
|
| NAMES = {"black": (20, 20, 20), "dark grey": (70, 70, 70), "grey": (140, 140, 140), "light grey": (200, 200, 200), |
| "white": (245, 245, 245), "red": (220, 40, 40), "dark red": (130, 20, 20), "orange": (240, 140, 30), |
| "yellow": (240, 220, 40), "brown": (130, 80, 40), "green": (50, 170, 60), "dark green": (20, 90, 30), |
| "light green": (150, 220, 120), "teal": (30, 160, 160), "light blue": (140, 200, 240), "blue": (40, 90, 220), |
| "dark blue": (20, 30, 90), "purple": (130, 60, 190), "pink": (240, 120, 180), "beige": (235, 220, 190)} |
|
|
|
|
| def colour_name(rgb): |
| return min(NAMES, key=lambda k: sum((a - b) ** 2 for a, b in zip(NAMES[k], rgb))) |
|
|
|
|
| def pixel_outline(png_bytes, png2_bytes=None): |
| """For canvas / WebGL pages: the background colour and the coloured regions of the screenshot (connected blobs on |
| a 64x48 grid), with position, size and colour; and which regions moved between two screenshots.""" |
| import io |
|
|
| import numpy as np |
| from PIL import Image |
|
|
| def grid(b): |
| return np.asarray(Image.open(io.BytesIO(b)).convert("RGB").resize((64, 48), Image.BILINEAR)).astype(int) |
|
|
| a = grid(png_bytes) |
| flat = a.reshape(-1, 3) |
| vals, counts = np.unique((flat // 24) * 24, axis=0, return_counts=True) |
| bg = vals[counts.argmax()] + 12 |
| fg = np.abs(a - bg).sum(axis=2) > 60 |
| out = [f"background {colour_name(bg)}; {100 * fg.mean():.0f}% of the screen is covered by drawn content"] |
| seen, blobs = np.zeros_like(fg), [] |
| for y in range(48): |
| for x in range(64): |
| if fg[y, x] and not seen[y, x]: |
| stack, cells = [(y, x)], [] |
| seen[y, x] = True |
| while stack: |
| cy, cx = stack.pop() |
| cells.append((cy, cx)) |
| for ny, nx in ((cy + 1, cx), (cy - 1, cx), (cy, cx + 1), (cy, cx - 1)): |
| if 0 <= ny < 48 and 0 <= nx < 64 and fg[ny, nx] and not seen[ny, nx]: |
| seen[ny, nx] = True |
| stack.append((ny, nx)) |
| blobs.append(cells) |
| blobs.sort(key=len, reverse=True) |
|
|
| def pos(cy, cx): |
| v = "top" if cy < 16 else "bottom" if cy >= 32 else "middle" |
| h = "left" if cx < 21 else "right" if cx >= 43 else "centre" |
| return "centre" if (v, h) == ("middle", "centre") else f"{v} {h}" if v != "middle" else f"middle {h}" |
|
|
| for cells in blobs[:10]: |
| if len(cells) < 4: |
| break |
| ys, xs = [c[0] for c in cells], [c[1] for c in cells] |
| col = np.mean([a[c] for c in cells], axis=0) |
| out.append(f"{colour_name(col)} shape at the {pos(np.mean(ys), np.mean(xs))}, about {100 * (max(xs) - min(xs) + 1) / 64:.0f}% " |
| f"of the width x {100 * (max(ys) - min(ys) + 1) / 48:.0f}% of the height ({100 * len(cells) / 3072:.1f}% of the screen)") |
| small = sum(1 for c in blobs if len(c) < 4) |
| if small: |
| out.append(f"{small} small dots or specks") |
| if png2_bytes is not None: |
| moved = (np.abs(grid(png2_bytes) - a).sum(axis=2) > 60).mean() |
| out.append(f"{100 * moved:.1f}% of the screen changed in 0.7 seconds" + (" (nothing moves)" if moved < 0.002 else "")) |
| return out |
|
|
