"""Text 'eyes' without any model: what a rendered page shows, read from the browser's layout (DOM) and, for drawings, from the geometry of the SVG elements. Used as feedback for the model, which cannot see its result. outline(page) -> lines like "first screen: navbar 'Golden Crust | Home About Shop'", "y 820-1100 (below the first screen): form with inputs Email, Password; checkbox 'Remember me'; button 'Sign in'". svg_outline(page) -> the largest shapes and all texts with their positions and colours, back to front. """ OUTLINE_JS = r""" () => { const H = innerHeight, out = [], seen = new Set(); const vis = (el) => { const r = el.getBoundingClientRect(), s = getComputedStyle(el); return r.width > 3 && r.height > 3 && s.visibility !== 'hidden' && s.display !== 'none' && +s.opacity > 0.05; }; const txt = (el, n = 60) => (el.innerText || el.value || el.placeholder || el.getAttribute('aria-label') || '') .replace(/\s+/g, ' ').trim().slice(0, n); const where = (r) => (r.top < H - 40 ? 'first screen' : `y ${Math.round(r.top + scrollY)} (below the first screen)`); const colour = (el) => getComputedStyle(el).backgroundColor; const sel = 'nav,header,footer,aside,h1,h2,h3,form,table,button,a[class*=btn],input,select,textarea,img,canvas,svg,' + '[class*=card],[class*=modal],[class*=hero],[class*=sidebar],details,blockquote,pre,ul,ol'; for (const el of document.querySelectorAll(sel)) { if (!vis(el)) continue; if ([...seen].some((p) => p.contains(el) && p.matches('form,table,ul,ol,nav,[class*=card],details,pre,blockquote'))) continue; const r = el.getBoundingClientRect(), tag = el.tagName.toLowerCase(), cls = (el.className && el.className.baseVal === undefined) ? String(el.className) : ''; let kind = tag, detail = txt(el); if (tag === 'form') { const parts = [...el.querySelectorAll('input,select,textarea,button')].filter(vis).map((x) => { const t = x.type || x.tagName.toLowerCase(), lab = x.labels && x.labels[0] ? x.labels[0].innerText.trim() : (x.placeholder || x.name || x.value || x.innerText || '').trim(); return `${t === 'submit' || x.tagName === 'BUTTON' ? 'button' : t} '${lab.slice(0, 30)}'`; }); detail = parts.join('; '); } else if (tag === 'table') { const rows = el.rows.length, head = [...(el.rows[0] || { cells: [] }).cells].map((c) => c.innerText.trim()).join(' | '); detail = `${rows} rows, header: ${head.slice(0, 80)}`; } else if (tag === 'ul' || tag === 'ol') { detail = `${el.children.length} items: ` + [...el.children].slice(0, 4).map((li) => li.innerText.trim().slice(0, 25)).join(' / '); } else if (/card/.test(cls)) { kind = 'card'; } else if (/modal/.test(cls)) { kind = 'modal (open, covers the page)'; } else if (/hero/.test(cls)) { kind = 'hero section'; detail = txt(el, 80); } else if (tag === 'img' || tag === 'canvas' || tag === 'svg') { detail = `${Math.round(r.width)}x${Math.round(r.height)} px`; } else if (tag === 'input') { detail = `${el.type} '${txt(el, 30)}'`; } seen.add(el); out.push(`${where(r)}: ${kind}${detail ? " '" + detail + "'" : ''} [${Math.round(r.left)},${Math.round(r.top + scrollY)} ${Math.round(r.width)}x${Math.round(r.height)}, bg ${colour(el)}]`); if (out.length >= 40) break; } const h = document.documentElement.scrollHeight; out.unshift(`page height ${h} px (the first screen is 1024x768); body background ${getComputedStyle(document.body).backgroundColor}`); return out; } """ SVG_JS = r""" () => { const svg = document.querySelector('svg'); if (!svg) return ['no element']; const vb = svg.viewBox && svg.viewBox.baseVal && svg.viewBox.baseVal.width ? svg.viewBox.baseVal : { x: 0, y: 0, width: svg.clientWidth, height: svg.clientHeight }; const out = [`viewBox ${vb.width}x${vb.height}`], items = []; svg.querySelectorAll('rect,circle,ellipse,path,polygon,polyline,line,text').forEach((el, i) => { let b; try { b = el.getBBox(); } catch (e) { return; } if (!b || (b.width < 0.5 && b.height < 0.5)) return; const s = getComputedStyle(el), fill = s.fill, stroke = s.stroke; const area = b.width * b.height, outside = b.x + b.width < vb.x || b.y + b.height < vb.y || b.x > vb.x + vb.width || b.y > vb.y + vb.height; items.push({ i, tag: el.tagName, x: Math.round(b.x), y: Math.round(b.y), w: Math.round(b.width), h: Math.round(b.height), fill, stroke, text: el.tagName === 'text' ? el.textContent.trim().slice(0, 30) : '', area, outside }); }); const big = items.filter((o) => o.tag !== 'text').sort((a, b) => b.area - a.area).slice(0, 18).sort((a, b) => a.i - b.i); for (const o of big) out.push(`${o.tag} at x ${o.x}-${o.x + o.w}, y ${o.y}-${o.y + o.h}, fill ${o.fill}${o.stroke !== 'none' ? ', stroke ' + o.stroke : ''}${o.outside ? ' (OUTSIDE the picture)' : ''}`); for (const o of items.filter((o) => o.tag === 'text')) out.push(`text '${o.text}' at x ${o.x}-${o.x + o.w}, y ${o.y}-${o.y + o.h}`); out.push(`${items.length} shapes in total (back to front in the order listed; y grows downwards)`); return out; } """ def outline(page): return page.evaluate(OUTLINE_JS) def svg_outline(page): return page.evaluate(SVG_JS) NAMES = {"black": (20, 20, 20), "dark grey": (70, 70, 70), "grey": (140, 140, 140), "light grey": (200, 200, 200), "white": (245, 245, 245), "red": (220, 40, 40), "dark red": (130, 20, 20), "orange": (240, 140, 30), "yellow": (240, 220, 40), "brown": (130, 80, 40), "green": (50, 170, 60), "dark green": (20, 90, 30), "light green": (150, 220, 120), "teal": (30, 160, 160), "light blue": (140, 200, 240), "blue": (40, 90, 220), "dark blue": (20, 30, 90), "purple": (130, 60, 190), "pink": (240, 120, 180), "beige": (235, 220, 190)} def colour_name(rgb): return min(NAMES, key=lambda k: sum((a - b) ** 2 for a, b in zip(NAMES[k], rgb))) def pixel_outline(png_bytes, png2_bytes=None): """For canvas / WebGL pages: the background colour and the coloured regions of the screenshot (connected blobs on a 64x48 grid), with position, size and colour; and which regions moved between two screenshots.""" import io import numpy as np from PIL import Image def grid(b): return np.asarray(Image.open(io.BytesIO(b)).convert("RGB").resize((64, 48), Image.BILINEAR)).astype(int) a = grid(png_bytes) flat = a.reshape(-1, 3) vals, counts = np.unique((flat // 24) * 24, axis=0, return_counts=True) bg = vals[counts.argmax()] + 12 fg = np.abs(a - bg).sum(axis=2) > 60 out = [f"background {colour_name(bg)}; {100 * fg.mean():.0f}% of the screen is covered by drawn content"] seen, blobs = np.zeros_like(fg), [] for y in range(48): for x in range(64): if fg[y, x] and not seen[y, x]: stack, cells = [(y, x)], [] seen[y, x] = True while stack: cy, cx = stack.pop() cells.append((cy, cx)) for ny, nx in ((cy + 1, cx), (cy - 1, cx), (cy, cx + 1), (cy, cx - 1)): if 0 <= ny < 48 and 0 <= nx < 64 and fg[ny, nx] and not seen[ny, nx]: seen[ny, nx] = True stack.append((ny, nx)) blobs.append(cells) blobs.sort(key=len, reverse=True) def pos(cy, cx): v = "top" if cy < 16 else "bottom" if cy >= 32 else "middle" h = "left" if cx < 21 else "right" if cx >= 43 else "centre" return "centre" if (v, h) == ("middle", "centre") else f"{v} {h}" if v != "middle" else f"middle {h}" for cells in blobs[:10]: if len(cells) < 4: break ys, xs = [c[0] for c in cells], [c[1] for c in cells] col = np.mean([a[c] for c in cells], axis=0) out.append(f"{colour_name(col)} shape at the {pos(np.mean(ys), np.mean(xs))}, about {100 * (max(xs) - min(xs) + 1) / 64:.0f}% " f"of the width x {100 * (max(ys) - min(ys) + 1) / 48:.0f}% of the height ({100 * len(cells) / 3072:.1f}% of the screen)") small = sum(1 for c in blobs if len(c) < 4) if small: out.append(f"{small} small dots or specks") if png2_bytes is not None: moved = (np.abs(grid(png2_bytes) - a).sum(axis=2) > 60).mean() out.append(f"{100 * moved:.1f}% of the screen changed in 0.7 seconds" + (" (nothing moves)" if moved < 0.002 else "")) return out