File size: 8,461 Bytes
67ebcef | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 | """Text 'eyes' without any model: what a rendered page shows, read from the browser's layout (DOM) and, for
drawings, from the geometry of the SVG elements. Used as feedback for the model, which cannot see its result.
outline(page) -> lines like "first screen: navbar 'Golden Crust | Home About Shop'", "y 820-1100 (below the first
screen): form with inputs Email, Password; checkbox 'Remember me'; button 'Sign in'".
svg_outline(page) -> the largest shapes and all texts with their positions and colours, back to front.
"""
OUTLINE_JS = r"""
() => {
const H = innerHeight, out = [], seen = new Set();
const vis = (el) => { const r = el.getBoundingClientRect(), s = getComputedStyle(el);
return r.width > 3 && r.height > 3 && s.visibility !== 'hidden' && s.display !== 'none' && +s.opacity > 0.05; };
const txt = (el, n = 60) => (el.innerText || el.value || el.placeholder || el.getAttribute('aria-label') || '')
.replace(/\s+/g, ' ').trim().slice(0, n);
const where = (r) => (r.top < H - 40 ? 'first screen' : `y ${Math.round(r.top + scrollY)} (below the first screen)`);
const colour = (el) => getComputedStyle(el).backgroundColor;
const sel = 'nav,header,footer,aside,h1,h2,h3,form,table,button,a[class*=btn],input,select,textarea,img,canvas,svg,' +
'[class*=card],[class*=modal],[class*=hero],[class*=sidebar],details,blockquote,pre,ul,ol';
for (const el of document.querySelectorAll(sel)) {
if (!vis(el)) continue;
if ([...seen].some((p) => p.contains(el) && p.matches('form,table,ul,ol,nav,[class*=card],details,pre,blockquote'))) continue;
const r = el.getBoundingClientRect(), tag = el.tagName.toLowerCase(), cls = (el.className && el.className.baseVal === undefined) ? String(el.className) : '';
let kind = tag, detail = txt(el);
if (tag === 'form') {
const parts = [...el.querySelectorAll('input,select,textarea,button')].filter(vis).map((x) => {
const t = x.type || x.tagName.toLowerCase(), lab = x.labels && x.labels[0] ? x.labels[0].innerText.trim() : (x.placeholder || x.name || x.value || x.innerText || '').trim();
return `${t === 'submit' || x.tagName === 'BUTTON' ? 'button' : t} '${lab.slice(0, 30)}'`; });
detail = parts.join('; ');
} else if (tag === 'table') {
const rows = el.rows.length, head = [...(el.rows[0] || { cells: [] }).cells].map((c) => c.innerText.trim()).join(' | ');
detail = `${rows} rows, header: ${head.slice(0, 80)}`;
} else if (tag === 'ul' || tag === 'ol') {
detail = `${el.children.length} items: ` + [...el.children].slice(0, 4).map((li) => li.innerText.trim().slice(0, 25)).join(' / ');
} else if (/card/.test(cls)) { kind = 'card'; }
else if (/modal/.test(cls)) { kind = 'modal (open, covers the page)'; }
else if (/hero/.test(cls)) { kind = 'hero section'; detail = txt(el, 80); }
else if (tag === 'img' || tag === 'canvas' || tag === 'svg') { detail = `${Math.round(r.width)}x${Math.round(r.height)} px`; }
else if (tag === 'input') { detail = `${el.type} '${txt(el, 30)}'`; }
seen.add(el);
out.push(`${where(r)}: ${kind}${detail ? " '" + detail + "'" : ''} [${Math.round(r.left)},${Math.round(r.top + scrollY)} ${Math.round(r.width)}x${Math.round(r.height)}, bg ${colour(el)}]`);
if (out.length >= 40) break;
}
const h = document.documentElement.scrollHeight;
out.unshift(`page height ${h} px (the first screen is 1024x768); body background ${getComputedStyle(document.body).backgroundColor}`);
return out;
}
"""
SVG_JS = r"""
() => {
const svg = document.querySelector('svg'); if (!svg) return ['no <svg> element'];
const vb = svg.viewBox && svg.viewBox.baseVal && svg.viewBox.baseVal.width ? svg.viewBox.baseVal : { x: 0, y: 0, width: svg.clientWidth, height: svg.clientHeight };
const out = [`viewBox ${vb.width}x${vb.height}`], items = [];
svg.querySelectorAll('rect,circle,ellipse,path,polygon,polyline,line,text').forEach((el, i) => {
let b; try { b = el.getBBox(); } catch (e) { return; }
if (!b || (b.width < 0.5 && b.height < 0.5)) return;
const s = getComputedStyle(el), fill = s.fill, stroke = s.stroke;
const area = b.width * b.height, outside = b.x + b.width < vb.x || b.y + b.height < vb.y || b.x > vb.x + vb.width || b.y > vb.y + vb.height;
items.push({ i, tag: el.tagName, x: Math.round(b.x), y: Math.round(b.y), w: Math.round(b.width), h: Math.round(b.height),
fill, stroke, text: el.tagName === 'text' ? el.textContent.trim().slice(0, 30) : '', area, outside });
});
const big = items.filter((o) => o.tag !== 'text').sort((a, b) => b.area - a.area).slice(0, 18).sort((a, b) => a.i - b.i);
for (const o of big) out.push(`${o.tag} at x ${o.x}-${o.x + o.w}, y ${o.y}-${o.y + o.h}, fill ${o.fill}${o.stroke !== 'none' ? ', stroke ' + o.stroke : ''}${o.outside ? ' (OUTSIDE the picture)' : ''}`);
for (const o of items.filter((o) => o.tag === 'text')) out.push(`text '${o.text}' at x ${o.x}-${o.x + o.w}, y ${o.y}-${o.y + o.h}`);
out.push(`${items.length} shapes in total (back to front in the order listed; y grows downwards)`);
return out;
}
"""
def outline(page):
return page.evaluate(OUTLINE_JS)
def svg_outline(page):
return page.evaluate(SVG_JS)
NAMES = {"black": (20, 20, 20), "dark grey": (70, 70, 70), "grey": (140, 140, 140), "light grey": (200, 200, 200),
"white": (245, 245, 245), "red": (220, 40, 40), "dark red": (130, 20, 20), "orange": (240, 140, 30),
"yellow": (240, 220, 40), "brown": (130, 80, 40), "green": (50, 170, 60), "dark green": (20, 90, 30),
"light green": (150, 220, 120), "teal": (30, 160, 160), "light blue": (140, 200, 240), "blue": (40, 90, 220),
"dark blue": (20, 30, 90), "purple": (130, 60, 190), "pink": (240, 120, 180), "beige": (235, 220, 190)}
def colour_name(rgb):
return min(NAMES, key=lambda k: sum((a - b) ** 2 for a, b in zip(NAMES[k], rgb)))
def pixel_outline(png_bytes, png2_bytes=None):
"""For canvas / WebGL pages: the background colour and the coloured regions of the screenshot (connected blobs on
a 64x48 grid), with position, size and colour; and which regions moved between two screenshots."""
import io
import numpy as np
from PIL import Image
def grid(b):
return np.asarray(Image.open(io.BytesIO(b)).convert("RGB").resize((64, 48), Image.BILINEAR)).astype(int)
a = grid(png_bytes)
flat = a.reshape(-1, 3)
vals, counts = np.unique((flat // 24) * 24, axis=0, return_counts=True)
bg = vals[counts.argmax()] + 12
fg = np.abs(a - bg).sum(axis=2) > 60
out = [f"background {colour_name(bg)}; {100 * fg.mean():.0f}% of the screen is covered by drawn content"]
seen, blobs = np.zeros_like(fg), []
for y in range(48):
for x in range(64):
if fg[y, x] and not seen[y, x]:
stack, cells = [(y, x)], []
seen[y, x] = True
while stack:
cy, cx = stack.pop()
cells.append((cy, cx))
for ny, nx in ((cy + 1, cx), (cy - 1, cx), (cy, cx + 1), (cy, cx - 1)):
if 0 <= ny < 48 and 0 <= nx < 64 and fg[ny, nx] and not seen[ny, nx]:
seen[ny, nx] = True
stack.append((ny, nx))
blobs.append(cells)
blobs.sort(key=len, reverse=True)
def pos(cy, cx):
v = "top" if cy < 16 else "bottom" if cy >= 32 else "middle"
h = "left" if cx < 21 else "right" if cx >= 43 else "centre"
return "centre" if (v, h) == ("middle", "centre") else f"{v} {h}" if v != "middle" else f"middle {h}"
for cells in blobs[:10]:
if len(cells) < 4:
break
ys, xs = [c[0] for c in cells], [c[1] for c in cells]
col = np.mean([a[c] for c in cells], axis=0)
out.append(f"{colour_name(col)} shape at the {pos(np.mean(ys), np.mean(xs))}, about {100 * (max(xs) - min(xs) + 1) / 64:.0f}% "
f"of the width x {100 * (max(ys) - min(ys) + 1) / 48:.0f}% of the height ({100 * len(cells) / 3072:.1f}% of the screen)")
small = sum(1 for c in blobs if len(c) < 4)
if small:
out.append(f"{small} small dots or specks")
if png2_bytes is not None:
moved = (np.abs(grid(png2_bytes) - a).sum(axis=2) > 60).mean()
out.append(f"{100 * moved:.1f}% of the screen changed in 0.7 seconds" + (" (nothing moves)" if moved < 0.002 else ""))
return out
|