Download index.html from commoncrawl/web-graph-embeddings: direct link, hf CLI and curl.
- Browser
- Download file 68.9 kB
-
https://huggingface.co/spaces/commoncrawl/web-graph-embeddings/resolve/main/index.html
- Command line
-
hf download hf://spaces/commoncrawl/web-graph-embeddings/index.html
-
curl -L -o index.html https://huggingface.co/spaces/commoncrawl/web-graph-embeddings/resolve/main/index.html
68.9 kB
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8"> | |
| <meta name="viewport" content="width=device-width,initial-scale=1"> | |
| <title>Web graph embeddings β 2D map (streaming)</title> | |
| <!-- Common Crawl web fonts; every rule keeps a system fallback behind them. --> | |
| <link rel="preconnect" href="https://fonts.googleapis.com"> | |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> | |
| <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Libre+Franklin:wght@400;500;600;700&family=IBM+Plex+Mono:wght@400;500;600&display=swap"> | |
| <!-- Streaming (fetch-based) LOD viewer. Reads ./manifest.json + ./t/<id>.bin tiles produced by | |
| embed_2d_tile.py, lazy-loading tiles by viewport + zoom so the full ~53 M-node map fits without | |
| inlining anything. Serve its directory over HTTP (python -m http.server) and open this file. --> | |
| <style> | |
| /* Common Crawl theme, same values as the release figures (src/wgl/viz/svg.py): brand blue | |
| #0050bd, navy ink #152a47, amber #ff9220, and the site's cool neutral greys. Dark-first, since | |
| the map reads best on a dark field; --hot stays a 6-digit hex (drawPin parses it byte-wise). */ | |
| :root { | |
| --font: 'Libre Franklin', system-ui, -apple-system, 'Segoe UI', sans-serif; | |
| --mono: 'IBM Plex Mono', ui-monospace, SFMono-Regular, Menlo, monospace; | |
| --bg: #0d1929; --panel: rgba(21,42,71,.80); --solid: #152a47; --ink: #ffffff; | |
| --muted: #8d99b0; --line: #2a456d; --accent: #4a90e8; --hot: #ff9220; | |
| --shadow: 0 8px 30px rgba(0,0,0,.45); | |
| } | |
| @media (prefers-color-scheme: light) { | |
| :root { | |
| --bg: #f6f7fa; --panel: rgba(255,255,255,.86); --solid: #ffffff; --ink: #152a47; | |
| --muted: #6e7191; --line: #dde2e8; --accent: #0050bd; --hot: #ff9220; | |
| --shadow: 0 8px 26px rgba(21,42,71,.16); | |
| } | |
| } | |
| :root[data-theme="dark"] { | |
| --bg: #0d1929; --panel: rgba(21,42,71,.80); --solid: #152a47; --ink: #ffffff; | |
| --muted: #8d99b0; --line: #2a456d; --accent: #4a90e8; --hot: #ff9220; | |
| --shadow: 0 8px 30px rgba(0,0,0,.45); | |
| } | |
| :root[data-theme="light"] { | |
| --bg: #f6f7fa; --panel: rgba(255,255,255,.86); --solid: #ffffff; --ink: #152a47; | |
| --muted: #6e7191; --line: #dde2e8; --accent: #0050bd; --hot: #ff9220; | |
| --shadow: 0 8px 26px rgba(21,42,71,.16); | |
| } | |
| * { box-sizing: border-box; } | |
| html, body { margin: 0; height: 100%; } | |
| body { | |
| background: var(--bg); color: var(--ink); overflow: hidden; | |
| font-family: var(--font); | |
| -webkit-font-smoothing: antialiased; | |
| } | |
| #stage { position: fixed; inset: 0; } | |
| canvas { display: block; width: 100%; height: 100%; cursor: crosshair; touch-action: none; } | |
| .panel { | |
| position: fixed; background: var(--panel); border: 1px solid var(--line); | |
| border-radius: 12px; box-shadow: var(--shadow); | |
| backdrop-filter: blur(14px) saturate(1.2); -webkit-backdrop-filter: blur(14px) saturate(1.2); | |
| } | |
| #rail { top: 16px; left: 16px; width: 268px; padding: 14px 15px 15px; display: flex; | |
| flex-direction: column; gap: 13px; } | |
| .brand { display: flex; align-items: baseline; gap: 8px; } | |
| .brand h1 { margin: 0; font-size: 14px; font-weight: 650; letter-spacing: -.01em; } | |
| .brand .n { margin-left: auto; font: 600 11px/1 var(--mono); | |
| color: var(--accent); font-variant-numeric: tabular-nums; } | |
| .lede { margin: 0; font-size: 11.5px; line-height: 1.45; color: var(--muted); } | |
| .lede a { color: var(--accent); overflow-wrap: anywhere; word-break: break-word; } | |
| .grp { display: flex; flex-direction: column; gap: 6px; } | |
| .lbl { font-size: 10px; text-transform: uppercase; letter-spacing: .09em; color: var(--muted); | |
| font-weight: 600; } | |
| .seg { display: grid; grid-template-columns: 1fr 1fr; gap: 5px; } | |
| .seg button { | |
| appearance: none; border: 1px solid var(--line); background: transparent; color: var(--ink); | |
| padding: 7px 8px; border-radius: 8px; font: 500 12px/1 var(--font); cursor: pointer; | |
| transition: background .12s, border-color .12s; text-align: left; | |
| } | |
| .seg button:hover { background: color-mix(in srgb, var(--accent) 12%, transparent); } | |
| .seg button[aria-pressed="true"] { | |
| border-color: var(--accent); background: color-mix(in srgb, var(--accent) 20%, transparent); | |
| color: var(--ink); font-weight: 600; | |
| } | |
| .seg button:focus-visible { outline: 2px solid var(--accent); outline-offset: 1px; } | |
| .search { position: relative; } | |
| .search input { | |
| width: 100%; padding: 8px 10px; border-radius: 8px; border: 1px solid var(--line); | |
| background: var(--solid); color: var(--ink); font: 500 12.5px var(--mono); | |
| } | |
| .search input::placeholder { color: var(--muted); } | |
| .search input:focus-visible { outline: 2px solid var(--accent); outline-offset: 0; border-color: var(--accent); } | |
| #hits { list-style: none; margin: 5px 0 0; padding: 0; max-height: 176px; overflow-y: auto; | |
| border-radius: 8px; } | |
| #hits li { padding: 6px 9px; border-radius: 6px; cursor: pointer; | |
| font: 500 12px var(--mono); color: var(--ink); display: flex; gap: 8px; } | |
| #hits li:hover, #hits li[aria-selected="true"] { background: color-mix(in srgb, var(--hot) 22%, transparent); } | |
| #hits li .deg { margin-left: auto; color: var(--muted); font-variant-numeric: tabular-nums; } | |
| #legend { left: 16px; bottom: 16px; max-width: 268px; padding: 11px 13px; display: none; | |
| flex-direction: column; gap: 8px; } | |
| #legend.on { display: flex; } | |
| #legend .lbl { margin-bottom: 1px; } | |
| .swatches { display: flex; flex-direction: column; gap: 2px; max-height: 240px; overflow-y: auto; } | |
| .sw { display: flex; align-items: center; gap: 7px; font-size: 11px; color: var(--ink); | |
| appearance: none; border: 1px solid transparent; background: transparent; text-align: left; | |
| width: 100%; padding: 3px 6px; border-radius: 6px; cursor: pointer; font-family: inherit; | |
| transition: background .12s, border-color .12s; } | |
| .sw:hover { background: color-mix(in srgb, var(--accent) 12%, transparent); } | |
| .sw[aria-pressed="true"] { border-color: var(--accent); | |
| background: color-mix(in srgb, var(--accent) 18%, transparent); } | |
| .sw:focus-visible { outline: 2px solid var(--accent); outline-offset: 1px; } | |
| .sw i { width: 10px; height: 10px; border-radius: 3px; display: inline-block; flex: none; } | |
| .sw .nm { flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } | |
| .sw .ct { color: var(--muted); font-variant-numeric: tabular-nums; | |
| font: 500 10px var(--mono); } | |
| #tip { | |
| position: fixed; pointer-events: none; z-index: 40; padding: 8px 10px; border-radius: 9px; | |
| background: var(--solid); border: 1px solid var(--line); box-shadow: var(--shadow); | |
| font: 500 12px var(--mono); max-width: 300px; display: none; | |
| transform: translate(12px, 12px); | |
| } | |
| #tip .host { color: var(--accent); word-break: break-all; font-weight: 600; } | |
| #tip .row { color: var(--muted); margin-top: 3px; display: flex; gap: 7px; } | |
| #tip .row b { color: var(--ink); font-weight: 600; } | |
| #hud { right: 16px; bottom: 16px; padding: 8px 11px; display: flex; gap: 12px; align-items: center; | |
| font: 500 11px var(--mono); color: var(--muted); font-variant-numeric: tabular-nums; } | |
| #hud b { color: var(--ink); font-weight: 600; } | |
| #hud button { appearance: none; border: 1px solid var(--line); background: transparent; color: var(--muted); | |
| border-radius: 7px; padding: 4px 8px; cursor: pointer; font: inherit; } | |
| #hud button:hover { color: var(--ink); border-color: var(--accent); } | |
| #hud button[aria-pressed="true"] { color: var(--ink); border-color: var(--accent); | |
| background: color-mix(in srgb, var(--accent) 18%, transparent); } | |
| #spin { position: fixed; top: 16px; right: 16px; z-index: 45; width: 17px; height: 17px; | |
| border-radius: 50%; border: 2.5px solid var(--line); border-top-color: var(--accent); | |
| box-shadow: var(--shadow); opacity: 0; pointer-events: none; transition: opacity .2s; | |
| animation: spin .7s linear infinite; } | |
| #spin.on { opacity: .95; } | |
| @keyframes spin { to { transform: rotate(360deg); } } | |
| #err { position: fixed; inset: 0; display: none; place-items: center; padding: 24px; text-align: center; } | |
| #err div { max-width: 400px; color: var(--muted); font-size: 13px; line-height: 1.5; } | |
| /* ββ region select βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ | |
| #toprail { position: fixed; top: 16px; right: 16px; z-index: 45; display: flex; | |
| align-items: center; gap: 10px; pointer-events: none; } | |
| #spin { position: relative; top: auto; right: auto; flex: none; } /* was fixed; now in the rail */ | |
| #selpill { position: relative; display: none; pointer-events: auto; align-items: stretch; | |
| border-radius: 999px; overflow: hidden; } | |
| #selpill.on { display: flex; } | |
| #selpill.open { border-color: var(--accent); } | |
| #seln, #selclr { appearance: none; border: 0; background: transparent; color: var(--ink); | |
| cursor: pointer; font: 600 11.5px/1.35 var(--mono); | |
| font-variant-numeric: tabular-nums; } | |
| #seln { position: relative; padding: 6px 11px; overflow: hidden; } | |
| #seln:hover { background: color-mix(in srgb, var(--accent) 12%, transparent); } | |
| #selpill.open #seln { background: color-mix(in srgb, var(--accent) 18%, transparent); } | |
| #selclr { padding: 5px 11px 6px; color: var(--muted); font-size: 15px; line-height: 1.1; | |
| border-left: 1px solid var(--line); } | |
| #selclr:hover { color: var(--ink); background: color-mix(in srgb, var(--hot) 22%, transparent); } | |
| /* the pill clips to its radius, so focus rings have to sit inside the button */ | |
| #seln:focus-visible, #selclr:focus-visible { outline: 2px solid var(--accent); | |
| outline-offset: -2px; } | |
| #seln::after { content: ""; position: absolute; left: 0; bottom: 0; height: 2px; width: 0; | |
| background: var(--accent); transition: width .15s; } | |
| #seln.scanning::after { width: calc(var(--p, 0) * 100%); } | |
| #selbox { position: fixed; z-index: 30; display: none; pointer-events: none; border-radius: 3px; | |
| border: 1.5px dashed var(--accent); background: color-mix(in srgb, var(--accent) 13%, transparent); } | |
| #selbox.on { display: block; } | |
| #selbox.set { border-style: solid; } | |
| #sel { display: none; right: 16px; top: 58px; bottom: 16px; width: 452px; padding: 0; | |
| flex-direction: column; overflow: hidden; z-index: 44; } | |
| #sel.on { display: flex; } | |
| #sel header { display: flex; align-items: baseline; gap: 8px; padding: 12px 14px 9px; | |
| border-bottom: 1px solid var(--line); } | |
| #sel h2 { margin: 0; font-size: 13px; font-weight: 650; letter-spacing: -.01em; } | |
| #sel .sub { color: var(--muted); font-size: 10.5px; padding: 8px 14px 0; line-height: 1.45; } | |
| #selx, #helpx { margin-left: auto; appearance: none; border: 1px solid var(--line); background: transparent; | |
| color: var(--muted); border-radius: 7px; padding: 1px 8px 3px; cursor: pointer; font: inherit; } | |
| #selx:hover, #helpx:hover { color: var(--ink); border-color: var(--accent); } | |
| #selx:focus-visible, #helpx:focus-visible { outline: 2px solid var(--accent); outline-offset: 1px; } | |
| .selrow { display: grid; grid-template-columns: 1fr 58px 132px; gap: 9px; padding: 5px 14px; | |
| align-items: baseline; font: 500 11.5px var(--mono); } | |
| .selrow > span { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } | |
| .selrow .d { color: var(--muted); text-align: right; font-variant-numeric: tabular-nums; } | |
| .selrow .c { color: var(--muted); font-size: 10.5px; } | |
| .selhd { color: var(--muted); text-transform: uppercase; letter-spacing: .08em; font-size: 9.5px; | |
| font-weight: 600; padding: 9px 14px 7px; border-bottom: 1px solid var(--line); } | |
| #selrows { flex: 1; overflow-y: auto; overscroll-behavior: contain; } | |
| #selrows .selrow { cursor: pointer; } | |
| #selrows .selrow:nth-child(2n) { background: color-mix(in srgb, var(--ink) 5%, transparent); } | |
| #selrows .selrow:hover { background: color-mix(in srgb, var(--accent) 16%, transparent); } | |
| #selrows .selrow:focus-visible { outline: 2px solid var(--accent); outline-offset: -2px; } | |
| #selrows .empty { color: var(--muted); padding: 16px 14px; font-size: 12px; } | |
| #selfoot { border-top: 1px solid var(--line); padding: 9px 14px; display: flex; gap: 10px; | |
| align-items: center; flex-wrap: wrap; | |
| font: 500 10.5px var(--mono); color: var(--muted); } | |
| #selfoot .warn { color: var(--hot); } | |
| #selfoot button { appearance: none; border: 1px solid var(--line); background: transparent; | |
| color: var(--muted); border-radius: 7px; padding: 3px 8px; cursor: pointer; font: inherit; | |
| margin-left: auto; } | |
| #selfoot button:hover { color: var(--ink); border-color: var(--accent); } | |
| /* the sheet would otherwise sit on top of the HUD, putting select/reset out of reach */ | |
| body.sheet #hud { right: 484px; } | |
| /* help overlay: one paragraph per colourable property, opened from the "?" beside "Color by" */ | |
| #helpbtn { appearance: none; width: 14px; height: 14px; padding: 0; margin-left: 4px; | |
| border: 1px solid var(--line); border-radius: 50%; background: transparent; color: var(--muted); | |
| font: 600 9px/1 var(--font); cursor: pointer; vertical-align: 1px; } | |
| #helpbtn:hover { color: var(--ink); border-color: var(--accent); } | |
| #helpbtn:focus-visible { outline: 2px solid var(--accent); outline-offset: 1px; } | |
| #scrim { position: fixed; inset: 0; z-index: 40; background: rgba(3, 10, 20, .55); } | |
| /* opaque, unlike the floating .panel chrome: a modal over the scrim should not show the map | |
| through it, and backdrop-filter over a full-screen scrim composites unreliably. */ | |
| #help { position: fixed; z-index: 41; top: 50%; left: 50%; transform: translate(-50%, -50%); | |
| width: min(560px, calc(100vw - 32px)); max-height: min(78vh, 720px); | |
| display: flex; flex-direction: column; | |
| background: var(--solid); backdrop-filter: none; -webkit-backdrop-filter: none; } | |
| #help header { display: flex; align-items: baseline; gap: 10px; | |
| padding: 12px 16px 10px; border-bottom: 1px solid var(--line); } | |
| #help h2 { margin: 0; font-size: 13px; } | |
| #helpbody { overflow-y: auto; overscroll-behavior: contain; padding: 2px 16px 14px; } | |
| #helpbody h3 { margin: 14px 0 3px; font-size: 11.5px; color: var(--ink); text-transform: capitalize; } | |
| #helpbody p { margin: 0; font-size: 11.5px; line-height: 1.5; color: var(--muted); } | |
| #helpbody a, #help footer a { color: var(--accent); } | |
| #help footer { border-top: 1px solid var(--line); padding: 10px 16px; | |
| font-size: 11px; line-height: 1.5; color: var(--muted); } | |
| [hidden] { display: none ; } | |
| @media (max-width: 560px) { | |
| #rail { width: calc(100vw - 32px); } | |
| #legend { max-width: calc(100vw - 32px); } | |
| } | |
| @media (max-width: 560px) { | |
| #sel { left: 16px; width: auto; } | |
| body.sheet #hud { right: 16px; } /* full-width sheet: nothing to dodge */ | |
| } | |
| @media (prefers-reduced-motion: reduce) { * { transition: none ; } #spin { animation: none; } } | |
| </style> | |
| </head> | |
| <body> | |
| <div id="stage"><canvas id="cv" aria-label="2D map of host embeddings"></canvas></div> | |
| <section id="rail" class="panel" aria-label="Controls"> | |
| <div class="brand"><h1>Web graph embeddings</h1><span class="n" id="nnodes">β</span></div> | |
| <p class="lede" id="lede">The link-prediction embeddings, projected to 2D and streamed by tile. | |
| Structure the model learned is the shape you see. Recolor to ask which properties it separates.</p> | |
| <div class="grp"> | |
| <span class="lbl">Color by <button id="helpbtn" type="button" | |
| aria-label="What the colours mean" title="What the colours mean">?</button></span> | |
| <div class="seg" id="colorby" role="group" aria-label="Color by"></div> | |
| </div> | |
| <div class="grp search"> | |
| <span class="lbl" id="findlbl">Find a top-20k host</span> | |
| <input id="q" type="search" placeholder="search hostnameβ¦" autocomplete="off" spellcheck="false" /> | |
| <ul id="hits" role="listbox" aria-label="Search results"></ul> | |
| </div> | |
| </section> | |
| <aside id="legend" class="panel" aria-label="Legend"></aside> | |
| <div id="toprail"> | |
| <div id="selpill" class="panel"> | |
| <button id="seln" type="button" aria-expanded="false" aria-controls="sel"></button> | |
| <button id="selclr" type="button" title="Clear selection (Esc)" | |
| aria-label="Clear selection">×</button> | |
| </div> | |
| <div id="spin" role="status" aria-label="Loading tiles" aria-hidden="true"></div> | |
| </div> | |
| <div id="selbox" aria-hidden="true"></div> | |
| <section id="sel" class="panel" role="dialog" aria-labelledby="seltitle"> | |
| <header><h2 id="seltitle">Selected hosts</h2><button id="selx" type="button" | |
| aria-label="Close list">×</button></header> | |
| <p class="sub" id="selsub"></p> | |
| <div class="selrow selhd" id="selhd"></div> | |
| <div id="selrows"></div> | |
| <footer id="selfoot"></footer> | |
| </section> | |
| <div id="scrim" hidden></div> | |
| <section id="help" class="panel" role="dialog" aria-modal="true" aria-labelledby="helptitle" hidden> | |
| <header><h2 id="helptitle">What the colours mean</h2><button id="helpx" type="button" | |
| aria-label="Close">×</button></header> | |
| <div id="helpbody"></div> | |
| <footer>The model never saw any of these properties — only anonymous host ids and the links | |
| between them. Any structure here was learned from links alone. | |
| <a href="https://huggingface.co/datasets/commoncrawl/web-graph-embeddings" | |
| target="_blank" rel="noopener">Dataset</a> · | |
| <a href="https://github.com/commoncrawl/web-graph-embeddings" | |
| target="_blank" rel="noopener">Code</a></footer> | |
| </section> | |
| <div id="tip" role="tooltip"></div> | |
| <div id="hud" class="panel"> | |
| <span><b id="fps">β</b> fps</span><span><b id="tiles">0</b> tiles</span> | |
| <span>zoom <b id="zoom">1.0Γ</b></span> | |
| <button id="selarm" aria-pressed="false" | |
| title="Drag a rectangle to select hosts (or hold Shift and drag)">select</button> | |
| <button id="reset" title="Reset the view, the selection and the pinned host">reset</button> | |
| </div> | |
| <div id="err"><div id="errmsg"></div></div> | |
| <script> | |
| (function () { | |
| "use strict"; | |
| const $ = (id) => document.getElementById(id); | |
| function fail(msg) { $("errmsg").textContent = msg; $("err").style.display = "grid"; } | |
| const params = new URLSearchParams(location.search); | |
| const BASE = (params.get("base") || ".").replace(/\/$/, ""); // tile root; default = page dir | |
| // ββ palettes ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| // 24-colour categorical palette, greedy farthest-point ordered so the most frequent classes | |
| // (rank 0,1,2,β¦) are the most distinct; legible on both dark and light grounds. | |
| const CAT = [ | |
| [85, 158, 231], [237, 70, 29], [29, 237, 29], [240, 76, 208], | |
| [190, 211, 105], [29, 237, 153], [142, 76, 240], [133, 237, 29], | |
| [200, 65, 119], [237, 174, 29], [223, 144, 152], [105, 211, 115], | |
| [29, 237, 237], [184, 144, 223], [65, 92, 200], [144, 207, 223], | |
| [200, 119, 65], [90, 226, 185], [65, 200, 65], [29, 133, 237], | |
| [244, 123, 244], [29, 237, 91], [184, 218, 47], [186, 65, 200], | |
| ]; | |
| const UNLAB = [110, 120, 135]; | |
| function rampColor(t) { | |
| t = Math.max(0, Math.min(1, t)); | |
| const stops = [[36, 52, 96], [50, 130, 190], [86, 199, 245], [255, 180, 84]]; | |
| const s = t * (stops.length - 1); const i = Math.floor(s), f = s - i; | |
| const a = stops[i], b = stops[Math.min(i + 1, stops.length - 1)]; | |
| return [a[0] + (b[0] - a[0]) * f, a[1] + (b[1] - a[1]) * f, a[2] + (b[2] - a[2]) * f]; | |
| } | |
| const DIM = 26; // dimmed alpha (~10%) for filtered-out nodes | |
| let DEG_BUCKETS = 6; | |
| function degBucket(b) { return Math.min(DEG_BUCKETS - 1, Math.floor(b / (256 / DEG_BUCKETS))); } | |
| function fmtN(n) { | |
| n = Math.round(n); | |
| if (n >= 1e6) return (n / 1e6).toFixed(1) + "M"; | |
| if (n >= 1e3) return (n / 1e3).toFixed(n >= 1e4 ? 0 : 1) + "k"; | |
| return "" + n; | |
| } | |
| // ββ WebGL βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| const cv = $("cv"); | |
| const gl = cv.getContext("webgl", { antialias: true, alpha: false, premultipliedAlpha: false }); | |
| if (!gl) { fail("This viewer needs WebGL, which your browser has disabled."); return; } | |
| function sh(type, src) { | |
| const s = gl.createShader(type); gl.shaderSource(s, src); gl.compileShader(s); | |
| if (!gl.getShaderParameter(s, gl.COMPILE_STATUS)) throw new Error(gl.getShaderInfoLog(s)); | |
| return s; | |
| } | |
| const prog = gl.createProgram(); | |
| gl.attachShader(prog, sh(gl.VERTEX_SHADER, ` | |
| attribute vec2 a_xy; attribute vec3 a_col; attribute float a_deg; attribute float a_alpha; | |
| uniform vec2 u_scale; uniform vec2 u_off; uniform float u_pt; uniform float u_ptMin; uniform float u_matchOnly; | |
| varying vec3 v_col; varying float v_alpha; | |
| void main() { | |
| vec2 p = a_xy * u_scale + u_off; | |
| gl_Position = vec4(p, 0.0, 1.0); | |
| gl_PointSize = max(u_ptMin, u_pt * (0.65 + 1.9 * a_deg)); | |
| v_col = a_col; v_alpha = a_alpha; | |
| // foreground pass: draw only the filter-matched points (alpha==1), cull the dimmed rest | |
| if (u_matchOnly > 0.5 && a_alpha < 0.99) { gl_PointSize = 0.0; gl_Position = vec4(2.0, 2.0, 2.0, 1.0); } | |
| }`)); | |
| gl.attachShader(prog, sh(gl.FRAGMENT_SHADER, ` | |
| precision mediump float; varying vec3 v_col; varying float v_alpha; uniform float u_alpha; | |
| void main() { | |
| vec2 d = gl_PointCoord - 0.5; float r = dot(d, d); | |
| if (r > 0.25) discard; | |
| float a = smoothstep(0.25, 0.14, r); | |
| gl_FragColor = vec4(v_col, a * u_alpha * v_alpha); | |
| }`)); | |
| gl.linkProgram(prog); | |
| if (!gl.getProgramParameter(prog, gl.LINK_STATUS)) { fail("Shader link failed."); return; } | |
| gl.useProgram(prog); | |
| const A = { | |
| xy: gl.getAttribLocation(prog, "a_xy"), col: gl.getAttribLocation(prog, "a_col"), | |
| deg: gl.getAttribLocation(prog, "a_deg"), alpha: gl.getAttribLocation(prog, "a_alpha"), | |
| }; | |
| const U = { | |
| scale: gl.getUniformLocation(prog, "u_scale"), off: gl.getUniformLocation(prog, "u_off"), | |
| pt: gl.getUniformLocation(prog, "u_pt"), alpha: gl.getUniformLocation(prog, "u_alpha"), | |
| ptMin: gl.getUniformLocation(prog, "u_ptMin"), matchOnly: gl.getUniformLocation(prog, "u_matchOnly"), | |
| }; | |
| gl.enableVertexAttribArray(A.xy); gl.enableVertexAttribArray(A.col); | |
| gl.enableVertexAttribArray(A.deg); gl.enableVertexAttribArray(A.alpha); | |
| // ββ state βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| let META = null, N = 0, nCat = 0, active = 0, filterVal = -1, defaultSet = 0; | |
| let view = { s: 0.9, cx: 0, cy: 0 }; | |
| let asp = 1, dpr = Math.min(window.devicePixelRatio || 1, 2), dirty = true; | |
| const tileMeta = new Map(); // id -> manifest tile {id,z,n,bbox} | |
| const colorRank = {}; // setIdx -> [classIdx -> frequency rank] for CAT color | |
| const ready = new Map(); // id -> loaded tile object | |
| let activeIds = new Set(); | |
| const MAXKEEP = 600; // evict non-visible tiles beyond this many | |
| let inflight = 0; const MAXFETCH = 8; const queue = []; | |
| let pin = null; // {id, li} pinned/searched node | |
| // ββ region-select state βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| // The rect lives in WORLD coords, so a selection survives pan/zoom untouched. ``epoch`` is the | |
| // cancellation token: every await in the scan re-reads it and bails if a newer rect superseded it. | |
| const Q = 32767; // int16 quantum β matches embed_2d_tile._quantize_xy | |
| const EPS = 1e-4; // bbox slack; see classify() | |
| const sel = { rect: null, cls: null, count: 0, done: 0, todo: 0, errs: 0, | |
| exact: false, epoch: 0, partN: new Map() }; | |
| let scanSig = null; // AbortController for the in-flight scan | |
| let band = null; // live rubber-band drag, screen px | |
| let selArmed = false, sheetOpen = false; | |
| const list = { band: 0, buf: null, tiles: [], pos: 0, rows: 0, | |
| eof: false, busy: false, approx: false }; | |
| // Scan tiles are cached SEPARATELY from ``ready``: that path uploads 4 GL buffers per tile and | |
| // evicts at MAXKEEP, so routing a scan through it would evict the map and thrash GPU memory. | |
| const scanCache = new Map(), nameCache = new Map(); // Map order == LRU (touch = delete + set) | |
| const scanWait = new Map(), nameWait = new Map(); // in-flight dedupe | |
| let scanBytes = 0, nameBytes = 0; | |
| const SCAN_BYTES_MAX = 96 << 20, NAME_BYTES_MAX = 48 << 20; | |
| const SCAN_FETCH = 6; // < MAXFETCH so rendering keeps fetch priority | |
| const PAGE = 200, LIST_MAX = 10000, BAND_MAX = 3e6; | |
| // ββ boot: fetch manifest ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| fetch(BASE + "/manifest.json").then((r) => { | |
| if (!r.ok) throw new Error("manifest.json " + r.status); | |
| return r.json(); | |
| }).then((m) => { META = m; boot(); }).catch((e) => | |
| fail("Could not load manifest.json from " + BASE + " β serve this folder over HTTP " | |
| + "(python -m http.server) and open the page from there. (" + e.message + ")")); | |
| function boot() { | |
| N = META.n; DEG_BUCKETS = META.degBuckets || 6; | |
| nCat = META.labelSets.filter((l) => l.kind === "cat").length; | |
| META.tiles.forEach((t) => tileMeta.set(t.id, t)); | |
| // colors follow frequency rank (most frequent class -> CAT[0]) so the top classes are distinct | |
| META.labelSets.forEach((ls, i) => { | |
| if (ls.kind !== "cat") return; | |
| const cnt = ls.counts || ls.classes.map(() => 0); | |
| const rank = new Array(ls.classes.length); | |
| ls.classes.map((_, k) => k).sort((a, b) => cnt[b] - cnt[a]).forEach((k, pos) => { rank[k] = pos; }); | |
| colorRank[i] = rank; | |
| }); | |
| $("nnodes").textContent = N >= 1e6 ? (N / 1e6).toFixed(N % 1e6 ? 1 : 0) + "M" | |
| : (N / 1e3).toFixed(0) + "k"; | |
| if (META.lede) $("lede").innerHTML = META.lede; // trusted manifest string (may contain a link) | |
| defaultSet = Math.max(0, META.labelSets.findIndex((l) => l.name === "language")); // color by language on load | |
| const nSearch = (META.search || []).length; // label the search box with its actual coverage | |
| if (nSearch) $("findlbl").textContent = "Find a top-" + fmtN(nSearch) + " host"; | |
| buildColorBy(); buildSearch(); buildSelHead(); | |
| resize(); recolor(defaultSet); requestAnimationFrame(loop); | |
| } | |
| // ββ tile loading ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| function pump() { | |
| while (inflight < MAXFETCH && queue.length) { const id = queue.shift(); doLoad(id); } | |
| } | |
| function startLoad(id) { | |
| if (ready.has(id) || queue.includes(id)) return; | |
| ready.set(id, { id, status: "queued" }); queue.push(id); pump(); | |
| } | |
| function doLoad(id) { | |
| const t = ready.get(id); const meta = tileMeta.get(id); if (!t || !meta) return; | |
| // tiles are sharded into t/<d>/ subdirs (manifest ``d``); fall back to flat t/ for old manifests | |
| const tdir = "/t/" + (meta.d != null ? meta.d + "/" : "") + id; | |
| t.status = "loading"; inflight++; | |
| fetch(BASE + tdir + ".bin").then((r) => r.arrayBuffer()).then((buf) => { | |
| const K = meta.n; let o = 0; | |
| t.n = K; | |
| t.xy = new Int16Array(buf, o, K * 2); o += K * 4; | |
| t.labs = []; | |
| for (let c = 0; c < nCat; c++) { t.labs.push(new Uint8Array(buf, o, K)); o += K; } | |
| t.deg = new Uint8Array(buf, o, K); o += K; | |
| t.bXY = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, t.bXY); | |
| gl.bufferData(gl.ARRAY_BUFFER, t.xy, gl.STATIC_DRAW); | |
| t.bDeg = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, t.bDeg); | |
| gl.bufferData(gl.ARRAY_BUFFER, t.deg, gl.STATIC_DRAW); | |
| t.bCol = gl.createBuffer(); t.bAlpha = gl.createBuffer(); | |
| t.names = null; | |
| colorTile(t); alphaTile(t); | |
| t.status = "ready"; dirty = true; | |
| if (META.hasNames) fetch(BASE + tdir + ".nm").then((r) => r.text()) | |
| .then((txt) => { t.names = txt.length ? txt.split("\n") : []; }); | |
| }).catch(() => { ready.delete(id); }).finally(() => { inflight--; pump(); }); | |
| } | |
| function evictTile(t) { | |
| if (t.bXY) gl.deleteBuffer(t.bXY); if (t.bDeg) gl.deleteBuffer(t.bDeg); | |
| if (t.bCol) gl.deleteBuffer(t.bCol); if (t.bAlpha) gl.deleteBuffer(t.bAlpha); | |
| } | |
| function colorTile(t) { | |
| if (t.status === "queued" || !t.deg) return; | |
| const set = META.labelSets[active], K = t.n, buf = new Uint8Array(K * 3); | |
| if (set.kind === "degree") { | |
| for (let i = 0; i < K; i++) { const c = rampColor(t.deg[i] / 255), j = i * 3; | |
| buf[j] = c[0]; buf[j + 1] = c[1]; buf[j + 2] = c[2]; } | |
| } else { | |
| const lab = t.labs[active], rank = colorRank[active]; | |
| for (let i = 0; i < K; i++) { const v = lab[i], c = v === 255 ? UNLAB : CAT[rank[v] % CAT.length]; | |
| const j = i * 3; buf[j] = c[0]; buf[j + 1] = c[1]; buf[j + 2] = c[2]; } | |
| } | |
| gl.bindBuffer(gl.ARRAY_BUFFER, t.bCol); gl.bufferData(gl.ARRAY_BUFFER, buf, gl.STATIC_DRAW); | |
| } | |
| function alphaTile(t) { | |
| if (t.status === "queued" || !t.deg) return; | |
| const set = META.labelSets[active], K = t.n, buf = new Uint8Array(K); | |
| if (filterVal < 0) { | |
| buf.fill(255); | |
| } else if (set.kind === "degree") { | |
| for (let i = 0; i < K; i++) buf[i] = degBucket(t.deg[i]) === filterVal ? 255 : DIM; | |
| } else { | |
| const lab = t.labs[active]; | |
| for (let i = 0; i < K; i++) buf[i] = lab[i] === filterVal ? 255 : DIM; | |
| } | |
| gl.bindBuffer(gl.ARRAY_BUFFER, t.bAlpha); gl.bufferData(gl.ARRAY_BUFFER, buf, gl.DYNAMIC_DRAW); | |
| } | |
| // ββ LOD + viewport β which tiles are active βββββββββββββββββββββββββββββββββββββββββββββββββ | |
| // LOD = highest degree band shown at the current zoom. The bands themselves are baked into the | |
| // tiles; this only controls WHEN each appears. LOD_RATE = bands revealed per zoom-doubling (lower | |
| // β low-degree hosts hold off until you zoom deeper); LOD_BIAS = bands the overview already shows. | |
| const LOD_RATE = 0.7, LOD_BIAS = 0.4; | |
| function lodLevel() { | |
| return Math.max(0, Math.min(META.zMax, Math.round(Math.log2(view.s) * LOD_RATE + LOD_BIAS))); | |
| } | |
| // one screen<->world projection for hover-pick and region-select (the wheel handler keeps its | |
| // own inline copy: it needs the intermediate NDC to anchor the zoom on the cursor) | |
| function toWorld(px, py) { | |
| const w = cv.clientWidth, h = cv.clientHeight; | |
| return { x: ((px / w) * 2 - 1) / (view.s * asp) + view.cx, | |
| y: (1 - (py / h) * 2) / view.s + view.cy }; | |
| } | |
| function toScreenX(wx) { return ((wx - view.cx) * view.s * asp + 1) * cv.clientWidth / 2; } | |
| function toScreenY(wy) { return (1 - (wy - view.cy) * view.s) * cv.clientHeight / 2; } | |
| function viewport() { | |
| const hw = 1 / (view.s * asp), hh = 1 / view.s, m = 0.25; // 25% prefetch margin | |
| return [view.cx - hw * (1 + m), view.cy - hh * (1 + m), | |
| view.cx + hw * (1 + m), view.cy + hh * (1 + m)]; | |
| } | |
| function updateActive() { | |
| // Tiles loaded = current LOD band + viewport, ALWAYS (a filter never changes this). Filtering | |
| // only dims the non-matching points among what this zoom already shows β it must not pull in the | |
| // low-degree hosts that are meant to stay hidden until you zoom in. | |
| const maxZ = lodLevel(), vp = viewport(); activeIds = new Set(); | |
| for (const t of META.tiles) { | |
| if (t.z > maxZ) continue; | |
| const b = t.bbox; | |
| if (b[0] < vp[2] && b[2] > vp[0] && b[1] < vp[3] && b[3] > vp[1]) { | |
| activeIds.add(t.id); if (!ready.has(t.id)) startLoad(t.id); | |
| } | |
| } | |
| if (ready.size > MAXKEEP) { // drop non-visible tiles to bound GPU memory | |
| for (const [id, t] of ready) { | |
| if (ready.size <= MAXKEEP) break; | |
| if (!activeIds.has(id) && t.status === "ready") { evictTile(t); ready.delete(id); } | |
| } | |
| } | |
| } | |
| // ββ draw ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| function drawTiles() { // bind + draw each ready active tile; returns count | |
| let drawn = 0; | |
| for (const id of activeIds) { | |
| const t = ready.get(id); if (!t || t.status !== "ready") continue; | |
| gl.bindBuffer(gl.ARRAY_BUFFER, t.bXY); gl.vertexAttribPointer(A.xy, 2, gl.SHORT, true, 0, 0); | |
| gl.bindBuffer(gl.ARRAY_BUFFER, t.bCol); gl.vertexAttribPointer(A.col, 3, gl.UNSIGNED_BYTE, true, 0, 0); | |
| gl.bindBuffer(gl.ARRAY_BUFFER, t.bDeg); gl.vertexAttribPointer(A.deg, 1, gl.UNSIGNED_BYTE, true, 0, 0); | |
| gl.bindBuffer(gl.ARRAY_BUFFER, t.bAlpha); gl.vertexAttribPointer(A.alpha, 1, gl.UNSIGNED_BYTE, true, 0, 0); | |
| gl.drawArrays(gl.POINTS, 0, t.n); drawn++; | |
| } | |
| return drawn; | |
| } | |
| function draw() { | |
| const bg = getComputedStyle(document.body).backgroundColor.match(/\d+/g) || [11, 14, 21]; | |
| gl.clearColor(bg[0] / 255, bg[1] / 255, bg[2] / 255, 1); gl.clear(gl.COLOR_BUFFER_BIT); | |
| gl.enable(gl.BLEND); gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA); | |
| gl.uniform2f(U.scale, view.s * asp, view.s); | |
| gl.uniform2f(U.off, -view.cx * view.s * asp, -view.cy * view.s); | |
| // point size = a hump in log-zoom: grow through the hub-only overview (declutter), then SHRINK | |
| // once LOD reveals the dense low-degree crowd at high zoom (screen density peaks there, so smaller | |
| // dots keep the map readable). Peaks ~4Γ zoom, decays toward a 2px floor at 150β500Γ+. | |
| const L = Math.log2(view.s); | |
| gl.uniform1f(U.pt, Math.max(1.2, dpr * (2.0 + 5.0 * Math.exp(-(L - 2.0) * (L - 2.0) / 8.0)))); | |
| // floor on-screen size so points stay visible when zoomed way in: off through the good low/mid | |
| // zoom (β€~22Γ), then ramps in to a 10px minimum by ~128Γ (per-point max, so it lifts only the | |
| // too-small dots rather than inflating the whole map). | |
| gl.uniform1f(U.ptMin, dpr * 10 * Math.min(1, Math.max(0, (L - 4.6) / 2.4))); | |
| // pass 1: whole scene (filtered-out points already dimmed to ~10% via per-vertex alpha) | |
| gl.uniform1f(U.matchOnly, 0); | |
| gl.uniform1f(U.alpha, N > 4e6 ? 0.5 : 0.75); | |
| const drawn = drawTiles(); | |
| // pass 2 (filter active): redraw only the matched points, opaque and on top (foreground) | |
| if (filterVal >= 0) { | |
| gl.uniform1f(U.matchOnly, 1); | |
| gl.uniform1f(U.alpha, N > 4e6 ? 0.9 : 1.0); | |
| drawTiles(); | |
| gl.uniform1f(U.matchOnly, 0); | |
| } | |
| $("tiles").textContent = drawn; | |
| if (pin) drawPin(); | |
| syncSelBox(); | |
| } | |
| const bPin = gl.createBuffer(); | |
| function drawPin() { | |
| const t = ready.get(pin.id); if (!t || t.status !== "ready") return; | |
| gl.bindBuffer(gl.ARRAY_BUFFER, bPin); | |
| gl.bufferData(gl.ARRAY_BUFFER, new Int16Array([t.xy[pin.li * 2], t.xy[pin.li * 2 + 1]]), gl.STATIC_DRAW); | |
| gl.vertexAttribPointer(A.xy, 2, gl.SHORT, true, 0, 0); | |
| const hot = getComputedStyle(document.body).getPropertyValue("--hot").trim(); | |
| const m = hot.match(/\w\w/g).map((h) => parseInt(h, 16)); | |
| gl.disableVertexAttribArray(A.col); gl.disableVertexAttribArray(A.deg); | |
| gl.disableVertexAttribArray(A.alpha); | |
| gl.vertexAttrib3f(A.col, m[0] / 255, m[1] / 255, m[2] / 255); | |
| gl.vertexAttrib1f(A.deg, 1); gl.vertexAttrib1f(A.alpha, 1); | |
| gl.uniform1f(U.pt, 16 * dpr); gl.uniform1f(U.alpha, 1); | |
| gl.drawArrays(gl.POINTS, 0, 1); | |
| gl.enableVertexAttribArray(A.col); gl.enableVertexAttribArray(A.deg); | |
| gl.enableVertexAttribArray(A.alpha); | |
| } | |
| // ββ recolor + legend ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| function recolor(idx) { | |
| active = idx; filterVal = -1; | |
| for (const t of ready.values()) { colorTile(t); alphaTile(t); } | |
| drawLegend(idx); dirty = true; | |
| } | |
| function buildColorBy() { | |
| const seg = $("colorby"); | |
| META.labelSets.forEach((ls, i) => { | |
| const b = document.createElement("button"); b.textContent = ls.name; | |
| b.setAttribute("aria-pressed", i === defaultSet ? "true" : "false"); | |
| b.onclick = () => { [...seg.children].forEach((c) => c.setAttribute("aria-pressed", "false")); | |
| b.setAttribute("aria-pressed", "true"); recolor(i); }; | |
| seg.appendChild(b); | |
| }); | |
| } | |
| function degLabel(b) { return Math.round(Math.pow(10, (b / 255) * (META.degLogMax || 4))); } | |
| function drawLegend(idx) { | |
| const ls = META.labelSets[idx], el = $("legend"); | |
| el.innerHTML = ""; | |
| const head = document.createElement("span"); head.className = "lbl"; | |
| head.textContent = ls.name + (ls.note ? " Β· " + ls.note : ""); | |
| el.appendChild(head); | |
| let entries; | |
| if (ls.kind === "degree") { | |
| const w = 256 / DEG_BUCKETS, c = META.degCounts || []; entries = []; | |
| for (let k = 0; k < DEG_BUCKETS; k++) { | |
| const lo = degLabel(Math.round(k * w)), hi = degLabel(Math.round(Math.min(255, (k + 1) * w))); | |
| entries.push({ val: k, color: rampColor((k + 0.5) * w / 255), | |
| text: fmtN(lo) + "β" + fmtN(hi), count: c[k] || 0 }); | |
| } | |
| } else { | |
| const c = ls.counts || ls.classes.map(() => 0), rank = colorRank[idx]; | |
| entries = ls.classes.map((cl, k) => ({ val: k, color: CAT[rank[k] % CAT.length], text: cl, | |
| count: c[k] || 0 })).filter((e) => e.count > 0).sort((a, b) => b.count - a.count); | |
| const na = N - c.reduce((s, v) => s + (v || 0), 0); // unlabelled = 255 sentinel | |
| if (na > 0) entries.push({ val: 255, color: UNLAB, text: "N/A", count: na }); | |
| } | |
| const wrap = document.createElement("div"); wrap.className = "swatches"; | |
| entries.forEach((e) => { | |
| const b = document.createElement("button"); b.type = "button"; b.className = "sw"; | |
| b.setAttribute("aria-pressed", filterVal === e.val ? "true" : "false"); | |
| b.title = "Show only these β click again to clear"; | |
| const c = e.color; | |
| const sw = document.createElement("i"); | |
| sw.style.background = `rgb(${c[0] | 0},${c[1] | 0},${c[2] | 0})`; | |
| const nm = document.createElement("span"); nm.className = "nm"; nm.textContent = e.text; | |
| const ct = document.createElement("span"); ct.className = "ct"; ct.textContent = fmtN(e.count); | |
| b.append(sw, nm, ct); | |
| b.onclick = () => { | |
| filterVal = (filterVal === e.val) ? -1 : e.val; | |
| for (const t of ready.values()) alphaTile(t); | |
| drawLegend(idx); dirty = true; | |
| }; | |
| wrap.appendChild(b); | |
| }); | |
| el.appendChild(wrap); el.classList.add("on"); | |
| } | |
| // ββ region select: exact count over ALL degree bands βββββββββββββββββββββββββββββββββββββββββ | |
| // A tile's bbox IS its quadtree cell and the tiler assigns points by cell, so a tile sitting | |
| // inside the rect contributes its whole ``n`` with zero bytes fetched; only tiles straddling an | |
| // edge are read. That is what makes an exact count over 52.9 M points affordable β a whole-map | |
| // rect costs 0 fetches, a 10 % box ~8 MB. | |
| // | |
| // The EPS slack is not cosmetic. _quantize_xy stores round(unit * 32767), but cell membership was | |
| // decided on the pre-rounding ``unit``, so a point can land up to one quantum (6.1e-5) outside its | |
| // own cell β measured worst case 1.4e-5, concentrated at deep quadtree depths where cells are | |
| // narrower than the quantum. Inflating every bbox by EPS before classifying makes the count exact | |
| // again; clamping the inflated box back to the map keeps the whole-map case free instead of | |
| // turning the border into 123 straddlers. | |
| const clamp1 = (v) => Math.max(-1, Math.min(1, v)); | |
| function classify(r) { // r is in world coords, NOT clamped to the map | |
| const full = [], part = []; let freeN = 0; | |
| for (const t of META.tiles) { | |
| const b = t.bbox; | |
| const x0 = clamp1(b[0] - EPS), y0 = clamp1(b[1] - EPS); | |
| const x1 = clamp1(b[2] + EPS), y1 = clamp1(b[3] + EPS); | |
| if (x0 > r.x1 || x1 < r.x0 || y0 > r.y1 || y1 < r.y0) continue; // disjoint | |
| if (x0 >= r.x0 && x1 <= r.x1 && y0 >= r.y0 && y1 <= r.y1) { full.push(t); freeN += t.n; } | |
| else part.push(t); | |
| } | |
| return { full, part, freeN }; | |
| } | |
| // Bounds are inclusive throughout: _quantize_xy clips outliers to exactly Β±1.0, so points really | |
| // do sit on the outermost edges and a strict test would silently drop them. | |
| function quantRect(r) { return [r.x0 * Q, r.y0 * Q, r.x1 * Q, r.y1 * Q]; } | |
| function hits(xy, K, q, out) { | |
| const ax = q[0], ay = q[1], bx = q[2], by = q[3]; let c = 0; | |
| for (let i = 0; i < K; i++) { | |
| const x = xy[i * 2], y = xy[i * 2 + 1]; | |
| if (x >= ax && x <= bx && y >= ay && y <= by) { if (out) out[c] = i; c++; } | |
| } | |
| return c; | |
| } | |
| // ββ scan tile cache (deliberately not the render path) βββββββββββββββββββββββββββββββββββββββ | |
| function tdirOf(t) { return BASE + "/t/" + (t.d != null ? t.d + "/" : "") + t.id; } | |
| function lru(cache, max, cur) { | |
| for (const [k, v] of cache) { if (cur <= max) break; cache.delete(k); cur -= v.bytes; } | |
| return cur; | |
| } | |
| function scanTile(t, sig) { | |
| const hit = scanCache.get(t.id); | |
| if (hit) { scanCache.delete(t.id); scanCache.set(t.id, hit); return Promise.resolve(hit); } | |
| if (scanWait.has(t.id)) return scanWait.get(t.id); | |
| const p = fetch(tdirOf(t) + ".bin", { signal: sig }).then((r) => { | |
| if (!r.ok) throw new Error(t.id + " " + r.status); | |
| return r.arrayBuffer(); | |
| }).then((buf) => { | |
| const K = t.n; let o = 0; // same layout as doLoad(), minus the GL upload | |
| const v = { n: K, xy: new Int16Array(buf, o, K * 2), labs: [], deg: null, | |
| bytes: buf.byteLength }; | |
| o += K * 4; | |
| for (let c = 0; c < nCat; c++) { v.labs.push(new Uint8Array(buf, o, K)); o += K; } | |
| v.deg = new Uint8Array(buf, o, K); | |
| scanCache.set(t.id, v); scanBytes = lru(scanCache, SCAN_BYTES_MAX, scanBytes + v.bytes); | |
| return v; | |
| }).finally(() => scanWait.delete(t.id)); | |
| scanWait.set(t.id, p); return p; | |
| } | |
| function scanNames(t, sig) { | |
| const hit = nameCache.get(t.id); | |
| if (hit) { nameCache.delete(t.id); nameCache.set(t.id, hit); return Promise.resolve(hit); } | |
| if (nameWait.has(t.id)) return nameWait.get(t.id); | |
| const p = fetch(tdirOf(t) + ".nm", { signal: sig }).then((r) => { | |
| if (!r.ok) throw new Error(t.id + " " + r.status); | |
| return r.text(); | |
| }).then((txt) => { | |
| const v = { names: txt.length ? txt.split("\n") : [], bytes: txt.length }; | |
| nameCache.set(t.id, v); nameBytes = lru(nameCache, NAME_BYTES_MAX, nameBytes + v.bytes); | |
| return v; | |
| }).finally(() => nameWait.delete(t.id)); | |
| nameWait.set(t.id, p); return p; | |
| } | |
| function pool(items, k, fn) { // bounded-concurrency map | |
| let i = 0; | |
| const run = async () => { while (i < items.length) { const j = i++; await fn(items[j]); } }; | |
| return Promise.all(Array.from({ length: Math.min(k, items.length) || 1 }, run)); | |
| } | |
| // ββ commit a selection and scan the straddling tiles βββββββββββββββββββββββββββββββββββββββββ | |
| function cancelScan() { | |
| sel.epoch++; | |
| if (scanSig) { scanSig.abort(); scanSig = null; } | |
| } | |
| function setSelection(r) { | |
| cancelScan(); listReset(); | |
| Object.assign(sel, { rect: r, cls: null, count: 0, done: 0, todo: 0, errs: 0, | |
| exact: false, partN: new Map() }); | |
| if (!r) { | |
| $("selpill").classList.remove("on"); closeSheet(); syncSelBox(); dirty = true; return; | |
| } | |
| const ep = sel.epoch, ac = new AbortController(); scanSig = ac; | |
| const cls = sel.cls = classify(r); | |
| sel.count = cls.freeN; sel.todo = cls.part.length; sel.exact = !cls.part.length; | |
| $("selpill").classList.add("on"); paintBadge(); syncSelBox(); dirty = true; | |
| if (!cls.part.length) return; | |
| const q = quantRect(r); | |
| cls.part.sort((a, b) => a.z - b.z); // band order, so the list can start on band 0 early | |
| pool(cls.part, SCAN_FETCH, async (t) => { | |
| if (ep !== sel.epoch) return; | |
| let v; | |
| try { v = await scanTile(t, ac.signal); } catch { if (ep === sel.epoch) sel.errs++; return; } | |
| if (ep !== sel.epoch) return; | |
| const c = hits(v.xy, t.n, q, null); | |
| sel.partN.set(t.id, c); sel.count += c; sel.done++; paintBadge(); | |
| }).then(() => { | |
| if (ep !== sel.epoch) return; | |
| sel.exact = !sel.errs; paintBadge(); | |
| }); | |
| } | |
| let badgeRaf = 0; | |
| function paintBadge() { | |
| if (badgeRaf) return; | |
| badgeRaf = requestAnimationFrame(() => { | |
| badgeRaf = 0; const b = $("seln"), scanning = !sel.exact && sel.done < sel.todo; | |
| b.textContent = fmtFull(sel.count) + (sel.exact ? " selected" : "+ scanningβ¦"); | |
| b.title = sel.exact | |
| ? "Hosts inside the rectangle, across all degree bands β not just the points drawn at this " | |
| + "zoom. Click to list them, highest degree first." | |
| : "reading " + sel.done + " / " + sel.todo + " edge tiles" | |
| + (sel.errs ? " Β· " + sel.errs + " failed" : ""); | |
| b.style.setProperty("--p", sel.todo ? sel.done / sel.todo : 1); | |
| b.classList.toggle("scanning", scanning); | |
| b.setAttribute("aria-live", sel.exact ? "polite" : "off"); | |
| if (sheetOpen) { listSub(); listFoot(); } | |
| }); | |
| } | |
| function fmtFull(v) { return v.toLocaleString("en-US"); } | |
| // ββ rubber band (a DOM overlay: the shader is a point program with no line primitive, and a | |
| // band updated at pointer rate through GL would force a full redraw of up to 600 tiles) βββββββ | |
| function syncSelBox() { | |
| const el = $("selbox"); | |
| let x0, y0, x1, y1, set = false; | |
| if (band) { | |
| x0 = Math.min(band.x0, band.x1); x1 = Math.max(band.x0, band.x1); | |
| y0 = Math.min(band.y0, band.y1); y1 = Math.max(band.y0, band.y1); | |
| } else if (sel.rect) { | |
| const r = sel.rect, C = 4000; // a rect held while zooming 1000x is megapixels wide | |
| x0 = Math.max(-C, toScreenX(r.x0)); x1 = Math.min(C, toScreenX(r.x1)); | |
| y0 = Math.max(-C, toScreenY(r.y1)); y1 = Math.min(C, toScreenY(r.y0)); // screen y flips | |
| set = true; | |
| if (x1 <= x0 || y1 <= y0) { el.classList.remove("on"); return; } | |
| } else { el.classList.remove("on"); return; } | |
| el.style.left = x0 + "px"; el.style.top = y0 + "px"; | |
| el.style.width = (x1 - x0) + "px"; el.style.height = (y1 - y0) + "px"; | |
| el.classList.toggle("set", set); el.classList.add("on"); | |
| } | |
| // ββ selected-host list: "next page" IS "next by degree" ββββββββββββββββββββββββββββββββββββββ | |
| // ``z`` is a global degree-rank band (band 0 = the top 16 384 hosts by degree, band 1 the next | |
| // 65 536, β¦), assigned in embed_2d_tile._degree_levels by searchsorted over the degree rank. So | |
| // ascending bands == descending degree, exactly, and a 256-bucket counting sort on deg8 refines | |
| // it within a band. The list is therefore lazy in its own sort key: page 1 of a whole-map | |
| // selection costs one 128 KB tile, and band z+1 is never touched until band z runs out. | |
| function listReset() { | |
| Object.assign(list, { band: 0, buf: null, tiles: [], pos: 0, rows: 0, | |
| eof: false, busy: false, approx: false }); | |
| const rows = $("selrows"); | |
| rows.textContent = ""; rows.scrollTop = 0; | |
| listSub(); listFoot(); | |
| } | |
| let scratch = new Uint32Array(0); | |
| async function nextBand(ep) { | |
| while (list.band <= META.zMax) { | |
| const z = list.band++; | |
| const ts = []; | |
| for (const t of sel.cls.full) if (t.z === z) ts.push(t); | |
| for (const t of sel.cls.part) if (t.z === z) ts.push(t); | |
| if (!ts.length) continue; | |
| let want = 0; | |
| for (const t of ts) want += sel.partN.has(t.id) ? sel.partN.get(t.id) : t.n; | |
| if (!want) continue; | |
| const cap = Math.min(want, BAND_MAX); | |
| const li = new Uint16Array(cap), ti = new Uint16Array(cap), dg = new Uint8Array(cap); | |
| const lb = new Uint8Array(cap * nCat); // labels are COPIED so a row survives tile eviction | |
| const slot = []; let m = 0; | |
| await pool(ts, SCAN_FETCH, async (t) => { | |
| if (ep !== sel.epoch || m >= cap) return; | |
| let v; | |
| try { v = await scanTile(t, scanSig && scanSig.signal); } catch { return; } | |
| if (ep !== sel.epoch) return; | |
| // everything below is synchronous, so one shared scratch is safe across the pool | |
| if (scratch.length < t.n) scratch = new Uint32Array(t.n); | |
| const c = hits(v.xy, t.n, quantRect(sel.rect), scratch); | |
| if (!c) return; | |
| const s = slot.push(t) - 1; | |
| for (let i = 0; i < c && m < cap; i++, m++) { | |
| const j = scratch[i]; | |
| ti[m] = s; li[m] = j; dg[m] = v.deg[j]; | |
| for (let k = 0; k < nCat; k++) lb[m * nCat + k] = v.labs[k][j]; | |
| } | |
| }); | |
| if (ep !== sel.epoch) return false; | |
| if (m >= cap && want > cap) list.approx = true; | |
| // counting sort on deg8, descending: O(m), 256 buckets, no comparator | |
| const cnt = new Uint32Array(256); | |
| for (let i = 0; i < m; i++) cnt[dg[i]]++; | |
| const at = new Uint32Array(256); let acc = 0; | |
| for (let v = 255; v >= 0; v--) { at[v] = acc; acc += cnt[v]; } | |
| const ord = new Uint32Array(m); | |
| for (let i = 0; i < m; i++) ord[at[dg[i]]++] = i; | |
| list.buf = { li, ti, dg, lb, ord, m }; list.tiles = slot; list.pos = 0; | |
| if (m) return true; | |
| } | |
| list.eof = true; return false; | |
| } | |
| // Hostnames come from the crawl and are attacker-influenceable; this table renders up to LIST_MAX | |
| // of them, so every cell is filled with textContent, which cannot introduce markup. | |
| function rowEl(t, j, deg8, lb, i) { | |
| const el = document.createElement("div"); | |
| el.className = "selrow"; el.tabIndex = 0; el.title = "Show on the map"; | |
| const host = document.createElement("span"); host.className = "h"; host.textContent = "β¦"; | |
| const deg = document.createElement("span"); deg.className = "d"; | |
| deg.textContent = "~" + fmtN(degLabel(deg8)); | |
| el.append(host, deg); | |
| // One adaptive label column rather than one per set: across the full corpus language is 68 % | |
| // populated but topic is 0.3 % and quality 1.3 %, so per-set columns would be em-dashes almost | |
| // every row. Joining whatever *is* known keeps the information and drops the dead space. | |
| const known = [], full = []; | |
| let k2 = 0; | |
| META.labelSets.forEach((ls) => { | |
| if (ls.kind === "degree") return; // labs is indexed by CAT position, not absolute | |
| const v = lb[i * nCat + k2++]; | |
| if (v === 255) return; | |
| known.push(ls.classes[v]); | |
| full.push(ls.name + ": " + ls.classes[v]); | |
| }); | |
| const sp = document.createElement("span"); sp.className = "c"; | |
| sp.textContent = known.length ? known.join(" Β· ") : "β"; | |
| // the cell is narrow, so which set a value came from lives in the tooltip | |
| sp.title = full.join(" Β· ") || "no labels"; | |
| el.appendChild(sp); | |
| const go = () => flyTo([host.textContent, t.id, j, deg8]); | |
| el.onclick = go; | |
| el.onkeydown = (ev) => { | |
| if (ev.key === "Enter" || ev.key === " ") { ev.preventDefault(); go(); } | |
| }; | |
| return { el, host }; | |
| } | |
| async function listFill() { | |
| if (list.busy || list.eof || list.rows >= LIST_MAX || !sel.rect) return; | |
| list.busy = true; const ep = sel.epoch; | |
| try { | |
| while (!list.buf || list.pos >= list.buf.m) { | |
| list.buf = null; | |
| const more = await nextBand(ep); | |
| if (ep !== sel.epoch) return; | |
| if (!more) { listEmpty(); listFoot(); return; } | |
| } | |
| const b = list.buf, take = Math.min(PAGE, b.m - list.pos, LIST_MAX - list.rows); | |
| const frag = document.createDocumentFragment(), need = new Map(); | |
| for (let k = 0; k < take; k++) { | |
| const i = b.ord[list.pos + k], t = list.tiles[b.ti[i]], j = b.li[i]; | |
| const r = rowEl(t, j, b.dg[i], b.lb, i); | |
| if (!need.has(t)) need.set(t, []); | |
| need.get(t).push([r.host, j]); | |
| frag.appendChild(r.el); | |
| } | |
| list.pos += take; list.rows += take; | |
| $("selrows").appendChild(frag); listFoot(); | |
| // .nm is ~2x the bytes of .bin, so names are fetched only for the tiles this page shows β | |
| // the same lazy-name discipline doLoad() uses once a tile is already drawable. | |
| pool([...need.keys()], 4, async (t) => { | |
| let v; | |
| try { v = await scanNames(t, scanSig && scanSig.signal); } catch { v = null; } | |
| if (ep !== sel.epoch) return; | |
| for (const pair of need.get(t)) { | |
| pair[0].textContent = (v && v.names[pair[1]]) || "(unnamed)"; | |
| } | |
| }); | |
| // a short page may not overflow the scroller, so ``scroll`` would never fire again | |
| const e = $("selrows"); | |
| if (e.scrollHeight <= e.clientHeight && !list.eof && list.rows < LIST_MAX) { | |
| requestAnimationFrame(listFill); | |
| } | |
| } finally { list.busy = false; } | |
| } | |
| function listEmpty() { | |
| if (list.rows) return; | |
| const d = document.createElement("div"); d.className = "empty"; | |
| d.textContent = sel.count | |
| ? "Nothing to list β the tiles for this region could not be read." | |
| : "No hosts in this region."; | |
| $("selrows").appendChild(d); | |
| } | |
| function buildSelHead() { | |
| const hd = $("selhd"); hd.textContent = ""; | |
| const cats = META.labelSets.filter((ls) => ls.kind !== "degree").map((ls) => ls.name); | |
| ["host", "degree", "labels"].forEach((c, k) => { | |
| const s = document.createElement("span"); s.textContent = c; | |
| if (k === 2) s.title = cats.join(", ") + " β whichever are known for that host"; | |
| hd.appendChild(s); | |
| }); | |
| } | |
| function listSub() { | |
| if (!sel.rect) { $("selsub").textContent = ""; return; } | |
| $("selsub").textContent = | |
| "Every host in the rectangle, across all degree bands β not just the points drawn at this " | |
| + "zoom. Highest degree first. Click a row to fly to it."; | |
| } | |
| function note(cls, text) { | |
| const s = document.createElement("span"); s.className = cls; s.textContent = text; return s; | |
| } | |
| function listFoot() { | |
| const f = $("selfoot"); f.textContent = ""; | |
| f.appendChild(note("", fmtFull(list.rows) + " of " + fmtFull(sel.count) | |
| + (sel.exact ? "" : "+") + " shown")); | |
| if (list.rows >= LIST_MAX) { | |
| f.appendChild(note("warn", "capped at " + fmtFull(LIST_MAX) | |
| + " rows β shrink the rectangle to reach further down")); | |
| } | |
| if (list.approx) { | |
| f.appendChild(note("warn", "order approximate past " + fmtFull(BAND_MAX) | |
| + " rows in one band")); | |
| } | |
| if (sel.errs) f.appendChild(note("warn", sel.errs + " tile(s) failed to load")); | |
| const c = document.createElement("button"); | |
| c.type = "button"; c.textContent = "clear"; | |
| c.onclick = () => setSelection(null); | |
| f.appendChild(c); | |
| } | |
| function openSheet() { | |
| if (!sel.rect) return; | |
| sheetOpen = true; | |
| $("sel").classList.add("on"); $("seln").setAttribute("aria-expanded", "true"); | |
| $("selpill").classList.add("open"); | |
| document.body.classList.add("sheet"); | |
| listSub(); $("selx").focus(); listFill(); | |
| } | |
| function closeSheet() { | |
| if (!sheetOpen) return; | |
| sheetOpen = false; | |
| $("sel").classList.remove("on"); $("seln").setAttribute("aria-expanded", "false"); | |
| $("selpill").classList.remove("open"); | |
| document.body.classList.remove("sheet"); | |
| $("seln").focus(); | |
| } | |
| // ββ hover pick (linear scan over visible tiles) βββββββββββββββββββββββββββββββββββββββββββββ | |
| const PICK_PX = 15; | |
| function pick(px, py) { | |
| const w = cv.clientWidth, h = cv.clientHeight; | |
| const p = toWorld(px, py), sx = p.x * Q, sy = p.y * Q; | |
| const supx = (2 / w) / (view.s * asp) * Q, supy = (2 / h) / view.s * Q; | |
| let best = null, bd = PICK_PX * PICK_PX; | |
| for (const id of activeIds) { | |
| const t = ready.get(id); if (!t || t.status !== "ready") continue; | |
| const xy = t.xy; | |
| for (let i = 0; i < t.n; i++) { | |
| const dxp = (xy[i * 2] - sx) / supx, dyp = (xy[i * 2 + 1] - sy) / supy; | |
| const d = dxp * dxp + dyp * dyp; | |
| if (d < bd) { bd = d; best = { id, li: i, t }; } | |
| } | |
| } | |
| return best; | |
| } | |
| function hover(px, py) { | |
| const hit = pick(px, py); const tip = $("tip"); | |
| if (!hit) { tip.style.display = "none"; return; } | |
| const t = hit.t, i = hit.li; | |
| const name = t.names ? (t.names[i] || null) : null; | |
| // Hostnames are crawl-derived, so they go in as text, never as markup. | |
| tip.textContent = ""; | |
| const h = document.createElement("div"); h.className = "host"; | |
| h.textContent = name || "(loading nameβ¦)"; | |
| tip.appendChild(h); | |
| tip.appendChild(tipRow("degree", "~" + fmtN(degLabel(t.deg[i])))); | |
| META.labelSets.forEach((ls, k) => { | |
| if (ls.kind === "degree") return; | |
| const v = t.labs[k][i]; | |
| tip.appendChild(tipRow(ls.name, v === 255 ? "β" : ls.classes[v])); | |
| }); | |
| tip.style.left = px + "px"; tip.style.top = py + "px"; tip.style.display = "block"; | |
| } | |
| function tipRow(label, value) { | |
| const row = document.createElement("span"); row.className = "row"; | |
| const b = document.createElement("b"); b.textContent = label; | |
| const v = document.createElement("span"); v.textContent = value; | |
| row.append(b, v); return row; | |
| } | |
| // ββ search (global top-degree index from manifest) ββββββββββββββββββββββββββββββββββββββββββ | |
| function buildSearch() { | |
| const idxList = META.search || []; | |
| const lo = idxList.map((e) => e[0].toLowerCase()); | |
| const q = $("q"), hits = $("hits"); | |
| q.addEventListener("input", () => { | |
| const term = q.value.trim().toLowerCase(); hits.innerHTML = ""; | |
| if (!term) return; | |
| const out = []; | |
| for (let k = 0; k < idxList.length && out.length < 30; k++) | |
| if (lo[k].includes(term)) out.push(idxList[k]); | |
| out.forEach((e) => { | |
| const li = document.createElement("li"); li.setAttribute("role", "option"); | |
| const nm = document.createElement("span"); nm.textContent = e[0]; | |
| const dg = document.createElement("span"); dg.className = "deg"; | |
| dg.textContent = "~" + fmtN(degLabel(e[3])); | |
| li.append(nm, dg); | |
| li.onclick = () => flyTo(e); hits.appendChild(li); | |
| }); | |
| }); | |
| } | |
| function flyTo(entry) { // entry = [name, tileId, localIdx, deg8] | |
| const [, tid, liLocal] = entry; const meta = tileMeta.get(tid); if (!meta) return; | |
| startLoad(tid); | |
| const b = meta.bbox, tx = (b[0] + b[2]) / 2, ty = (b[1] + b[3]) / 2; // fallback: tile centre | |
| const finish = () => { | |
| const t = ready.get(tid); | |
| const px = (t && t.xy) ? t.xy[liLocal * 2] / Q : tx; | |
| const py = (t && t.xy) ? t.xy[liLocal * 2 + 1] / Q : ty; | |
| pin = { id: tid, li: liLocal }; | |
| const s0 = view.s, cx0 = view.cx, cy0 = view.cy, ts = Math.max(40, view.s), t0 = performance.now(), D = 620; | |
| const reduce = matchMedia("(prefers-reduced-motion: reduce)").matches; | |
| (function step(now) { | |
| const p = reduce ? 1 : Math.min(1, (now - t0) / D), e = 1 - Math.pow(1 - p, 3); | |
| view.s = s0 + (ts - s0) * e; view.cx = cx0 + (px - cx0) * e; view.cy = cy0 + (py - cy0) * e; | |
| dirty = true; if (p < 1) requestAnimationFrame(step); | |
| })(t0); | |
| }; | |
| const t = ready.get(tid); | |
| if (t && t.status === "ready") finish(); | |
| else { const iv = setInterval(() => { const tt = ready.get(tid); | |
| if (tt && tt.status === "ready") { clearInterval(iv); finish(); } }, 60); } | |
| } | |
| // ββ interaction βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| let drag = null; | |
| // Shift (or the armed ``select`` toggle) at pointerdown latches rect-select for the WHOLE gesture: | |
| // releasing Shift mid-drag must not turn a rubber band into a pan, or vice versa. | |
| cv.addEventListener("pointerdown", (e) => { | |
| cv.setPointerCapture(e.pointerId); | |
| if (e.shiftKey || selArmed) { | |
| band = { x0: e.clientX, y0: e.clientY, x1: e.clientX, y1: e.clientY }; | |
| $("tip").style.display = "none"; syncSelBox(); return; | |
| } | |
| drag = { x: e.clientX, y: e.clientY }; | |
| }); | |
| function endBand(commit) { | |
| if (!band) return; | |
| const b = band; band = null; | |
| if (!commit || Math.abs(b.x1 - b.x0) < 4 || Math.abs(b.y1 - b.y0) < 4) { | |
| setSelection(null); return; // a shift-*click* clears the selection | |
| } | |
| // The rect is NOT clamped to the map: only the tile bboxes are (in classify). That way a drag | |
| // past every edge still counts everything for free, while a drag entirely off the map stays a | |
| // real, empty rectangle instead of collapsing onto the border. | |
| const p = toWorld(b.x0, b.y0), q = toWorld(b.x1, b.y1); | |
| setSelection({ x0: Math.min(p.x, q.x), y0: Math.min(p.y, q.y), | |
| x1: Math.max(p.x, q.x), y1: Math.max(p.y, q.y) }); | |
| } | |
| cv.addEventListener("pointerup", () => { endBand(true); drag = null; }); | |
| cv.addEventListener("pointercancel", () => { endBand(false); drag = null; }); | |
| cv.addEventListener("pointermove", (e) => { | |
| if (band) { band.x1 = e.clientX; band.y1 = e.clientY; syncSelBox(); return; } | |
| if (drag) { | |
| const w = cv.clientWidth, h = cv.clientHeight; | |
| view.cx -= ((e.clientX - drag.x) / w) * 2 / (view.s * asp); | |
| view.cy += ((e.clientY - drag.y) / h) * 2 / view.s; | |
| drag.x = e.clientX; drag.y = e.clientY; dirty = true; $("tip").style.display = "none"; | |
| } else { hover(e.clientX, e.clientY); } | |
| }); | |
| cv.addEventListener("pointerleave", () => { $("tip").style.display = "none"; }); | |
| // ββ colour-property help ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| // One short paragraph per colourable property, keyed by label-set name so the dialog lists only | |
| // the sets this build actually shipped. Rendered once, on first open. | |
| const HELP = { | |
| topic: ["What the sites are about, in 24 categories from the WebOrganizer corpus. Labels are " + | |
| "assigned per page and rolled up to the host, so only a small share of hosts carry one \u2014 " + | |
| "but topic is the property these embeddings separate best.", | |
| [["https://huggingface.co/datasets/WebOrganizer/Corpus-200B", "WebOrganizer/Corpus-200B"]]], | |
| language: ["The main content language of a host's pages, across 120 languages. Detected per " + | |
| "page with CLD2 and read from Common Crawl's columnar URL index. This has the widest " + | |
| "coverage of the four. Sites tend to link to sites in the same language, so it separates " + | |
| "cleanly once a host has enough links.", | |
| [["https://github.com/commoncrawl/language-detection-cld2", "Language detection (CLD2)"], | |
| ["https://commoncrawl.org/blog/index-to-warc-files-and-urls-in-columnar-format", | |
| "Columnar URL index"]]], | |
| quality: ["A four-way content-quality annotation from GneissWeb. Its source crawls predate this " + | |
| "graph, so coverage is thin and tilted toward long-lived sites.", | |
| [["https://huggingface.co/datasets/commoncrawl/gneissweb-annotation-host-testing-v1", | |
| "gneissweb-annotation-host-testing-v1"]]], | |
| degree: ["How many links a host has in the web graph, counting both directions. The only one of " + | |
| "the four that is not a label \u2014 it is read straight off the graph, so every host has a " + | |
| "value. High-degree hubs pull toward the dense centre.", | |
| [["https://commoncrawl.org/web-graphs", "Common Crawl web graphs"]]], | |
| }; | |
| let helpBuilt = false; | |
| function buildHelp() { | |
| if (helpBuilt) return; | |
| helpBuilt = true; | |
| const seen = new Set(META.labelSets.map((ls) => ls.name)); | |
| $("helpbody").innerHTML = Object.keys(HELP).filter((k) => seen.has(k)).map((k) => { | |
| const [text, links] = HELP[k]; | |
| const hrefs = links | |
| .map(([href, label]) => `<a href="${href}" target="_blank" rel="noopener">${label}</a>`) | |
| .join(" · "); | |
| return `<h3>${k}</h3><p>${text} ${hrefs}</p>`; | |
| }).join(""); | |
| } | |
| function openHelp() { | |
| buildHelp(); | |
| $("scrim").hidden = false; $("help").hidden = false; | |
| $("helpx").focus(); | |
| } | |
| function closeHelp() { | |
| $("scrim").hidden = true; $("help").hidden = true; | |
| $("helpbtn").focus(); | |
| } | |
| $("helpbtn").onclick = () => ($("help").hidden ? openHelp() : closeHelp()); | |
| $("helpx").onclick = closeHelp; | |
| $("scrim").onclick = closeHelp; | |
| $("seln").onclick = () => (sheetOpen ? closeSheet() : openSheet()); | |
| $("selclr").onclick = () => setSelection(null); | |
| $("selx").onclick = () => closeSheet(); | |
| $("selarm").onclick = () => { | |
| selArmed = !selArmed; | |
| $("selarm").setAttribute("aria-pressed", selArmed ? "true" : "false"); | |
| cv.style.cursor = selArmed ? "cell" : "crosshair"; | |
| }; | |
| window.addEventListener("keydown", (e) => { | |
| if (e.key !== "Escape" || e.target === $("q")) return; // let the search box clear itself | |
| if (!$("help").hidden) closeHelp(); | |
| else if (sheetOpen) closeSheet(); else if (sel.rect) setSelection(null); | |
| }); | |
| $("selrows").addEventListener("scroll", () => { | |
| const e = $("selrows"); | |
| if (e.scrollTop + e.clientHeight > e.scrollHeight - 400) listFill(); | |
| }, { passive: true }); | |
| cv.addEventListener("wheel", (e) => { | |
| e.preventDefault(); | |
| const w = cv.clientWidth, h = cv.clientHeight; | |
| const ndcx = (e.clientX / w) * 2 - 1, ndcy = 1 - (e.clientY / h) * 2; | |
| const wx = (ndcx / (view.s * asp)) + view.cx, wy = (ndcy / view.s) + view.cy; | |
| const f = Math.exp(-e.deltaY * 0.0016); view.s = Math.max(0.15, Math.min(1200, view.s * f)); | |
| view.cx = wx - (ndcx / (view.s * asp)); view.cy = wy - (ndcy / view.s); dirty = true; | |
| }, { passive: false }); | |
| // "reset" means everything the user can leave behind: camera, rubber-band selection, and the | |
| // pinned host a row click or a search hit dropped on the map. | |
| $("reset").onclick = () => { | |
| view = { s: 0.9, cx: 0, cy: 0 }; pin = null; setSelection(null); dirty = true; | |
| }; | |
| function resize() { | |
| const w = cv.clientWidth, h = cv.clientHeight; | |
| cv.width = Math.round(w * dpr); cv.height = Math.round(h * dpr); | |
| gl.viewport(0, 0, cv.width, cv.height); asp = h / w; dirty = true; | |
| } | |
| window.addEventListener("resize", resize); | |
| // ββ render loop βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| let fFrames = 0, fT = performance.now(), spinOn = false, spinUntil = 0; | |
| function syncSpinner(now) { // ring visible while any tile is queued/fetching | |
| const busy = inflight > 0 || queue.length > 0; // ...held β₯350ms so fast local loads still flash | |
| if (busy) spinUntil = now + 350; | |
| const show = busy || now < spinUntil; | |
| if (show !== spinOn) { spinOn = show; $("spin").classList.toggle("on", show); } | |
| } | |
| function loop(now) { | |
| syncSpinner(now); | |
| if (dirty) { | |
| updateActive(); draw(); dirty = false; | |
| $("zoom").textContent = view.s.toFixed(view.s < 10 ? 1 : 0) + "Γ"; | |
| } | |
| fFrames++; | |
| if (now - fT > 500) { $("fps").textContent = Math.round((fFrames * 1000) / (now - fT)); fFrames = 0; fT = now; } | |
| requestAnimationFrame(loop); | |
| } | |
| })(); | |
| </script> | |
| </body> | |
| </html> | |