Spaces:
Running
Running
File size: 17,492 Bytes
27e064c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 | // RuleChef Live: replays data/replay.json (written by sim.py) and runs the rules in the browser (engine.js).
(function () {
"use strict";
const E = window.RCEngine;
const $ = (s) => document.querySelector(s);
const fmt = (n) => n.toLocaleString("en-US");
const pct = (a, b) => (b ? (100 * a / b).toFixed(1) + " %" : "–");
const esc = (s) => String(s).replace(/[&<>"]/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """ }[c]));
const NAMES = { DATETIME: "date/time", CODE: "code", QUANTITY: "quantity" };
let R, REQ = {}, EV, VERS, GUARD, META, N, AT, S = {}, LAB = [], HOLD = [];
let k = 0, timer = null;
function cum(xs) { const out = [0]; let s = 0; for (const x of xs) { s += x; out.push(s); } return out; }
function versionAt(i) { let lo = 0; for (let j = 0; j < AT.length; j++) if (AT[j] <= i) lo = j; return VERS[lo]; }
function blockedAt(i) { return new Set(GUARD.filter((g) => g.at < i).map((g) => g.key)); }
function activeRules(v, i) { const off = blockedAt(i); return v.rules.filter((r) => r.trusted && !off.has(r.key)); }
function card(t, v, s) { return `<div class="kpi"><div class="kt">${t}</div><div class="kv">${v}</div><div class="ks">${s}</div></div>`; }
function renderKpis() {
const v = versionAt(k);
const nr = S.rules[k], ns = S.spans[k], nc = S.check[k], na = S.agree[k], calls = S.call[k];
const lat = k ? S.lat[k] / k : 0, tlat = k ? S.tlat[k] / k : 0;
const rm = EV.slice(0, k).filter((e) => e.by === "rules").map((e) => e.rule_ms).sort((a, b) => a - b);
const med = rm.length ? rm[rm.length >> 1].toFixed(2) + " ms (median)" : "–";
$("#kpis").innerHTML = [
card("Requests", `${fmt(k)} / ${fmt(N)}`, `rule set v${v.v}`),
card("Answered by rules", pct(nr, k), `${fmt(nr)}: ${fmt(ns)} with spans, ${fmt(nr - ns)} with nothing to extract`),
card("Model calls", fmt(calls), `${fmt(k - calls)} saved (${pct(k - calls, k)}); spot checks included`),
card("Mean latency", `${Math.round(lat)} ms`, `teacher on every request: ${Math.round(tlat)} ms; a rule answer: ${med}`),
card("Spot-check agreement", pct(na, nc), `${na} of ${nc} checked rule answers equal the teacher's`),
card("Rules answering", String(activeRules(v, k).length), `of ${v.rules.length} in v${v.v}; ${blockedAt(k).size} switched off by the guard`),
].join("");
}
// ------------------------------------------------------------------ charts (SVG, two series each, crosshair tooltip)
const W = 360, H = 190, P = { l: 40, r: 10, t: 8, b: 22 };
function lineChart(el, cfg) {
const { series, yMax, yTicks, yFmt, markers } = cfg;
const x = (v) => P.l + (W - P.l - P.r) * v / N, y = (v) => H - P.b - (H - P.t - P.b) * v / yMax;
let g = `<svg viewBox="0 0 ${W} ${H}" role="img" aria-label="${esc(cfg.label)}">`;
for (const t of yTicks) g += `<line x1="${P.l}" x2="${W - P.r}" y1="${y(t)}" y2="${y(t)}" stroke="var(--grid)"/><g class="axis"><text x="${P.l - 5}" y="${y(t) + 3}" text-anchor="end">${yFmt(t)}</text></g>`;
for (let t = 0; t <= N; t += 1000) g += `<g class="axis"><text x="${x(t)}" y="${H - 6}" text-anchor="middle">${fmt(t)}</text></g>`;
for (const m of markers || []) if (m.at <= k && m.at > 0) g += `<line x1="${x(m.at)}" x2="${x(m.at)}" y1="${P.t}" y2="${H - P.b}" stroke="var(--line)" stroke-dasharray="2 3"/>`;
g += `<line x1="${P.l}" x2="${W - P.r}" y1="${H - P.b}" y2="${H - P.b}" stroke="var(--line)"/>`;
for (const s of series) {
if (s.pts.length < 2) continue;
g += `<polyline fill="none" stroke="${s.color}" stroke-width="2" stroke-linejoin="round" stroke-linecap="round" points="${s.pts.map((p) => x(p[0]).toFixed(1) + "," + y(p[1]).toFixed(1)).join(" ")}"/>`;
}
g += `<line class="xh" x1="0" x2="0" y1="${P.t}" y2="${H - P.b}" stroke="var(--muted)" visibility="hidden"/>`;
g += `<rect x="${P.l}" y="${P.t}" width="${W - P.l - P.r}" height="${H - P.t - P.b}" fill="transparent"/></svg>`;
g += `<div class="legend">${series.map((s) => `<span><i style="background:${s.color}"></i>${esc(s.name)}</span>`).join("")}</div>`;
el.innerHTML = g;
const svg = el.querySelector("svg"), xh = el.querySelector(".xh"), tip = $("#tip");
svg.addEventListener("mousemove", (ev) => {
const b = svg.getBoundingClientRect();
const vx = (ev.clientX - b.left) / b.width * W;
const req = Math.max(0, Math.min(k, Math.round((vx - P.l) / (W - P.l - P.r) * N)));
const rows = series.map((s) => { let best = null; for (const p of s.pts) if (!best || Math.abs(p[0] - req) < Math.abs(best[0] - req)) best = p; return [s, best]; }).filter((r) => r[1]);
if (!rows.length) return;
xh.setAttribute("x1", x(rows[0][1][0])); xh.setAttribute("x2", x(rows[0][1][0])); xh.setAttribute("visibility", "visible");
tip.innerHTML = `request ${fmt(rows[0][1][0])}<br>` + rows.map(([s, p]) => `<span style="color:${s.color}">▬</span> ${esc(s.name)}: <b>${cfg.tipFmt(p[1])}</b>`).join("<br>");
tip.hidden = false; tip.style.left = Math.min(ev.clientX + 12, window.innerWidth - 230) + "px"; tip.style.top = (ev.clientY + 12) + "px";
});
svg.addEventListener("mouseleave", () => { tip.hidden = true; xh.setAttribute("visibility", "hidden"); });
}
function windowPts(arr, w = 200, step = 20) {
const pts = [];
for (let i = step; i <= k; i += step) { const lo = Math.max(0, i - w); pts.push([i, (arr[i] - arr[lo]) / (i - lo)]); }
return pts;
}
const C1 = () => getComputedStyle(document.documentElement).getPropertyValue("--s1").trim();
const C2 = () => getComputedStyle(document.documentElement).getPropertyValue("--s2").trim();
function renderCharts() {
const markers = VERS.slice(1);
lineChart($("#c-cov"), { label: "Share of requests answered by rules", yMax: 1, yTicks: [0, 0.25, 0.5, 0.75, 1], yFmt: (t) => Math.round(t * 100) + "%",
tipFmt: (v) => (100 * v).toFixed(1) + " %", markers,
series: [{ name: "answered by rules", color: C1(), pts: windowPts(S.rules) }, { name: "rules found spans", color: C2(), pts: windowPts(S.spans) }] });
const step = 20, a = [], b = [];
for (let i = 0; i <= k; i += step) { a.push([i, S.call[i]]); b.push([i, i]); }
lineChart($("#c-calls"), { label: "Model calls", yMax: N, yTicks: [0, 1000, 2000, 3000], yFmt: (t) => fmt(t), tipFmt: (v) => fmt(v), markers,
series: [{ name: "with rules", color: C1(), pts: a }, { name: "teacher on every request", color: C2(), pts: b }] });
lineChart($("#c-lat"), { label: "Latency per request", yMax: 500, yTicks: [0, 100, 200, 300, 400, 500], yFmt: (t) => String(t), tipFmt: (v) => Math.round(v) + " ms", markers,
series: [{ name: "cascade", color: C1(), pts: windowPts(S.lat) }, { name: "teacher on every request", color: C2(), pts: windowPts(S.tlat) }] });
}
// ------------------------------------------------------------------ tables
function renderVersions() {
const cur = versionAt(k).v;
const head = `<tr><th>version</th><th class="num">from request</th><th class="num">labelled examples</th><th class="num">rules answering / learned</th>
<th class="num">synthesis s</th><th class="num">holdout answered by rules</th><th class="num">same answer as teacher</th>
<th class="num">rule spans P / R vs teacher</th><th class="num">cascade F1 vs teacher</th><th class="num">TAB annotation F1: teacher / cascade</th></tr>`;
const rows = VERS.filter((v) => v.at <= k).map((v) => {
const h = v.holdout, ra = h.rules_answer;
return `<tr class="${v.v === cur ? "current" : ""}"><td>v${v.v}</td><td class="num">${fmt(v.at)}</td><td class="num">${fmt(v.n_labelled)}</td>
<td class="num">${v.rules.filter((r) => r.trusted).length} / ${v.rules.length}</td><td class="num">${v.synth_s.toFixed(1)}</td>
<td class="num">${pct(ra, h.n)}</td><td class="num">${ra ? pct(h.exact_agree, ra) : "–"}</td>
<td class="num">${ra ? h.routed_t.P.toFixed(2) + " / " + h.routed_t.R.toFixed(2) : "–"}</td>
<td class="num">${h.casc_t.F1.toFixed(3)}</td><td class="num">${h.teach_g.F1.toFixed(3)} / ${h.casc_g.F1.toFixed(3)}</td></tr>`;
});
$("#t-vers").innerHTML = `<thead>${head}</thead><tbody>${rows.join("")}</tbody>`;
}
function renderRules() {
const v = versionAt(k), off = blockedAt(k), used = {};
for (let i = 0; i < k; i++) if (EV[i].by === "rules") for (const key of EV[i].fired) used[key] = (used[key] || 0) + 1;
const rows = v.rules.map((r) => {
const n = r.tp + r.fp;
const st = off.has(r.key) ? ["cut", "switched off"] : r.trusted ? ["on", "answering"] : ["off", "not trusted"];
return { r, st, n, used: used[r.key] || 0 };
}).sort((a, b) => (a.st[0] !== "on") - (b.st[0] !== "on") || b.used - a.used || b.n - a.n);
$("#rules-help").textContent = v.v === 0
? "No rule set yet: every request goes to the teacher until 50 examples are labelled."
: `v${v.v}, learned from ${fmt(v.n_labelled)} labelled examples in ${v.synth_s.toFixed(1)} s. "Answering" rules matched at least ${META.trust_min} times on the labelled data with precision at least ${META.trust_prec}; the others are kept but do not answer. The last column counts requests this exact rule answered so far.`;
const head = `<thead><tr><th>status</th><th>type</th><th>regex</th><th>output</th><th class="num">labelled matches (precision)</th><th class="num">requests answered</th></tr></thead>`;
const tr = ({ r, st, n, used: u }) => `<tr><td><span class="st ${st[0]}">${st[1]}</span></td><td>${r.type}</td><td class="rx">${esc(r.regex)}</td>
<td class="mono">${esc(r.template)}</td><td class="num">${n ? `${n} (${(r.tp / n).toFixed(2)})` : "0"}</td><td class="num">${u}</td></tr>`;
const on = rows.filter((x) => x.st[0] !== "off"), rest = rows.filter((x) => x.st[0] === "off");
$("#t-rules").innerHTML = head + "<tbody>" + (on.map(tr).join("") || '<tr><td colspan="6">No answering rules.</td></tr>') + "</tbody>";
const det = $("#rules-rest");
const open = det.open;
det.innerHTML = rest.length ? `<summary>${rest.length} learned rules that do not answer (precision below ${META.trust_prec} or too few matches)</summary><div class="tablewrap"><table class="rules">${head}<tbody>${rest.map(tr).join("")}</tbody></table></div>` : "";
det.open = open;
const g = v.gate;
let html = "";
if (g) {
html += `<p><b>Gate of v${v.v}.</b> A request goes to the rules only if every trigger token is covered by an answering rule or sits in a skip context.
Trigger tokens: anything with a digit, plus ${g.trigger_words.length} learned words: ${g.trigger_words.map((w) => `<code>${esc(w)}</code>`).join(", ")}.</p>
<p>Skip contexts (previous word + digit shape, <code>9</code> = digits) that the teacher labels at most ${Math.round(META.skip_ratio * 100)} % of the time:
${g.skip.map(([p, s]) => `<code>${esc(p || "(start)")} ${esc(s)}</code>`).join(", ") || "none"}.</p>`;
}
const gl = GUARD.filter((x) => x.at < k);
html += gl.length
? `<p><b>Drift guard.</b> ${gl.map((x) => `at request ${x.at}: <code>${esc(x.key.split("\t")[1])}</code> switched off (spot checks ${x.tp} right, ${x.fp} wrong)`).join("; ")}.</p>`
: `<p><b>Drift guard.</b> No rule switched off so far. A rule is switched off when its spot-check precision drops below ${META.guard_prec} after ${META.guard_min} checked spans; in this run none did.</p>`;
$("#gate").innerHTML = html;
}
function highlight(text, spans) {
const ss = spans.slice().sort((a, b) => a.start - b.start || b.end - a.end);
let out = "", pos = 0;
for (const s of ss) {
if (s.start < pos) continue;
out += esc(text.slice(pos, s.start)) + `<mark class="${s.type}">${esc(text.slice(s.start, s.end))}<sup>${NAMES[s.type] || s.type}</sup></mark>`;
pos = s.end;
}
return out + esc(text.slice(pos));
}
function renderRecent() {
const items = [];
for (let i = k - 1; i >= Math.max(0, k - 6); i--) {
const e = EV[i], q = REQ[e.id];
let spans, by;
if (e.by === "rules") {
spans = E.applyRules(activeRules(versionAt(i), i).map((r) => r.dict), q.text);
by = `<span class="by-rules">rules</span> (v${e.v}, ${e.rule_ms.toFixed(2)} ms)` + (e.checked ? `, spot-checked: ${e.agree ? "same as the teacher" : "differs from the teacher"}` : "");
} else {
spans = q.teacher;
by = `<span class="by-teacher">teacher</span> (${Math.round(e.teacher_ms)} ms), answer added to the labelled examples`;
}
items.push(`<li value="${i + 1}"><div class="meta">#${fmt(i + 1)} answered by ${by}</div><div class="hl-inline">${highlight(q.text, spans)}${spans.length ? "" : ' <span class="meta">(nothing to extract)</span>'}</div></li>`);
}
$("#recent").innerHTML = items.join("") || "<li>No requests yet. Press Play.</li>";
}
function render() {
$("#pos").value = k;
$("#posout").textContent = `${fmt(k)} requests`;
renderKpis(); renderCharts(); renderVersions(); renderRules(); renderRecent();
}
// ------------------------------------------------------------------ playback
function stop() { if (timer) clearTimeout(timer); timer = null; $("#play").textContent = "Play"; }
function tick() {
k = Math.min(N, k + parseInt($("#speed").value, 10));
render();
if (k >= N) { stop(); return; }
timer = setTimeout(tick, 280);
}
$("#play").addEventListener("click", () => {
if (timer) { stop(); return; }
if (k >= N) k = 0;
$("#play").textContent = "Pause";
tick();
});
$("#pos").addEventListener("input", (ev) => { stop(); k = parseInt(ev.target.value, 10); render(); });
// ------------------------------------------------------------------ try your own text
function tryText(text, recorded) {
text = (text || "").trim().slice(0, 2000);
if (!text) { $("#status").textContent = "Type or paste a sentence."; return; }
let v = versionAt(k);
if (!v.gate) v = VERS[VERS.length - 1];
const rules = activeRules(v, Math.max(k, v.at)).map((r) => r.dict);
const t0 = performance.now();
const spans = E.applyRules(rules, text);
const un = E.unexplained(v.gate, text, spans);
const ms = performance.now() - t0;
const rec = recorded || HOLD.find((q) => q.text === text);
if (!un.length) {
$("#status").innerHTML = `<b>Answered by rules</b> (v${v.v}, ${rules.length} answering rules) in ${ms.toFixed(2)} ms in your browser. No model call.`;
$("#out").innerHTML = `<span class="lab">Answer (rules)</span>${highlight(text, spans)}`;
$("#out-teacher").innerHTML = rec ? `<span class="lab">Teacher's recorded answer for this holdout sentence, for comparison</span>${highlight(rec.text, rec.teacher)}` : "";
} else {
$("#status").innerHTML = `<b>The gate sends this request to the teacher</b>: uncovered trigger tokens ${un.slice(0, 12).map((u) => `<code>${esc(u)}</code>`).join(", ")}. ` +
`Rules took ${ms.toFixed(2)} ms. The teacher does not run on this static page` + (rec ? "; its recorded answer for this holdout sentence is shown." : ".");
$("#out").innerHTML = rec ? `<span class="lab">Answer (teacher, recorded offline)</span>${highlight(rec.text, rec.teacher)}` : "";
$("#out-teacher").innerHTML = `<span class="lab">What the rules found</span>${highlight(text, spans)}`;
}
}
$("#go").addEventListener("click", () => tryText($("#inp").value));
$("#pick").addEventListener("click", () => { const q = HOLD[Math.floor(Math.random() * HOLD.length)]; $("#inp").value = q.text; tryText(q.text, q); });
document.querySelectorAll("button.ex").forEach((b) => b.addEventListener("click", () => { $("#inp").value = b.dataset.ex; tryText(b.dataset.ex); }));
window.matchMedia("(prefers-color-scheme: dark)").addEventListener("change", () => R && renderCharts());
// ------------------------------------------------------------------ load
Promise.all([fetch("data/replay.json").then((r) => r.json()), fetch("data/requests.jsonl").then((r) => r.text())]).then(([rep, txt]) => {
R = rep; META = R.meta; VERS = R.versions; EV = R.events; GUARD = R.guard; N = EV.length; AT = VERS.map((v) => v.at);
for (const line of txt.trim().split("\n")) { const q = JSON.parse(line); REQ[q.id] = q; if (q.part === "holdout") HOLD.push(q); }
S.rules = cum(EV.map((e) => +(e.by === "rules")));
S.spans = cum(EV.map((e) => +(e.by === "rules" && e.fired.length > 0)));
S.check = cum(EV.map((e) => +!!e.checked));
S.agree = cum(EV.map((e) => +!!e.agree));
S.call = cum(EV.map((e) => +(e.by === "teacher" || !!e.checked)));
S.lat = cum(EV.map((e) => (e.by === "rules" ? e.rule_ms : e.rule_ms + e.teacher_ms)));
S.tlat = cum(EV.map((e) => e.teacher_ms));
document.querySelectorAll("[data-n-stream]").forEach((x) => (x.textContent = fmt(META.n_stream)));
document.querySelectorAll("[data-n-holdout]").forEach((x) => (x.textContent = fmt(META.n_holdout)));
const set = (sel, v) => document.querySelectorAll(sel).forEach((x) => (x.textContent = v));
set("[data-trust-min]", META.trust_min); set("[data-trust-prec]", META.trust_prec); set("[data-spot]", META.spot_every);
set("[data-guard-prec]", META.guard_prec); set("[data-guard-min]", META.guard_min);
$("#pos").max = N; k = N;
render();
tryText($("#inp").value);
}).catch((err) => { $("#kpis").textContent = "Could not load the replay data: " + err; });
})();
|