File size: 17,492 Bytes
27e064c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
// RuleChef Live: replays data/replay.json (written by sim.py) and runs the rules in the browser (engine.js).
(function () {
  "use strict";
  const E = window.RCEngine;
  const $ = (s) => document.querySelector(s);
  const fmt = (n) => n.toLocaleString("en-US");
  const pct = (a, b) => (b ? (100 * a / b).toFixed(1) + " %" : "–");
  const esc = (s) => String(s).replace(/[&<>"]/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" }[c]));
  const NAMES = { DATETIME: "date/time", CODE: "code", QUANTITY: "quantity" };

  let R, REQ = {}, EV, VERS, GUARD, META, N, AT, S = {}, LAB = [], HOLD = [];
  let k = 0, timer = null;

  function cum(xs) { const out = [0]; let s = 0; for (const x of xs) { s += x; out.push(s); } return out; }

  function versionAt(i) { let lo = 0; for (let j = 0; j < AT.length; j++) if (AT[j] <= i) lo = j; return VERS[lo]; }
  function blockedAt(i) { return new Set(GUARD.filter((g) => g.at < i).map((g) => g.key)); }
  function activeRules(v, i) { const off = blockedAt(i); return v.rules.filter((r) => r.trusted && !off.has(r.key)); }

  function card(t, v, s) { return `<div class="kpi"><div class="kt">${t}</div><div class="kv">${v}</div><div class="ks">${s}</div></div>`; }

  function renderKpis() {
    const v = versionAt(k);
    const nr = S.rules[k], ns = S.spans[k], nc = S.check[k], na = S.agree[k], calls = S.call[k];
    const lat = k ? S.lat[k] / k : 0, tlat = k ? S.tlat[k] / k : 0;
    const rm = EV.slice(0, k).filter((e) => e.by === "rules").map((e) => e.rule_ms).sort((a, b) => a - b);
    const med = rm.length ? rm[rm.length >> 1].toFixed(2) + " ms (median)" : "–";
    $("#kpis").innerHTML = [
      card("Requests", `${fmt(k)} / ${fmt(N)}`, `rule set v${v.v}`),
      card("Answered by rules", pct(nr, k), `${fmt(nr)}: ${fmt(ns)} with spans, ${fmt(nr - ns)} with nothing to extract`),
      card("Model calls", fmt(calls), `${fmt(k - calls)} saved (${pct(k - calls, k)}); spot checks included`),
      card("Mean latency", `${Math.round(lat)} ms`, `teacher on every request: ${Math.round(tlat)} ms; a rule answer: ${med}`),
      card("Spot-check agreement", pct(na, nc), `${na} of ${nc} checked rule answers equal the teacher's`),
      card("Rules answering", String(activeRules(v, k).length), `of ${v.rules.length} in v${v.v}; ${blockedAt(k).size} switched off by the guard`),
    ].join("");
  }

  // ------------------------------------------------------------------ charts (SVG, two series each, crosshair tooltip)
  const W = 360, H = 190, P = { l: 40, r: 10, t: 8, b: 22 };
  function lineChart(el, cfg) {
    const { series, yMax, yTicks, yFmt, markers } = cfg;
    const x = (v) => P.l + (W - P.l - P.r) * v / N, y = (v) => H - P.b - (H - P.t - P.b) * v / yMax;
    let g = `<svg viewBox="0 0 ${W} ${H}" role="img" aria-label="${esc(cfg.label)}">`;
    for (const t of yTicks) g += `<line x1="${P.l}" x2="${W - P.r}" y1="${y(t)}" y2="${y(t)}" stroke="var(--grid)"/><g class="axis"><text x="${P.l - 5}" y="${y(t) + 3}" text-anchor="end">${yFmt(t)}</text></g>`;
    for (let t = 0; t <= N; t += 1000) g += `<g class="axis"><text x="${x(t)}" y="${H - 6}" text-anchor="middle">${fmt(t)}</text></g>`;
    for (const m of markers || []) if (m.at <= k && m.at > 0) g += `<line x1="${x(m.at)}" x2="${x(m.at)}" y1="${P.t}" y2="${H - P.b}" stroke="var(--line)" stroke-dasharray="2 3"/>`;
    g += `<line x1="${P.l}" x2="${W - P.r}" y1="${H - P.b}" y2="${H - P.b}" stroke="var(--line)"/>`;
    for (const s of series) {
      if (s.pts.length < 2) continue;
      g += `<polyline fill="none" stroke="${s.color}" stroke-width="2" stroke-linejoin="round" stroke-linecap="round" points="${s.pts.map((p) => x(p[0]).toFixed(1) + "," + y(p[1]).toFixed(1)).join(" ")}"/>`;
    }
    g += `<line class="xh" x1="0" x2="0" y1="${P.t}" y2="${H - P.b}" stroke="var(--muted)" visibility="hidden"/>`;
    g += `<rect x="${P.l}" y="${P.t}" width="${W - P.l - P.r}" height="${H - P.t - P.b}" fill="transparent"/></svg>`;
    g += `<div class="legend">${series.map((s) => `<span><i style="background:${s.color}"></i>${esc(s.name)}</span>`).join("")}</div>`;
    el.innerHTML = g;
    const svg = el.querySelector("svg"), xh = el.querySelector(".xh"), tip = $("#tip");
    svg.addEventListener("mousemove", (ev) => {
      const b = svg.getBoundingClientRect();
      const vx = (ev.clientX - b.left) / b.width * W;
      const req = Math.max(0, Math.min(k, Math.round((vx - P.l) / (W - P.l - P.r) * N)));
      const rows = series.map((s) => { let best = null; for (const p of s.pts) if (!best || Math.abs(p[0] - req) < Math.abs(best[0] - req)) best = p; return [s, best]; }).filter((r) => r[1]);
      if (!rows.length) return;
      xh.setAttribute("x1", x(rows[0][1][0])); xh.setAttribute("x2", x(rows[0][1][0])); xh.setAttribute("visibility", "visible");
      tip.innerHTML = `request ${fmt(rows[0][1][0])}<br>` + rows.map(([s, p]) => `<span style="color:${s.color}">&#9644;</span> ${esc(s.name)}: <b>${cfg.tipFmt(p[1])}</b>`).join("<br>");
      tip.hidden = false; tip.style.left = Math.min(ev.clientX + 12, window.innerWidth - 230) + "px"; tip.style.top = (ev.clientY + 12) + "px";
    });
    svg.addEventListener("mouseleave", () => { tip.hidden = true; xh.setAttribute("visibility", "hidden"); });
  }
  function windowPts(arr, w = 200, step = 20) {
    const pts = [];
    for (let i = step; i <= k; i += step) { const lo = Math.max(0, i - w); pts.push([i, (arr[i] - arr[lo]) / (i - lo)]); }
    return pts;
  }
  const C1 = () => getComputedStyle(document.documentElement).getPropertyValue("--s1").trim();
  const C2 = () => getComputedStyle(document.documentElement).getPropertyValue("--s2").trim();
  function renderCharts() {
    const markers = VERS.slice(1);
    lineChart($("#c-cov"), { label: "Share of requests answered by rules", yMax: 1, yTicks: [0, 0.25, 0.5, 0.75, 1], yFmt: (t) => Math.round(t * 100) + "%",
      tipFmt: (v) => (100 * v).toFixed(1) + " %", markers,
      series: [{ name: "answered by rules", color: C1(), pts: windowPts(S.rules) }, { name: "rules found spans", color: C2(), pts: windowPts(S.spans) }] });
    const step = 20, a = [], b = [];
    for (let i = 0; i <= k; i += step) { a.push([i, S.call[i]]); b.push([i, i]); }
    lineChart($("#c-calls"), { label: "Model calls", yMax: N, yTicks: [0, 1000, 2000, 3000], yFmt: (t) => fmt(t), tipFmt: (v) => fmt(v), markers,
      series: [{ name: "with rules", color: C1(), pts: a }, { name: "teacher on every request", color: C2(), pts: b }] });
    lineChart($("#c-lat"), { label: "Latency per request", yMax: 500, yTicks: [0, 100, 200, 300, 400, 500], yFmt: (t) => String(t), tipFmt: (v) => Math.round(v) + " ms", markers,
      series: [{ name: "cascade", color: C1(), pts: windowPts(S.lat) }, { name: "teacher on every request", color: C2(), pts: windowPts(S.tlat) }] });
  }

  // ------------------------------------------------------------------ tables
  function renderVersions() {
    const cur = versionAt(k).v;
    const head = `<tr><th>version</th><th class="num">from request</th><th class="num">labelled examples</th><th class="num">rules answering / learned</th>
      <th class="num">synthesis s</th><th class="num">holdout answered by rules</th><th class="num">same answer as teacher</th>
      <th class="num">rule spans P / R vs teacher</th><th class="num">cascade F1 vs teacher</th><th class="num">TAB annotation F1: teacher / cascade</th></tr>`;
    const rows = VERS.filter((v) => v.at <= k).map((v) => {
      const h = v.holdout, ra = h.rules_answer;
      return `<tr class="${v.v === cur ? "current" : ""}"><td>v${v.v}</td><td class="num">${fmt(v.at)}</td><td class="num">${fmt(v.n_labelled)}</td>
        <td class="num">${v.rules.filter((r) => r.trusted).length} / ${v.rules.length}</td><td class="num">${v.synth_s.toFixed(1)}</td>
        <td class="num">${pct(ra, h.n)}</td><td class="num">${ra ? pct(h.exact_agree, ra) : "–"}</td>
        <td class="num">${ra ? h.routed_t.P.toFixed(2) + " / " + h.routed_t.R.toFixed(2) : "–"}</td>
        <td class="num">${h.casc_t.F1.toFixed(3)}</td><td class="num">${h.teach_g.F1.toFixed(3)} / ${h.casc_g.F1.toFixed(3)}</td></tr>`;
    });
    $("#t-vers").innerHTML = `<thead>${head}</thead><tbody>${rows.join("")}</tbody>`;
  }

  function renderRules() {
    const v = versionAt(k), off = blockedAt(k), used = {};
    for (let i = 0; i < k; i++) if (EV[i].by === "rules") for (const key of EV[i].fired) used[key] = (used[key] || 0) + 1;
    const rows = v.rules.map((r) => {
      const n = r.tp + r.fp;
      const st = off.has(r.key) ? ["cut", "switched off"] : r.trusted ? ["on", "answering"] : ["off", "not trusted"];
      return { r, st, n, used: used[r.key] || 0 };
    }).sort((a, b) => (a.st[0] !== "on") - (b.st[0] !== "on") || b.used - a.used || b.n - a.n);
    $("#rules-help").textContent = v.v === 0
      ? "No rule set yet: every request goes to the teacher until 50 examples are labelled."
      : `v${v.v}, learned from ${fmt(v.n_labelled)} labelled examples in ${v.synth_s.toFixed(1)} s. "Answering" rules matched at least ${META.trust_min} times on the labelled data with precision at least ${META.trust_prec}; the others are kept but do not answer. The last column counts requests this exact rule answered so far.`;
    const head = `<thead><tr><th>status</th><th>type</th><th>regex</th><th>output</th><th class="num">labelled matches (precision)</th><th class="num">requests answered</th></tr></thead>`;
    const tr = ({ r, st, n, used: u }) => `<tr><td><span class="st ${st[0]}">${st[1]}</span></td><td>${r.type}</td><td class="rx">${esc(r.regex)}</td>
        <td class="mono">${esc(r.template)}</td><td class="num">${n ? `${n} (${(r.tp / n).toFixed(2)})` : "0"}</td><td class="num">${u}</td></tr>`;
    const on = rows.filter((x) => x.st[0] !== "off"), rest = rows.filter((x) => x.st[0] === "off");
    $("#t-rules").innerHTML = head + "<tbody>" + (on.map(tr).join("") || '<tr><td colspan="6">No answering rules.</td></tr>') + "</tbody>";
    const det = $("#rules-rest");
    const open = det.open;
    det.innerHTML = rest.length ? `<summary>${rest.length} learned rules that do not answer (precision below ${META.trust_prec} or too few matches)</summary><div class="tablewrap"><table class="rules">${head}<tbody>${rest.map(tr).join("")}</tbody></table></div>` : "";
    det.open = open;
    const g = v.gate;
    let html = "";
    if (g) {
      html += `<p><b>Gate of v${v.v}.</b> A request goes to the rules only if every trigger token is covered by an answering rule or sits in a skip context.
        Trigger tokens: anything with a digit, plus ${g.trigger_words.length} learned words: ${g.trigger_words.map((w) => `<code>${esc(w)}</code>`).join(", ")}.</p>
        <p>Skip contexts (previous word + digit shape, <code>9</code> = digits) that the teacher labels at most ${Math.round(META.skip_ratio * 100)} % of the time:
        ${g.skip.map(([p, s]) => `<code>${esc(p || "(start)")} ${esc(s)}</code>`).join(", ") || "none"}.</p>`;
    }
    const gl = GUARD.filter((x) => x.at < k);
    html += gl.length
      ? `<p><b>Drift guard.</b> ${gl.map((x) => `at request ${x.at}: <code>${esc(x.key.split("\t")[1])}</code> switched off (spot checks ${x.tp} right, ${x.fp} wrong)`).join("; ")}.</p>`
      : `<p><b>Drift guard.</b> No rule switched off so far. A rule is switched off when its spot-check precision drops below ${META.guard_prec} after ${META.guard_min} checked spans; in this run none did.</p>`;
    $("#gate").innerHTML = html;
  }

  function highlight(text, spans) {
    const ss = spans.slice().sort((a, b) => a.start - b.start || b.end - a.end);
    let out = "", pos = 0;
    for (const s of ss) {
      if (s.start < pos) continue;
      out += esc(text.slice(pos, s.start)) + `<mark class="${s.type}">${esc(text.slice(s.start, s.end))}<sup>${NAMES[s.type] || s.type}</sup></mark>`;
      pos = s.end;
    }
    return out + esc(text.slice(pos));
  }

  function renderRecent() {
    const items = [];
    for (let i = k - 1; i >= Math.max(0, k - 6); i--) {
      const e = EV[i], q = REQ[e.id];
      let spans, by;
      if (e.by === "rules") {
        spans = E.applyRules(activeRules(versionAt(i), i).map((r) => r.dict), q.text);
        by = `<span class="by-rules">rules</span> (v${e.v}, ${e.rule_ms.toFixed(2)} ms)` + (e.checked ? `, spot-checked: ${e.agree ? "same as the teacher" : "differs from the teacher"}` : "");
      } else {
        spans = q.teacher;
        by = `<span class="by-teacher">teacher</span> (${Math.round(e.teacher_ms)} ms), answer added to the labelled examples`;
      }
      items.push(`<li value="${i + 1}"><div class="meta">#${fmt(i + 1)} answered by ${by}</div><div class="hl-inline">${highlight(q.text, spans)}${spans.length ? "" : ' <span class="meta">(nothing to extract)</span>'}</div></li>`);
    }
    $("#recent").innerHTML = items.join("") || "<li>No requests yet. Press Play.</li>";
  }

  function render() {
    $("#pos").value = k;
    $("#posout").textContent = `${fmt(k)} requests`;
    renderKpis(); renderCharts(); renderVersions(); renderRules(); renderRecent();
  }

  // ------------------------------------------------------------------ playback
  function stop() { if (timer) clearTimeout(timer); timer = null; $("#play").textContent = "Play"; }
  function tick() {
    k = Math.min(N, k + parseInt($("#speed").value, 10));
    render();
    if (k >= N) { stop(); return; }
    timer = setTimeout(tick, 280);
  }
  $("#play").addEventListener("click", () => {
    if (timer) { stop(); return; }
    if (k >= N) k = 0;
    $("#play").textContent = "Pause";
    tick();
  });
  $("#pos").addEventListener("input", (ev) => { stop(); k = parseInt(ev.target.value, 10); render(); });

  // ------------------------------------------------------------------ try your own text
  function tryText(text, recorded) {
    text = (text || "").trim().slice(0, 2000);
    if (!text) { $("#status").textContent = "Type or paste a sentence."; return; }
    let v = versionAt(k);
    if (!v.gate) v = VERS[VERS.length - 1];
    const rules = activeRules(v, Math.max(k, v.at)).map((r) => r.dict);
    const t0 = performance.now();
    const spans = E.applyRules(rules, text);
    const un = E.unexplained(v.gate, text, spans);
    const ms = performance.now() - t0;
    const rec = recorded || HOLD.find((q) => q.text === text);
    if (!un.length) {
      $("#status").innerHTML = `<b>Answered by rules</b> (v${v.v}, ${rules.length} answering rules) in ${ms.toFixed(2)} ms in your browser. No model call.`;
      $("#out").innerHTML = `<span class="lab">Answer (rules)</span>${highlight(text, spans)}`;
      $("#out-teacher").innerHTML = rec ? `<span class="lab">Teacher's recorded answer for this holdout sentence, for comparison</span>${highlight(rec.text, rec.teacher)}` : "";
    } else {
      $("#status").innerHTML = `<b>The gate sends this request to the teacher</b>: uncovered trigger tokens ${un.slice(0, 12).map((u) => `<code>${esc(u)}</code>`).join(", ")}. ` +
        `Rules took ${ms.toFixed(2)} ms. The teacher does not run on this static page` + (rec ? "; its recorded answer for this holdout sentence is shown." : ".");
      $("#out").innerHTML = rec ? `<span class="lab">Answer (teacher, recorded offline)</span>${highlight(rec.text, rec.teacher)}` : "";
      $("#out-teacher").innerHTML = `<span class="lab">What the rules found</span>${highlight(text, spans)}`;
    }
  }
  $("#go").addEventListener("click", () => tryText($("#inp").value));
  $("#pick").addEventListener("click", () => { const q = HOLD[Math.floor(Math.random() * HOLD.length)]; $("#inp").value = q.text; tryText(q.text, q); });
  document.querySelectorAll("button.ex").forEach((b) => b.addEventListener("click", () => { $("#inp").value = b.dataset.ex; tryText(b.dataset.ex); }));
  window.matchMedia("(prefers-color-scheme: dark)").addEventListener("change", () => R && renderCharts());

  // ------------------------------------------------------------------ load
  Promise.all([fetch("data/replay.json").then((r) => r.json()), fetch("data/requests.jsonl").then((r) => r.text())]).then(([rep, txt]) => {
    R = rep; META = R.meta; VERS = R.versions; EV = R.events; GUARD = R.guard; N = EV.length; AT = VERS.map((v) => v.at);
    for (const line of txt.trim().split("\n")) { const q = JSON.parse(line); REQ[q.id] = q; if (q.part === "holdout") HOLD.push(q); }
    S.rules = cum(EV.map((e) => +(e.by === "rules")));
    S.spans = cum(EV.map((e) => +(e.by === "rules" && e.fired.length > 0)));
    S.check = cum(EV.map((e) => +!!e.checked));
    S.agree = cum(EV.map((e) => +!!e.agree));
    S.call = cum(EV.map((e) => +(e.by === "teacher" || !!e.checked)));
    S.lat = cum(EV.map((e) => (e.by === "rules" ? e.rule_ms : e.rule_ms + e.teacher_ms)));
    S.tlat = cum(EV.map((e) => e.teacher_ms));
    document.querySelectorAll("[data-n-stream]").forEach((x) => (x.textContent = fmt(META.n_stream)));
    document.querySelectorAll("[data-n-holdout]").forEach((x) => (x.textContent = fmt(META.n_holdout)));
    const set = (sel, v) => document.querySelectorAll(sel).forEach((x) => (x.textContent = v));
    set("[data-trust-min]", META.trust_min); set("[data-trust-prec]", META.trust_prec); set("[data-spot]", META.spot_every);
    set("[data-guard-prec]", META.guard_prec); set("[data-guard-min]", META.guard_min);
    $("#pos").max = N; k = N;
    render();
    tryText($("#inp").value);
  }).catch((err) => { $("#kpis").textContent = "Could not load the replay data: " + err; });
})();