Spaces:
Running
Running
| /* Results tab: every number below is taken from the SchemeArena paper (arXiv:2609.08126). */ | |
| (function () { | |
| "use strict"; | |
| var MODELS = [ | |
| { key: "q8", name: "Qwen3-8B", color: "var(--m-q8)", hex: "#a78bdb" }, | |
| { key: "q32", name: "Qwen3-32B", color: "var(--m-q32)", hex: "#5b3fb8" }, | |
| { key: "o4", name: "o4-mini", color: "var(--m-o4)", hex: "#23a585" }, | |
| { key: "o1", name: "o1", color: "var(--m-o1)", hex: "#0c6a52" }, | |
| { key: "claude", name: "Claude-3.7-Sonnet", color: "var(--m-claude)", hex: "#d2714f" } | |
| ]; | |
| // Figure 4: scheming propensity (%), Setups 1-4. | |
| var PROP_SETUPS = [ | |
| { name: "Setup 1", desc: "All factors off", hex: "#d9d5ce" }, | |
| { name: "Setup 2", desc: "Pressure only", hex: "#ebbf98" }, | |
| { name: "Setup 3", desc: "Goal only", hex: "#d9783d" }, | |
| { name: "Setup 4", desc: "Goal + pressure", hex: "#97320f" } | |
| ]; | |
| var PROP = { | |
| q8: [31.9, 37.2, 81.2, 84.8], | |
| q32: [29.6, 38.1, 79.0, 82.9], | |
| o4: [31.1, 38.3, 75.4, 76.7], | |
| o1: [29.1, 34.5, 74.4, 78.8], | |
| claude: [21.7, 23.7, 75.6, 76.8] | |
| }; | |
| // Section 5.3 text: capability (%). null = not reported numerically in the text. | |
| var CAP = { | |
| q8: [99.47, null, null], | |
| q32: [98.94, null, null], | |
| o4: [77.04, 81.89, 96.43], | |
| o1: [70.85, 76.46, 97.98], | |
| claude: [63.12, 68.57, 68.68] | |
| }; | |
| // Figure 6: % trajectories with scheming reasoning but no scheming action. | |
| var GAP_COLS = ["Setup 1<br>All off", "Setup 2<br>Pressure", "Setup 3<br>Goal", "Setup 4<br>Goal + Pres.", "Setup 5<br>Hint", "Setup 6<br>w/ Action", "Setup 6<br>w/ All", "Setup 7<br>Detailed hint"]; | |
| var GAP = { | |
| q8: [55.3, 53.9, 48.0, 44.0, 5.0, 6.0, 5.7, 0.3], | |
| q32: [42.4, 52.9, 39.2, 35.1, 4.3, 4.7, 4.8, 0.5], | |
| o4: [47.2, 57.9, 49.2, 50.3, 50.2, 46.9, 49.6, 13.2], | |
| o1: [62.4, 59.4, 50.7, 53.9, 62.9, 49.0, 57.7, 18.0], | |
| claude: [44.2, 64.3, 50.4, 50.3, 34.3, 32.3, 44.2, 19.6] | |
| }; | |
| var tip; | |
| function showTip(e, html) { tip.innerHTML = html; tip.style.left = e.clientX + "px"; tip.style.top = e.clientY + "px"; tip.style.opacity = 1; } | |
| function hideTip() { tip.style.opacity = 0; } | |
| function propChart() { | |
| var el = document.getElementById("propChart"); | |
| var W = 960, H = 380, m = { t: 26, r: 8, b: 44, l: 44 }; | |
| var iw = W - m.l - m.r, ih = H - m.t - m.b; | |
| var gw = iw / MODELS.length, pad = gw * 0.16, bw = (gw - 2 * pad) / 4; | |
| var y = function (v) { return m.t + ih - (v / 100) * ih; }; | |
| var s = '<svg viewBox="0 0 ' + W + " " + H + '" role="img" aria-label="Grouped bar chart of scheming propensity for Setups 1 to 4 across five models">'; | |
| for (var t = 0; t <= 100; t += 20) { | |
| s += '<line x1="' + m.l + '" x2="' + (W - m.r) + '" y1="' + y(t) + '" y2="' + y(t) + '" stroke="#ebe9e4" />'; | |
| s += '<text x="' + (m.l - 8) + '" y="' + (y(t) + 4) + '" text-anchor="end" font-size="12" fill="#8a8f98">' + t + "</text>"; | |
| } | |
| s += '<text transform="translate(12,' + (m.t + ih / 2) + ') rotate(-90)" text-anchor="middle" font-size="12" fill="#6b7079">Scheming rate (%)</text>'; | |
| MODELS.forEach(function (md, i) { | |
| var gx = m.l + i * gw + pad; | |
| PROP[md.key].forEach(function (v, j) { | |
| var x = gx + j * bw, h = y(0) - y(v); | |
| s += '<rect class="bar" data-tip="' + md.name + " · " + PROP_SETUPS[j].name + " (" + PROP_SETUPS[j].desc + "): <b>" + v.toFixed(1) + '%</b>" x="' + (x + 2) + '" y="' + y(v) + '" width="' + (bw - 4) + '" height="' + h + '" rx="4" fill="' + PROP_SETUPS[j].hex + '" />'; | |
| s += '<text x="' + (x + bw / 2) + '" y="' + (y(v) - 6) + '" text-anchor="middle" font-size="11" font-weight="600" fill="#3a3f47">' + v.toFixed(1) + "</text>"; | |
| }); | |
| s += '<text x="' + (m.l + i * gw + gw / 2) + '" y="' + (H - 16) + '" text-anchor="middle" font-size="13" font-weight="600" fill="#15171b">' + md.name + "</text>"; | |
| }); | |
| s += "</svg>"; | |
| el.innerHTML = s; | |
| el.querySelectorAll(".bar").forEach(function (b) { | |
| b.addEventListener("mousemove", function (e) { showTip(e, b.getAttribute("data-tip")); }); | |
| b.addEventListener("mouseleave", hideTip); | |
| }); | |
| document.getElementById("propLegend").innerHTML = PROP_SETUPS.map(function (p) { | |
| return '<span><i style="background:' + p.hex + '"></i>' + p.name + ": " + p.desc + "</span>"; | |
| }).join(""); | |
| } | |
| function dot(md) { return '<span class="mdot" style="background:' + md.hex + '"></span>' + md.name; } | |
| function propTable() { | |
| var h = "<colgroup><col class='lab' /></colgroup><thead><tr><th>Model</th>" + PROP_SETUPS.map(function (p) { return "<th>" + p.name + "<br><span style='text-transform:none;letter-spacing:0'>" + p.desc + "</span></th>"; }).join("") + "<th>Goal effect<br><span style='text-transform:none;letter-spacing:0'>Setup 3 − 1</span></th></tr></thead><tbody>"; | |
| MODELS.forEach(function (md) { | |
| var r = PROP[md.key]; | |
| h += "<tr><td>" + dot(md) + "</td>" + r.map(function (v) { return "<td>" + v.toFixed(1) + "%</td>"; }).join("") + | |
| '<td><span class="delta up">+' + (r[2] - r[0]).toFixed(1) + " pts</span></td></tr>"; | |
| }); | |
| document.getElementById("propTable").innerHTML = h + "</tbody>"; | |
| } | |
| function capTable() { | |
| var cols = ["Setup 5<br><span style='text-transform:none;letter-spacing:0'>Light hint</span>", "Setup 6<br><span style='text-transform:none;letter-spacing:0'>Light hint + action-only oversight</span>", "Setup 7<br><span style='text-transform:none;letter-spacing:0'>Detailed hint</span>"]; | |
| var h = "<colgroup><col class='lab' /></colgroup><thead><tr><th>Model</th>" + cols.map(function (c) { return "<th>" + c + "</th>"; }).join("") + "</tr></thead><tbody>"; | |
| MODELS.forEach(function (md) { | |
| var r = CAP[md.key]; | |
| h += "<tr><td>" + dot(md) + "</td>"; | |
| r.forEach(function (v, j) { | |
| if (v === null) { h += '<td class="na">—</td>'; return; } | |
| var d = (j > 0 && r[0] !== null) ? ' <span class="delta up">+' + (v - r[0]).toFixed(2) + "</span>" : ""; | |
| h += "<td>" + v.toFixed(2) + "%" + d + "</td>"; | |
| }); | |
| h += "</tr>"; | |
| }); | |
| document.getElementById("capTable").innerHTML = h + "</tbody>"; | |
| } | |
| function heatColor(v) { | |
| // white -> burnt orange | |
| var t = Math.max(0, Math.min(1, v / 65)); | |
| var a = [253, 250, 246], b = [185, 71, 15]; | |
| var c = a.map(function (x, i) { return Math.round(x + (b[i] - x) * t); }); | |
| return "rgb(" + c.join(",") + ")"; | |
| } | |
| function gapTable() { | |
| var h = "<thead><tr><th>Model</th>" + GAP_COLS.map(function (c) { return "<th style='text-transform:none;letter-spacing:0;font-size:.74rem'>" + c + "</th>"; }).join("") + "</tr></thead><tbody>"; | |
| MODELS.forEach(function (md) { | |
| h += "<tr><td>" + dot(md) + "</td>" + GAP[md.key].map(function (v) { | |
| return '<td class="cell" style="background:' + heatColor(v) + ";color:" + (v > 38 ? "#fff" : "#3a3f47") + '">' + v.toFixed(1) + "</td>"; | |
| }).join("") + "</tr>"; | |
| }); | |
| document.getElementById("gapTable").innerHTML = h + "</tbody>"; | |
| } | |
| var done = false; | |
| window.renderResults = function () { | |
| if (done) return; | |
| done = true; | |
| tip = document.getElementById("tooltip"); | |
| propChart(); propTable(); capTable(); gapTable(); | |
| }; | |
| // Setup table on the overview page (Figure 3a). | |
| window.renderSetupTable = function () { | |
| var rows = [ | |
| { label: "Goal", cls: "goal", v: ["", "", "on", "on", "on", "on", "on"] }, | |
| { label: "Pressure", cls: "pressure", v: ["", "on", "", "on", "on", "on", "on"] }, | |
| { label: "Oversight", cls: "oversight", v: ["", "", "", "", "", "on", ""] }, | |
| { label: "Hint", cls: "hint", v: ["", "", "", "", "Light", "Light", "Detailed"] } | |
| ]; | |
| document.getElementById("setupBody").innerHTML = rows.map(function (r) { | |
| var style = "--c:var(--" + r.cls + ");--soft:var(--" + r.cls + "-soft)"; | |
| return '<tr style="' + style + '"><th><i></i>' + r.label + "</th>" + r.v.map(function (x) { | |
| return x ? '<td class="on"><span>' + x + "</span></td>" : '<td class="off">–</td>'; | |
| }).join("") + "</tr>"; | |
| }).join(""); | |
| }; | |
| })(); | |