File size: 21,959 Bytes
3ed78a4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6853af9
3ed78a4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>RRSI — Main Results</title>
<style>
  :root{
    --evolve:#6366f1; --held:#0ea5e9; --ood:#10b981; --base:#94a3b8;
    --ink:#0f172a; --muted:#64748b; --line:#e2e8f0; --bg:#f8fafc; --card:#ffffff;
  }
  *{box-sizing:border-box}
  body{
    margin:0; background:var(--bg); color:var(--ink);
    font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif;
    line-height:1.55; -webkit-font-smoothing:antialiased;
  }
  .wrap{max-width:1080px;margin:0 auto;padding:40px 22px 70px}
  header h1{font-size:26px;margin:0 0 6px;letter-spacing:-.02em}
  header .sub{color:var(--muted);font-size:14px;margin:0 0 4px}
  header a{color:#2563eb;text-decoration:none}
  header a:hover{text-decoration:underline}
  .note{
    margin-top:14px;font-size:13px;color:var(--muted);
    background:#fff;border:1px solid var(--line);border-radius:10px;padding:10px 14px;
  }
  section{margin-top:34px}
  h2{font-size:17px;margin:0 0 4px;letter-spacing:-.01em}
  h2 .tag{
    font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:.06em;
    color:#475569;background:#eef2ff;border:1px solid #e0e7ff;border-radius:999px;
    padding:2px 8px;margin-right:8px;vertical-align:2px;
  }
  .card{background:var(--card);border:1px solid var(--line);border-radius:14px;padding:20px 22px;
        box-shadow:0 1px 2px rgba(15,23,42,.04)}
  .cap{font-size:13px;color:var(--muted);margin:12px 0 0}
  .legend{display:flex;gap:18px;flex-wrap:wrap;font-size:13px;color:#334155;margin:6px 0 14px}
  .legend i{display:inline-block;width:12px;height:12px;border-radius:3px;margin-right:6px;vertical-align:-1px}
  svg{display:block;width:100%;height:auto;overflow:visible}
  /* bar chart */
  .domains{display:flex;gap:26px;flex-wrap:wrap;align-items:flex-end}
  .dgroup{flex:1 1 300px;min-width:260px}
  .dgroup h4{margin:0 0 10px;font-size:13px;letter-spacing:.04em;text-transform:uppercase;color:#475569}
  .bars{display:flex;gap:14px;align-items:flex-end;border-bottom:2px solid var(--line);padding-bottom:0;min-height:230px}
  .bar{flex:1 1 0;display:flex;flex-direction:column;justify-content:flex-end;align-items:center;gap:6px;min-width:48px}
  .bar .val{font-size:13px;font-weight:650;font-variant-numeric:tabular-nums}
  .bar .col{width:100%;border-radius:6px 6px 0 0;transition:height .3s ease}
  .bar .nm{font-size:11px;color:var(--muted);text-align:center;line-height:1.3;min-height:30px}
  .bar .nm b{display:block;color:#334155;font-weight:600}
  /* token cost */
  .tokens{display:flex;gap:22px;flex-wrap:wrap;margin-top:6px}
  .tok{flex:1 1 200px;background:#fff;border:1px solid var(--line);border-radius:12px;padding:14px 16px}
  .tok .lab{font-size:12px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em}
  .tok .big{font-size:26px;font-weight:700;font-variant-numeric:tabular-nums;margin:2px 0 0}
  .tok .big span{font-size:13px;font-weight:500;color:var(--muted)}
  .tok .meter{height:8px;border-radius:6px;background:#eef2f7;margin-top:10px;overflow:hidden}
  .tok .meter i{display:block;height:100%;border-radius:6px}
  /* tables */
  table{width:100%;border-collapse:collapse;font-size:13.5px;font-variant-numeric:tabular-nums}
  th,td{padding:8px 10px;text-align:right;border-bottom:1px solid var(--line)}
  th:first-child,td:first-child{text-align:left}
  thead th{font-size:11.5px;text-transform:uppercase;letter-spacing:.05em;color:#475569;
           border-bottom:2px solid var(--line);background:#fbfcfe}
  thead tr:last-child th{background:#f4f6fb}
  tr.ours td{background:#ecfdf5;font-weight:650}
  tr.h0 td{color:#475569}
  .tblnotes{font-size:12.5px;color:var(--muted);margin-top:10px}
  .win{color:#047857;font-weight:650}
  .lose{color:#b91c1c;font-weight:650}
  ul.mech{margin:6px 0 0;padding-left:20px;font-size:14px;color:#1e293b}
  ul.mech li{margin-bottom:5px}
  ul.mech b{color:#0f172a}
  .grid2{display:grid;grid-template-columns:1fr 1fr;gap:18px}
  @media(max-width:760px){.grid2{grid-template-columns:1fr}}
</style>
</head>
<body>
<div class="wrap">

<header>
  <h1>RRSI: Regularized Recursive Self-Improvement of Agent Harnesses</h1>
  <p class="sub">Peng Xia, Rujun Han, Zifeng Wang, Yanfei Chen, Yufan Zhang, Yoonho Lee, Chengsong Huang, Han Yu, Zhongying CuiZhu, Yifei Ming, Huaxiu Yao, Burak Gokturk, Tomas Pfister, Chen-Yu Lee — Google Cloud AI Research</p>
  <p class="sub"><a href="https://arxiv.org/abs/2609.24972">arXiv:2609.24972</a></p>
  <p class="note">Charts below re-plot numbers reported in the paper (Figures 1 &amp; 3, Tables 1–3). Policy frozen at Claude Opus 4.8; the harness is evolved on one suite per domain and then evaluated <em>unchanged</em> on held-out and out-of-distribution (OOD) benchmarks.</p>
</header>

<section>
  <h2><span class="tag">Figure 1a</span>Evolve-set gain vs. out-of-distribution gain</h2>
  <p class="cap" style="margin-top:0">Agentic workspace instance. Points far to the upper-left are the goal: small evolve-set gain, large transfer. Points left of the dashed line at 0 fall <em>below</em> the unevolved harness H₀.</p>
  <div class="card">
    <div class="legend">
      <span><i style="background:var(--evolve)"></i>Evolve-set gain (Harvey LAB)</span>
      <span><i style="background:var(--ood)"></i>OOD gain (mean of JobBench, GDPval, APEX-Agents)</span>
    </div>
    <div id="scatter"></div>
    <p class="cap">Reading it: Meta-Harness has the biggest evolve-set gain (+3.6) but keeps only +0.9 OOD. AHE and TTHE are negative OOD (−0.5 and −1.7) — their evolution actively hurt tasks they were never scored on. RRSI has the <em>smallest</em> evolve gain (+1.1) and the largest transfer (+3.9), the exact trade the regularizers are designed to make.</p>
  </div>
</section>

<section>
  <h2><span class="tag">Figure 3</span>Main results in all three domains (Δ vs. H₀)</h2>
  <div class="card">
    <div class="legend">
      <span><i style="background:var(--evolve)"></i>Evolve split</span>
      <span><i style="background:var(--held)"></i>In-distribution held-out</span>
      <span><i style="background:var(--ood)"></i>OOD held-out (never scored during search)</span>
    </div>
    <div class="domains" id="domains"></div>
    <p class="cap"><b>Every</b> held-out split improves and none regresses. Frontier-Eng is measured in medal points: +4.3 is a 24.3% relative gain. The engineering-design domain matters because its tasks are graded by frozen deterministic simulators, not a judge model — so the transfer can't be explained by a harness learning to write the way a judge rewards.</p>
  </div>
</section>

<section>
  <h2><span class="tag">Efficiency</span>Policy tokens per trial</h2>
  <div class="card">
    <div class="tokens" id="tokens"></div>
    <p class="cap">Regularization is <em>cheaper</em>, not more expensive: RRSI spends 2.42M tokens/trial against 3.80M for unregularized evolution (≈36% less), and only 1.6× the unevolved baseline. Unregularized selection spends most of its accepted edits on noise and on context rather than on mechanism — it buys score with compute, and the compute doesn't transfer.</p>
  </div>
</section>

<section>
  <h2><span class="tag">Table 1</span>Comparison with prior harness evolution methods</h2>
  <div class="card">
    <table>
      <thead>
        <tr>
          <th rowspan="2">Method</th>
          <th colspan="2">In-distribution</th>
          <th colspan="3">Out-of-distribution</th>
          <th rowspan="2">OOD avg.</th>
        </tr>
        <tr>
          <th>Harvey LAB (evolve)</th>
          <th>Harvey LAB (ID held-out)</th>
          <th>JobBench</th>
          <th>GDPval</th>
          <th>APEX-Agents</th>
        </tr>
      </thead>
      <tbody id="t1"></tbody>
    </table>
    <p class="tblnotes">OOD avg. is computed as the mean of JobBench, GDPval and APEX-Agents (the paper reports only the H₀ and RRSI averages explicitly, 39.7 and 43.6; the baseline averages shown for the others are recomputed from the table's own columns). Same H₀, same evolve split, same candidate budget for every row.</p>
  </div>
</section>

<section>
  <h2><span class="tag">Table 2</span>Ablation: is it the regularizers or the score?</h2>
  <div class="card">
    <table>
      <thead>
        <tr>
          <th>Variant</th>
          <th>Harvey LAB (evolve)</th>
          <th>Harvey LAB (ID held-out)</th>
          <th>OOD avg.</th>
          <th>Tokens/trial (M) ↓</th>
        </tr>
      </thead>
      <tbody id="t2"></tbody>
    </table>
    <p class="tblnotes">The strongest single signal in the paper: evolve-set score and transfer move in <em>opposite</em> directions. Dropping both regularizer groups yields the highest evolve score of any arm (92.8) and an OOD average within a point of the unevolved harness. Dropping only the acceptance constraints raises the evolve score by 1.0 pt, costs 2.6 pts OOD, and inflates token cost by half.</p>
  </div>
</section>

<section>
  <h2><span class="tag">Table 3</span>Robustness to the backbone policy (coding)</h2>
  <div class="card">
    <table>
      <thead>
        <tr><th>Policy</th><th>Benchmark</th><th>H₀</th><th>RRSI</th><th>Δ</th></tr>
      </thead>
      <tbody id="t3"></tbody>
    </table>
    <p class="tblnotes">Evolution is run independently per policy on Terminal-Bench 2.1 only; the resulting harness is then evaluated unchanged on SWE-bench Verified. Gemini 3.5 Flash gains more on the evolve split (+14.1 — the paper's headline number) while the stronger Claude Opus 4.8 sits closer to both ceilings and gains less.</p>
  </div>
</section>

<section>
  <h2><span class="tag">Figure 2</span>What is actually regularized</h2>
  <div class="card grid2">
    <div>
      <h4 style="margin:0 0 6px;font-size:13px;letter-spacing:.04em;text-transform:uppercase;color:#475569">Proposal side — how search capacity is spent</h4>
      <ul class="mech">
        <li><b>L₀-style annealed edit budget:</b> b_t = ⌈b_min + (b_max−b_min)·½(1+cos(πt/T))⌉. Early rounds bundle coordinated edits to find mechanisms; late rounds become sparse and attributable.</li>
        <li><b>Evidence-aware credit assignment:</b> every candidate's component, hypothesis, diff, score/cost delta and accept/reject outcome are recorded; falsified hypotheses stay negative evidence.</li>
        <li><b>Structured exploration:</b> when progress over w rounds stays inside the noise band δ, part of the budget is reserved for components never yet exercised.</li>
      </ul>
    </div>
    <div>
      <h4 style="margin:0 0 6px;font-size:13px;letter-spacing:.04em;text-transform:uppercase;color:#475569">Selection side — what becomes permanent state</h4>
      <ul class="mech">
        <li><b>Leakage screening:</b> a critic rejects diffs encoding task/entity names, task-specific values, answers, or inert machinery — <em>before</em> evaluation, so a leaking candidate never banks an inflated score.</li>
        <li><b>Stability-aware acceptance:</b> Ŝ(H′) ≥ S⋆ − δ, with δ estimated from repeated runs of the unchanged base harness. Blocks slow downhill walks of individually noise-sized regressions.</li>
        <li><b>L₂-style complexity-aware acceptance:</b> for ΔS &gt; δ, require ΔC ≤ β₀ + β₁ΔS — extra tokens must be justified by measured improvement.</li>
        <li><b>L₁-style structural pruning:</b> components with no strictly positive measured gain over a window are handed back to the proposer as deletion targets.</li>
      </ul>
    </div>
  </div>
</section>

</div>

<script>
const C = {evolve:'#6366f1', held:'#0ea5e9', ood:'#10b981', base:'#94a3b8'};

/* ---------- Figure 1a scatter ---------- */
(function(){
  const W=660,H=430,m={l:66,r:30,t:26,b:56};
  const xMin=-0.5,xMax=4.3,yMin=-2.5,yMax=4.7;
  const X=v=>m.l+(v-xMin)/(xMax-xMin)*(W-m.l-m.r);
  const Y=v=>m.t+(yMax-v)/(yMax-yMin)*(H-m.t-m.b);

  const pts=[
    {n:'H₀ (base)',x:0,y:0,c:C.base,r:6,ax:-6,ay:-13,anchor:'end'},
    {n:'Meta-Harness',x:3.6,y:0.93,c:C.evolve,r:7,ax:-10,ay:-12,anchor:'end'},
    {n:'HarnessX',x:2.4,y:0.00,c:C.evolve,r:7,ax:10,ay:5,anchor:'start'},
    {n:'AHE',x:1.3,y:-0.53,c:C.evolve,r:7,ax:11,ay:4,anchor:'start'},
    {n:'TTHE',x:1.7,y:-1.73,c:C.evolve,r:7,ax:11,ay:4,anchor:'start'},
    {n:'RRSI (ours)',x:1.1,y:3.90,c:C.ood,r:9,ax:0,ay:-16,anchor:'middle',bold:true}
  ];

  let s=`<svg viewBox="0 0 ${W} ${H}" role="img" aria-label="Evolve-set gain versus out-of-distribution gain">`;
  // horizontal gridlines
  for(let gy=-2;gy<=4;gy++){
    s+=`<line x1="${m.l}" y1="${Y(gy)}" x2="${W-m.r}" y2="${Y(gy)}" stroke="${gy===0?'#cbd5e1':'#eef2f7'}" stroke-width="${gy===0?1.5:1}" ${gy===0?'stroke-dasharray="5 4"':''}/>`;
    s+=`<text x="${m.l-10}" y="${Y(gy)+4}" font-size="11.5" fill="#94a3b8" text-anchor="end">${gy>0?'+':''}${gy}</text>`;
  }
  for(let gx=0;gx<=4;gx++){
    s+=`<line x1="${X(gx)}" y1="${m.t}" x2="${X(gx)}" y2="${H-m.b}" stroke="#eef2f7" stroke-width="1"/>`;
    s+=`<text x="${X(gx)}" y="${H-m.b+20}" font-size="11.5" fill="#94a3b8" text-anchor="middle">${gx===0?'0':'+'+gx}</text>`;
  }
  // axes
  s+=`<line x1="${m.l}" y1="${H-m.b}" x2="${W-m.r}" y2="${H-m.b}" stroke="#cbd5e1" stroke-width="1.5"/>`;
  s+=`<line x1="${m.l}" y1="${m.t}" x2="${m.l}" y2="${H-m.b}" stroke="#cbd5e1" stroke-width="1.5"/>`;
  s+=`<text x="${(m.l+W-m.r)/2}" y="${H-14}" font-size="12.5" fill="#475569" text-anchor="middle">Evolve-set gain (points vs H₀)</text>`;
  s+=`<text transform="translate(18 ${(m.t+H-m.b)/2}) rotate(-90)" font-size="12.5" fill="#475569" text-anchor="middle">Out-of-distribution gain (points vs H₀)</text>`;
  // y = x reference
  s+=`<line x1="${X(0)}" y1="${Y(0)}" x2="${X(xMax)}" y2="${Y(xMax)}" stroke="#94a3b8" stroke-width="1.2" stroke-dasharray="3 4"/>`;
  s+=`<text x="${X(3.55)}" y="${Y(3.35)}" font-size="11" fill="#94a3b8" text-anchor="end">y = x (gain fully retained)</text>`;
  s+=`<text x="${X(0)+6}" y="${Y(0)-7}" font-size="11" fill="#94a3b8">H₀ line — below here, evolution made things worse</text>`;
  // points
  pts.forEach(p=>{
    s+=`<circle cx="${X(p.x)}" cy="${Y(p.y)}" r="${p.r}" fill="${p.c}" stroke="#fff" stroke-width="2"/>`;
    s+=`<text x="${X(p.x)+p.ax}" y="${Y(p.y)+p.ay}" font-size="12.5" fill="#1e293b" text-anchor="${p.anchor}" ${p.bold?'font-weight="700"':''}>${p.n}</text>`;
  });
  s+=`</svg>`;
  document.getElementById('scatter').innerHTML=s;
})();

/* ---------- Figure 3 bars ---------- */
(function(){
  const groups=[
    {label:'Coding',bars:[
      {b:'Terminal-Bench 2.1',k:'evolve',v:6.0,t:'evolve'},
      {b:'SWE-bench Verified',k:'ood',v:1.8,t:'OOD'}
    ]},
    {label:'Agentic workspace',bars:[
      {b:'Harvey LAB',k:'evolve',v:1.1,t:'evolve'},
      {b:'Harvey LAB held-out',k:'held',v:2.3,t:'ID held-out'},
      {b:'JobBench',k:'ood',v:4.7,t:'OOD'},
      {b:'GDPval',k:'ood',v:3.5,t:'OOD'},
      {b:'APEX-Agents',k:'ood',v:3.7,t:'OOD'}
    ]},
    {label:'Engineering design',bars:[
      {b:'EngDesign',k:'evolve',v:4.9,t:'evolve'},
      {b:'Frontier-Eng',k:'ood',v:4.3,t:'OOD'}
    ]}
  ];
  const MAX=6.0, PX=200;
  let html='';
  groups.forEach(g=>{
    html+=`<div class="dgroup"><h4>${g.label}</h4><div class="bars">`;
    g.bars.forEach(x=>{
      const h=Math.max(6,Math.round(x.v/MAX*PX));
      html+=`<div class="bar">
        <div class="val">+${x.v.toFixed(1)}</div>
        <div class="col" style="height:${h}px;background:${C[x.k]}"></div>
        <div class="nm"><b>${x.b}</b>${x.t}</div>
      </div>`;
    });
    html+=`</div></div>`;
  });
  document.getElementById('domains').innerHTML=html;
})();

/* ---------- efficiency ---------- */
(function(){
  const rows=[
    {lab:'H₀ — no evolution',v:1.56,c:C.base},
    {lab:'RRSI (ours)',v:2.42,c:C.ood},
    {lab:'Unregularized evolution',v:3.80,c:C.evolve}
  ];
  const MAX=4.0;
  let html='';
  rows.forEach(r=>{
    html+=`<div class="tok">
      <div class="lab">${r.lab}</div>
      <div class="big">${r.v.toFixed(2)}<span> M policy tokens / trial</span></div>
      <div class="meter"><i style="width:${r.v/MAX*100}%;background:${r.c}"></i></div>
    </div>`;
  });
  document.getElementById('tokens').innerHTML=html;
})();

/* ---------- tables ---------- */
(function(){
  const t1=[
    {m:'H₀ (no evolution)',e:89.4,h:86.9,j:36.0,g:48.8,a:34.2,cls:'h0'},
    {m:'Meta-Harness',e:93.0,h:89.2,j:37.1,g:49.1,a:35.7},
    {m:'AHE',e:90.7,h:88.7,j:37.2,g:47.2,a:33.1},
    {m:'TTHE',e:91.1,h:88.5,j:35.2,g:47.0,a:31.7},
    {m:'HarnessX',e:91.8,h:89.1,j:36.3,g:48.5,a:34.3},
    {m:'RRSI (ours)',e:90.5,h:89.2,j:40.7,g:52.3,a:37.9,cls:'ours'}
  ];
  document.getElementById('t1').innerHTML=t1.map(r=>{
    const avg=(r.j+r.g+r.a)/3;
    const cell=(v,i)=>`<td>${i===1&&v<40?'<span class="lose">'+v.toFixed(1)+'</span>':v.toFixed(1)}</td>`;
    return `<tr class="${r.cls||''}"><td>${r.m}</td>`+
      [r.e,r.h,r.j,r.g,r.a].map(cell).join('')+
      `<td>${avg.toFixed(1)}</td></tr>`;
  }).join('');

  const t2=[
    {m:'H₀ (no evolution)',e:89.4,h:86.9,o:39.7,k:1.56,cls:'h0'},
    {m:'Unregularized evolution',e:92.8,h:88.9,o:40.3,k:3.80},
    {m:'w/o proposal regularizers',e:90.7,h:88.8,o:41.9,k:2.69},
    {m:'w/o acceptance regularizers',e:91.5,h:88.7,o:41.0,k:3.59},
    {m:'RRSI (ours)',e:90.5,h:89.2,o:43.6,k:2.42,cls:'ours'}
  ];
  document.getElementById('t2').innerHTML=t2.map(r=>
    `<tr class="${r.cls||''}"><td>${r.m}</td><td>${r.e.toFixed(1)}</td><td>${r.h.toFixed(1)}</td><td>${r.o.toFixed(1)}</td><td>${r.k.toFixed(2)}</td></tr>`
  ).join('');

  const t3=[
    {p:'Claude Opus 4.8',b:'Terminal-Bench 2.1 (evolve)',h0:74.2,r:80.2,d:6.0},
    {p:'',b:'SWE-bench Verified (OOD)',h0:82.0,r:83.8,d:1.8},
    {p:'Gemini 3.5 Flash',b:'Terminal-Bench 2.1 (evolve)',h0:64.6,r:78.7,d:14.1},
    {p:'',b:'SWE-bench Verified (OOD)',h0:76.8,r:79.0,d:2.2}
  ];
  document.getElementById('t3').innerHTML=t3.map(r=>
    `<tr><td>${r.p?'<b>'+r.p+'</b>':''}</td><td>${r.b}</td><td>${r.h0.toFixed(1)}</td><td>${r.r.toFixed(1)}</td><td class="win">+${r.d.toFixed(1)}</td></tr>`
  ).join('');
})();
</script>
</body>
</html>
<script data-huggingchat-badge="script">
(function () {
	try {
		if (document.querySelector("[data-huggingchat-badge]:not(script)")) return;
		var host = document.createElement("div");
		host.setAttribute("data-huggingchat-badge", "");
		var styles = [["all","initial"],["position","fixed"],["right","12px"],["bottom","12px"],["z-index","2147483647"],["display","block"],["visibility","visible"],["opacity","1"],["pointer-events","auto"],["width","auto"],["height","auto"],["margin","0"],["padding","0"],["max-width","none"],["max-height","none"],["transform","none"],["filter","none"],["clip-path","none"],["color-scheme","light"]];
		for (var i = 0; i < styles.length; i++) {
			host.style.setProperty(styles[i][0], styles[i][1], "important");
		}
		host.attachShadow({ mode: "closed" }).innerHTML = "<style>\n:host { all: initial; }\na {\n  display: flex;\n  align-items: center;\n  gap: 5px;\n  box-sizing: border-box;\n  height: 26px;\n  padding: 0 9px 0 7px;\n  border-radius: 999px;\n  border: 1px solid rgba(0, 0, 0, 0.07);\n  background: rgba(255, 255, 255, 0.92);\n  -webkit-backdrop-filter: saturate(180%) blur(8px);\n  backdrop-filter: saturate(180%) blur(8px);\n  box-shadow: 0 1px 2px rgba(0, 0, 0, 0.08), 0 6px 16px rgba(0, 0, 0, 0.06);\n  color: #111827;\n  font: 500 11px/1 ui-sans-serif, system-ui, -apple-system, \"Segoe UI\", sans-serif;\n  text-decoration: none;\n  white-space: nowrap;\n  opacity: 0.72;\n  transition: opacity 0.15s ease, transform 0.15s ease;\n}\na:hover, a:focus-visible { opacity: 1; transform: translateY(-1px); }\nsvg { width: 14px; height: 14px; flex: none; }\n@media print { a { display: none; } }\n<\/style><a href=\"https://huggingface.co/chat\" target=\"_blank\" rel=\"noopener noreferrer nofollow\"><svg viewBox=\"0 0 32 32\" fill=\"none\" aria-hidden=\"true\"><path d=\"M16.0006 25.9992C13.8266 25.999 11.7118 25.2901 9.97686 23.9799C8.2419 22.6698 6.98127 20.8298 6.38599 18.7388C5.79071 16.6478 5.89323 14.4198 6.678 12.3923C7.46278 10.3648 8.88705 8.64837 10.735 7.50308C12.5829 6.35779 14.7538 5.84606 16.9187 6.04544C19.0837 6.24481 21.1246 7.14442 22.7323 8.60795C24.34 10.0715 25.4268 12.0192 25.8281 14.1559C26.2293 16.2926 25.9232 18.5019 24.9561 20.449C24.7703 20.8042 24.7223 21.2155 24.8211 21.604L25.4211 23.8316C25.4803 24.0518 25.4805 24.2837 25.4216 24.5039C25.3627 24.7242 25.2468 24.925 25.0856 25.0862C24.9244 25.2474 24.7235 25.3633 24.5033 25.4222C24.283 25.4811 24.0512 25.4809 23.831 25.4217L21.6034 24.8217C21.2172 24.7248 20.809 24.7729 20.4558 24.9567C19.0683 25.6467 17.5457 26.0068 16.0006 26.0068V25.9992Z\" fill=\"currentColor\"/><path d=\"M9.62598 16.0013C9.62598 15.3799 10.1294 14.8765 10.7508 14.8765C11.3721 14.8765 11.8756 15.3799 11.8756 16.0013C11.8756 17.0953 12.3102 18.1448 13.0838 18.9184C13.8574 19.692 14.9069 20.1266 16.001 20.1267C17.095 20.1267 18.1445 19.692 18.9181 18.9184C19.6918 18.1448 20.1264 17.0953 20.1264 16.0013C20.1264 15.3799 20.6299 14.8765 21.2512 14.8765C21.8725 14.8765 22.3759 15.3799 22.3759 16.0013C22.3759 17.6921 21.7046 19.3137 20.509 20.5093C19.3134 21.7049 17.6918 22.3762 16.001 22.3762C14.3102 22.3762 12.6885 21.7049 11.4929 20.5093C10.2974 19.3137 9.62598 17.6921 9.62598 16.0013Z\" fill=\"#fff\"/><\/svg><span>Made with HuggingChat<\/span><\/a>";
		function attach() {
			var parent = document.body || document.documentElement;
			if (parent && host.parentNode !== parent) parent.appendChild(host);
		}
		attach();
		// Artifacts that rewrite document.body wholesale would drop the badge.
		if (document.body && typeof MutationObserver === "function") {
			new MutationObserver(attach).observe(document.body, { childList: true });
		} else {
			document.addEventListener("DOMContentLoaded", attach);
		}
	} catch (e) {}
})();
</script>