| <!doctype html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"> |
| <title>OpenGuardrailsBench — AI guardrails, measured in context</title> |
| <style> |
| :root{--ink:#152f2b;--muted:#61716b;--line:#dce4db;--paper:#f5f7f0;--green:#116c52;--lime:#deefaf}*{box-sizing:border-box}body{margin:0;background:var(--paper);color:var(--ink);font:15px/1.6 ui-sans-serif,system-ui,-apple-system,sans-serif}a{color:inherit;text-underline-offset:4px}header,main,footer{max-width:1260px;margin:auto;padding:24px 42px}header{display:flex;align-items:center;justify-content:space-between;border-bottom:1px solid var(--line)}.brand{font-weight:850;font-size:20px;letter-spacing:-1px}.mark{display:inline-flex;background:var(--ink);color:var(--lime);width:32px;height:36px;align-items:center;justify-content:center;border-radius:6px 6px 14px 14px;margin-right:9px;font-size:18px}nav{display:flex;gap:24px;font-size:13px}nav a{text-decoration:none}.hero{display:grid;grid-template-columns:1.6fr 1fr;gap:60px;padding:46px 0 32px}.eyebrow{font:11px ui-monospace,monospace;letter-spacing:2px;text-transform:uppercase;color:var(--green)}h1{font-size:clamp(38px,5vw,64px);line-height:1.08;letter-spacing:-3px;margin:19px 0 22px;max-width:740px}h1 em{font-style:normal;color:var(--green)}.lead{font-size:17px;color:var(--muted);max-width:620px}.edition{display:inline-block;border:1px solid #bad1b1;padding:3px 10px;border-radius:24px;background:#edf4df;font-size:11px;font-weight:700;letter-spacing:1px}.formula{align-self:center;background:#e9eee4;border:1px solid #d6dfd0;border-radius:12px;padding:25px}.formula strong{display:block;font-size:29px;letter-spacing:-1px}.formula .bar{display:flex;height:10px;gap:4px;margin:20px 0}.bar i:first-child{width:60%;background:var(--green)}.bar i:last-child{width:40%;background:#a5bc6d}.mini{font-size:12px;color:var(--muted)}.notice{border-left:3px solid #a7ad50;padding:12px 18px;background:#eff0db;color:#596039;font-size:13px;margin:0 0 30px}.stats{display:grid;grid-template-columns:repeat(4,1fr);border-top:1px solid var(--line);border-bottom:1px solid var(--line);margin:20px 0 38px}.stat{padding:18px 20px;border-right:1px solid var(--line)}.stat:first-child{padding-left:0}.stat:last-child{border:0}.stat b{font-size:29px;letter-spacing:-1px;display:block}.stat span{font-size:12px;color:var(--muted)}.section-top{display:flex;justify-content:space-between;align-items:end;margin-bottom:17px}h2{font-size:24px;letter-spacing:-.7px;margin:0}.controls{display:flex;align-items:center;gap:12px;flex-wrap:wrap}.tabs{display:flex;gap:3px;background:#e6ebe0;padding:4px;border-radius:7px}button{border:0;background:transparent;padding:7px 13px;color:var(--muted);font:inherit;font-size:12px;cursor:pointer;border-radius:4px}button.active{color:var(--ink);background:white;box-shadow:0 1px 3px #0001}input{font:inherit;font-size:12px;border:1px solid var(--line);background:#fff;padding:10px 12px;border-radius:6px;width:190px}table{border-collapse:collapse;width:100%;font-size:13px}th{text-align:right;color:var(--muted);font-size:10px;letter-spacing:.7px;font-weight:650;text-transform:uppercase;padding:12px;border-block:1px solid var(--line);white-space:nowrap}th:nth-child(2),td:nth-child(2){text-align:left}td{padding:16px 12px;border-bottom:1px solid var(--line);text-align:right}td:first-child,th:first-child{width:35px;text-align:left;color:var(--muted)}tbody tr{cursor:pointer}tbody tr:hover{background:#edf1e7}.name{font-weight:700;font-size:14px;display:block}.sub{font-size:10px;color:var(--muted)}.score{font-size:22px;font-weight:750;letter-spacing:-.7px;color:var(--green)}.tag{display:inline-block;padding:1px 6px;border:1px solid var(--line);border-radius:4px;font-size:10px}.run-status{font-size:12px;min-width:200px}.ineligible{font-weight:650;color:#826827}.sub{display:block}.table-wrap{overflow:auto;min-height:160px}.empty{padding:30px;text-align:center;color:var(--muted)}.details{padding:18px 0;display:none}.details.show{display:block}.areas{display:grid;grid-template-columns:repeat(4,1fr);gap:12px}.area{background:#fff;border:1px solid var(--line);border-radius:7px;padding:14px}.area b{display:block;font-size:23px}.docs{display:grid;grid-template-columns:repeat(3,1fr);gap:22px;margin:45px 0 20px}.card{border-top:2px solid var(--ink);padding-top:17px}.card h3{font-size:15px}.card p{color:var(--muted);font-size:13px}.card .num{font:11px ui-monospace,monospace;color:var(--green)}footer{border-top:1px solid var(--line);font-size:12px;color:var(--muted);display:flex;justify-content:space-between;margin-top:30px}code{background:#e6ebe0;padding:2px 5px;border-radius:3px;font-size:11px}@media(max-width:800px){header,main,footer{padding:20px}header{flex-wrap:wrap;gap:14px}nav{gap:12px;font-size:11px}.hero{grid-template-columns:1fr;gap:15px;padding-top:30px}h1{letter-spacing:-2px}.stats{grid-template-columns:repeat(2,1fr)}.stat:nth-child(2){border:0}.stat:nth-child(3){padding-left:0}.section-top{display:block}.controls{margin-top:15px}.docs,.areas{grid-template-columns:1fr}.formula{padding:20px}footer{gap:20px}} |
| </style> |
| </head> |
| <body> |
| <header><div class="brand"><span class="mark" style="font-size:10px">OGB</span>OpenGuardrailsBench</div><nav><a href="https://huggingface.co/datasets/openguardrails/OpenGuardrailsBench">Dataset ↗</a><a href="https://github.com/openguardrails/OpenGuardrailsBench">Code & methodology ↗</a></nav></header> |
| <main> |
| <div class="hero"><div><span class="edition">EDITION 0.1 · SYNTHETIC PILOT</span><h1>AI guardrails,<br>measured <em>in context.</em></h1><p class="lead">Can a guardrail recognize the violation — and let the legitimate request through? Paired tests across safety, security, privacy, and policy.</p></div><aside class="formula"><span class="eyebrow">Full index · held-out evaluation</span><strong>60% same skills<br>40% new domains</strong><div class="bar"><i></i><i></i></div><div class="mini">Public tests validate submissions. Private tests measure transfer. A complete public run is required.</div></aside></div> |
| <div class="notice" id="notice">Synthetic pilot. Machine-authored labels; independent human adjudication pending. Scores apply to this suite and adapter, and do not establish production reliability.</div> |
| <div class="stats"><div class="stat"><b id="model-count">—</b><span>Evaluated baselines</span></div><div class="stat"><b id="case-count">—</b><span>Public cases · paired & balanced</span></div><div class="stat"><b>12</b><span>Public guardrail skills</span></div><div class="stat"><b>EN + 中文</b><span>Context and policy in two languages</span></div></div> |
| <section><div class="section-top"><div><span class="eyebrow">Reproducible receipts. Explicit coverage.</span><h2>Model results</h2></div><div class="controls"><div class="tabs" role="group" aria-label="Evaluation partition"><button class="active" data-part="full">Full index</button><button data-part="public">Public</button><button data-part="private_same">Private · same</button><button data-part="private_novel">Private · novel</button></div><input id="search" type="search" placeholder="Search models…" aria-label="Search models"></div></div> |
| <p class="mini">Pilot scores are shown without competitive ranks. Select a model for public area scores. Four-bit NF4 runs are marked. Fixed-taxonomy guards use a published policy/context wrapper. Full view shows public diagnostic metrics; a Full confidence interval is not computed. Status distinguishes a completed zero score from invalid answers.</p> |
| <div class="table-wrap"><table><thead><tr><th>#</th><th>Model / adapter</th><th>Run status</th><th>Index ↑</th><th id="ci-label">Full 95% CI</th><th>Valid coverage</th><th id="recall-label">Public recall</th><th id="fpr-label">Public false positives ↓</th><th id="pair-label">Public pair exact ↑</th></tr></thead><tbody id="body"><tr><td colspan="9" class="empty">Loading published results…</td></tr></tbody></table></div><div class="details" id="details"></div> |
| </section> |
| <div class="docs"><article class="card"><span class="num">01 / MEASURE BOTH SIDES</span><h3>Blocking everything scores zero.</h3><p>The index uses chance-corrected balanced accuracy. Every unsafe test has a legitimate counterpart. Errors and missing outputs lose credit; coverage stays visible.</p></article><article class="card"><span class="num">02 / KEEP THE TEST HONEST</span><h3>Public recipe. Private evidence.</h3><p>Open cases and predictions make runs reproducible. Held-out cases and their raw outputs stay private. Only aggregate private scores reach this page.</p></article><article class="card"><span class="num">03 / REPRODUCE A RESULT</span><h3>Bring your model.</h3><p>Run <code>python -m ogb</code> against an HTTP endpoint or a pinned model checkpoint. Outputs include raw predictions, configuration, hashes, and scoring receipts.</p></article></div> |
| </main><footer><span>OpenGuardrailsBench · OpenGuardrails · Apache-2.0</span><span>Inspired by <a href="https://github.com/apolinario/decision-index">Decision Index</a>. Independent tasks and scoring.</span></footer> |
| <script> |
| let models=[],part='full'; |
| const el=id=>document.getElementById(id); |
| const fmt=(n,d=1)=>typeof n==='number'?n.toFixed(d):'Unavailable'; |
| const pct=n=>typeof n==='number'?(100*n).toFixed(1)+'%':'Unavailable'; |
| function view(m,selected){ |
| const c=m.completion; |
| const p=m.parts[selected==='full'?'public':selected]; |
| const count=c.invalid+c.errors; |
| const status=c.state==='complete'?'Completed':c.state==='finished_with_invalid'?`Finished · ${count} invalid/error`:'In progress'; |
| let note=`${c.valid}/${c.total} valid · ${c.attempted} attempted`; |
| if(c.all_predictions_safe)note+=' · All predictions safe → index 0'; |
| if(c.output_limit_failures)note+=` · ${c.output_limit_failures} invalid response reached output limit`; |
| return {status,note,score:selected==='full'?m.full_index:p?.index, |
| scoreReason:selected==='full'&&m.full_index===null?'Full requires valid answers on the public and both private partitions.':'', |
| ci:selected==='full'?'Not computed':p?.ci95?p.ci95.map(x=>fmt(x)).join(' – '):'Unavailable', |
| coverage:selected==='full'?c.valid/c.total:p?.coverage, |
| recall:p?.recall,fpr:p?.false_positive_rate,pair:p?.pair_exact}; |
| } |
| function cell(text,cls){const t=document.createElement('td');t.textContent=text;if(cls)t.className=cls;return t} |
| function render(){ |
| const query=el('search').value.toLowerCase(); |
| const list=models.filter(m=>(m.model+' '+m.adapter).toLowerCase().includes(query)); |
| list.sort((a,b)=>((part==='full'?b.full_index:b.parts[part]?.index)??-1)-((part==='full'?a.full_index:a.parts[part]?.index)??-1)); |
| el('ci-label').textContent=part==='full'?'Full 95% CI':'95% CI'; |
| el('recall-label').textContent=part==='full'?'Public recall':'Recall'; |
| el('fpr-label').textContent=part==='full'?'Public false positives ↓':'False positives ↓'; |
| el('pair-label').textContent=part==='full'?'Public pair exact ↑':'Pair exact ↑'; |
| el('body').replaceChildren(); |
| list.forEach(m=>{ |
| const v=view(m,part),tr=document.createElement('tr');tr.tabIndex=0; |
| tr.setAttribute('aria-label','Show public area scores for '+m.model);tr.append(cell('·')); |
| const td=document.createElement('td'),name=document.createElement('span'),sub=document.createElement('span'); |
| name.className='name';name.textContent=m.model;sub.className='sub'; |
| sub.textContent=(m.adapter||'baseline')+' · '+(m.quantize_nf4?'NF4 4-bit':'BF16 / baseline')+(m.offload_embeddings?' · CPU offload':'')+(m.affiliation?' · MAINTAINER-OWNED':'')+' · '+(m.revision||'unversioned').slice(0,10);td.append(name,sub); |
| const status=cell(v.status,'run-status'),detail=document.createElement('span');detail.className='sub';detail.textContent=v.note;status.append(detail); |
| const score=cell(typeof v.score==='number'?fmt(v.score,2):'Not eligible',typeof v.score==='number'?'score':'ineligible');score.title=v.scoreReason; |
| tr.append(td,status,score,cell(v.ci),cell(pct(v.coverage)),cell(pct(v.recall)),cell(pct(v.fpr)),cell(pct(v.pair))); |
| const show=()=>showDetails(m);tr.onclick=show;tr.onkeydown=e=>{if(e.key==='Enter'||e.key===' '){e.preventDefault();show()}};el('body').append(tr); |
| }); |
| if(!list.length){const tr=document.createElement('tr'),td=cell('No published results match this selection.','empty');td.colSpan=9;tr.append(td);el('body').append(tr)} |
| } |
| function showDetails(m){ |
| const d=el('details');d.replaceChildren(); |
| const title=document.createElement('h3');title.textContent=m.model+' · public areas';d.append(title); |
| const note=document.createElement('p'),v=view(m,'full');note.className='mini';note.textContent=v.status+' · '+v.note+(v.scoreReason?' · '+v.scoreReason:'');d.append(note); |
| if(m.model==='OpenGuardrails'){const p=document.createElement('p');p.className='mini';p.textContent='OpenGuardrails is for builders. AIRS is the enterprise edition; both share the same safety capability. This entry is maintainer-owned and self-evaluated.';d.append(p)} |
| const cards=document.createElement('div');cards.className='areas'; |
| Object.entries(m.parts.public?.areas||{}).forEach(([k,v])=>{const c=document.createElement('div');c.className='area';const label=document.createElement('span'),value=document.createElement('b');label.textContent=k.replaceAll('_',' ');value.textContent=fmt(v.score,2);c.append(label,value);cards.append(c)});d.append(cards);d.classList.add('show'); |
| } |
| document.querySelectorAll('[data-part]').forEach(b=>b.onclick=()=>{part=b.dataset.part;document.querySelectorAll('[data-part]').forEach(t=>t.classList.toggle('active',t===b));render()});el('search').oninput=render; |
| fetch('leaderboard.json').then(r=>{if(!r.ok)throw Error('HTTP '+r.status);return r.json()}).then(d=>{models=d.models||[];el('model-count').textContent=models.length;el('case-count').textContent=models[0]?.parts.public?.n??'192';if(d.notice)el('notice').textContent=d.notice;render()}).catch(e=>{el('body').replaceChildren();const tr=document.createElement('tr'),td=cell('Results could not be loaded. Please reload.','empty');td.colSpan=9;tr.append(td);el('body').append(tr)}); |
| </script> |
| </body></html> |
|
|
