Spaces:
Running
Running
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <meta name="description" content="Reranker Benchmark is a vendor-neutral framework for evaluating reranker models using nDCG, MRR, Recall, latency, throughput and deployment criteria."> | |
| <title>Reranker Benchmark</title> | |
| <style> | |
| :root{--bg:#f7f9fc;--card:#fff;--text:#101828;--muted:#667085;--line:#e4e7ec;--blue:#2563eb;--indigo:#4f46e5;--shadow:0 18px 50px rgba(16,24,40,.08)} | |
| *{box-sizing:border-box}body{margin:0;font-family:Inter,system-ui,sans-serif;color:var(--text);background:radial-gradient(circle at 10% 0%,rgba(37,99,235,.10),transparent 30%),radial-gradient(circle at 95% 8%,rgba(79,70,229,.09),transparent 26%),var(--bg)} | |
| .wrap{max-width:1120px;margin:auto;padding:0 22px}header{padding:64px 0 30px}.badge{display:inline-flex;align-items:center;gap:8px;padding:8px 12px;border:1px solid #c7d7fe;background:#f5f8ff;border-radius:999px;font-size:13px;font-weight:800;color:#344054}.dot{width:8px;height:8px;border-radius:50%;background:linear-gradient(135deg,var(--blue),var(--indigo))} | |
| h1{font-size:clamp(42px,7vw,72px);line-height:1;letter-spacing:-.05em;margin:22px 0 16px}.gradient{background:linear-gradient(90deg,var(--blue),var(--indigo));-webkit-background-clip:text;background-clip:text;color:transparent}.lead{font-size:19px;line-height:1.7;color:#475467;max-width:880px}.note{margin-top:20px;padding:15px 17px;border:1px solid #d0d5dd;border-radius:15px;background:rgba(255,255,255,.82);font-size:13px;line-height:1.55;color:#475467} | |
| nav{position:sticky;top:0;z-index:20;background:rgba(247,249,252,.88);backdrop-filter:blur(16px);border-top:1px solid rgba(228,231,236,.7);border-bottom:1px solid rgba(228,231,236,.9)}nav .inner{display:flex;gap:8px;overflow:auto;padding:11px 22px;max-width:1120px;margin:auto}nav a{white-space:nowrap;text-decoration:none;color:#344054;font-size:13px;font-weight:800;padding:8px 11px;border-radius:999px}nav a:hover{background:#fff} | |
| section{padding:48px 0}.section-title{display:flex;justify-content:space-between;gap:20px;align-items:end;margin-bottom:22px}.section-title h2{font-size:34px;letter-spacing:-.035em;margin:0}.section-title p{margin:0;color:var(--muted);line-height:1.6;max-width:650px}.card{background:rgba(255,255,255,.96);border:1px solid var(--line);border-radius:22px;box-shadow:var(--shadow)} | |
| .grid3{display:grid;grid-template-columns:repeat(3,1fr);gap:16px}.metric{padding:20px}.metric h3{margin:0 0 7px;font-size:18px}.metric p{margin:0;color:var(--muted);font-size:13px;line-height:1.6}.metric code{display:inline-block;margin-top:12px;padding:6px 9px;border-radius:9px;background:#f2f4f7;color:#344054} | |
| .bench{padding:28px}.rows{display:grid;gap:14px}.model-row{display:grid;grid-template-columns:1.25fr repeat(6,1fr) 44px;gap:8px;align-items:center}.model-row.head{font-size:11px;font-weight:900;color:#667085;text-transform:uppercase;letter-spacing:.06em}.model-row input{width:100%;border:1px solid #d0d5dd;border-radius:11px;padding:10px 9px;background:#fff;font-size:13px;outline:none}.model-row input:focus{border-color:#84adff;box-shadow:0 0 0 3px #eff4ff}.remove{border:1px solid #d0d5dd;background:#fff;border-radius:11px;height:38px;cursor:pointer;font-weight:800;color:#667085} | |
| .controls{display:flex;gap:10px;flex-wrap:wrap;margin-top:18px}button{border:1px solid #d0d5dd;background:#fff;color:#344054;border-radius:12px;padding:11px 14px;font-size:13px;font-weight:800;cursor:pointer}button.primary{border:none;color:#fff;background:linear-gradient(90deg,var(--blue),var(--indigo))} | |
| .weights{display:grid;grid-template-columns:repeat(4,1fr);gap:12px;margin-top:20px}.weight-card{padding:16px;border:1px solid var(--line);border-radius:16px;background:#fff}.weight-card label{display:flex;justify-content:space-between;gap:12px;font-size:12px;font-weight:800;margin-bottom:10px}.weight-card input{width:100%} | |
| .results{margin-top:22px}.result-row{display:grid;grid-template-columns:1fr 90px 2fr;gap:12px;align-items:center;padding:12px 0;border-bottom:1px solid #eaecf0}.score{font-weight:900}.bar{height:10px;border-radius:999px;background:#eaecf0;overflow:hidden}.bar span{display:block;height:100%;background:linear-gradient(90deg,var(--blue),var(--indigo))} | |
| .workflow{display:grid;grid-template-columns:repeat(5,1fr);gap:12px}.step{padding:18px}.step b{font-size:11px;color:var(--blue)}.step h3{margin:7px 0 6px;font-size:16px}.step p{margin:0;color:var(--muted);font-size:12px;line-height:1.55}.table-wrap{overflow:auto}table{width:100%;border-collapse:collapse;background:#fff;border-radius:18px;overflow:hidden}th,td{padding:14px 15px;border-bottom:1px solid #eaecf0;text-align:left;font-size:13px;vertical-align:top}th{background:#f9fafb;color:#344054}td{color:#475467} | |
| .footer{padding:34px 0 65px;color:#667085;font-size:13px}.footer .inner{border-top:1px solid var(--line);padding-top:22px;display:flex;justify-content:space-between;gap:20px;flex-wrap:wrap} | |
| @media(max-width:950px){.grid3{grid-template-columns:repeat(2,1fr)}.workflow{grid-template-columns:repeat(2,1fr)}.model-row{grid-template-columns:1fr 1fr 1fr}.model-row.head{display:none}.weights{grid-template-columns:repeat(2,1fr)}}@media(max-width:620px){.grid3,.workflow,.weights{grid-template-columns:1fr}.wrap{padding:0 16px}.section-title{display:block}.section-title p{margin-top:10px}} | |
| </style> | |
| </head> | |
| <body> | |
| <header><div class="wrap"><span class="badge"><span class="dot"></span> Reranker · Evaluation</span><h1>Benchmark <span class="gradient">rerankers reproducibly.</span></h1><p class="lead">Compare reranker candidates using retrieval quality, ranking metrics and operational measurements instead of relying on a single leaderboard number.</p><div class="note"><strong>Important:</strong> This tool does not contain pre-filled vendor scores. Enter measurements from your own benchmark runs. The composite score is a configurable planning aid, not an objective universal ranking.</div></div></header> | |
| <nav><div class="inner"><a href="#metrics">Metrics</a><a href="#benchmark">Benchmark Tool</a><a href="#workflow">Workflow</a><a href="#guidance">Guidance</a></div></nav> | |
| <main> | |
| <section id="metrics"><div class="wrap"><div class="section-title"><h2>Core reranker metrics</h2><p>A useful reranker benchmark combines ranking quality with latency and throughput under realistic candidate-set sizes.</p></div><div class="grid3"> | |
| <div class="card metric"><h3>nDCG@k</h3><p>Measures ranking quality while giving more value to highly relevant results placed near the top.</p><code>Higher is better</code></div> | |
| <div class="card metric"><h3>MRR@k</h3><p>Measures how early the first relevant result appears. Useful for search and question-answering workflows.</p><code>Higher is better</code></div> | |
| <div class="card metric"><h3>Recall@k</h3><p>Measures how much of the relevant set survives retrieval and reranking at the chosen cutoff.</p><code>Higher is better</code></div> | |
| <div class="card metric"><h3>Latency</h3><p>Measures reranking time for a defined query, document length and candidate-set size.</p><code>Lower is better</code></div> | |
| <div class="card metric"><h3>Throughput</h3><p>Measures how many query-document pairs or requests can be processed per unit time.</p><code>Higher is better</code></div> | |
| <div class="card metric"><h3>Memory</h3><p>Tracks accelerator or system memory required for the target batch size and sequence length.</p><code>Lower is better</code></div> | |
| </div></div></section> | |
| <section id="benchmark"><div class="wrap"><div class="section-title"><h2>Interactive benchmark comparison</h2><p>Enter normalized quality metrics between 0 and 1, plus measured latency, throughput and memory. Adjust weighting to match your own production priorities.</p></div> | |
| <div class="card bench"><div class="rows" id="rows"><div class="model-row head"><span>Candidate</span><span>nDCG</span><span>MRR</span><span>Recall</span><span>Latency ms</span><span>Throughput</span><span>Memory GB</span><span></span></div></div> | |
| <div class="controls"><button id="add">Add candidate</button><button class="primary" id="calculate">Calculate comparison</button><button id="example">Load neutral example</button></div> | |
| <div class="weights"><div class="weight-card"><label><span>Quality weight</span><span id="wqv">45%</span></label><input id="wq" type="range" min="0" max="100" value="45"></div><div class="weight-card"><label><span>Latency weight</span><span id="wlv">25%</span></label><input id="wl" type="range" min="0" max="100" value="25"></div><div class="weight-card"><label><span>Throughput weight</span><span id="wtv">20%</span></label><input id="wt" type="range" min="0" max="100" value="20"></div><div class="weight-card"><label><span>Memory weight</span><span id="wmv">10%</span></label><input id="wm" type="range" min="0" max="100" value="10"></div></div> | |
| <div class="results" id="results"></div></div></div></section> | |
| <section id="workflow"><div class="wrap"><div class="section-title"><h2>A reproducible reranker benchmark workflow</h2><p>Comparisons are only useful when the dataset, candidate generation and runtime conditions are documented.</p></div><div class="workflow"> | |
| <div class="card step"><b>01</b><h3>Define tasks</h3><p>Select representative queries, documents, languages and domains.</p></div><div class="card step"><b>02</b><h3>Freeze retrieval</h3><p>Use the same first-stage candidate set for every reranker.</p></div><div class="card step"><b>03</b><h3>Measure quality</h3><p>Compute nDCG, MRR, Recall or task-specific ranking metrics.</p></div><div class="card step"><b>04</b><h3>Measure serving</h3><p>Record latency, throughput, memory, batch size and hardware.</p></div><div class="card step"><b>05</b><h3>Version everything</h3><p>Record model revision, tokenizer, runtime, precision and dataset version.</p></div> | |
| </div></div></section> | |
| <section id="guidance"><div class="wrap"><div class="section-title"><h2>Benchmark design guidance</h2><p>A reranker should be measured as part of a retrieval system, not in isolation from candidate generation and downstream use.</p></div><div class="card table-wrap"><table><thead><tr><th>Variable</th><th>Why it matters</th><th>What to report</th></tr></thead><tbody> | |
| <tr><td>Candidate-set size</td><td>Reranking 20 documents is very different from reranking 1,000.</td><td>Top-N input candidates and final top-k output.</td></tr> | |
| <tr><td>Document length</td><td>Longer inputs can increase latency and truncation risk.</td><td>Average / percentile token lengths and truncation policy.</td></tr> | |
| <tr><td>Language</td><td>Multilingual quality can vary substantially.</td><td>Language distribution and per-language metrics.</td></tr> | |
| <tr><td>Domain</td><td>General models may behave differently on legal, medical, code or enterprise data.</td><td>Dataset domain and relevance-labeling method.</td></tr> | |
| <tr><td>Hardware</td><td>Operational performance depends on accelerator, CPU and runtime.</td><td>Device, precision, batch size and serving framework.</td></tr> | |
| <tr><td>First-stage retriever</td><td>A reranker cannot recover documents that were never retrieved.</td><td>Retriever model, index settings and initial recall.</td></tr> | |
| <tr><td>Downstream RAG quality</td><td>Better ranking should improve the final application, not only an offline metric.</td><td>Answer quality, citation quality or task success where relevant.</td></tr> | |
| </tbody></table></div></div></section> | |
| </main> | |
| <footer class="footer"><div class="wrap inner"><div><strong>Reranker Benchmark</strong><br>Vendor-neutral evaluation framework for retrieval and RAG systems.</div><div>Cooperations & benchmark contributions:<br><strong>agenten@magenta.de</strong></div></div></footer> | |
| <script> | |
| const rows=document.getElementById("rows");let counter=0; | |
| function addRow(name=""){counter++;const r=document.createElement("div");r.className="model-row data-row";r.innerHTML=`<input class="name" placeholder="Candidate ${counter}" value="${name}"><input class="ndcg" type="number" min="0" max="1" step="0.001" placeholder="0–1"><input class="mrr" type="number" min="0" max="1" step="0.001" placeholder="0–1"><input class="recall" type="number" min="0" max="1" step="0.001" placeholder="0–1"><input class="latency" type="number" min="0" step="0.1" placeholder="ms"><input class="throughput" type="number" min="0" step="0.1" placeholder="pairs/s"><input class="memory" type="number" min="0" step="0.1" placeholder="GB"><button class="remove">×</button>`;r.querySelector(".remove").onclick=()=>r.remove();rows.appendChild(r)} | |
| addRow("Candidate A");addRow("Candidate B");document.getElementById("add").onclick=()=>addRow(); | |
| [["wq","wqv"],["wl","wlv"],["wt","wtv"],["wm","wmv"]].forEach(([a,b])=>{const el=document.getElementById(a),out=document.getElementById(b);el.oninput=()=>out.textContent=el.value+"%"}); | |
| document.getElementById("example").onclick=()=>{[...document.querySelectorAll(".data-row")].forEach(r=>r.remove());const examples=[["Example X",.72,.66,.83,31,720,1.9],["Example Y",.79,.73,.85,54,430,2.8],["Example Z",.75,.70,.82,24,880,1.5]];examples.forEach(e=>{addRow(e[0]);const r=[...document.querySelectorAll(".data-row")].slice(-1)[0];["ndcg","mrr","recall","latency","throughput","memory"].forEach((c,i)=>r.querySelector("."+c).value=e[i+1])})}; | |
| function safeNum(el){const v=Number(el.value);return Number.isFinite(v)?v:0}function normHigh(v,min,max){return max===min?1:(v-min)/(max-min)}function normLow(v,min,max){return max===min?1:1-(v-min)/(max-min)} | |
| document.getElementById("calculate").onclick=()=>{const data=[...document.querySelectorAll(".data-row")].map(r=>({name:r.querySelector(".name").value||"Unnamed candidate",ndcg:safeNum(r.querySelector(".ndcg")),mrr:safeNum(r.querySelector(".mrr")),recall:safeNum(r.querySelector(".recall")),latency:safeNum(r.querySelector(".latency")),throughput:safeNum(r.querySelector(".throughput")),memory:safeNum(r.querySelector(".memory"))})).filter(x=>x.ndcg||x.mrr||x.recall||x.latency||x.throughput||x.memory);const results=document.getElementById("results");if(data.length<2){results.innerHTML="<p style='color:#667085'>Enter measurements for at least two candidates.</p>";return}const quality=data.map(x=>(x.ndcg+x.mrr+x.recall)/3),lat=data.map(x=>x.latency),thr=data.map(x=>x.throughput),mem=data.map(x=>x.memory);const qmin=Math.min(...quality),qmax=Math.max(...quality),lmin=Math.min(...lat),lmax=Math.max(...lat),tmin=Math.min(...thr),tmax=Math.max(...thr),mmin=Math.min(...mem),mmax=Math.max(...mem);let wq=Number(document.getElementById("wq").value),wl=Number(document.getElementById("wl").value),wt=Number(document.getElementById("wt").value),wm=Number(document.getElementById("wm").value);const total=wq+wl+wt+wm||1;wq/=total;wl/=total;wt/=total;wm/=total;const scored=data.map((x,i)=>{const s=normHigh(quality[i],qmin,qmax)*wq+normLow(x.latency,lmin,lmax)*wl+normHigh(x.throughput,tmin,tmax)*wt+normLow(x.memory,mmin,mmax)*wm;return {...x,score:Math.round(s*1000)/10}}).sort((a,b)=>b.score-a.score);results.innerHTML="<h3 style='margin:0 0 10px'>Configurable comparison result</h3><p style='margin:0 0 14px;color:#667085;font-size:13px;line-height:1.55'>Scores are normalized only within the candidates entered here and depend on your chosen weights. They are not universal benchmark scores.</p>";scored.forEach(x=>{const row=document.createElement("div");row.className="result-row";row.innerHTML=`<strong>${x.name}</strong><span class="score">${x.score}</span><div class="bar"><span style="width:${x.score}%"></span></div>`;results.appendChild(row)})}; | |
| </script> | |
| </body> | |
| </html> |