Spaces:
Running
Running
Fix Studio diagnostics, Fréchet clamp, flip-count drift, synthetic demo labels, and canonical citation
Browse files- index.html +34 -25
index.html
CHANGED
|
@@ -3,6 +3,13 @@
|
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
<title>PACE Studio · Parity-Aware Compute Evaluation</title>
|
| 7 |
<style>
|
| 8 |
:root{
|
|
@@ -404,7 +411,7 @@ footer a:hover{text-decoration:underline}
|
|
| 404 |
</section>
|
| 405 |
|
| 406 |
<div class="cli-strip">
|
| 407 |
-
<div><strong>Run PACE on AutoGen / LangGraph / CrewAI logs:</strong> <code>git clone https://github.com/mahmoudsajjadi/pace && pace import-logs --format autogen --input trace.json --output trace.jsonl && pace card
|
| 408 |
<div style="display:flex;align-items:center;gap:8px">
|
| 409 |
<button id="copyCli" class="teal" type="button">Copy CLI</button>
|
| 410 |
<span id="cliToast" class="toast"></span>
|
|
@@ -428,18 +435,18 @@ let currentRows=[];
|
|
| 428 |
let currentContrasts=[];
|
| 429 |
const $=id=>document.getElementById(id);
|
| 430 |
|
| 431 |
-
const BIBTEX=`@article{
|
| 432 |
title = {More Agents, More Samples, or More Thinking? Untangling Coordination, Ensembling, and Test-Time Compute in Multi-Agent LLMs},
|
| 433 |
author = {Sajjadi Mohammadabadi, Seyed Mahmoud},
|
| 434 |
year = {2026},
|
| 435 |
eprint = {2611.17610},
|
| 436 |
archivePrefix = {arXiv},
|
| 437 |
-
primaryClass = {cs.
|
| 438 |
-
doi = {10.
|
| 439 |
url = {https://arxiv.org/abs/2611.17610}
|
| 440 |
}`;
|
| 441 |
|
| 442 |
-
const CLI_CMD='git clone https://github.com/mahmoudsajjadi/pace && cd pace && PYTHONPATH=code python -m pace import-logs --format autogen --input trace.json --output trace.jsonl && PYTHONPATH=code python -m pace card
|
| 443 |
|
| 444 |
function esc(s){
|
| 445 |
return String(s??'').replace(/[&<>"']/g,c=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]));
|
|
@@ -611,9 +618,9 @@ function markdown(c){
|
|
| 611 |
}
|
| 612 |
|
| 613 |
function latex(c){
|
| 614 |
-
return '% Generated by PACE Studio (https://arxiv.org/abs/2611.17610) — \\cite{
|
| 615 |
Object.entries(c.resource_vector).map(([k,v])=>k.replaceAll('_','\\_')+' & '+Number(v).toFixed(3)+'\\\\').join('\n')+
|
| 616 |
-
'\n\\bottomrule\n\\end{tabular}\n\\end{
|
| 617 |
}
|
| 618 |
|
| 619 |
function download(text,ext){
|
|
@@ -656,7 +663,7 @@ function poolSize(){
|
|
| 656 |
'<div class="banner"><strong>Protocol Checklist:</strong> To claim a Category C gain over single-agent compute for K='+K+', T='+T+' ('+esc(top)+'), evaluate a single-agent self-consistency baseline with <strong>k = '+recommended+' samples</strong> on <strong>n = '+nTarget+' items</strong> (detectable difference ≥ ±'+mdeTarget.toFixed(1)+' pp at 80% power).</div>'+
|
| 657 |
'<div class="kpi-grid">'+
|
| 658 |
'<div class="kpi"><div class="k-label">MDE (80% power, n='+nTarget+')</div><div class="k-val">±'+mdeTarget.toFixed(1)+' pp</div><div class="k-sub">at p<sub>0</sub>='+p0.toFixed(2)+', φ='+phi.toFixed(2)+'</div></div>'+
|
| 659 |
-
'<div class="kpi"><div class="k-label">MDE Reference (n=500)</div><div class="k-val">±'+(2.8*se500*100).toFixed(1)+' pp</div><div class="k-sub">CLI: <code>pace power --n '+nTarget+' --phi '+phi.toFixed(2)+'</code></div></div>'+
|
| 660 |
'</div>';
|
| 661 |
|
| 662 |
return 'k_tok = '+tok.toFixed(3)+'; recommended pool = '+recommended+'; r = '+r.toFixed(3);
|
|
@@ -706,9 +713,10 @@ function noise(rows,phi){
|
|
| 706 |
if(Math.abs(d)>1.96*se)band96++;
|
| 707 |
if(Math.abs(d)>se)band1++;
|
| 708 |
hs.push(2*Math.asin(Math.sqrt(Math.max(0,m)))-2*Math.asin(Math.sqrt(Math.max(0,b))));
|
| 709 |
-
const
|
| 710 |
-
const
|
| 711 |
-
const
|
|
|
|
| 712 |
if(clamped)clamp++;
|
| 713 |
plotItems.push({study:r.study||'study',d,se});
|
| 714 |
const status=Math.abs(d)<=1.96*se?'<span class="pill warn">Within Noise Band</span>':(d>0?'<span class="pill ok">Above +1.96 SE</span>':'<span class="pill bad">Below -1.96 SE</span>');
|
|
@@ -717,7 +725,9 @@ function noise(rows,phi){
|
|
| 717 |
const meanH=hs.reduce((a,b)=>a+b,0)/hs.length;
|
| 718 |
const summary='Noise band (φ='+pCorr.toFixed(2)+'): '+band96+'/'+valid.length+' outside +-1.96 SE; '+band1+'/'+valid.length+' outside +-1.0 SE; mean Cohen h = '+meanH.toFixed(3)+'; Frechet clamp count = '+clamp;
|
| 719 |
|
|
|
|
| 720 |
$('corpusLab').innerHTML=
|
|
|
|
| 721 |
'<div class="kpi-grid">'+
|
| 722 |
'<div class="kpi"><div class="k-label">Outside ±1.96 SE</div><div class="k-val">'+band96+' / '+valid.length+'</div><div class="k-sub">Paired 95% noise band</div></div>'+
|
| 723 |
'<div class="kpi"><div class="k-label">Outside ±1.0 SE</div><div class="k-val">'+band1+' / '+valid.length+'</div><div class="k-sub">At φ = '+pCorr.toFixed(2)+'</div></div>'+
|
|
@@ -755,12 +765,12 @@ function influence(rows){
|
|
| 755 |
const ps=(ti+tc)?ti/(ti+tc):0;
|
| 756 |
const gap=Math.abs((ps-p1)-((r*w*(fic-fci))/((fic*r+fci*w)*(r+w)||1)));
|
| 757 |
|
| 758 |
-
const diagCls=rounds.length<=1?'warn':(
|
| 759 |
const diagText=rounds.length<=1
|
| 760 |
?'Single-round trace: no multi-round peer influence transitions observed.'
|
| 761 |
-
:(
|
| 762 |
-
?'<strong>Net Positive Correction (
|
| 763 |
-
:'<strong>
|
| 764 |
|
| 765 |
const perRound=rounds.map(rd=>{
|
| 766 |
const sub=rows.filter(x=>(numOrNull(x.round)??0)===rd);
|
|
@@ -804,7 +814,6 @@ function render(rows){
|
|
| 804 |
['rungA','rungB','rungC','rungD'].forEach(id=>$(id).classList.remove('on'));
|
| 805 |
if(card.baseline==='single_pass')$('rungA').classList.add('on');
|
| 806 |
else if(card.baseline==='sampled')$('rungB').classList.add('on');
|
| 807 |
-
else if(card.baseline==='single_context')$('rungC').classList.add('on');
|
| 808 |
|
| 809 |
$('vector').innerHTML=Object.entries(card.resource_vector).map(([k,v])=>
|
| 810 |
'<div class="axis"><span class="axis-name"><strong>'+esc(k)+'</strong><span class="axis-desc">'+(AXIS_DESC[k]||'')+'</span></span><strong>'+Number(v).toFixed(2)+'</strong></div>'
|
|
@@ -873,15 +882,15 @@ for(let item=0;item<8;item++){
|
|
| 873 |
}
|
| 874 |
|
| 875 |
const demoContrastRows=[
|
| 876 |
-
{study:'
|
| 877 |
-
{study:'
|
| 878 |
-
{study:'
|
| 879 |
-
{study:'
|
| 880 |
-
{study:'
|
| 881 |
-
{study:'
|
| 882 |
-
{study:'
|
| 883 |
-
{study:'
|
| 884 |
-
{study:'
|
| 885 |
];
|
| 886 |
|
| 887 |
function loadDemo(kind,silent){
|
|
|
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 6 |
+
<meta property="og:title" content="PACE Studio · Parity-Aware Compute Evaluation">
|
| 7 |
+
<meta property="og:description" content="Audit multi-agent LLM traces against compute-matched single-agent baselines, inspect debate transition dynamics, and size self-consistency pools (arXiv:2611.17610).">
|
| 8 |
+
<meta property="og:type" content="website">
|
| 9 |
+
<meta property="og:url" content="https://huggingface.co/spaces/sajjadi/pace-studio">
|
| 10 |
+
<meta name="twitter:card" content="summary">
|
| 11 |
+
<meta name="twitter:title" content="PACE Studio · Parity-Aware Compute Evaluation">
|
| 12 |
+
<meta name="twitter:description" content="Audit multi-agent LLM traces against compute-matched single-agent baselines (arXiv:2611.17610).">
|
| 13 |
<title>PACE Studio · Parity-Aware Compute Evaluation</title>
|
| 14 |
<style>
|
| 15 |
:root{
|
|
|
|
| 411 |
</section>
|
| 412 |
|
| 413 |
<div class="cli-strip">
|
| 414 |
+
<div><strong>Run PACE on AutoGen / LangGraph / CrewAI logs:</strong> <code>git clone https://github.com/mahmoudsajjadi/pace && pace import-logs --format autogen --input trace.json --output trace.jsonl && pace card trace.jsonl</code></div>
|
| 415 |
<div style="display:flex;align-items:center;gap:8px">
|
| 416 |
<button id="copyCli" class="teal" type="button">Copy CLI</button>
|
| 417 |
<span id="cliToast" class="toast"></span>
|
|
|
|
| 435 |
let currentContrasts=[];
|
| 436 |
const $=id=>document.getElementById(id);
|
| 437 |
|
| 438 |
+
const BIBTEX=`@article{sajjadi2026more,
|
| 439 |
title = {More Agents, More Samples, or More Thinking? Untangling Coordination, Ensembling, and Test-Time Compute in Multi-Agent LLMs},
|
| 440 |
author = {Sajjadi Mohammadabadi, Seyed Mahmoud},
|
| 441 |
year = {2026},
|
| 442 |
eprint = {2611.17610},
|
| 443 |
archivePrefix = {arXiv},
|
| 444 |
+
primaryClass = {cs.AI},
|
| 445 |
+
doi = {10.48550/arXiv.2611.17610},
|
| 446 |
url = {https://arxiv.org/abs/2611.17610}
|
| 447 |
}`;
|
| 448 |
|
| 449 |
+
const CLI_CMD='git clone https://github.com/mahmoudsajjadi/pace && cd pace && PYTHONPATH=code python -m pace import-logs --format autogen --input trace.json --output trace.jsonl && PYTHONPATH=code python -m pace card trace.jsonl';
|
| 450 |
|
| 451 |
function esc(s){
|
| 452 |
return String(s??'').replace(/[&<>"']/g,c=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]));
|
|
|
|
| 618 |
}
|
| 619 |
|
| 620 |
function latex(c){
|
| 621 |
+
return '% Generated by PACE Studio (https://arxiv.org/abs/2611.17610) — \\cite{sajjadi2026more}\n\\begin{table}[t]\n\\centering\n\\caption{PACE Evaluation Card (Baseline: '+c.baseline.replaceAll('_','\\_')+', '+c.n_items+' items).}\n\\begin{tabular}{lr}\n\\toprule\nAxis & Value\\\\\n\\midrule\n'+
|
| 622 |
Object.entries(c.resource_vector).map(([k,v])=>k.replaceAll('_','\\_')+' & '+Number(v).toFixed(3)+'\\\\').join('\n')+
|
| 623 |
+
'\n\\bottomrule\n\\end{tabular}\n\\end{table}\n';
|
| 624 |
}
|
| 625 |
|
| 626 |
function download(text,ext){
|
|
|
|
| 663 |
'<div class="banner"><strong>Protocol Checklist:</strong> To claim a Category C gain over single-agent compute for K='+K+', T='+T+' ('+esc(top)+'), evaluate a single-agent self-consistency baseline with <strong>k = '+recommended+' samples</strong> on <strong>n = '+nTarget+' items</strong> (detectable difference ≥ ±'+mdeTarget.toFixed(1)+' pp at 80% power).</div>'+
|
| 664 |
'<div class="kpi-grid">'+
|
| 665 |
'<div class="kpi"><div class="k-label">MDE (80% power, n='+nTarget+')</div><div class="k-val">±'+mdeTarget.toFixed(1)+' pp</div><div class="k-sub">at p<sub>0</sub>='+p0.toFixed(2)+', φ='+phi.toFixed(2)+'</div></div>'+
|
| 666 |
+
'<div class="kpi"><div class="k-label">MDE Reference (n=500)</div><div class="k-val">±'+(2.8*se500*100).toFixed(1)+' pp</div><div class="k-sub">CLI: <code>pace power --p-base '+p0.toFixed(2)+' --delta 0.05 --n-items '+nTarget+' --phi '+phi.toFixed(2)+'</code></div></div>'+
|
| 667 |
'</div>';
|
| 668 |
|
| 669 |
return 'k_tok = '+tok.toFixed(3)+'; recommended pool = '+recommended+'; r = '+r.toFixed(3);
|
|
|
|
| 713 |
if(Math.abs(d)>1.96*se)band96++;
|
| 714 |
if(Math.abs(d)>se)band1++;
|
| 715 |
hs.push(2*Math.asin(Math.sqrt(Math.max(0,m)))-2*Math.asin(Math.sqrt(Math.max(0,b))));
|
| 716 |
+
const denom=b*(1-m);
|
| 717 |
+
const o=denom>0?(m*(1-b))/denom:Infinity;
|
| 718 |
+
const phiMax=(o>0&&isFinite(o))?Math.min(Math.sqrt(o),Math.sqrt(1/o)):0;
|
| 719 |
+
const clamped=pCorr>phiMax;
|
| 720 |
if(clamped)clamp++;
|
| 721 |
plotItems.push({study:r.study||'study',d,se});
|
| 722 |
const status=Math.abs(d)<=1.96*se?'<span class="pill warn">Within Noise Band</span>':(d>0?'<span class="pill ok">Above +1.96 SE</span>':'<span class="pill bad">Below -1.96 SE</span>');
|
|
|
|
| 725 |
const meanH=hs.reduce((a,b)=>a+b,0)/hs.length;
|
| 726 |
const summary='Noise band (φ='+pCorr.toFixed(2)+'): '+band96+'/'+valid.length+' outside +-1.96 SE; '+band1+'/'+valid.length+' outside +-1.0 SE; mean Cohen h = '+meanH.toFixed(3)+'; Frechet clamp count = '+clamp;
|
| 727 |
|
| 728 |
+
const isSyntheticDemo=rows===demoContrastRows;
|
| 729 |
$('corpusLab').innerHTML=
|
| 730 |
+
(isSyntheticDemo?'<div class="banner warn"><strong>Synthetic Illustrative Demo:</strong> Placeholder rows (Studies A–I) for UI demonstration only. Upload <code>baseline_tier_contrasts.csv</code> to audit the extracted 9-study corpus.</div>':'')+
|
| 731 |
'<div class="kpi-grid">'+
|
| 732 |
'<div class="kpi"><div class="k-label">Outside ±1.96 SE</div><div class="k-val">'+band96+' / '+valid.length+'</div><div class="k-sub">Paired 95% noise band</div></div>'+
|
| 733 |
'<div class="kpi"><div class="k-label">Outside ±1.0 SE</div><div class="k-val">'+band1+' / '+valid.length+'</div><div class="k-sub">At φ = '+pCorr.toFixed(2)+'</div></div>'+
|
|
|
|
| 765 |
const ps=(ti+tc)?ti/(ti+tc):0;
|
| 766 |
const gap=Math.abs((ps-p1)-((r*w*(fic-fci))/((fic*r+fci*w)*(r+w)||1)));
|
| 767 |
|
| 768 |
+
const diagCls=rounds.length<=1?'warn':(fic>fci?'ok':'bad');
|
| 769 |
const diagText=rounds.length<=1
|
| 770 |
?'Single-round trace: no multi-round peer influence transitions observed.'
|
| 771 |
+
:(fic>fci
|
| 772 |
+
?'<strong>Net Positive Correction (f<sub>ic</sub> > f<sub>ci</sub>):</strong> Wrong-to-right flips ('+fic+') exceed right-to-wrong flips ('+fci+'), moving toward p* = '+(ps*100).toFixed(1)+'%.'
|
| 773 |
+
:'<strong>Accuracy Fell or Stalled Across Rounds (f<sub>ci</sub> ≥ f<sub>ic</sub>):</strong> Right-to-wrong flips ('+fci+') equal or exceed wrong-to-right flips ('+fic+'); attribution requires a no-communication control.');
|
| 774 |
|
| 775 |
const perRound=rounds.map(rd=>{
|
| 776 |
const sub=rows.filter(x=>(numOrNull(x.round)??0)===rd);
|
|
|
|
| 814 |
['rungA','rungB','rungC','rungD'].forEach(id=>$(id).classList.remove('on'));
|
| 815 |
if(card.baseline==='single_pass')$('rungA').classList.add('on');
|
| 816 |
else if(card.baseline==='sampled')$('rungB').classList.add('on');
|
|
|
|
| 817 |
|
| 818 |
$('vector').innerHTML=Object.entries(card.resource_vector).map(([k,v])=>
|
| 819 |
'<div class="axis"><span class="axis-name"><strong>'+esc(k)+'</strong><span class="axis-desc">'+(AXIS_DESC[k]||'')+'</span></span><strong>'+Number(v).toFixed(2)+'</strong></div>'
|
|
|
|
| 882 |
}
|
| 883 |
|
| 884 |
const demoContrastRows=[
|
| 885 |
+
{study:'Synthetic Study A',benchmark:'Math / MMLU',category:'B',p_mas:0.780,p_base:0.810,n:200},
|
| 886 |
+
{study:'Synthetic Study B',benchmark:'Commonsense / Math',category:'B',p_mas:0.640,p_base:0.690,n:150},
|
| 887 |
+
{study:'Synthetic Study C',benchmark:'Reasoning',category:'C',p_mas:0.715,p_base:0.730,n:180},
|
| 888 |
+
{study:'Synthetic Study D',benchmark:'QA / Math',category:'B',p_mas:0.742,p_base:0.768,n:220},
|
| 889 |
+
{study:'Synthetic Study E',benchmark:'MATH / BBH',category:'B',p_mas:0.590,p_base:0.645,n:300},
|
| 890 |
+
{study:'Synthetic Study F',benchmark:'MMLU-Pro',category:'D',p_mas:0.620,p_base:0.600,n:250},
|
| 891 |
+
{study:'Synthetic Study G',benchmark:'MMLU',category:'B',p_mas:0.695,p_base:0.735,n:200},
|
| 892 |
+
{study:'Synthetic Study H',benchmark:'MMLU / ARC',category:'B',p_mas:0.660,p_base:0.705,n:180},
|
| 893 |
+
{study:'Synthetic Study I',benchmark:'Multi-step',category:'C',p_mas:0.540,p_base:0.555,n:160}
|
| 894 |
];
|
| 895 |
|
| 896 |
function loadDemo(kind,silent){
|