sajjadi commited on
Commit
dbc70a9
·
verified ·
1 Parent(s): b95d311

Fix Studio diagnostics, Fréchet clamp, flip-count drift, synthetic demo labels, and canonical citation

Browse files
Files changed (1) hide show
  1. index.html +34 -25
index.html CHANGED
@@ -3,6 +3,13 @@
3
  <head>
4
  <meta charset="utf-8">
5
  <meta name="viewport" content="width=device-width, initial-scale=1">
 
 
 
 
 
 
 
6
  <title>PACE Studio · Parity-Aware Compute Evaluation</title>
7
  <style>
8
  :root{
@@ -404,7 +411,7 @@ footer a:hover{text-decoration:underline}
404
  </section>
405
 
406
  <div class="cli-strip">
407
- <div><strong>Run PACE on AutoGen / LangGraph / CrewAI logs:</strong> <code>git clone https://github.com/mahmoudsajjadi/pace &amp;&amp; pace import-logs --format autogen --input trace.json --output trace.jsonl &amp;&amp; pace card --input trace.jsonl</code></div>
408
  <div style="display:flex;align-items:center;gap:8px">
409
  <button id="copyCli" class="teal" type="button">Copy CLI</button>
410
  <span id="cliToast" class="toast"></span>
@@ -428,18 +435,18 @@ let currentRows=[];
428
  let currentContrasts=[];
429
  const $=id=>document.getElementById(id);
430
 
431
- const BIBTEX=`@article{sajjadi2026pace,
432
  title = {More Agents, More Samples, or More Thinking? Untangling Coordination, Ensembling, and Test-Time Compute in Multi-Agent LLMs},
433
  author = {Sajjadi Mohammadabadi, Seyed Mahmoud},
434
  year = {2026},
435
  eprint = {2611.17610},
436
  archivePrefix = {arXiv},
437
- primaryClass = {cs.CL},
438
- doi = {10.5281/zenodo.24286397},
439
  url = {https://arxiv.org/abs/2611.17610}
440
  }`;
441
 
442
- const CLI_CMD='git clone https://github.com/mahmoudsajjadi/pace && cd pace && PYTHONPATH=code python -m pace import-logs --format autogen --input trace.json --output trace.jsonl && PYTHONPATH=code python -m pace card --input trace.jsonl';
443
 
444
  function esc(s){
445
  return String(s??'').replace(/[&<>"']/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));
@@ -611,9 +618,9 @@ function markdown(c){
611
  }
612
 
613
  function latex(c){
614
- return '% Generated by PACE Studio (https://arxiv.org/abs/2611.17610) — \\cite{sajjadi2026pace}\n\\begin{tcolorbox}[title={PACE Evaluation Card}]\n\\textbf{Baseline:} '+c.baseline.replaceAll('_','\\_')+' ('+c.n_items+' items)\\par\n\\begin{tabular}{lr}\n\\toprule\nAxis & Value\\\\\n\\midrule\n'+
615
  Object.entries(c.resource_vector).map(([k,v])=>k.replaceAll('_','\\_')+' & '+Number(v).toFixed(3)+'\\\\').join('\n')+
616
- '\n\\bottomrule\n\\end{tabular}\n\\end{tcolorbox}\n';
617
  }
618
 
619
  function download(text,ext){
@@ -656,7 +663,7 @@ function poolSize(){
656
  '<div class="banner"><strong>Protocol Checklist:</strong> To claim a Category C gain over single-agent compute for K='+K+', T='+T+' ('+esc(top)+'), evaluate a single-agent self-consistency baseline with <strong>k = '+recommended+' samples</strong> on <strong>n = '+nTarget+' items</strong> (detectable difference &ge; &plusmn;'+mdeTarget.toFixed(1)+' pp at 80% power).</div>'+
657
  '<div class="kpi-grid">'+
658
  '<div class="kpi"><div class="k-label">MDE (80% power, n='+nTarget+')</div><div class="k-val">&plusmn;'+mdeTarget.toFixed(1)+' pp</div><div class="k-sub">at p<sub>0</sub>='+p0.toFixed(2)+', &phi;='+phi.toFixed(2)+'</div></div>'+
659
- '<div class="kpi"><div class="k-label">MDE Reference (n=500)</div><div class="k-val">&plusmn;'+(2.8*se500*100).toFixed(1)+' pp</div><div class="k-sub">CLI: <code>pace power --n '+nTarget+' --phi '+phi.toFixed(2)+'</code></div></div>'+
660
  '</div>';
661
 
662
  return 'k_tok = '+tok.toFixed(3)+'; recommended pool = '+recommended+'; r = '+r.toFixed(3);
@@ -706,9 +713,10 @@ function noise(rows,phi){
706
  if(Math.abs(d)>1.96*se)band96++;
707
  if(Math.abs(d)>se)band1++;
708
  hs.push(2*Math.asin(Math.sqrt(Math.max(0,m)))-2*Math.asin(Math.sqrt(Math.max(0,b))));
709
- const p11=Math.min(m,b),p10=m-p11,p01=b-p11,p00=1-m-b+p11;
710
- const logor=p10*p01?Math.log(Math.max(1e-300,p11*p00/(p10*p01))):Infinity;
711
- const clamped=pCorr>(isFinite(logor)?Math.exp(-Math.abs(logor)/2):0);
 
712
  if(clamped)clamp++;
713
  plotItems.push({study:r.study||'study',d,se});
714
  const status=Math.abs(d)<=1.96*se?'<span class="pill warn">Within Noise Band</span>':(d>0?'<span class="pill ok">Above +1.96 SE</span>':'<span class="pill bad">Below -1.96 SE</span>');
@@ -717,7 +725,9 @@ function noise(rows,phi){
717
  const meanH=hs.reduce((a,b)=>a+b,0)/hs.length;
718
  const summary='Noise band (&phi;='+pCorr.toFixed(2)+'): '+band96+'/'+valid.length+' outside +-1.96 SE; '+band1+'/'+valid.length+' outside +-1.0 SE; mean Cohen h = '+meanH.toFixed(3)+'; Frechet clamp count = '+clamp;
719
 
 
720
  $('corpusLab').innerHTML=
 
721
  '<div class="kpi-grid">'+
722
  '<div class="kpi"><div class="k-label">Outside &plusmn;1.96 SE</div><div class="k-val">'+band96+' / '+valid.length+'</div><div class="k-sub">Paired 95% noise band</div></div>'+
723
  '<div class="kpi"><div class="k-label">Outside &plusmn;1.0 SE</div><div class="k-val">'+band1+' / '+valid.length+'</div><div class="k-sub">At &phi; = '+pCorr.toFixed(2)+'</div></div>'+
@@ -755,12 +765,12 @@ function influence(rows){
755
  const ps=(ti+tc)?ti/(ti+tc):0;
756
  const gap=Math.abs((ps-p1)-((r*w*(fic-fci))/((fic*r+fci*w)*(r+w)||1)));
757
 
758
- const diagCls=rounds.length<=1?'warn':(ti>tc?'ok':'bad');
759
  const diagText=rounds.length<=1
760
  ?'Single-round trace: no multi-round peer influence transitions observed.'
761
- :(ti>tc
762
- ?'<strong>Net Positive Correction (&tau;<sub>ic</sub> &gt; &tau;<sub>ci</sub>):</strong> Agents flip from wrong to right more often than right to wrong, converging toward p* = '+(ps*100).toFixed(1)+'%.'
763
- :'<strong>Sycophancy / Drift Risk (&tau;<sub>ci</sub> &ge; &tau;<sub>ic</sub>):</strong> Correct agents are flipping to wrong answers at least as fast as wrong agents are corrected.');
764
 
765
  const perRound=rounds.map(rd=>{
766
  const sub=rows.filter(x=>(numOrNull(x.round)??0)===rd);
@@ -804,7 +814,6 @@ function render(rows){
804
  ['rungA','rungB','rungC','rungD'].forEach(id=>$(id).classList.remove('on'));
805
  if(card.baseline==='single_pass')$('rungA').classList.add('on');
806
  else if(card.baseline==='sampled')$('rungB').classList.add('on');
807
- else if(card.baseline==='single_context')$('rungC').classList.add('on');
808
 
809
  $('vector').innerHTML=Object.entries(card.resource_vector).map(([k,v])=>
810
  '<div class="axis"><span class="axis-name"><strong>'+esc(k)+'</strong><span class="axis-desc">'+(AXIS_DESC[k]||'')+'</span></span><strong>'+Number(v).toFixed(2)+'</strong></div>'
@@ -873,15 +882,15 @@ for(let item=0;item<8;item++){
873
  }
874
 
875
  const demoContrastRows=[
876
- {study:'Du et al. (2024)',benchmark:'GSM8K / MMLU',category:'B',p_mas:0.780,p_base:0.810,n:200},
877
- {study:'Liang et al. (2024)',benchmark:'Commonsense / Math',category:'B',p_mas:0.640,p_base:0.690,n:150},
878
- {study:'Wang et al. (2024 MoA)',benchmark:'AlpacaEval / Reasoning',category:'C',p_mas:0.715,p_base:0.730,n:180},
879
- {study:'Chen et al. (2024 ReConcile)',benchmark:'StrategyQA / GSM8K',category:'B',p_mas:0.742,p_base:0.768,n:220},
880
- {study:'Zhang et al. (2025)',benchmark:'MATH / BBH',category:'B',p_mas:0.590,p_base:0.645,n:300},
881
- {study:'Wunderlich et al. (2025)',benchmark:'MMLU-Pro',category:'A/B',p_mas:0.620,p_base:0.600,n:250},
882
- {study:'Smit et al. (2024)',benchmark:'MMLU',category:'B',p_mas:0.695,p_base:0.735,n:200},
883
- {study:'Estornell & Liu (2024)',benchmark:'MMLU / ARC',category:'B',p_mas:0.660,p_base:0.705,n:180},
884
- {study:'Cemri et al. (2025)',benchmark:'Multi-step Benchmarks',category:'C',p_mas:0.540,p_base:0.555,n:160}
885
  ];
886
 
887
  function loadDemo(kind,silent){
 
3
  <head>
4
  <meta charset="utf-8">
5
  <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <meta property="og:title" content="PACE Studio · Parity-Aware Compute Evaluation">
7
+ <meta property="og:description" content="Audit multi-agent LLM traces against compute-matched single-agent baselines, inspect debate transition dynamics, and size self-consistency pools (arXiv:2611.17610).">
8
+ <meta property="og:type" content="website">
9
+ <meta property="og:url" content="https://huggingface.co/spaces/sajjadi/pace-studio">
10
+ <meta name="twitter:card" content="summary">
11
+ <meta name="twitter:title" content="PACE Studio · Parity-Aware Compute Evaluation">
12
+ <meta name="twitter:description" content="Audit multi-agent LLM traces against compute-matched single-agent baselines (arXiv:2611.17610).">
13
  <title>PACE Studio · Parity-Aware Compute Evaluation</title>
14
  <style>
15
  :root{
 
411
  </section>
412
 
413
  <div class="cli-strip">
414
+ <div><strong>Run PACE on AutoGen / LangGraph / CrewAI logs:</strong> <code>git clone https://github.com/mahmoudsajjadi/pace &amp;&amp; pace import-logs --format autogen --input trace.json --output trace.jsonl &amp;&amp; pace card trace.jsonl</code></div>
415
  <div style="display:flex;align-items:center;gap:8px">
416
  <button id="copyCli" class="teal" type="button">Copy CLI</button>
417
  <span id="cliToast" class="toast"></span>
 
435
  let currentContrasts=[];
436
  const $=id=>document.getElementById(id);
437
 
438
+ const BIBTEX=`@article{sajjadi2026more,
439
  title = {More Agents, More Samples, or More Thinking? Untangling Coordination, Ensembling, and Test-Time Compute in Multi-Agent LLMs},
440
  author = {Sajjadi Mohammadabadi, Seyed Mahmoud},
441
  year = {2026},
442
  eprint = {2611.17610},
443
  archivePrefix = {arXiv},
444
+ primaryClass = {cs.AI},
445
+ doi = {10.48550/arXiv.2611.17610},
446
  url = {https://arxiv.org/abs/2611.17610}
447
  }`;
448
 
449
+ const CLI_CMD='git clone https://github.com/mahmoudsajjadi/pace && cd pace && PYTHONPATH=code python -m pace import-logs --format autogen --input trace.json --output trace.jsonl && PYTHONPATH=code python -m pace card trace.jsonl';
450
 
451
  function esc(s){
452
  return String(s??'').replace(/[&<>"']/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));
 
618
  }
619
 
620
  function latex(c){
621
+ return '% Generated by PACE Studio (https://arxiv.org/abs/2611.17610) — \\cite{sajjadi2026more}\n\\begin{table}[t]\n\\centering\n\\caption{PACE Evaluation Card (Baseline: '+c.baseline.replaceAll('_','\\_')+', '+c.n_items+' items).}\n\\begin{tabular}{lr}\n\\toprule\nAxis & Value\\\\\n\\midrule\n'+
622
  Object.entries(c.resource_vector).map(([k,v])=>k.replaceAll('_','\\_')+' & '+Number(v).toFixed(3)+'\\\\').join('\n')+
623
+ '\n\\bottomrule\n\\end{tabular}\n\\end{table}\n';
624
  }
625
 
626
  function download(text,ext){
 
663
  '<div class="banner"><strong>Protocol Checklist:</strong> To claim a Category C gain over single-agent compute for K='+K+', T='+T+' ('+esc(top)+'), evaluate a single-agent self-consistency baseline with <strong>k = '+recommended+' samples</strong> on <strong>n = '+nTarget+' items</strong> (detectable difference &ge; &plusmn;'+mdeTarget.toFixed(1)+' pp at 80% power).</div>'+
664
  '<div class="kpi-grid">'+
665
  '<div class="kpi"><div class="k-label">MDE (80% power, n='+nTarget+')</div><div class="k-val">&plusmn;'+mdeTarget.toFixed(1)+' pp</div><div class="k-sub">at p<sub>0</sub>='+p0.toFixed(2)+', &phi;='+phi.toFixed(2)+'</div></div>'+
666
+ '<div class="kpi"><div class="k-label">MDE Reference (n=500)</div><div class="k-val">&plusmn;'+(2.8*se500*100).toFixed(1)+' pp</div><div class="k-sub">CLI: <code>pace power --p-base '+p0.toFixed(2)+' --delta 0.05 --n-items '+nTarget+' --phi '+phi.toFixed(2)+'</code></div></div>'+
667
  '</div>';
668
 
669
  return 'k_tok = '+tok.toFixed(3)+'; recommended pool = '+recommended+'; r = '+r.toFixed(3);
 
713
  if(Math.abs(d)>1.96*se)band96++;
714
  if(Math.abs(d)>se)band1++;
715
  hs.push(2*Math.asin(Math.sqrt(Math.max(0,m)))-2*Math.asin(Math.sqrt(Math.max(0,b))));
716
+ const denom=b*(1-m);
717
+ const o=denom>0?(m*(1-b))/denom:Infinity;
718
+ const phiMax=(o>0&&isFinite(o))?Math.min(Math.sqrt(o),Math.sqrt(1/o)):0;
719
+ const clamped=pCorr>phiMax;
720
  if(clamped)clamp++;
721
  plotItems.push({study:r.study||'study',d,se});
722
  const status=Math.abs(d)<=1.96*se?'<span class="pill warn">Within Noise Band</span>':(d>0?'<span class="pill ok">Above +1.96 SE</span>':'<span class="pill bad">Below -1.96 SE</span>');
 
725
  const meanH=hs.reduce((a,b)=>a+b,0)/hs.length;
726
  const summary='Noise band (&phi;='+pCorr.toFixed(2)+'): '+band96+'/'+valid.length+' outside +-1.96 SE; '+band1+'/'+valid.length+' outside +-1.0 SE; mean Cohen h = '+meanH.toFixed(3)+'; Frechet clamp count = '+clamp;
727
 
728
+ const isSyntheticDemo=rows===demoContrastRows;
729
  $('corpusLab').innerHTML=
730
+ (isSyntheticDemo?'<div class="banner warn"><strong>Synthetic Illustrative Demo:</strong> Placeholder rows (Studies A&ndash;I) for UI demonstration only. Upload <code>baseline_tier_contrasts.csv</code> to audit the extracted 9-study corpus.</div>':'')+
731
  '<div class="kpi-grid">'+
732
  '<div class="kpi"><div class="k-label">Outside &plusmn;1.96 SE</div><div class="k-val">'+band96+' / '+valid.length+'</div><div class="k-sub">Paired 95% noise band</div></div>'+
733
  '<div class="kpi"><div class="k-label">Outside &plusmn;1.0 SE</div><div class="k-val">'+band1+' / '+valid.length+'</div><div class="k-sub">At &phi; = '+pCorr.toFixed(2)+'</div></div>'+
 
765
  const ps=(ti+tc)?ti/(ti+tc):0;
766
  const gap=Math.abs((ps-p1)-((r*w*(fic-fci))/((fic*r+fci*w)*(r+w)||1)));
767
 
768
+ const diagCls=rounds.length<=1?'warn':(fic>fci?'ok':'bad');
769
  const diagText=rounds.length<=1
770
  ?'Single-round trace: no multi-round peer influence transitions observed.'
771
+ :(fic>fci
772
+ ?'<strong>Net Positive Correction (f<sub>ic</sub> &gt; f<sub>ci</sub>):</strong> Wrong-to-right flips ('+fic+') exceed right-to-wrong flips ('+fci+'), moving toward p* = '+(ps*100).toFixed(1)+'%.'
773
+ :'<strong>Accuracy Fell or Stalled Across Rounds (f<sub>ci</sub> &ge; f<sub>ic</sub>):</strong> Right-to-wrong flips ('+fci+') equal or exceed wrong-to-right flips ('+fic+'); attribution requires a no-communication control.');
774
 
775
  const perRound=rounds.map(rd=>{
776
  const sub=rows.filter(x=>(numOrNull(x.round)??0)===rd);
 
814
  ['rungA','rungB','rungC','rungD'].forEach(id=>$(id).classList.remove('on'));
815
  if(card.baseline==='single_pass')$('rungA').classList.add('on');
816
  else if(card.baseline==='sampled')$('rungB').classList.add('on');
 
817
 
818
  $('vector').innerHTML=Object.entries(card.resource_vector).map(([k,v])=>
819
  '<div class="axis"><span class="axis-name"><strong>'+esc(k)+'</strong><span class="axis-desc">'+(AXIS_DESC[k]||'')+'</span></span><strong>'+Number(v).toFixed(2)+'</strong></div>'
 
882
  }
883
 
884
  const demoContrastRows=[
885
+ {study:'Synthetic Study A',benchmark:'Math / MMLU',category:'B',p_mas:0.780,p_base:0.810,n:200},
886
+ {study:'Synthetic Study B',benchmark:'Commonsense / Math',category:'B',p_mas:0.640,p_base:0.690,n:150},
887
+ {study:'Synthetic Study C',benchmark:'Reasoning',category:'C',p_mas:0.715,p_base:0.730,n:180},
888
+ {study:'Synthetic Study D',benchmark:'QA / Math',category:'B',p_mas:0.742,p_base:0.768,n:220},
889
+ {study:'Synthetic Study E',benchmark:'MATH / BBH',category:'B',p_mas:0.590,p_base:0.645,n:300},
890
+ {study:'Synthetic Study F',benchmark:'MMLU-Pro',category:'D',p_mas:0.620,p_base:0.600,n:250},
891
+ {study:'Synthetic Study G',benchmark:'MMLU',category:'B',p_mas:0.695,p_base:0.735,n:200},
892
+ {study:'Synthetic Study H',benchmark:'MMLU / ARC',category:'B',p_mas:0.660,p_base:0.705,n:180},
893
+ {study:'Synthetic Study I',benchmark:'Multi-step',category:'C',p_mas:0.540,p_base:0.555,n:160}
894
  ];
895
 
896
  function loadDemo(kind,silent){