sajjadi commited on
Commit
71770c2
·
verified ·
1 Parent(s): 5651c6c

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. README.md +14 -6
  2. index.html +54 -19
README.md CHANGED
@@ -1,12 +1,20 @@
1
  ---
2
- title: Pace Studio
3
- emoji: 🏢
4
- colorFrom: indigo
5
- colorTo: red
6
  sdk: static
7
  pinned: false
8
  license: apache-2.0
9
- short_description: Interactive PACE (Parity-Aware Compute Evaluation) Workbench
 
 
 
10
  ---
11
 
12
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: PACE Studio
 
 
 
3
  sdk: static
4
  pinned: false
5
  license: apache-2.0
6
+ tags:
7
+ - arxiv:2611.17610
8
+ - multi-agent
9
+ - evaluation
10
  ---
11
 
12
+ # PACE Studio (Parity-Aware Compute Evaluation)
13
+
14
+ Interactive browser workbench for **More Agents, More Samples, or More Thinking? Untangling Coordination, Ensembling, and Test-Time Compute in Multi-Agent LLMs**.
15
+
16
+ - **Paper (arXiv):** https://arxiv.org/abs/2611.17610
17
+ - **HTML Version:** https://arxiv.org/html/2611.17610v1
18
+ - **GitHub Repository:** https://github.com/mahmoudsajjadi/pace
19
+ - **Contrast Dataset:** https://huggingface.co/datasets/sajjadi/pace-contrasts
20
+ - **Hidden Profile Dataset:** https://huggingface.co/datasets/sajjadi/hidden-profile-control
index.html CHANGED
@@ -1,19 +1,54 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <meta charset="utf-8" />
5
- <meta name="viewport" content="width=device-width" />
6
- <title>My static Space</title>
7
- <link rel="stylesheet" href="style.css" />
8
- </head>
9
- <body>
10
- <div class="card">
11
- <h1>Welcome to your static Space!</h1>
12
- <p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
13
- <p>
14
- Also don't forget to check the
15
- <a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
16
- </p>
17
- </div>
18
- </body>
19
- </html>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <title>PACE Studio</title>
7
+ <style>
8
+ :root{--ink:#17212b;--muted:#607080;--paper:#f5f0e8;--panel:#fffdf8;--line:#d8d0c3;--accent:#bb4d2e;--accent2:#1d6b68}
9
+ *{box-sizing:border-box}body{margin:0;color:var(--ink);font:15px Georgia,serif;background:radial-gradient(circle at 85% 0,#e7cdb7 0,transparent 28%),var(--paper)}
10
+ header{padding:42px max(24px,calc((100vw - 1120px)/2));border-bottom:1px solid var(--line)}h1{font-size:clamp(2rem,5vw,4.5rem);line-height:.95;margin:0 0 12px;letter-spacing:0}header p{max-width:680px;color:var(--muted);font-size:1.05rem}
11
+ main{max-width:1120px;margin:28px auto;padding:0 24px}.drop{padding:28px;border:2px dashed var(--accent);background:var(--panel);text-align:center;cursor:pointer}.drop.drag{background:#f3dfd2}.drop input{display:none}.actions{display:flex;flex-wrap:wrap;gap:8px;margin:18px 0}.actions button{border:1px solid var(--ink);background:var(--ink);color:white;padding:10px 14px;cursor:pointer}.actions button.secondary{background:transparent;color:var(--ink)}
12
+ .grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(230px,1fr));gap:14px}.panel{background:var(--panel);border-top:4px solid var(--accent2);padding:18px;box-shadow:0 8px 24px #6d594018}.panel h2{font-size:1.1rem;margin:0 0 14px}.value{font-size:2rem;color:var(--accent)}.axis{display:flex;justify-content:space-between;padding:8px 0;border-bottom:1px solid var(--line)}table{width:100%;border-collapse:collapse;font-size:.9rem}th,td{text-align:left;padding:8px;border-bottom:1px solid var(--line)}.empty{color:var(--muted)}pre{white-space:pre-wrap;overflow:auto;background:#202b32;color:#f8eee3;padding:16px;min-height:80px}
13
+ @media(max-width:600px){header{padding:30px 20px}main{padding:0 16px}.actions button{flex:1 1 40%}}
14
+ .tabs{display:flex;flex-wrap:wrap;gap:6px;margin-bottom:16px}.tab{border:1px solid var(--line);background:var(--panel);padding:10px;cursor:pointer}.tab.active{background:var(--accent2);color:white}.tab-panel{display:none}.tab-panel.active{display:block}.demo-bar{border-bottom:1px solid var(--line);padding-bottom:12px}
15
+ </style>
16
+ </head>
17
+ <body>
18
+ <header><h1>PACE Studio</h1><p>Inspect a multi-agent trace locally. Your file stays in this browser tab.</p></header>
19
+ <main>
20
+ <nav class="tabs" aria-label="PACE Studio tabs"><button class="tab active" data-tab="trace">Trace &amp; Resource Auditor</button><button class="tab" data-tab="dynamics">Debate Dynamics</button><button class="tab" data-tab="corpus">Corpus &amp; Noise-Band Explorer</button><button class="tab" data-tab="planner">Experiment Planner</button></nav>
21
+ <div class="actions demo-bar"><button id="demoDebate">Load Demo: Debate JSONL</button><button id="demoHidden">Load Demo: Hidden-Profile Split vs Shared</button><button id="demoContrasts">Load Demo: 9-Study Contrasts CSV</button></div>
22
+ <label class="drop" id="drop">Drop a PACE JSONL log or contrast CSV here, or choose a file<input id="file" type="file" accept=".jsonl,.json,.csv"></label>
23
+ <div class="actions"><button id="md">Export Markdown</button><button id="tex">Export LaTeX</button><button id="json" class="secondary">Export JSON</button></div>
24
+ <section class="tab-panel active" data-panel="trace"><section class="grid"><article class="panel"><h2>Baseline</h2><div id="baseline" class="value">Waiting</div><p id="status" class="empty">Load a scored trace to calculate its resource vector.</p></article><article class="panel"><h2>Resource Vector R</h2><div id="vector" class="empty">No data</div></article><article class="panel"><h2>Directional feasibility</h2><div id="feasible" class="empty">No data</div></article></section></section>
25
+ <section class="panel" style="margin-top:14px"><h2>Round transitions</h2><div id="transitions" class="empty">No data</div></section>
26
+ <section class="tab-panel" data-panel="dynamics"><section class="panel" style="margin-top:14px"><h2>Debate Dynamics, Flow Identity &amp; Peer-Influence Lab</h2><div id="dynamicsLab" class="empty">Load a debate log.</div></section></section>
27
+ <section class="tab-panel" data-panel="corpus"><section class="panel" style="margin-top:14px"><h2>Corpus &amp; Noise-Band Explorer</h2><label>phi <input id="phiSlider" type="range" min="0" max=".9" step=".01" value=".5"></label><div id="corpusLab" class="empty">Load contrast CSV.</div></section></section>
28
+ <section class="tab-panel" data-panel="planner"><section class="panel" style="margin-top:14px"><h2>Prospective Experiment Planner</h2><div class="actions"><label>K <input id="pk" type="number" value="3" min="1"></label><label>T <input id="pt" type="number" value="2" min="1"></label><label>c0 <input id="pc" type="number" value="100" min="0"></label><label>l <input id="pl" type="number" value="20" min="0"></label><select id="topology"><option>last_round_peers</option><option>last_round_all</option><option>full_history_echo</option></select><button id="calcPool">Calculate k_tok</button></div><div id="theory" class="empty">Configure a budget.</div><div id="powerLab" class="empty">Power calculator available through pace power.</div></section></section>
29
+ <section class="panel" style="margin-top:14px"><h2>PACE Card preview</h2><pre id="preview">Load a file to render the card.</pre></section>
30
+ </main>
31
+ <script>
32
+ let card=null;
33
+ const $=id=>document.getElementById(id);
34
+ function csv(text){const lines=text.trim().split(/\r?\n/);const head=lines.shift().split(',');return lines.filter(Boolean).map(line=>{const cells=line.split(',');return Object.fromEntries(head.map((key,i)=>[key.trim(),cells[i]===undefined?'':cells[i].trim()]))})}
35
+ function bool(v){return v===true||v===1||v==='1'||String(v).toLowerCase()==='true'||String(v).toLowerCase()==='correct'}
36
+ function parseRows(text,name){if(name.toLowerCase().endsWith('.csv'))return csv(text);return text.trim().split(/\r?\n/).filter(Boolean).map(JSON.parse)}
37
+ function compute(rows){const items=new Set(rows.map(r=>String(r.item??r.item_id)));const agents=new Set(rows.map(r=>String(r.agent??r.agent_id)));const rounds=rows.map(r=>Number(r.round));const tokens=rows.filter(r=>r.prompt_tokens!==undefined&&r.completion_tokens!==undefined).map(r=>Number(r.prompt_tokens)+Number(r.completion_tokens));const R={r_token:tokens.reduce((a,b)=>a+b,0)/Math.max(1,items.size),r_call:rows.length/Math.max(1,items.size),r_round:Math.max(...rounds)-Math.min(...rounds)+1,r_sample:agents.size,r_context:Math.max(0,rows.length-items.size),r_scaff:rows.filter(r=>r.scaffold||r.tool_calls).length};let baseline=R.r_round<=1&&R.r_sample<=1&&R.r_context<=0?'single_pass':R.r_round<=1&&R.r_context<=0?'sampled':R.r_sample<=1?'single_context':'frontier';const by={};rows.forEach(r=>{const k=String(r.item??r.item_id)+'\u0000'+String(r.agent??r.agent_id);(by[k]??={})[Number(r.round)]=bool(r.correct)});const counts={};Object.keys(by).forEach(k=>{const rs=Object.keys(by[k]).map(Number).sort((a,b)=>a-b);for(let i=0;i<rs.length-1;i++){const own=by[k][rs[i]]?1:0;const next=by[k][rs[i+1]]?1:0;const peers=Object.keys(by).filter(x=>x.startsWith(k.split('\u0000')[0]+'\u0000')).map(x=>by[x][rs[i]]).filter(x=>x!==undefined).reduce((a,b)=>a+(b?1:0),0)-own;const key='own='+own+',peers_right='+peers;(counts[key]??={good:0,n:0});counts[key].good+=next;counts[key].n++}});const T=Object.fromEntries(Object.entries(counts).map(([k,v])=>[k,{p_next:v.good/v.n,n:v.n}]));return {schema:'pace-card/v1',n_records:rows.length,n_items:items.size,resource_vector:R,baseline,feasibility:{status:Object.values(R).some(v=>v>0)?'feasible':'insufficient_data',active_axes:Object.keys(R).filter(k=>R[k]>0),direction:'increasing resources'},transition_matrix:T}}
38
+ function markdown(c){return '# PACE Card\n\nBaseline: **'+c.baseline+'**\n\n| Axis | Value |\n|---|---:|\n'+Object.entries(c.resource_vector).map(([k,v])=>'| '+k+' | '+Number(v).toFixed(3)+' |').join('\n')+'\n\nTransition matrix\n\n| State | P(correct at t+1) | n |\n|---|---:|---:|\n'+Object.entries(c.transition_matrix).map(([k,v])=>'| '+k+' | '+v.p_next.toFixed(3)+' | '+v.n+' |').join('\n')+'\n'}
39
+ function latex(c){return '\\begin{tcolorbox}[title={PACE Card}]\n\\textbf{Baseline:} '+c.baseline+'\\par\n\\begin{tabular}{lr}\n\\toprule\nAxis & Value\\\\\n\\midrule\n'+Object.entries(c.resource_vector).map(([k,v])=>k.replaceAll('_','\\_')+' & '+Number(v).toFixed(3)+'\\\\').join('\n')+'\n\\bottomrule\n\\end{tabular}\n\\end{tcolorbox}\n'}
40
+ function download(text,ext){const a=document.createElement('a');a.href=URL.createObjectURL(new Blob([text],{type:'text/plain'}));a.download='pace-card.'+ext;a.click();URL.revokeObjectURL(a.href)}
41
+ function poolSize(){const K=Number($('pk').value),T=Number($('pt').value),c=Number($('pc').value),l=Number($('pl').value),r=l/(c+l||1),top=$('topology').value;const coefficient=top==='last_round_peers'?(T-1)*K*(K-1):top==='last_round_all'?(T-1)*K*K:.5*K*K*T*(T-1);const tok=K*T+coefficient*r;let recommended=8;while(recommended<Math.ceil(tok))recommended*=2;return 'k_tok = '+tok.toFixed(3)+'; recommended pool = '+recommended+'; r = '+r.toFixed(3)}
42
+ function noise(rows){const valid=rows.filter(r=>r.p_mas!==undefined&&r.p_base!==undefined);if(!valid.length)return '';let band96=0,band1=0,clamp=0;const hs=[];valid.forEach(r=>{const m=Number(r.p_mas),b=Number(r.p_base),n=Number(r.n||1),se=Math.sqrt(Math.max(0,(m*(1-m)+b*(1-b)-2*.5*Math.sqrt(Math.max(0,m*(1-m)*b*(1-b))))/n)),d=m-b;if(Math.abs(d)>1.96*se)band96++;if(Math.abs(d)>se)band1++;hs.push(2*Math.asin(Math.sqrt(Math.max(0,m)))-2*Math.asin(Math.sqrt(Math.max(0,b))));const p11=Math.min(m,b),p10=m-p11,p01=b-p11,p00=1-m-b+p11;const logor=p10*p01?Math.log(Math.max(1e-300,p11*p00/(p10*p01))):Infinity;if(.5>(isFinite(logor)?Math.exp(-Math.abs(logor)/2):0))clamp++});return 'Noise band: '+band96+'/'+valid.length+' outside +-1.96 SE; '+band1+'/'+valid.length+' outside +-1.0 SE; mean Cohen h = '+(hs.reduce((a,b)=>a+b,0)/hs.length).toFixed(3)+'; Frechet clamp count = '+clamp}
43
+ function influence(rows){if(!rows.length||rows[0].correct===undefined)return '';const first=Math.min(...rows.map(r=>Number(r.round))),last=Math.max(...rows.map(r=>Number(r.round)));const by={};rows.forEach(r=>{const k=r.item+'\u0000'+r.agent;(by[k]??={})[Number(r.round)]=!!r.correct});let w=0,r=0,fic=0,fci=0;Object.values(by).forEach(s=>{if(s[first]!==undefined){if(s[first])r++;else w++}if(s[first]!==undefined&&s[last]!==undefined){if(!s[first]&&s[last])fic++;if(s[first]&&!s[last])fci++}});const p1=r/(r+w),ti=w?fic/w:0,tc=r?fci/r:0,ps=ti+tc?ti/(ti+tc):0;return 'Flow identity: gap = '+Math.abs((ps-p1)-((r*w*(fic-fci))/((fic*r+fci*w)*(r+w)||1))).toExponential(2)+'; best-fitting peer-influence model is reported by the Python API.'}
44
+ function render(rows){if(!card)return;$('baseline').textContent=card.baseline;$('status').textContent=card.feasibility.status+'; active axes: '+card.feasibility.active_axes.join(', ');$('vector').innerHTML=Object.entries(card.resource_vector).map(([k,v])=>'<div class="axis"><span>'+k+'</span><strong>'+Number(v).toFixed(3)+'</strong></div>').join('');$('feasible').innerHTML='<p>'+card.feasibility.direction+'</p><p>'+card.feasibility.active_axes.length+' active axes</p>';$('transitions').innerHTML='<table><tr><th>Own state, correct peers</th><th>P(correct next)</th><th>n</th></tr>'+Object.entries(card.transition_matrix).map(([k,v])=>'<tr><td>'+k+'</td><td>'+v.p_next.toFixed(3)+'</td><td>'+v.n+'</td></tr>').join('')+'</table>';$('theory').textContent=[poolSize(),noise(rows),influence(rows)].filter(Boolean).join(' | ');$('preview').textContent=markdown(card)}
45
+ function showTab(name){document.querySelectorAll('.tab').forEach(button=>button.classList.toggle('active',button.dataset.tab===name));document.querySelectorAll('.tab-panel').forEach(panel=>panel.classList.toggle('active',panel.dataset.panel===name))}
46
+ const demoRows=Array.from({length:15},(_,i)=>({item:'demo_'+Math.floor(i/5),agent:'agent_'+(i%5),round:Math.floor(i/5)+1,correct:i%3!==0,answer:i%3!==0?'A':'B',prompt_tokens:100,completion_tokens:20}));
47
+ function loadDemo(kind){if(kind==='contrasts'){$('theory').textContent=noise(Array.from({length:9},(_,i)=>({p_mas:.55+i*.02,p_base:.5,n:20,study:'study'+i,benchmark:'demo',model:'model',category:'C'})));showTab('corpus');return}card=compute(demoRows);render(demoRows);showTab(kind==='hidden'?'trace':'dynamics')}
48
+ function load(file){const reader=new FileReader();reader.onload=()=>{try{const rows=parseRows(reader.result,file.name);if(file.name.toLowerCase().endsWith('.csv')){$('theory').textContent=[poolSize(),noise(rows)].filter(Boolean).join(' | ');$('preview').textContent='Contrast diagnostics loaded. Drop a PACE JSONL log to render the full card.';return}card=compute(rows);render(rows)}catch(e){$('preview').textContent='Could not parse file: '+e.message}};reader.readAsText(file)}
49
+ $('file').onchange=e=>e.target.files[0]&&load(e.target.files[0]);$('drop').ondragover=e=>{e.preventDefault();$('drop').classList.add('drag')};$('drop').ondragleave=()=> $('drop').classList.remove('drag');$('drop').ondrop=e=>{e.preventDefault();$('drop').classList.remove('drag');if(e.dataTransfer.files[0])load(e.dataTransfer.files[0])};$('md').onclick=()=>card&&download(markdown(card),'md');$('tex').onclick=()=>card&&download(latex(card),'tex');$('json').onclick=()=>card&&download(JSON.stringify(card,null,2),'json');
50
+ $('calcPool').onclick=()=>{$('theory').textContent=poolSize()};
51
+ document.querySelectorAll('.tab').forEach(button=>button.onclick=()=>showTab(button.dataset.tab));$('demoDebate').onclick=()=>loadDemo('debate');$('demoHidden').onclick=()=>loadDemo('hidden');$('demoContrasts').onclick=()=>loadDemo('contrasts');$('phiSlider').oninput=()=>{$('corpusLab').textContent='Assumed paired correlation phi = '+$('phiSlider').value};
52
+ </script>
53
+ </body>
54
+ </html>