Spaces:
Running
Running
Draw the charts in the paper figures' kit style
Browse filesTakeaway: the requirement-family dot plot becomes the kit's grouped bars (Spatial Composition in the agent's company colour, the other families grey, logos and rotated names), with hover details; the mismatch chart and the judge's checks take the kit's neutrals; the evidence scene gets a company-colour chip. Leaderboard: 'Text-to-Scene' qualifier and the Qwen glyph. Pareto: y axis is the Text-to-Scene score; effort in title case.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
- assets/buildingbench.css +19 -0
- assets/chart-interactions.js +4 -0
- assets/leaderboard-bars.js +2 -2
- assets/pareto.js +3 -3
- assets/takeaway-bars.js +65 -0
- index.html +4 -4
- scripts/kit_charts.py +292 -0
assets/buildingbench.css
CHANGED
|
@@ -41,3 +41,22 @@ section,section.alt,section.dark{padding:34px 0;background:transparent;color:var
|
|
| 41 |
.featured.featured-video{height:auto;align-self:center;scroll-margin-top:88px}.featured-video video{display:block;width:100%;height:auto;aspect-ratio:16/9;object-fit:contain;background:#171816}.featured-video-caption{padding:12px 16px;display:flex;flex-wrap:wrap;align-items:baseline;justify-content:space-between;gap:5px 12px}.featured-video-caption strong{font-size:13px;color:var(--ink)}.featured-video-caption span{font-size:11px;color:var(--muted)}
|
| 42 |
|
| 43 |
.scene-texture-status{position:absolute;top:10px;right:10px;z-index:2;background:#fffefbed;color:#53655b;border-radius:5px;padding:5px 8px;font:10px/1.4 Inter,sans-serif;pointer-events:none}.scene-texture-status button{display:block;pointer-events:auto;margin-top:4px;border:1px solid #b4c1b9;border-radius:4px;background:white;padding:4px 6px;font:inherit;color:#35687e;cursor:pointer}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
.featured.featured-video{height:auto;align-self:center;scroll-margin-top:88px}.featured-video video{display:block;width:100%;height:auto;aspect-ratio:16/9;object-fit:contain;background:#171816}.featured-video-caption{padding:12px 16px;display:flex;flex-wrap:wrap;align-items:baseline;justify-content:space-between;gap:5px 12px}.featured-video-caption strong{font-size:13px;color:var(--ink)}.featured-video-caption span{font-size:11px;color:var(--muted)}
|
| 42 |
|
| 43 |
.scene-texture-status{position:absolute;top:10px;right:10px;z-index:2;background:#fffefbed;color:#53655b;border-radius:5px;padding:5px 8px;font:10px/1.4 Inter,sans-serif;pointer-events:none}.scene-texture-status button{display:block;pointer-events:auto;margin-top:4px;border:1px solid #b4c1b9;border-radius:4px;background:white;padding:4px 6px;font:inherit;color:#35687e;cursor:pointer}
|
| 44 |
+
|
| 45 |
+
/* kit-style charts (scripts/kit_charts.py) */
|
| 46 |
+
.bar-heading h3 span.q{font:inherit!important;color:#8d8d8d!important;max-width:none!important;text-align:left!important}
|
| 47 |
+
.tk figure{background:#fff;min-width:0}
|
| 48 |
+
.tk .figs{grid-template-columns:minmax(0,1fr)}.tk .figs.two{grid-template-columns:minmax(0,1.15fr) minmax(0,1fr)}
|
| 49 |
+
@media(max-width:900px){.tk .figs.two{grid-template-columns:minmax(0,1fr)}}
|
| 50 |
+
.tk figure .fh{font:400 22px/1.25 'Source Serif 4',Georgia,serif;color:#8d8d8d;margin-bottom:12px;letter-spacing:-.3px}
|
| 51 |
+
.kit-legend{display:flex;flex-wrap:wrap;gap:6px 26px;margin:0 0 6px;font:400 13px/1.5 Inter,sans-serif;color:#262626}
|
| 52 |
+
.kit-legend>span{display:inline-flex;align-items:center;gap:7px}.kit-legend b{font-weight:700}
|
| 53 |
+
.kit-legend i{width:12px;height:12px;border-radius:3px;display:inline-block;flex:none}
|
| 54 |
+
.kit-legend em{font:500 12.5px 'IBM Plex Mono',monospace;color:#8d8d8d;font-style:normal;margin-left:2px}
|
| 55 |
+
svg.kit-ch{display:block;width:100%;height:auto;overflow:visible}
|
| 56 |
+
.tk .chip{font:500 12.5px Inter,sans-serif;padding:5px 12px;border-radius:999px}
|
| 57 |
+
.tk .chip.ok{color:#262626;background:#f1f1ee;border-color:#e4e4e2}
|
| 58 |
+
.tk .chip.no{color:#fff;background:#111;border-color:#111}
|
| 59 |
+
.tk .chip small{font:500 12px 'IBM Plex Mono',monospace;color:#8d8d8d}.tk .chip.no small{color:#bdbdbd}
|
| 60 |
+
.agent-chip{position:absolute;z-index:4;top:12px;left:12px;display:inline-flex;align-items:center;gap:6px;padding:5px 10px;border-radius:6px;color:#fff;font:700 12.5px Inter,sans-serif;pointer-events:none}
|
| 61 |
+
.agent-chip svg{width:14px;height:14px}
|
| 62 |
+
@media(max-width:650px){.kit-legend{font-size:11px;gap:4px 14px}.kit-legend em{font-size:10.5px}.tk figure .fh{font-size:18px}.agent-chip{font-size:11px;top:8px;left:8px}}
|
assets/chart-interactions.js
CHANGED
|
@@ -15,6 +15,10 @@ function show(node,plot,record,anchorSelector){
|
|
| 15 |
}
|
| 16 |
export async function bindChartTooltip(node,plot,key,mode,anchorSelector='.bar,.chip'){
|
| 17 |
const data=await detailPromise,record=data[mode].find(d=>d.key===key||d.name===key);if(!record)return;
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
node.classList.add('chart-node');node.setAttribute('tabindex','0');node.setAttribute('role','button');node.setAttribute('aria-label',record.name+'; '+record.metrics.map(m=>m.label+' '+(m.currency?'$':'')+m.value).join('; '));
|
| 19 |
node.addEventListener('pointerenter',()=>{pinned=false;show(node,plot,record,anchorSelector);});
|
| 20 |
node.addEventListener('pointerleave',()=>{if(!pinned&&active?.node===node)hide();});
|
|
|
|
| 15 |
}
|
| 16 |
export async function bindChartTooltip(node,plot,key,mode,anchorSelector='.bar,.chip'){
|
| 17 |
const data=await detailPromise,record=data[mode].find(d=>d.key===key||d.name===key);if(!record)return;
|
| 18 |
+
bindRecordTooltip(node,plot,record,anchorSelector);
|
| 19 |
+
}
|
| 20 |
+
// the same behaviour for a chart whose record is not a leaderboard row (the takeaway's family scores)
|
| 21 |
+
export function bindRecordTooltip(node,plot,record,anchorSelector='.bar,.chip'){
|
| 22 |
node.classList.add('chart-node');node.setAttribute('tabindex','0');node.setAttribute('role','button');node.setAttribute('aria-label',record.name+'; '+record.metrics.map(m=>m.label+' '+(m.currency?'$':'')+m.value).join('; '));
|
| 23 |
node.addEventListener('pointerenter',()=>{pinned=false;show(node,plot,record,anchorSelector);});
|
| 24 |
node.addEventListener('pointerleave',()=>{if(!pinned&&active?.node===node)hide();});
|
assets/leaderboard-bars.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
| 1 |
// Bar layout and name wrapping adapted from BuildingBench figure-kit/bar-static/build.py.
|
| 2 |
// Data is exclusively the existing Code4Scene data/site.js paper board; no reference benchmark scores.
|
| 3 |
(async function(){
|
| 4 |
-
const {bindChartTooltip}=await import("./chart-interactions.js?v=
|
| 5 |
await Promise.all([document.fonts.load('400 13px Inter'),document.fonts.load('700 13px Inter'),document.fonts.load('500 15.5px "IBM Plex Mono"')]);
|
| 6 |
const ALL = {"t2s": [{"name": "Claude Fable 5.1 (max)", "org": "Anthropic", "score": 0.78775, "colour": "#D97757", "ink": "#ffffff", "label": "79", "new": false}, {"name": "GPT-6 Astra (max)", "org": "OpenAI", "score": 0.72382, "colour": "#10A37F", "ink": "#ffffff", "label": "72", "new": false}, {"name": "Claude Opus 5 (max)", "org": "Anthropic", "score": 0.7181599999999999, "colour": "#D97757", "ink": "#ffffff", "label": "72", "new": false}, {"name": "GPT-5.6 Sol (high)", "org": "OpenAI", "score": 0.70704, "colour": "#10A37F", "ink": "#ffffff", "label": "71", "new": false}, {"name": "Gemini 3.8 Flash (high)", "org": "Google", "score": 0.65689, "colour": "#7b1fa2", "ink": "#ffffff", "label": "66", "new": false}, {"name": "Muse Spark 1.3 (medium)", "org": "Meta", "score": 0.646195, "colour": "#42a5f5", "ink": "#111111", "label": "65", "new": false}, {"name": "Grok 4.6 (high)", "org": "xAI", "score": 0.56721, "colour": "#111111", "ink": "#ffffff", "label": "57", "new": false}, {"name": "Qwen 3.8 27B (thinking off)", "org": "Alibaba", "score": 0.55666, "colour": "#fe7016", "ink": "#111111", "label": "56", "new": false}, {"name": "Qwen 3.8 27B (thinking on)", "org": "Alibaba", "score": 0.51571, "colour": "#fe7016", "ink": "#111111", "label": "52", "new": false}, {"name": "GLM-5.3 Flash (max)", "org": "Z.ai", "score": 0.5094299999999999, "colour": "#96650b", "ink": "#ffffff", "label": "51", "new": false}, {"name": "Gemma 4 31B (thinking on)", "org": "Google", "score": 0.470705, "colour": "#7b1fa2", "ink": "#ffffff", "label": "47", "new": false}, {"name": "Gemma 4 31B (thinking off)", "org": "Google", "score": 0.45494, "colour": "#7b1fa2", "ink": "#ffffff", "label": "45", "new": false}, {"name": "Inkling (high)", "org": "Thinking Machines", "score": 0.42498500000000006, "colour": "#686868", "ink": "#ffffff", "label": "42", "new": false}, {"name": "DeepSeek V4.1 Flash (high)", "org": "DeepSeek", "score": 0.24269, "colour": "#3f51e0", "ink": "#ffffff", "label": "24", "new": false}]};
|
| 7 |
-
const LOGOS = {"Anthropic": {"icon": "anthropic", "fill": "#141413", "viewBox": "0 0 24 24", "inner": "<path d=\"M13.827 3.52h3.603L24 20h-3.603l-6.57-16.48zm-7.258 0h3.767L16.906 20h-3.674l-1.343-3.461H5.017l-1.344 3.46H0L6.57 3.522zm4.132 9.959L8.453 7.687 6.205 13.48H10.7z\"></path>", "rule": "evenodd"}, "OpenAI": {"icon": "openai", "fill": "#141413", "viewBox": "0 0 24 24", "inner": "<path d=\"M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z\"></path>", "rule": "evenodd"}, "Meta": {"icon": "meta-color", "viewBox": "0 0 24 24", "inner": "<path d=\"M6.897 4h-.024l-.031 2.615h.022c1.715 0 3.046 1.357 5.94 6.246l.175.297.012.02 1.62-2.438-.012-.019a48.763 48.763 0 00-1.098-1.716 28.01 28.01 0 00-1.175-1.629C10.413 4.932 8.812 4 6.896 4z\" fill=\"url(#lobe-icons-meta-0-_R_0_)\"></path><path d=\"M6.873 4C4.95 4.01 3.247 5.258 2.02 7.17a4.352 4.352 0 00-.01.017l2.254 1.231.011-.017c.718-1.083 1.61-1.774 2.568-1.785h.021L6.896 4h-.023z\" fill=\"url(#lobe-icons-meta-1-_R_0_)\"></path><path d=\"M2.019 7.17l-.011.017C1.2 8.447.598 9.995.274 11.664l-.005.022 2.534.6.004-.022c.27-1.467.786-2.828 1.456-3.845l.011-.017L2.02 7.17z\" fill=\"url(#lobe-icons-meta-2-_R_0_)\"></path><path d=\"M2.807 12.264l-2.533-.6-.005.022c-.177.918-.267 1.851-.269 2.786v.023l2.598.233v-.023a12.591 12.591 0 01.21-2.44z\" fill=\"url(#lobe-icons-meta-3-_R_0_)\"></path><path d=\"M2.677 15.537a5.462 5.462 0 01-.079-.813v-.022L0 14.468v.024a8.89 8.89 0 00.146 1.652l2.535-.585a4.106 4.106 0 01-.004-.022z\" fill=\"url(#lobe-icons-meta-4-_R_0_)\"></path><path d=\"M3.27 16.89c-.284-.31-.484-.756-.589-1.328l-.004-.021-2.535.585.004.021c.192 1.01.568 1.85 1.106 2.487l.014.017 2.018-1.745a2.106 2.106 0 01-.015-.016z\" fill=\"url(#lobe-icons-meta-5-_R_0_)\"></path><path d=\"M10.78 9.654c-1.528 2.35-2.454 3.825-2.454 3.825-2.035 3.2-2.739 3.917-3.871 3.917a1.545 1.545 0 01-1.186-.508l-2.017 1.744.014.017C2.01 19.518 3.058 20 4.356 20c1.963 0 3.374-.928 5.884-5.33l1.766-3.13a41.283 41.283 0 00-1.227-1.886z\" fill=\"#0082FB\"></path><path d=\"M13.502 5.946l-.016.016c-.4.43-.786.908-1.16 1.416.378.483.768 1.024 1.175 1.63.48-.743.928-1.345 1.367-1.807l.016-.016-1.382-1.24z\" fill=\"url(#lobe-icons-meta-6-_R_0_)\"></path><path d=\"M20.918 5.713C19.853 4.633 18.583 4 17.225 4c-1.432 0-2.637.787-3.723 1.944l-.016.016 1.382 1.24.016-.017c.715-.747 1.408-1.12 2.176-1.12.826 0 1.6.39 2.27 1.075l.015.016 1.589-1.425-.016-.016z\" fill=\"#0082FB\"></path><path d=\"M23.998 14.125c-.06-3.467-1.27-6.566-3.064-8.396l-.016-.016-1.588 1.424.015.016c1.35 1.392 2.277 3.98 2.361 6.971v.023h2.292v-.022z\" fill=\"url(#lobe-icons-meta-7-_R_0_)\"></path><path d=\"M23.998 14.15v-.023h-2.292v.022c.004.14.006.282.006.424 0 .815-.121 1.474-.368 1.95l-.011.022 1.708 1.782.013-.02c.62-.96.946-2.293.946-3.91 0-.083 0-.165-.002-.247z\" fill=\"url(#lobe-icons-meta-8-_R_0_)\"></path><path d=\"M21.344 16.52l-.011.02c-.214.402-.519.67-.917.787l.778 2.462a3.493 3.493 0 00.438-.182 3.558 3.558 0 001.366-1.218l.044-.065.012-.02-1.71-1.784z\" fill=\"url(#lobe-icons-meta-9-_R_0_)\"></path><path d=\"M19.92 17.393c-.262 0-.492-.039-.718-.14l-.798 2.522c.449.153.927.222 1.46.222.492 0 .943-.073 1.352-.215l-.78-2.462c-.167.05-.341.075-.517.073z\" fill=\"url(#lobe-icons-meta-10-_R_0_)\"></path><path d=\"M18.323 16.534l-.014-.017-1.836 1.914.016.017c.637.682 1.246 1.105 1.937 1.337l.797-2.52c-.291-.125-.573-.353-.9-.731z\" fill=\"url(#lobe-icons-meta-11-_R_0_)\"></path><path d=\"M18.309 16.515c-.55-.642-1.232-1.712-2.303-3.44l-1.396-2.336-.011-.02-1.62 2.438.012.02.989 1.668c.959 1.61 1.74 2.774 2.493 3.585l.016.016 1.834-1.914a2.353 2.353 0 01-.014-.017z\" fill=\"url(#lobe-icons-meta-12-_R_0_)\"></path><defs><linearGradient id=\"lobe-icons-meta-0-_R_0_\" x1=\"75.897%\" x2=\"26.312%\" y1=\"89.199%\" y2=\"12.194%\"><stop offset=\".06%\" stop-color=\"#0867DF\"></stop><stop offset=\"45.39%\" stop-color=\"#0668E1\"></stop><stop offset=\"85.91%\" stop-color=\"#0064E0\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-1-_R_0_\" x1=\"21.67%\" x2=\"97.068%\" y1=\"75.874%\" y2=\"23.985%\"><stop offset=\"13.23%\" stop-color=\"#0064DF\"></stop><stop offset=\"99.88%\" stop-color=\"#0064E0\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-2-_R_0_\" x1=\"38.263%\" x2=\"60.895%\" y1=\"89.127%\" y2=\"16.131%\"><stop offset=\"1.47%\" stop-color=\"#0072EC\"></stop><stop offset=\"68.81%\" stop-color=\"#0064DF\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-3-_R_0_\" x1=\"47.032%\" x2=\"52.15%\" y1=\"90.19%\" y2=\"15.745%\"><stop offset=\"7.31%\" stop-color=\"#007CF6\"></stop><stop offset=\"99.43%\" stop-color=\"#0072EC\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-4-_R_0_\" x1=\"52.155%\" x2=\"47.591%\" y1=\"58.301%\" y2=\"37.004%\"><stop offset=\"7.31%\" stop-color=\"#007FF9\"></stop><stop offset=\"100%\" stop-color=\"#007CF6\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-5-_R_0_\" x1=\"37.689%\" x2=\"61.961%\" y1=\"12.502%\" y2=\"63.624%\"><stop offset=\"7.31%\" stop-color=\"#007FF9\"></stop><stop offset=\"100%\" stop-color=\"#0082FB\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-6-_R_0_\" x1=\"34.808%\" x2=\"62.313%\" y1=\"68.859%\" y2=\"23.174%\"><stop offset=\"27.99%\" stop-color=\"#007FF8\"></stop><stop offset=\"91.41%\" stop-color=\"#0082FB\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-7-_R_0_\" x1=\"43.762%\" x2=\"57.602%\" y1=\"6.235%\" y2=\"98.514%\"><stop offset=\"0%\" stop-color=\"#0082FB\"></stop><stop offset=\"99.95%\" stop-color=\"#0081FA\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-8-_R_0_\" x1=\"60.055%\" x2=\"39.88%\" y1=\"4.661%\" y2=\"69.077%\"><stop offset=\"6.19%\" stop-color=\"#0081FA\"></stop><stop offset=\"100%\" stop-color=\"#0080F9\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-9-_R_0_\" x1=\"30.282%\" x2=\"61.081%\" y1=\"59.32%\" y2=\"33.244%\"><stop offset=\"0%\" stop-color=\"#027AF3\"></stop><stop offset=\"100%\" stop-color=\"#0080F9\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-10-_R_0_\" x1=\"20.433%\" x2=\"82.112%\" y1=\"50.001%\" y2=\"50.001%\"><stop offset=\"0%\" stop-color=\"#0377EF\"></stop><stop offset=\"99.94%\" stop-color=\"#0279F1\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-11-_R_0_\" x1=\"40.303%\" x2=\"72.394%\" y1=\"35.298%\" y2=\"57.811%\"><stop offset=\".19%\" stop-color=\"#0471E9\"></stop><stop offset=\"100%\" stop-color=\"#0377EF\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-12-_R_0_\" x1=\"32.254%\" x2=\"68.003%\" y1=\"19.719%\" y2=\"84.908%\"><stop offset=\"27.65%\" stop-color=\"#0867DF\"></stop><stop offset=\"100%\" stop-color=\"#0471E9\"></stop></linearGradient></defs>", "rule": null}, "Google": {"icon": "google-color", "viewBox": "0 0 24 24", "inner": "<path d=\"M23 12.245c0-.905-.075-1.565-.236-2.25h-10.54v4.083h6.186c-.124 1.014-.797 2.542-2.294 3.569l-.021.136 3.332 2.53.23.022C21.779 18.417 23 15.593 23 12.245z\" fill=\"#4285F4\"></path><path d=\"M12.225 23c3.03 0 5.574-.978 7.433-2.665l-3.542-2.688c-.948.648-2.22 1.1-3.891 1.1a6.745 6.745 0 01-6.386-4.572l-.132.011-3.465 2.628-.045.124C4.043 20.531 7.835 23 12.225 23z\" fill=\"#34A853\"></path><path d=\"M5.84 14.175A6.65 6.65 0 015.463 12c0-.758.138-1.491.361-2.175l-.006-.147-3.508-2.67-.115.054A10.831 10.831 0 001 12c0 1.772.436 3.447 1.197 4.938l3.642-2.763z\" fill=\"#FBBC05\"></path><path d=\"M12.225 5.253c2.108 0 3.529.892 4.34 1.638l3.167-3.031C17.787 2.088 15.255 1 12.225 1 7.834 1 4.043 3.469 2.197 7.062l3.63 2.763a6.77 6.77 0 016.398-4.572z\" fill=\"#EB4335\"></path>", "rule": null}, "DeepSeek": {"icon": "deepseek-color", "viewBox": "0 0 24 24", "inner": "<path d=\"M23.748 4.482c-.254-.124-.364.113-.512.234-.051.039-.094.09-.137.136-.372.397-.806.657-1.373.626-.829-.046-1.537.214-2.163.848-.133-.782-.575-1.248-1.247-1.548-.352-.156-.708-.311-.955-.65-.172-.241-.219-.51-.305-.774-.055-.16-.11-.323-.293-.35-.2-.031-.278.136-.356.276-.313.572-.434 1.202-.422 1.84.027 1.436.633 2.58 1.838 3.393.137.093.172.187.129.323-.082.28-.18.552-.266.833-.055.179-.137.217-.329.14a5.526 5.526 0 01-1.736-1.18c-.857-.828-1.631-1.742-2.597-2.458a11.365 11.365 0 00-.689-.471c-.985-.957.13-1.743.388-1.836.27-.098.093-.432-.779-.428-.872.004-1.67.295-2.687.684a3.055 3.055 0 01-.465.137 9.597 9.597 0 00-2.883-.102c-1.885.21-3.39 1.102-4.497 2.623C.082 8.606-.231 10.684.152 12.85c.403 2.284 1.569 4.175 3.36 5.653 1.858 1.533 3.997 2.284 6.438 2.14 1.482-.085 3.133-.284 4.994-1.86.47.234.962.327 1.78.397.63.059 1.236-.03 1.705-.128.735-.156.684-.837.419-.961-2.155-1.004-1.682-.595-2.113-.926 1.096-1.296 2.746-2.642 3.392-7.003.05-.347.007-.565 0-.845-.004-.17.035-.237.23-.256a4.173 4.173 0 001.545-.475c1.396-.763 1.96-2.015 2.093-3.517.02-.23-.004-.467-.247-.588zM11.581 18c-2.089-1.642-3.102-2.183-3.52-2.16-.392.024-.321.471-.235.763.09.288.207.486.371.739.114.167.192.416-.113.603-.673.416-1.842-.14-1.897-.167-1.361-.802-2.5-1.86-3.301-3.307-.774-1.393-1.224-2.887-1.298-4.482-.02-.386.093-.522.477-.592a4.696 4.696 0 011.529-.039c2.132.312 3.946 1.265 5.468 2.774.868.86 1.525 1.887 2.202 2.891.72 1.066 1.494 2.082 2.48 2.914.348.292.625.514.891.677-.802.09-2.14.11-3.054-.614zm1-6.44a.306.306 0 01.415-.287.302.302 0 01.2.288.306.306 0 01-.31.307.303.303 0 01-.304-.308zm3.11 1.596c-.2.081-.399.151-.59.16a1.245 1.245 0 01-.798-.254c-.274-.23-.47-.358-.552-.758a1.73 1.73 0 01.016-.588c.07-.327-.008-.537-.239-.727-.187-.156-.426-.199-.688-.199a.559.559 0 01-.254-.078c-.11-.054-.2-.19-.114-.358.028-.054.16-.186.192-.21.356-.202.767-.136 1.146.016.352.144.618.408 1.001.782.391.451.462.576.685.914.176.265.336.537.445.848.067.195-.019.354-.25.452z\" fill=\"#4D6BFE\"></path>", "rule": null}, "xAI": {"icon": "xai", "tile": "#111111", "shape": "square", "viewBox": "0 0 24 24", "inner": "<path d=\"M6.469 8.776L16.512 23h-4.464L2.005 8.776H6.47zm-.004 7.9l2.233 3.164L6.467 23H2l4.465-6.324zM22 2.582V23h-3.659V7.764L22 2.582zM22 1l-9.952 14.095-2.233-3.163L17.533 1H22z\"></path>", "rule": "evenodd"}, "Z.ai": {"icon": "zai", "tile": "#111111", "shape": "square", "viewBox": "0 0 24 24", "inner": "<path d=\"M12.105 2L9.927 4.953H.653L2.83 2h9.276zM23.254 19.048L21.078 22h-9.242l2.174-2.952h9.244zM24 2L9.264 22H0L14.736 2H24z\"></path>", "rule": "evenodd"}, "Moonshot": {"icon": "kimi", "tile": "#111111", "shape": "square", "viewBox": "0 0 24 24", "inner": "<path d=\"M21.846 0a1.923 1.923 0 110 3.846H20.15a.226.226 0 01-.227-.226V1.923C19.923.861 20.784 0 21.846 0z\"></path><path d=\"M11.065 11.199l7.257-7.2c.137-.136.06-.41-.116-.41H14.3a.164.164 0 00-.117.051l-7.82 7.756c-.122.12-.302.013-.302-.179V3.82c0-.127-.083-.23-.185-.23H3.186c-.103 0-.186.103-.186.23V19.77c0 .128.083.23.186.23h2.69c.103 0 .186-.102.186-.23v-3.25c0-.069.025-.135.069-.178l2.424-2.406a.158.158 0 01.205-.023l6.484 4.772a7.677 7.677 0 003.453 1.283c.108.012.2-.095.2-.23v-3.06c0-.117-.07-.212-.164-.227a5.028 5.028 0 01-2.027-.807l-5.613-4.064c-.117-.078-.132-.279-.028-.381z\"></path>", "rule": "evenodd"}, "StepFun": {"icon": "stepfun", "tile": "#01d9cf", "shape": "circle", "viewBox": "0 0 24 24", "inner": "<path d=\"M12 0C5.372 0 0 5.372 0 12s5.372 12 12 12 12-5.372 12-12S18.628 0 12 0zM8.762 19.614H4.376v-4.386h4.386v4.386zm5.423 0H9.798v-4.386h4.387v4.386zm0-5.42H9.798V9.81h4.387v4.386zm0-5.418H9.798V4.39h4.387v4.386zm5.422-.004h-4.386V4.386h4.386v4.386z\"></path>", "rule": "evenodd"}, "Undisclosed": {"icon": "unionalpha-color", "viewBox": "0 0 24 24", "inner": "<path d=\"M12.5709 13.3706C12.6556 13.4345 12.8442 13.7666 12.9123 13.875L13.4693 14.7624C13.7592 15.2257 14.0526 15.6895 14.3386 16.1555C14.4141 16.2951 14.4587 16.3467 14.3445 16.48C13.9818 16.9035 13.6097 17.321 13.2426 17.7406C12.6047 18.4698 11.5795 19.0999 10.6086 19.2277C10.5721 19.2324 10.5355 19.2366 10.4988 19.2405C10.3741 19.2219 10.2359 19.2226 10.1067 19.2077C9.62828 19.1531 9.13925 18.9833 8.73437 18.7229C8.58284 18.6255 8.20429 18.365 8.13135 18.2015C8.7166 18.0941 8.9971 17.6275 9.36375 17.2093C9.46647 17.0921 9.56558 16.9681 9.66524 16.8481C10.2482 16.1439 10.8249 15.4346 11.3952 14.7202C11.5532 14.5228 12.4352 13.3557 12.5709 13.3706Z\" fill=\"url(#lobe-icons-union-alpha-0-_R_0_)\"></path><path d=\"M23.9999 4.03711C23.9427 4.17731 23.8557 4.29083 23.7726 4.41626C23.5172 4.80162 23.2489 5.18017 22.9952 5.56645C21.7031 7.52279 20.3758 9.45572 19.0141 11.3643L18.2195 12.4575C18.0914 12.6376 17.8772 12.9025 17.8103 13.1017C17.5635 12.8289 17.3373 12.3638 17.1271 12.052C16.7507 11.4939 16.4007 10.8942 16.0072 10.3494C15.964 10.2787 15.9232 10.1649 15.8967 10.0866C15.9509 9.95674 16.2386 9.57362 16.3367 9.43732L17.184 8.26053L19.1886 5.47039C19.3731 5.21667 20.0937 4.17696 20.2819 4.03711H23.9999Z\" fill=\"url(#lobe-icons-union-alpha-1-_R_0_)\"></path><path d=\"M5.67537 4.61919C5.91433 4.59978 6.08153 4.64306 6.2388 4.65534C7.31097 4.73906 7.93546 5.78123 8.52073 6.55699C8.55486 6.60222 8.67307 6.77678 8.67756 6.81818C8.50211 6.78591 8.24062 6.87736 8.10425 6.98766C7.77451 7.25438 7.47306 7.60773 7.16738 7.90822C6.82713 8.24411 6.49119 8.58431 6.15964 8.92879C5.96629 9.12818 5.68162 9.40902 5.50821 9.61426C4.95769 10.264 4.58519 11.0455 4.42712 11.8823C4.38673 12.1047 4.33596 12.4789 4.34474 12.6979C4.38003 13.5788 4.68343 14.4541 5.20029 15.1681C5.44662 15.5083 5.69406 15.7534 5.97744 16.05C6.42647 16.5145 6.87971 16.9749 7.33712 17.4311C8.17604 18.2704 8.83084 19.0614 10.1068 19.2072C10.2359 19.2221 10.3741 19.2214 10.4989 19.24C10.4563 19.2438 10.4156 19.2437 10.3732 19.2456C9.63946 19.2779 8.9074 19.2593 8.17299 19.2571L6.81265 19.2561C6.34052 19.2551 5.82434 19.2555 5.36063 19.1766C4.33929 19.003 3.55816 18.5177 2.78058 17.8624C1.69732 16.9495 0.760791 15.9389 0.316835 14.5629C0.188539 14.1652 0.0845513 13.7688 0.0423572 13.352C0.0341296 13.2704 0.025306 13.1714 0 13.0938V12.1601C0.024482 12.0989 0.0539558 11.7751 0.0701206 11.682C0.118265 11.3996 0.185469 11.1208 0.271272 10.8475C0.464403 10.234 0.818387 9.53555 1.19037 9.01281C1.67633 8.32991 2.33983 7.69834 2.92227 7.09354L3.99839 5.99473C4.54749 5.44174 4.97245 4.98537 5.67537 4.61919Z\" fill=\"url(#lobe-icons-union-alpha-2-_R_0_)\"></path><path d=\"M16.0072 10.3494C16.4008 10.8942 16.7508 11.4939 17.1271 12.052C17.3373 12.3638 17.5635 12.8289 17.8104 13.1017C17.8351 13.1633 18.024 13.4562 18.0701 13.5307L18.6905 14.5371C19.1466 15.2881 19.6091 16.0351 20.078 16.7781C20.4399 17.3561 20.7699 17.9692 21.2574 18.4512C21.5622 18.7524 21.9167 18.9416 22.3073 19.1101C22.4041 19.1518 22.5241 19.1723 22.6131 19.2159L22.5999 19.2374C22.543 19.2593 22.4118 19.2355 22.3393 19.2386C21.4806 19.2765 20.6139 19.2542 19.7541 19.2467C19.2053 19.2419 18.6361 19.2653 18.0899 19.2461C17.9536 19.241 17.8182 19.2222 17.6855 19.1901C17.2306 19.0825 16.8145 18.8188 16.4914 18.4847C16.2155 18.1993 16.0406 17.8931 15.8347 17.558C15.6277 17.2199 15.423 16.8802 15.2208 16.5391L13.5134 13.7217C13.602 13.5224 14.0393 12.9594 14.2007 12.7419L16.0072 10.3494Z\" fill=\"url(#lobe-icons-union-alpha-3-_R_0_)\"></path><path d=\"M11.3662 4.03706C11.4355 4.04736 11.5279 4.0639 11.5825 4.11302C11.7409 4.25538 11.8717 4.45147 12.0009 4.62253C13.1498 6.14388 14.2121 7.73081 15.2749 9.31286C15.3244 9.38658 15.4528 9.56569 15.4748 9.646C15.3946 9.79571 15.0698 10.1847 14.9522 10.3329L13.8456 11.7243C13.647 11.9762 13.455 12.2345 13.2586 12.4878C13.1566 12.6194 13.0064 12.8305 12.8889 12.9425C12.6125 12.5563 12.3581 12.1531 12.0887 11.7623L9.66701 8.22137C9.43592 7.88192 9.20191 7.54448 8.96502 7.20905C8.92518 7.15337 8.69584 6.86111 8.67758 6.81865C8.67307 6.77723 8.55489 6.60269 8.52077 6.55747C7.9355 5.7817 7.311 4.73956 6.23883 4.65583C6.08162 4.64356 5.91434 4.61968 5.67542 4.61968C6.15935 4.35771 6.84659 4.17302 7.38673 4.09926C8.69114 3.92117 10.0523 4.03706 11.3662 4.03706Z\" fill=\"url(#lobe-icons-union-alpha-4-_R_0_)\"></path><path d=\"M15.3619 8.08232L15.874 8.87602L19.2024 4.16504L15.3619 8.08232Z\" fill=\"url(#lobe-icons-union-alpha-5-_R_0_)\"></path><defs><linearGradient gradientUnits=\"userSpaceOnUse\" id=\"lobe-icons-union-alpha-0-_R_0_\" x1=\"21.5578\" x2=\"21.5578\" y1=\"4.03711\" y2=\"12.691\"><stop stop-color=\"currentColor\" stop-opacity=\"0.39\"></stop><stop offset=\"1\" stop-color=\"currentColor\" stop-opacity=\"0.22\"></stop></linearGradient><linearGradient gradientUnits=\"userSpaceOnUse\" id=\"lobe-icons-union-alpha-1-_R_0_\" x1=\"21.5578\" x2=\"21.5578\" y1=\"4.03711\" y2=\"12.691\"><stop stop-color=\"currentColor\" stop-opacity=\"0.39\"></stop><stop offset=\"1\" stop-color=\"currentColor\" stop-opacity=\"0.22\"></stop></linearGradient><linearGradient gradientUnits=\"userSpaceOnUse\" id=\"lobe-icons-union-alpha-2-_R_0_\" x1=\"5.27425\" x2=\"5.24944\" y1=\"5.41908\" y2=\"14.0217\"><stop stop-color=\"currentColor\" stop-opacity=\"0.15\"></stop><stop offset=\"1\" stop-color=\"currentColor\" stop-opacity=\"0.87\"></stop></linearGradient><linearGradient gradientUnits=\"userSpaceOnUse\" id=\"lobe-icons-union-alpha-3-_R_0_\" x1=\"14.1443\" x2=\"14.1443\" y1=\"14.4319\" y2=\"19.26\"><stop stop-color=\"currentColor\" stop-opacity=\"0.93\"></stop><stop offset=\"1\" stop-color=\"currentColor\" stop-opacity=\"0.57\"></stop></linearGradient><linearGradient gradientUnits=\"userSpaceOnUse\" id=\"lobe-icons-union-alpha-4-_R_0_\" x1=\"14.1443\" x2=\"14.1443\" y1=\"14.4319\" y2=\"19.26\"><stop stop-color=\"currentColor\" stop-opacity=\"0.93\"></stop><stop offset=\"1\" stop-color=\"currentColor\" stop-opacity=\"0.57\"></stop></linearGradient><linearGradient gradientUnits=\"userSpaceOnUse\" id=\"lobe-icons-union-alpha-5-_R_0_\" x1=\"17.2822\" x2=\"17.2822\" y1=\"4.16504\" y2=\"8.87602\"><stop stop-color=\"#D72725\"></stop><stop offset=\"1\" stop-color=\"#CD2920\"></stop></linearGradient></defs>", "rule": null}, "Cognition": {"icon": "devin-color", "viewBox": "0 0 24 24", "inner": "<path d=\"M2.033 9.867l2.554 1.483a.589.589 0 00.592 0l2.554-1.483.01-.008a.608.608 0 00.11-.084l.013-.015a.631.631 0 00.076-.1c.003-.005.008-.01.01-.016a.558.558 0 00.052-.125l.007-.028a.611.611 0 00.019-.14V7.868c0-.572.307-1.105.8-1.392a1.595 1.595 0 011.598 0l1.277.742a.54.54 0 00.129.053l.028.01c.044.01.088.015.133.016h.006l.013-.002a.587.587 0 00.27-.074l.011-.004 2.554-1.483a.596.596 0 00.297-.516V2.253a.595.595 0 00-.297-.516L12.293.257a.587.587 0 00-.591 0L9.148 1.737l-.01.01a.609.609 0 00-.109.083l-.014.015a.632.632 0 00-.076.1c-.003.005-.008.01-.01.016a.57.57 0 00-.052.124l-.007.028a.612.612 0 00-.018.14v1.483c0 .572-.307 1.105-.8 1.393a1.597 1.597 0 01-1.599 0l-1.276-.742a.603.603 0 00-.13-.053l-.028-.008a.658.658 0 00-.133-.018h-.02a.57.57 0 00-.269.074c-.003.002-.008.002-.012.005L2.033 5.872a.596.596 0 00-.297.515v2.966c0 .213.113.41.297.515z\" fill=\"#3969CA\"></path><path d=\"M15.943 10.607a1.596 1.596 0 011.599 0l1.276.74c.041.025.085.04.13.055l.028.008c.043.01.088.016.133.018h.005c.005 0 .01-.002.014-.003a.474.474 0 00.122-.016l.021-.005a.616.616 0 00.126-.052c.004-.002.009-.002.013-.005l2.554-1.482a.597.597 0 00.297-.516V6.383a.596.596 0 00-.297-.515l-2.552-1.483a.587.587 0 00-.592 0l-2.553 1.482-.011.008a.61.61 0 00-.108.084l-.014.016a.637.637 0 00-.076.1c-.003.005-.008.01-.01.016a.57.57 0 00-.052.124l-.007.029a.612.612 0 00-.018.14v1.482c0 .572-.307 1.105-.8 1.393a1.597 1.597 0 01-1.599 0l-1.276-.742a.584.584 0 00-.13-.053l-.028-.008a.62.62 0 00-.133-.018h-.02a.587.587 0 00-.269.074l-.012.004L9.15 10a.596.596 0 00-.296.516v2.966c0 .212.112.409.296.515l2.554 1.483s.008.002.012.005c.04.022.082.04.126.052l.02.004a.57.57 0 00.123.017l.014.002h.006c.054 0 .108-.01.16-.025a.587.587 0 00.13-.054l1.277-.741a1.597 1.597 0 012.398 1.392v1.482c0 .049.007.095.019.14l.007.028a.619.619 0 00.051.125c.004.006.008.01.01.016a.6.6 0 00.076.1l.014.015c.033.032.069.06.108.084.004.002.006.006.011.008l2.554 1.483a.59.59 0 00.593 0l2.554-1.483a.597.597 0 00.296-.516v-2.965a.595.595 0 00-.296-.516l-2.554-1.483s-.008-.002-.012-.005a.54.54 0 00-.126-.051c-.007-.003-.013-.003-.02-.005a.635.635 0 00-.125-.017h-.018a.557.557 0 00-.16.026.588.588 0 00-.13.053l-1.276.742a1.595 1.595 0 01-1.598 0 1.615 1.615 0 010-2.785l-.005-.001z\" fill=\"#21C19A\"></path><path d=\"M14.848 18.265l-2.554-1.482-.012-.005a.526.526 0 00-.126-.052c-.007-.002-.014-.002-.02-.005a.64.64 0 00-.124-.017h-.02a.56.56 0 00-.16.026.588.588 0 00-.13.053l-1.276.742a1.594 1.594 0 01-1.598 0c-.493-.286-.8-.82-.8-1.393V14.65a.563.563 0 00-.018-.14l-.008-.028a.604.604 0 00-.051-.124l-.01-.017a.603.603 0 00-.076-.1l-.014-.015a.596.596 0 00-.109-.084c-.003-.002-.005-.006-.01-.008L5.178 12.65a.587.587 0 00-.591 0l-2.554 1.483a.596.596 0 00-.297.516v2.965c0 .213.113.41.297.516l2.554 1.483.012.004a.618.618 0 00.267.074l.016.002h.007a.55.55 0 00.16-.026.584.584 0 00.129-.053l1.277-.742a1.597 1.597 0 012.398 1.393v1.482c0 .05.007.095.019.14l.007.028c.013.044.03.085.051.125l.01.016c.022.036.047.07.076.1l.014.015c.032.032.069.06.109.084l.01.008 2.554 1.483a.587.587 0 00.593 0l2.554-1.483a.596.596 0 00.296-.515v-2.966a.596.596 0 00-.296-.516h-.002z\" fill=\"#0294DE\"></path>", "rule": null}, "Fireworks": {"icon": "fireworks-color", "viewBox": "0 0 24 24", "inner": "<path clip-rule=\"evenodd\" d=\"M14.8 5l-2.801 6.795L9.195 5H7.397l3.072 7.428a1.64 1.64 0 003.038.002L16.598 5H14.8zm1.196 10.352l5.124-5.244-.699-1.669-5.596 5.739a1.664 1.664 0 00-.343 1.807 1.642 1.642 0 001.516 1.012L16 17l8-.02-.699-1.669-7.303.041h-.002zM2.88 10.104l.699-1.669 5.596 5.739c.468.479.603 1.189.343 1.807a1.643 1.643 0 01-1.516 1.012l-8-.018-.002.002.699-1.669 7.303.042-5.122-5.246z\" fill=\"#5019C5\" fill-rule=\"evenodd\"></path>", "rule": null}, "Stealth": {"wordmark": ["STEALTH"]}, "Thinking Machines": {"wordmark": ["THINKING", "MACHINES"]}, "Alibaba": {"wordmark": ["QWEN"]}};
|
| 8 |
for(const [key,DATA] of Object.entries(ALL)){
|
| 9 |
const svg=document.getElementById('bars-'+key), W=1200, H=420;
|
| 10 |
const NS='http://www.w3.org/2000/svg', SANS='Inter, Arial, sans-serif', MONO='"IBM Plex Mono", monospace';
|
|
|
|
| 1 |
// Bar layout and name wrapping adapted from BuildingBench figure-kit/bar-static/build.py.
|
| 2 |
// Data is exclusively the existing Code4Scene data/site.js paper board; no reference benchmark scores.
|
| 3 |
(async function(){
|
| 4 |
+
const {bindChartTooltip}=await import("./chart-interactions.js?v=20261004kit");
|
| 5 |
await Promise.all([document.fonts.load('400 13px Inter'),document.fonts.load('700 13px Inter'),document.fonts.load('500 15.5px "IBM Plex Mono"')]);
|
| 6 |
const ALL = {"t2s": [{"name": "Claude Fable 5.1 (max)", "org": "Anthropic", "score": 0.78775, "colour": "#D97757", "ink": "#ffffff", "label": "79", "new": false}, {"name": "GPT-6 Astra (max)", "org": "OpenAI", "score": 0.72382, "colour": "#10A37F", "ink": "#ffffff", "label": "72", "new": false}, {"name": "Claude Opus 5 (max)", "org": "Anthropic", "score": 0.7181599999999999, "colour": "#D97757", "ink": "#ffffff", "label": "72", "new": false}, {"name": "GPT-5.6 Sol (high)", "org": "OpenAI", "score": 0.70704, "colour": "#10A37F", "ink": "#ffffff", "label": "71", "new": false}, {"name": "Gemini 3.8 Flash (high)", "org": "Google", "score": 0.65689, "colour": "#7b1fa2", "ink": "#ffffff", "label": "66", "new": false}, {"name": "Muse Spark 1.3 (medium)", "org": "Meta", "score": 0.646195, "colour": "#42a5f5", "ink": "#111111", "label": "65", "new": false}, {"name": "Grok 4.6 (high)", "org": "xAI", "score": 0.56721, "colour": "#111111", "ink": "#ffffff", "label": "57", "new": false}, {"name": "Qwen 3.8 27B (thinking off)", "org": "Alibaba", "score": 0.55666, "colour": "#fe7016", "ink": "#111111", "label": "56", "new": false}, {"name": "Qwen 3.8 27B (thinking on)", "org": "Alibaba", "score": 0.51571, "colour": "#fe7016", "ink": "#111111", "label": "52", "new": false}, {"name": "GLM-5.3 Flash (max)", "org": "Z.ai", "score": 0.5094299999999999, "colour": "#96650b", "ink": "#ffffff", "label": "51", "new": false}, {"name": "Gemma 4 31B (thinking on)", "org": "Google", "score": 0.470705, "colour": "#7b1fa2", "ink": "#ffffff", "label": "47", "new": false}, {"name": "Gemma 4 31B (thinking off)", "org": "Google", "score": 0.45494, "colour": "#7b1fa2", "ink": "#ffffff", "label": "45", "new": false}, {"name": "Inkling (high)", "org": "Thinking Machines", "score": 0.42498500000000006, "colour": "#686868", "ink": "#ffffff", "label": "42", "new": false}, {"name": "DeepSeek V4.1 Flash (high)", "org": "DeepSeek", "score": 0.24269, "colour": "#3f51e0", "ink": "#ffffff", "label": "24", "new": false}]};
|
| 7 |
+
const LOGOS = {"Anthropic": {"icon": "anthropic", "fill": "#141413", "viewBox": "0 0 24 24", "inner": "<path d=\"M13.827 3.52h3.603L24 20h-3.603l-6.57-16.48zm-7.258 0h3.767L16.906 20h-3.674l-1.343-3.461H5.017l-1.344 3.46H0L6.57 3.522zm4.132 9.959L8.453 7.687 6.205 13.48H10.7z\"></path>", "rule": "evenodd"}, "OpenAI": {"icon": "openai", "fill": "#141413", "viewBox": "0 0 24 24", "inner": "<path d=\"M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z\"></path>", "rule": "evenodd"}, "Meta": {"icon": "meta-color", "viewBox": "0 0 24 24", "inner": "<path d=\"M6.897 4h-.024l-.031 2.615h.022c1.715 0 3.046 1.357 5.94 6.246l.175.297.012.02 1.62-2.438-.012-.019a48.763 48.763 0 00-1.098-1.716 28.01 28.01 0 00-1.175-1.629C10.413 4.932 8.812 4 6.896 4z\" fill=\"url(#lobe-icons-meta-0-_R_0_)\"></path><path d=\"M6.873 4C4.95 4.01 3.247 5.258 2.02 7.17a4.352 4.352 0 00-.01.017l2.254 1.231.011-.017c.718-1.083 1.61-1.774 2.568-1.785h.021L6.896 4h-.023z\" fill=\"url(#lobe-icons-meta-1-_R_0_)\"></path><path d=\"M2.019 7.17l-.011.017C1.2 8.447.598 9.995.274 11.664l-.005.022 2.534.6.004-.022c.27-1.467.786-2.828 1.456-3.845l.011-.017L2.02 7.17z\" fill=\"url(#lobe-icons-meta-2-_R_0_)\"></path><path d=\"M2.807 12.264l-2.533-.6-.005.022c-.177.918-.267 1.851-.269 2.786v.023l2.598.233v-.023a12.591 12.591 0 01.21-2.44z\" fill=\"url(#lobe-icons-meta-3-_R_0_)\"></path><path d=\"M2.677 15.537a5.462 5.462 0 01-.079-.813v-.022L0 14.468v.024a8.89 8.89 0 00.146 1.652l2.535-.585a4.106 4.106 0 01-.004-.022z\" fill=\"url(#lobe-icons-meta-4-_R_0_)\"></path><path d=\"M3.27 16.89c-.284-.31-.484-.756-.589-1.328l-.004-.021-2.535.585.004.021c.192 1.01.568 1.85 1.106 2.487l.014.017 2.018-1.745a2.106 2.106 0 01-.015-.016z\" fill=\"url(#lobe-icons-meta-5-_R_0_)\"></path><path d=\"M10.78 9.654c-1.528 2.35-2.454 3.825-2.454 3.825-2.035 3.2-2.739 3.917-3.871 3.917a1.545 1.545 0 01-1.186-.508l-2.017 1.744.014.017C2.01 19.518 3.058 20 4.356 20c1.963 0 3.374-.928 5.884-5.33l1.766-3.13a41.283 41.283 0 00-1.227-1.886z\" fill=\"#0082FB\"></path><path d=\"M13.502 5.946l-.016.016c-.4.43-.786.908-1.16 1.416.378.483.768 1.024 1.175 1.63.48-.743.928-1.345 1.367-1.807l.016-.016-1.382-1.24z\" fill=\"url(#lobe-icons-meta-6-_R_0_)\"></path><path d=\"M20.918 5.713C19.853 4.633 18.583 4 17.225 4c-1.432 0-2.637.787-3.723 1.944l-.016.016 1.382 1.24.016-.017c.715-.747 1.408-1.12 2.176-1.12.826 0 1.6.39 2.27 1.075l.015.016 1.589-1.425-.016-.016z\" fill=\"#0082FB\"></path><path d=\"M23.998 14.125c-.06-3.467-1.27-6.566-3.064-8.396l-.016-.016-1.588 1.424.015.016c1.35 1.392 2.277 3.98 2.361 6.971v.023h2.292v-.022z\" fill=\"url(#lobe-icons-meta-7-_R_0_)\"></path><path d=\"M23.998 14.15v-.023h-2.292v.022c.004.14.006.282.006.424 0 .815-.121 1.474-.368 1.95l-.011.022 1.708 1.782.013-.02c.62-.96.946-2.293.946-3.91 0-.083 0-.165-.002-.247z\" fill=\"url(#lobe-icons-meta-8-_R_0_)\"></path><path d=\"M21.344 16.52l-.011.02c-.214.402-.519.67-.917.787l.778 2.462a3.493 3.493 0 00.438-.182 3.558 3.558 0 001.366-1.218l.044-.065.012-.02-1.71-1.784z\" fill=\"url(#lobe-icons-meta-9-_R_0_)\"></path><path d=\"M19.92 17.393c-.262 0-.492-.039-.718-.14l-.798 2.522c.449.153.927.222 1.46.222.492 0 .943-.073 1.352-.215l-.78-2.462c-.167.05-.341.075-.517.073z\" fill=\"url(#lobe-icons-meta-10-_R_0_)\"></path><path d=\"M18.323 16.534l-.014-.017-1.836 1.914.016.017c.637.682 1.246 1.105 1.937 1.337l.797-2.52c-.291-.125-.573-.353-.9-.731z\" fill=\"url(#lobe-icons-meta-11-_R_0_)\"></path><path d=\"M18.309 16.515c-.55-.642-1.232-1.712-2.303-3.44l-1.396-2.336-.011-.02-1.62 2.438.012.02.989 1.668c.959 1.61 1.74 2.774 2.493 3.585l.016.016 1.834-1.914a2.353 2.353 0 01-.014-.017z\" fill=\"url(#lobe-icons-meta-12-_R_0_)\"></path><defs><linearGradient id=\"lobe-icons-meta-0-_R_0_\" x1=\"75.897%\" x2=\"26.312%\" y1=\"89.199%\" y2=\"12.194%\"><stop offset=\".06%\" stop-color=\"#0867DF\"></stop><stop offset=\"45.39%\" stop-color=\"#0668E1\"></stop><stop offset=\"85.91%\" stop-color=\"#0064E0\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-1-_R_0_\" x1=\"21.67%\" x2=\"97.068%\" y1=\"75.874%\" y2=\"23.985%\"><stop offset=\"13.23%\" stop-color=\"#0064DF\"></stop><stop offset=\"99.88%\" stop-color=\"#0064E0\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-2-_R_0_\" x1=\"38.263%\" x2=\"60.895%\" y1=\"89.127%\" y2=\"16.131%\"><stop offset=\"1.47%\" stop-color=\"#0072EC\"></stop><stop offset=\"68.81%\" stop-color=\"#0064DF\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-3-_R_0_\" x1=\"47.032%\" x2=\"52.15%\" y1=\"90.19%\" y2=\"15.745%\"><stop offset=\"7.31%\" stop-color=\"#007CF6\"></stop><stop offset=\"99.43%\" stop-color=\"#0072EC\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-4-_R_0_\" x1=\"52.155%\" x2=\"47.591%\" y1=\"58.301%\" y2=\"37.004%\"><stop offset=\"7.31%\" stop-color=\"#007FF9\"></stop><stop offset=\"100%\" stop-color=\"#007CF6\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-5-_R_0_\" x1=\"37.689%\" x2=\"61.961%\" y1=\"12.502%\" y2=\"63.624%\"><stop offset=\"7.31%\" stop-color=\"#007FF9\"></stop><stop offset=\"100%\" stop-color=\"#0082FB\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-6-_R_0_\" x1=\"34.808%\" x2=\"62.313%\" y1=\"68.859%\" y2=\"23.174%\"><stop offset=\"27.99%\" stop-color=\"#007FF8\"></stop><stop offset=\"91.41%\" stop-color=\"#0082FB\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-7-_R_0_\" x1=\"43.762%\" x2=\"57.602%\" y1=\"6.235%\" y2=\"98.514%\"><stop offset=\"0%\" stop-color=\"#0082FB\"></stop><stop offset=\"99.95%\" stop-color=\"#0081FA\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-8-_R_0_\" x1=\"60.055%\" x2=\"39.88%\" y1=\"4.661%\" y2=\"69.077%\"><stop offset=\"6.19%\" stop-color=\"#0081FA\"></stop><stop offset=\"100%\" stop-color=\"#0080F9\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-9-_R_0_\" x1=\"30.282%\" x2=\"61.081%\" y1=\"59.32%\" y2=\"33.244%\"><stop offset=\"0%\" stop-color=\"#027AF3\"></stop><stop offset=\"100%\" stop-color=\"#0080F9\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-10-_R_0_\" x1=\"20.433%\" x2=\"82.112%\" y1=\"50.001%\" y2=\"50.001%\"><stop offset=\"0%\" stop-color=\"#0377EF\"></stop><stop offset=\"99.94%\" stop-color=\"#0279F1\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-11-_R_0_\" x1=\"40.303%\" x2=\"72.394%\" y1=\"35.298%\" y2=\"57.811%\"><stop offset=\".19%\" stop-color=\"#0471E9\"></stop><stop offset=\"100%\" stop-color=\"#0377EF\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-12-_R_0_\" x1=\"32.254%\" x2=\"68.003%\" y1=\"19.719%\" y2=\"84.908%\"><stop offset=\"27.65%\" stop-color=\"#0867DF\"></stop><stop offset=\"100%\" stop-color=\"#0471E9\"></stop></linearGradient></defs>", "rule": null}, "Google": {"icon": "google-color", "viewBox": "0 0 24 24", "inner": "<path d=\"M23 12.245c0-.905-.075-1.565-.236-2.25h-10.54v4.083h6.186c-.124 1.014-.797 2.542-2.294 3.569l-.021.136 3.332 2.53.23.022C21.779 18.417 23 15.593 23 12.245z\" fill=\"#4285F4\"></path><path d=\"M12.225 23c3.03 0 5.574-.978 7.433-2.665l-3.542-2.688c-.948.648-2.22 1.1-3.891 1.1a6.745 6.745 0 01-6.386-4.572l-.132.011-3.465 2.628-.045.124C4.043 20.531 7.835 23 12.225 23z\" fill=\"#34A853\"></path><path d=\"M5.84 14.175A6.65 6.65 0 015.463 12c0-.758.138-1.491.361-2.175l-.006-.147-3.508-2.67-.115.054A10.831 10.831 0 001 12c0 1.772.436 3.447 1.197 4.938l3.642-2.763z\" fill=\"#FBBC05\"></path><path d=\"M12.225 5.253c2.108 0 3.529.892 4.34 1.638l3.167-3.031C17.787 2.088 15.255 1 12.225 1 7.834 1 4.043 3.469 2.197 7.062l3.63 2.763a6.77 6.77 0 016.398-4.572z\" fill=\"#EB4335\"></path>", "rule": null}, "DeepSeek": {"icon": "deepseek-color", "viewBox": "0 0 24 24", "inner": "<path d=\"M23.748 4.482c-.254-.124-.364.113-.512.234-.051.039-.094.09-.137.136-.372.397-.806.657-1.373.626-.829-.046-1.537.214-2.163.848-.133-.782-.575-1.248-1.247-1.548-.352-.156-.708-.311-.955-.65-.172-.241-.219-.51-.305-.774-.055-.16-.11-.323-.293-.35-.2-.031-.278.136-.356.276-.313.572-.434 1.202-.422 1.84.027 1.436.633 2.58 1.838 3.393.137.093.172.187.129.323-.082.28-.18.552-.266.833-.055.179-.137.217-.329.14a5.526 5.526 0 01-1.736-1.18c-.857-.828-1.631-1.742-2.597-2.458a11.365 11.365 0 00-.689-.471c-.985-.957.13-1.743.388-1.836.27-.098.093-.432-.779-.428-.872.004-1.67.295-2.687.684a3.055 3.055 0 01-.465.137 9.597 9.597 0 00-2.883-.102c-1.885.21-3.39 1.102-4.497 2.623C.082 8.606-.231 10.684.152 12.85c.403 2.284 1.569 4.175 3.36 5.653 1.858 1.533 3.997 2.284 6.438 2.14 1.482-.085 3.133-.284 4.994-1.86.47.234.962.327 1.78.397.63.059 1.236-.03 1.705-.128.735-.156.684-.837.419-.961-2.155-1.004-1.682-.595-2.113-.926 1.096-1.296 2.746-2.642 3.392-7.003.05-.347.007-.565 0-.845-.004-.17.035-.237.23-.256a4.173 4.173 0 001.545-.475c1.396-.763 1.96-2.015 2.093-3.517.02-.23-.004-.467-.247-.588zM11.581 18c-2.089-1.642-3.102-2.183-3.52-2.16-.392.024-.321.471-.235.763.09.288.207.486.371.739.114.167.192.416-.113.603-.673.416-1.842-.14-1.897-.167-1.361-.802-2.5-1.86-3.301-3.307-.774-1.393-1.224-2.887-1.298-4.482-.02-.386.093-.522.477-.592a4.696 4.696 0 011.529-.039c2.132.312 3.946 1.265 5.468 2.774.868.86 1.525 1.887 2.202 2.891.72 1.066 1.494 2.082 2.48 2.914.348.292.625.514.891.677-.802.09-2.14.11-3.054-.614zm1-6.44a.306.306 0 01.415-.287.302.302 0 01.2.288.306.306 0 01-.31.307.303.303 0 01-.304-.308zm3.11 1.596c-.2.081-.399.151-.59.16a1.245 1.245 0 01-.798-.254c-.274-.23-.47-.358-.552-.758a1.73 1.73 0 01.016-.588c.07-.327-.008-.537-.239-.727-.187-.156-.426-.199-.688-.199a.559.559 0 01-.254-.078c-.11-.054-.2-.19-.114-.358.028-.054.16-.186.192-.21.356-.202.767-.136 1.146.016.352.144.618.408 1.001.782.391.451.462.576.685.914.176.265.336.537.445.848.067.195-.019.354-.25.452z\" fill=\"#4D6BFE\"></path>", "rule": null}, "xAI": {"icon": "xai", "tile": "#111111", "shape": "square", "viewBox": "0 0 24 24", "inner": "<path d=\"M6.469 8.776L16.512 23h-4.464L2.005 8.776H6.47zm-.004 7.9l2.233 3.164L6.467 23H2l4.465-6.324zM22 2.582V23h-3.659V7.764L22 2.582zM22 1l-9.952 14.095-2.233-3.163L17.533 1H22z\"></path>", "rule": "evenodd"}, "Z.ai": {"icon": "zai", "tile": "#111111", "shape": "square", "viewBox": "0 0 24 24", "inner": "<path d=\"M12.105 2L9.927 4.953H.653L2.83 2h9.276zM23.254 19.048L21.078 22h-9.242l2.174-2.952h9.244zM24 2L9.264 22H0L14.736 2H24z\"></path>", "rule": "evenodd"}, "Moonshot": {"icon": "kimi", "tile": "#111111", "shape": "square", "viewBox": "0 0 24 24", "inner": "<path d=\"M21.846 0a1.923 1.923 0 110 3.846H20.15a.226.226 0 01-.227-.226V1.923C19.923.861 20.784 0 21.846 0z\"></path><path d=\"M11.065 11.199l7.257-7.2c.137-.136.06-.41-.116-.41H14.3a.164.164 0 00-.117.051l-7.82 7.756c-.122.12-.302.013-.302-.179V3.82c0-.127-.083-.23-.185-.23H3.186c-.103 0-.186.103-.186.23V19.77c0 .128.083.23.186.23h2.69c.103 0 .186-.102.186-.23v-3.25c0-.069.025-.135.069-.178l2.424-2.406a.158.158 0 01.205-.023l6.484 4.772a7.677 7.677 0 003.453 1.283c.108.012.2-.095.2-.23v-3.06c0-.117-.07-.212-.164-.227a5.028 5.028 0 01-2.027-.807l-5.613-4.064c-.117-.078-.132-.279-.028-.381z\"></path>", "rule": "evenodd"}, "StepFun": {"icon": "stepfun", "tile": "#01d9cf", "shape": "circle", "viewBox": "0 0 24 24", "inner": "<path d=\"M12 0C5.372 0 0 5.372 0 12s5.372 12 12 12 12-5.372 12-12S18.628 0 12 0zM8.762 19.614H4.376v-4.386h4.386v4.386zm5.423 0H9.798v-4.386h4.387v4.386zm0-5.42H9.798V9.81h4.387v4.386zm0-5.418H9.798V4.39h4.387v4.386zm5.422-.004h-4.386V4.386h4.386v4.386z\"></path>", "rule": "evenodd"}, "Undisclosed": {"icon": "unionalpha-color", "viewBox": "0 0 24 24", "inner": "<path d=\"M12.5709 13.3706C12.6556 13.4345 12.8442 13.7666 12.9123 13.875L13.4693 14.7624C13.7592 15.2257 14.0526 15.6895 14.3386 16.1555C14.4141 16.2951 14.4587 16.3467 14.3445 16.48C13.9818 16.9035 13.6097 17.321 13.2426 17.7406C12.6047 18.4698 11.5795 19.0999 10.6086 19.2277C10.5721 19.2324 10.5355 19.2366 10.4988 19.2405C10.3741 19.2219 10.2359 19.2226 10.1067 19.2077C9.62828 19.1531 9.13925 18.9833 8.73437 18.7229C8.58284 18.6255 8.20429 18.365 8.13135 18.2015C8.7166 18.0941 8.9971 17.6275 9.36375 17.2093C9.46647 17.0921 9.56558 16.9681 9.66524 16.8481C10.2482 16.1439 10.8249 15.4346 11.3952 14.7202C11.5532 14.5228 12.4352 13.3557 12.5709 13.3706Z\" fill=\"url(#lobe-icons-union-alpha-0-_R_0_)\"></path><path d=\"M23.9999 4.03711C23.9427 4.17731 23.8557 4.29083 23.7726 4.41626C23.5172 4.80162 23.2489 5.18017 22.9952 5.56645C21.7031 7.52279 20.3758 9.45572 19.0141 11.3643L18.2195 12.4575C18.0914 12.6376 17.8772 12.9025 17.8103 13.1017C17.5635 12.8289 17.3373 12.3638 17.1271 12.052C16.7507 11.4939 16.4007 10.8942 16.0072 10.3494C15.964 10.2787 15.9232 10.1649 15.8967 10.0866C15.9509 9.95674 16.2386 9.57362 16.3367 9.43732L17.184 8.26053L19.1886 5.47039C19.3731 5.21667 20.0937 4.17696 20.2819 4.03711H23.9999Z\" fill=\"url(#lobe-icons-union-alpha-1-_R_0_)\"></path><path d=\"M5.67537 4.61919C5.91433 4.59978 6.08153 4.64306 6.2388 4.65534C7.31097 4.73906 7.93546 5.78123 8.52073 6.55699C8.55486 6.60222 8.67307 6.77678 8.67756 6.81818C8.50211 6.78591 8.24062 6.87736 8.10425 6.98766C7.77451 7.25438 7.47306 7.60773 7.16738 7.90822C6.82713 8.24411 6.49119 8.58431 6.15964 8.92879C5.96629 9.12818 5.68162 9.40902 5.50821 9.61426C4.95769 10.264 4.58519 11.0455 4.42712 11.8823C4.38673 12.1047 4.33596 12.4789 4.34474 12.6979C4.38003 13.5788 4.68343 14.4541 5.20029 15.1681C5.44662 15.5083 5.69406 15.7534 5.97744 16.05C6.42647 16.5145 6.87971 16.9749 7.33712 17.4311C8.17604 18.2704 8.83084 19.0614 10.1068 19.2072C10.2359 19.2221 10.3741 19.2214 10.4989 19.24C10.4563 19.2438 10.4156 19.2437 10.3732 19.2456C9.63946 19.2779 8.9074 19.2593 8.17299 19.2571L6.81265 19.2561C6.34052 19.2551 5.82434 19.2555 5.36063 19.1766C4.33929 19.003 3.55816 18.5177 2.78058 17.8624C1.69732 16.9495 0.760791 15.9389 0.316835 14.5629C0.188539 14.1652 0.0845513 13.7688 0.0423572 13.352C0.0341296 13.2704 0.025306 13.1714 0 13.0938V12.1601C0.024482 12.0989 0.0539558 11.7751 0.0701206 11.682C0.118265 11.3996 0.185469 11.1208 0.271272 10.8475C0.464403 10.234 0.818387 9.53555 1.19037 9.01281C1.67633 8.32991 2.33983 7.69834 2.92227 7.09354L3.99839 5.99473C4.54749 5.44174 4.97245 4.98537 5.67537 4.61919Z\" fill=\"url(#lobe-icons-union-alpha-2-_R_0_)\"></path><path d=\"M16.0072 10.3494C16.4008 10.8942 16.7508 11.4939 17.1271 12.052C17.3373 12.3638 17.5635 12.8289 17.8104 13.1017C17.8351 13.1633 18.024 13.4562 18.0701 13.5307L18.6905 14.5371C19.1466 15.2881 19.6091 16.0351 20.078 16.7781C20.4399 17.3561 20.7699 17.9692 21.2574 18.4512C21.5622 18.7524 21.9167 18.9416 22.3073 19.1101C22.4041 19.1518 22.5241 19.1723 22.6131 19.2159L22.5999 19.2374C22.543 19.2593 22.4118 19.2355 22.3393 19.2386C21.4806 19.2765 20.6139 19.2542 19.7541 19.2467C19.2053 19.2419 18.6361 19.2653 18.0899 19.2461C17.9536 19.241 17.8182 19.2222 17.6855 19.1901C17.2306 19.0825 16.8145 18.8188 16.4914 18.4847C16.2155 18.1993 16.0406 17.8931 15.8347 17.558C15.6277 17.2199 15.423 16.8802 15.2208 16.5391L13.5134 13.7217C13.602 13.5224 14.0393 12.9594 14.2007 12.7419L16.0072 10.3494Z\" fill=\"url(#lobe-icons-union-alpha-3-_R_0_)\"></path><path d=\"M11.3662 4.03706C11.4355 4.04736 11.5279 4.0639 11.5825 4.11302C11.7409 4.25538 11.8717 4.45147 12.0009 4.62253C13.1498 6.14388 14.2121 7.73081 15.2749 9.31286C15.3244 9.38658 15.4528 9.56569 15.4748 9.646C15.3946 9.79571 15.0698 10.1847 14.9522 10.3329L13.8456 11.7243C13.647 11.9762 13.455 12.2345 13.2586 12.4878C13.1566 12.6194 13.0064 12.8305 12.8889 12.9425C12.6125 12.5563 12.3581 12.1531 12.0887 11.7623L9.66701 8.22137C9.43592 7.88192 9.20191 7.54448 8.96502 7.20905C8.92518 7.15337 8.69584 6.86111 8.67758 6.81865C8.67307 6.77723 8.55489 6.60269 8.52077 6.55747C7.9355 5.7817 7.311 4.73956 6.23883 4.65583C6.08162 4.64356 5.91434 4.61968 5.67542 4.61968C6.15935 4.35771 6.84659 4.17302 7.38673 4.09926C8.69114 3.92117 10.0523 4.03706 11.3662 4.03706Z\" fill=\"url(#lobe-icons-union-alpha-4-_R_0_)\"></path><path d=\"M15.3619 8.08232L15.874 8.87602L19.2024 4.16504L15.3619 8.08232Z\" fill=\"url(#lobe-icons-union-alpha-5-_R_0_)\"></path><defs><linearGradient gradientUnits=\"userSpaceOnUse\" id=\"lobe-icons-union-alpha-0-_R_0_\" x1=\"21.5578\" x2=\"21.5578\" y1=\"4.03711\" y2=\"12.691\"><stop stop-color=\"currentColor\" stop-opacity=\"0.39\"></stop><stop offset=\"1\" stop-color=\"currentColor\" stop-opacity=\"0.22\"></stop></linearGradient><linearGradient gradientUnits=\"userSpaceOnUse\" id=\"lobe-icons-union-alpha-1-_R_0_\" x1=\"21.5578\" x2=\"21.5578\" y1=\"4.03711\" y2=\"12.691\"><stop stop-color=\"currentColor\" stop-opacity=\"0.39\"></stop><stop offset=\"1\" stop-color=\"currentColor\" stop-opacity=\"0.22\"></stop></linearGradient><linearGradient gradientUnits=\"userSpaceOnUse\" id=\"lobe-icons-union-alpha-2-_R_0_\" x1=\"5.27425\" x2=\"5.24944\" y1=\"5.41908\" y2=\"14.0217\"><stop stop-color=\"currentColor\" stop-opacity=\"0.15\"></stop><stop offset=\"1\" stop-color=\"currentColor\" stop-opacity=\"0.87\"></stop></linearGradient><linearGradient gradientUnits=\"userSpaceOnUse\" id=\"lobe-icons-union-alpha-3-_R_0_\" x1=\"14.1443\" x2=\"14.1443\" y1=\"14.4319\" y2=\"19.26\"><stop stop-color=\"currentColor\" stop-opacity=\"0.93\"></stop><stop offset=\"1\" stop-color=\"currentColor\" stop-opacity=\"0.57\"></stop></linearGradient><linearGradient gradientUnits=\"userSpaceOnUse\" id=\"lobe-icons-union-alpha-4-_R_0_\" x1=\"14.1443\" x2=\"14.1443\" y1=\"14.4319\" y2=\"19.26\"><stop stop-color=\"currentColor\" stop-opacity=\"0.93\"></stop><stop offset=\"1\" stop-color=\"currentColor\" stop-opacity=\"0.57\"></stop></linearGradient><linearGradient gradientUnits=\"userSpaceOnUse\" id=\"lobe-icons-union-alpha-5-_R_0_\" x1=\"17.2822\" x2=\"17.2822\" y1=\"4.16504\" y2=\"8.87602\"><stop stop-color=\"#D72725\"></stop><stop offset=\"1\" stop-color=\"#CD2920\"></stop></linearGradient></defs>", "rule": null}, "Cognition": {"icon": "devin-color", "viewBox": "0 0 24 24", "inner": "<path d=\"M2.033 9.867l2.554 1.483a.589.589 0 00.592 0l2.554-1.483.01-.008a.608.608 0 00.11-.084l.013-.015a.631.631 0 00.076-.1c.003-.005.008-.01.01-.016a.558.558 0 00.052-.125l.007-.028a.611.611 0 00.019-.14V7.868c0-.572.307-1.105.8-1.392a1.595 1.595 0 011.598 0l1.277.742a.54.54 0 00.129.053l.028.01c.044.01.088.015.133.016h.006l.013-.002a.587.587 0 00.27-.074l.011-.004 2.554-1.483a.596.596 0 00.297-.516V2.253a.595.595 0 00-.297-.516L12.293.257a.587.587 0 00-.591 0L9.148 1.737l-.01.01a.609.609 0 00-.109.083l-.014.015a.632.632 0 00-.076.1c-.003.005-.008.01-.01.016a.57.57 0 00-.052.124l-.007.028a.612.612 0 00-.018.14v1.483c0 .572-.307 1.105-.8 1.393a1.597 1.597 0 01-1.599 0l-1.276-.742a.603.603 0 00-.13-.053l-.028-.008a.658.658 0 00-.133-.018h-.02a.57.57 0 00-.269.074c-.003.002-.008.002-.012.005L2.033 5.872a.596.596 0 00-.297.515v2.966c0 .213.113.41.297.515z\" fill=\"#3969CA\"></path><path d=\"M15.943 10.607a1.596 1.596 0 011.599 0l1.276.74c.041.025.085.04.13.055l.028.008c.043.01.088.016.133.018h.005c.005 0 .01-.002.014-.003a.474.474 0 00.122-.016l.021-.005a.616.616 0 00.126-.052c.004-.002.009-.002.013-.005l2.554-1.482a.597.597 0 00.297-.516V6.383a.596.596 0 00-.297-.515l-2.552-1.483a.587.587 0 00-.592 0l-2.553 1.482-.011.008a.61.61 0 00-.108.084l-.014.016a.637.637 0 00-.076.1c-.003.005-.008.01-.01.016a.57.57 0 00-.052.124l-.007.029a.612.612 0 00-.018.14v1.482c0 .572-.307 1.105-.8 1.393a1.597 1.597 0 01-1.599 0l-1.276-.742a.584.584 0 00-.13-.053l-.028-.008a.62.62 0 00-.133-.018h-.02a.587.587 0 00-.269.074l-.012.004L9.15 10a.596.596 0 00-.296.516v2.966c0 .212.112.409.296.515l2.554 1.483s.008.002.012.005c.04.022.082.04.126.052l.02.004a.57.57 0 00.123.017l.014.002h.006c.054 0 .108-.01.16-.025a.587.587 0 00.13-.054l1.277-.741a1.597 1.597 0 012.398 1.392v1.482c0 .049.007.095.019.14l.007.028a.619.619 0 00.051.125c.004.006.008.01.01.016a.6.6 0 00.076.1l.014.015c.033.032.069.06.108.084.004.002.006.006.011.008l2.554 1.483a.59.59 0 00.593 0l2.554-1.483a.597.597 0 00.296-.516v-2.965a.595.595 0 00-.296-.516l-2.554-1.483s-.008-.002-.012-.005a.54.54 0 00-.126-.051c-.007-.003-.013-.003-.02-.005a.635.635 0 00-.125-.017h-.018a.557.557 0 00-.16.026.588.588 0 00-.13.053l-1.276.742a1.595 1.595 0 01-1.598 0 1.615 1.615 0 010-2.785l-.005-.001z\" fill=\"#21C19A\"></path><path d=\"M14.848 18.265l-2.554-1.482-.012-.005a.526.526 0 00-.126-.052c-.007-.002-.014-.002-.02-.005a.64.64 0 00-.124-.017h-.02a.56.56 0 00-.16.026.588.588 0 00-.13.053l-1.276.742a1.594 1.594 0 01-1.598 0c-.493-.286-.8-.82-.8-1.393V14.65a.563.563 0 00-.018-.14l-.008-.028a.604.604 0 00-.051-.124l-.01-.017a.603.603 0 00-.076-.1l-.014-.015a.596.596 0 00-.109-.084c-.003-.002-.005-.006-.01-.008L5.178 12.65a.587.587 0 00-.591 0l-2.554 1.483a.596.596 0 00-.297.516v2.965c0 .213.113.41.297.516l2.554 1.483.012.004a.618.618 0 00.267.074l.016.002h.007a.55.55 0 00.16-.026.584.584 0 00.129-.053l1.277-.742a1.597 1.597 0 012.398 1.393v1.482c0 .05.007.095.019.14l.007.028c.013.044.03.085.051.125l.01.016c.022.036.047.07.076.1l.014.015c.032.032.069.06.109.084l.01.008 2.554 1.483a.587.587 0 00.593 0l2.554-1.483a.596.596 0 00.296-.515v-2.966a.596.596 0 00-.296-.516h-.002z\" fill=\"#0294DE\"></path>", "rule": null}, "Fireworks": {"icon": "fireworks-color", "viewBox": "0 0 24 24", "inner": "<path clip-rule=\"evenodd\" d=\"M14.8 5l-2.801 6.795L9.195 5H7.397l3.072 7.428a1.64 1.64 0 003.038.002L16.598 5H14.8zm1.196 10.352l5.124-5.244-.699-1.669-5.596 5.739a1.664 1.664 0 00-.343 1.807 1.642 1.642 0 001.516 1.012L16 17l8-.02-.699-1.669-7.303.041h-.002zM2.88 10.104l.699-1.669 5.596 5.739c.468.479.603 1.189.343 1.807a1.643 1.643 0 01-1.516 1.012l-8-.018-.002.002.699-1.669 7.303.042-5.122-5.246z\" fill=\"#5019C5\" fill-rule=\"evenodd\"></path>", "rule": null}, "Stealth": {"wordmark": ["STEALTH"]}, "Thinking Machines": {"wordmark": ["THINKING", "MACHINES"]}, "Alibaba": {"icon": "qwen-color", "viewBox": "0 0 24 24", "inner": "<path d=\"M12.604 1.34c.393.69.784 1.382 1.174 2.075a.18.18 0 00.157.091h5.552c.174 0 .322.11.446.327l1.454 2.57c.19.337.24.478.024.837-.26.43-.513.864-.76 1.3l-.367.658c-.106.196-.223.28-.04.512l2.652 4.637c.172.301.111.494-.043.77-.437.785-.882 1.564-1.335 2.34-.159.272-.352.375-.68.37-.777-.016-1.552-.01-2.327.016a.099.099 0 00-.081.05 575.097 575.097 0 01-2.705 4.74c-.169.293-.38.363-.725.364-.997.003-2.002.004-3.017.002a.537.537 0 01-.465-.271l-1.335-2.323a.09.09 0 00-.083-.049H4.982c-.285.03-.553-.001-.805-.092l-1.603-2.77a.543.543 0 01-.002-.54l1.207-2.12a.198.198 0 000-.197 550.951 550.951 0 01-1.875-3.272l-.79-1.395c-.16-.31-.173-.496.095-.965.465-.813.927-1.625 1.387-2.436.132-.234.304-.334.584-.335a338.3 338.3 0 012.589-.001.124.124 0 00.107-.063l2.806-4.895a.488.488 0 01.422-.246c.524-.001 1.053 0 1.583-.006L11.704 1c.341-.003.724.032.9.34zm-3.432.403a.06.06 0 00-.052.03L6.254 6.788a.157.157 0 01-.135.078H3.253c-.056 0-.07.025-.041.074l5.81 10.156c.025.042.013.062-.034.063l-2.795.015a.218.218 0 00-.2.116l-1.32 2.31c-.044.078-.021.118.068.118l5.716.008c.046 0 .08.02.104.061l1.403 2.454c.046.081.092.082.139 0l5.006-8.76.783-1.382a.055.055 0 01.096 0l1.424 2.53a.122.122 0 00.107.062l2.763-.02a.04.04 0 00.035-.02.041.041 0 000-.04l-2.9-5.086a.108.108 0 010-.113l.293-.507 1.12-1.977c.024-.041.012-.062-.035-.062H9.2c-.059 0-.073-.026-.043-.077l1.434-2.505a.107.107 0 000-.114L9.225 1.774a.06.06 0 00-.053-.031zm6.29 8.02c.046 0 .058.02.034.06l-.832 1.465-2.613 4.585a.056.056 0 01-.05.029.058.058 0 01-.05-.029L8.498 9.841c-.02-.034-.01-.052.028-.054l.216-.012 6.722-.012z\" fill=\"url(#lobe-icons-qwen-_R_0_)\" fill-rule=\"nonzero\"></path><defs><linearGradient id=\"lobe-icons-qwen-_R_0_\" x1=\"0%\" x2=\"100%\" y1=\"0%\" y2=\"0%\"><stop offset=\"0%\" stop-color=\"#6336E7\" stop-opacity=\".84\"></stop><stop offset=\"100%\" stop-color=\"#6F69F7\" stop-opacity=\".84\"></stop></linearGradient></defs>", "rule": null}};
|
| 8 |
for(const [key,DATA] of Object.entries(ALL)){
|
| 9 |
const svg=document.getElementById('bars-'+key), W=1200, H=420;
|
| 10 |
const NS='http://www.w3.org/2000/svg', SANS='Inter, Arial, sans-serif', MONO='"IBM Plex Mono", monospace';
|
assets/pareto.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
import {bindChartTooltip} from "./chart-interactions.js?v=
|
| 2 |
// Adapted from BuildingBench's figure-kit/pareto-frontier/render.py and live SVG.
|
| 3 |
// Domain, costs and scores are exclusively Code4Scene's historical paper evaluation.
|
| 4 |
const NS='http://www.w3.org/2000/svg';
|
|
@@ -19,13 +19,13 @@ for(const svg of document.querySelectorAll('.pareto-svg')){
|
|
| 19 |
for(const c of [100,50,20,10,5,2,1,.5,.2,.1,.05,.02,.01,.005,.002,.001])if(-Math.log10(c)>lo&&-Math.log10(c)<hi){add('line',{x1:x(c),x2:x(c),y1:25.2,y2:473.2,class:'grid vertical'});add('text',{x:x(c),y:499,class:'tick','text-anchor':'middle'},'$'+c);}
|
| 20 |
add('line',{x1:96,x2:1170,y1:473.2,y2:473.2,class:'axis'});
|
| 21 |
add('text',{x:633,y:539,class:'axis-title','text-anchor':'middle'},'Mean Cost per Case (USD) · cheaper →');
|
| 22 |
-
add('text',{x:27,y:249,transform:'rotate(-90 27 249)',class:'axis-title','text-anchor':'middle'},'
|
| 23 |
const path=front.map((d,i)=>(i?'L':'M')+x(d.cost)+','+y(d.score)).join(' ');
|
| 24 |
add('path',{d:path,class:'frontier-halo'});add('path',{d:path,class:'frontier-line'});
|
| 25 |
const points=rows.map(d=>({x:x(d.cost),y:y(d.score)}));const boxes=points.map(p=>[p.x-9,p.y-9,p.x+9,p.y+9]);
|
| 26 |
const linePoints=[];for(let i=1;i<front.length;i++){const a=front[i-1],b=front[i],n=Math.max(2,Math.ceil(Math.hypot(x(b.cost)-x(a.cost),y(b.score)-y(a.score))/5));for(let j=0;j<=n;j++)linePoints.push([x(a.cost)+(x(b.cost)-x(a.cost))*j/n,y(a.score)+(y(b.score)-y(a.score))*j/n]);}
|
| 27 |
for(const d of [...rows].sort((a,b)=>(on.has(b.key)-on.has(a.key))||b.score-a.score)){
|
| 28 |
-
const isFront=on.has(d.key),cx=x(d.cost),cy=y(d.score),font=isFront?16:12.5,name=d.name.replace(/^Claude /,'').replace(/\((.*?)\)/,(_,s)=>'('+s
|
| 29 |
measure.font=`${isFront?700:400} ${size}px Inter`;const tw=measure.measureText(name).width,th=size*.78,r=9,gap=5,pad=2.5,up=r+gap,dn=r+gap+th;
|
| 30 |
const candidates=[];for(let k=0;k<5;k++){const lift=k*th*1.55;candidates.push([0,-up-lift,'middle'],[0,dn+lift,'middle'],[r+gap,th*.4-lift,'start'],[-r-gap,th*.4-lift,'end'],[r+gap,th*.4+lift,'start'],[-r-gap,th*.4+lift,'end'],[r*.6,-up*.82-lift,'start'],[-r*.6,-up*.82-lift,'end'],[r*.6,dn*.9+lift,'start'],[-r*.6,dn*.9+lift,'end']);}
|
| 31 |
const hard=[...boxes,...linePoints.filter(p=>Math.hypot(p[0]-cx,p[1]-cy)>r*2.4).map(p=>[p[0]-4,p[1]-4,p[0]+4,p[1]+4])];let best,bestCost=Infinity;
|
|
|
|
| 1 |
+
import {bindChartTooltip} from "./chart-interactions.js?v=20261004kit";
|
| 2 |
// Adapted from BuildingBench's figure-kit/pareto-frontier/render.py and live SVG.
|
| 3 |
// Domain, costs and scores are exclusively Code4Scene's historical paper evaluation.
|
| 4 |
const NS='http://www.w3.org/2000/svg';
|
|
|
|
| 19 |
for(const c of [100,50,20,10,5,2,1,.5,.2,.1,.05,.02,.01,.005,.002,.001])if(-Math.log10(c)>lo&&-Math.log10(c)<hi){add('line',{x1:x(c),x2:x(c),y1:25.2,y2:473.2,class:'grid vertical'});add('text',{x:x(c),y:499,class:'tick','text-anchor':'middle'},'$'+c);}
|
| 20 |
add('line',{x1:96,x2:1170,y1:473.2,y2:473.2,class:'axis'});
|
| 21 |
add('text',{x:633,y:539,class:'axis-title','text-anchor':'middle'},'Mean Cost per Case (USD) · cheaper →');
|
| 22 |
+
add('text',{x:27,y:249,transform:'rotate(-90 27 249)',class:'axis-title','text-anchor':'middle'},'Text-to-Scene Score');
|
| 23 |
const path=front.map((d,i)=>(i?'L':'M')+x(d.cost)+','+y(d.score)).join(' ');
|
| 24 |
add('path',{d:path,class:'frontier-halo'});add('path',{d:path,class:'frontier-line'});
|
| 25 |
const points=rows.map(d=>({x:x(d.cost),y:y(d.score)}));const boxes=points.map(p=>[p.x-9,p.y-9,p.x+9,p.y+9]);
|
| 26 |
const linePoints=[];for(let i=1;i<front.length;i++){const a=front[i-1],b=front[i],n=Math.max(2,Math.ceil(Math.hypot(x(b.cost)-x(a.cost),y(b.score)-y(a.score))/5));for(let j=0;j<=n;j++)linePoints.push([x(a.cost)+(x(b.cost)-x(a.cost))*j/n,y(a.score)+(y(b.score)-y(a.score))*j/n]);}
|
| 27 |
for(const d of [...rows].sort((a,b)=>(on.has(b.key)-on.has(a.key))||b.score-a.score)){
|
| 28 |
+
const isFront=on.has(d.key),cx=x(d.cost),cy=y(d.score),font=isFront?16:12.5,name=d.name.replace(/^Claude /,'').replace(/\((.*?)\)/,(_,s)=>'('+s.replace(/\b\w/g,c=>c.toUpperCase())+')'),size=font;
|
| 29 |
measure.font=`${isFront?700:400} ${size}px Inter`;const tw=measure.measureText(name).width,th=size*.78,r=9,gap=5,pad=2.5,up=r+gap,dn=r+gap+th;
|
| 30 |
const candidates=[];for(let k=0;k<5;k++){const lift=k*th*1.55;candidates.push([0,-up-lift,'middle'],[0,dn+lift,'middle'],[r+gap,th*.4-lift,'start'],[-r-gap,th*.4-lift,'end'],[r+gap,th*.4+lift,'start'],[-r-gap,th*.4+lift,'end'],[r*.6,-up*.82-lift,'start'],[-r*.6,-up*.82-lift,'end'],[r*.6,dn*.9+lift,'start'],[-r*.6,dn*.9+lift,'end']);}
|
| 31 |
const hard=[...boxes,...linePoints.filter(p=>Math.hypot(p[0]-cx,p[1]-cy)>r*2.4).map(p=>[p[0]-4,p[1]-4,p[0]+4,p[1]+4])];let best,bestCost=Infinity;
|
assets/takeaway-bars.js
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Code4Scene takeaway: requirement-family scores per agent (paper Figure 14b), in the paper figures' kit style --
|
| 2 |
+
// grouped bars in Text-to-Scene rank order, Spatial Composition in the agent's company colour, the other families grey.
|
| 3 |
+
// Generated by scripts/kit_charts.py; the layout follows figure-kit/takeaway.py and the leaderboard bars.
|
| 4 |
+
import {bindRecordTooltip} from "./chart-interactions.js?v=20261004kit";
|
| 5 |
+
const D = {"rows": [{"name": "Claude Fable 5.1 (max)", "org": "Anthropic", "colour": "#D97757", "fam": [0.906, 0.738, 0.67, 0.538]}, {"name": "GPT-6 Astra (max)", "org": "OpenAI", "colour": "#10A37F", "fam": [0.875, 0.715, 0.643, 0.543]}, {"name": "Claude Opus 5 (max)", "org": "Anthropic", "colour": "#D97757", "fam": [0.803, 0.624, 0.573, 0.479]}, {"name": "GPT-5.6 Sol (high)", "org": "OpenAI", "colour": "#10A37F", "fam": [0.75, 0.66, 0.602, 0.469]}, {"name": "Gemini 3.8 Flash (high)", "org": "Google", "colour": "#7b1fa2", "fam": [0.814, 0.639, 0.572, 0.473]}, {"name": "Muse Spark 1.3 (medium)", "org": "Meta", "colour": "#42a5f5", "fam": [0.73, 0.641, 0.546, 0.393]}, {"name": "Grok 4.6 (high)", "org": "xAI", "colour": "#111111", "fam": [0.647, 0.558, 0.457, 0.334]}, {"name": "Qwen 3.8 27B (thinking off)", "org": "Alibaba", "colour": "#fe7016", "fam": [0.724, 0.52, 0.428, 0.3]}, {"name": "Qwen 3.8 27B (thinking on)", "org": "Alibaba", "colour": "#fe7016", "fam": [0.66, 0.525, 0.47, 0.307]}, {"name": "GLM-5.3 Flash (max)", "org": "Z.ai", "colour": "#96650b", "fam": [0.467, 0.421, 0.377, 0.299]}, {"name": "Gemma 4 31B (thinking on)", "org": "Google", "colour": "#7b1fa2", "fam": [0.561, 0.407, 0.387, 0.272]}, {"name": "Gemma 4 31B (thinking off)", "org": "Google", "colour": "#7b1fa2", "fam": [0.612, 0.453, 0.387, 0.252]}, {"name": "Inkling (high)", "org": "Thinking Machines", "colour": "#686868", "fam": [0.572, 0.394, 0.361, 0.193]}, {"name": "DeepSeek V4.1 Flash (high)", "org": "DeepSeek", "colour": "#3f51e0", "fam": [0.294, 0.215, 0.259, 0.12]}], "families": ["Identity & Environment", "Content & Quantity", "Attributes & Materials", "Spatial Composition"], "greys": ["#dedede", "#c6c6c6", "#a9a9a9"]};
|
| 6 |
+
const LOGOS = {"Anthropic": {"icon": "anthropic", "fill": "#141413", "viewBox": "0 0 24 24", "inner": "<path d=\"M13.827 3.52h3.603L24 20h-3.603l-6.57-16.48zm-7.258 0h3.767L16.906 20h-3.674l-1.343-3.461H5.017l-1.344 3.46H0L6.57 3.522zm4.132 9.959L8.453 7.687 6.205 13.48H10.7z\"></path>", "rule": "evenodd"}, "OpenAI": {"icon": "openai", "fill": "#141413", "viewBox": "0 0 24 24", "inner": "<path d=\"M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z\"></path>", "rule": "evenodd"}, "Meta": {"icon": "meta-color", "viewBox": "0 0 24 24", "inner": "<path d=\"M6.897 4h-.024l-.031 2.615h.022c1.715 0 3.046 1.357 5.94 6.246l.175.297.012.02 1.62-2.438-.012-.019a48.763 48.763 0 00-1.098-1.716 28.01 28.01 0 00-1.175-1.629C10.413 4.932 8.812 4 6.896 4z\" fill=\"url(#lobe-icons-meta-0-_R_0_-tk)\"></path><path d=\"M6.873 4C4.95 4.01 3.247 5.258 2.02 7.17a4.352 4.352 0 00-.01.017l2.254 1.231.011-.017c.718-1.083 1.61-1.774 2.568-1.785h.021L6.896 4h-.023z\" fill=\"url(#lobe-icons-meta-1-_R_0_-tk)\"></path><path d=\"M2.019 7.17l-.011.017C1.2 8.447.598 9.995.274 11.664l-.005.022 2.534.6.004-.022c.27-1.467.786-2.828 1.456-3.845l.011-.017L2.02 7.17z\" fill=\"url(#lobe-icons-meta-2-_R_0_-tk)\"></path><path d=\"M2.807 12.264l-2.533-.6-.005.022c-.177.918-.267 1.851-.269 2.786v.023l2.598.233v-.023a12.591 12.591 0 01.21-2.44z\" fill=\"url(#lobe-icons-meta-3-_R_0_-tk)\"></path><path d=\"M2.677 15.537a5.462 5.462 0 01-.079-.813v-.022L0 14.468v.024a8.89 8.89 0 00.146 1.652l2.535-.585a4.106 4.106 0 01-.004-.022z\" fill=\"url(#lobe-icons-meta-4-_R_0_-tk)\"></path><path d=\"M3.27 16.89c-.284-.31-.484-.756-.589-1.328l-.004-.021-2.535.585.004.021c.192 1.01.568 1.85 1.106 2.487l.014.017 2.018-1.745a2.106 2.106 0 01-.015-.016z\" fill=\"url(#lobe-icons-meta-5-_R_0_-tk)\"></path><path d=\"M10.78 9.654c-1.528 2.35-2.454 3.825-2.454 3.825-2.035 3.2-2.739 3.917-3.871 3.917a1.545 1.545 0 01-1.186-.508l-2.017 1.744.014.017C2.01 19.518 3.058 20 4.356 20c1.963 0 3.374-.928 5.884-5.33l1.766-3.13a41.283 41.283 0 00-1.227-1.886z\" fill=\"#0082FB\"></path><path d=\"M13.502 5.946l-.016.016c-.4.43-.786.908-1.16 1.416.378.483.768 1.024 1.175 1.63.48-.743.928-1.345 1.367-1.807l.016-.016-1.382-1.24z\" fill=\"url(#lobe-icons-meta-6-_R_0_-tk)\"></path><path d=\"M20.918 5.713C19.853 4.633 18.583 4 17.225 4c-1.432 0-2.637.787-3.723 1.944l-.016.016 1.382 1.24.016-.017c.715-.747 1.408-1.12 2.176-1.12.826 0 1.6.39 2.27 1.075l.015.016 1.589-1.425-.016-.016z\" fill=\"#0082FB\"></path><path d=\"M23.998 14.125c-.06-3.467-1.27-6.566-3.064-8.396l-.016-.016-1.588 1.424.015.016c1.35 1.392 2.277 3.98 2.361 6.971v.023h2.292v-.022z\" fill=\"url(#lobe-icons-meta-7-_R_0_-tk)\"></path><path d=\"M23.998 14.15v-.023h-2.292v.022c.004.14.006.282.006.424 0 .815-.121 1.474-.368 1.95l-.011.022 1.708 1.782.013-.02c.62-.96.946-2.293.946-3.91 0-.083 0-.165-.002-.247z\" fill=\"url(#lobe-icons-meta-8-_R_0_-tk)\"></path><path d=\"M21.344 16.52l-.011.02c-.214.402-.519.67-.917.787l.778 2.462a3.493 3.493 0 00.438-.182 3.558 3.558 0 001.366-1.218l.044-.065.012-.02-1.71-1.784z\" fill=\"url(#lobe-icons-meta-9-_R_0_-tk)\"></path><path d=\"M19.92 17.393c-.262 0-.492-.039-.718-.14l-.798 2.522c.449.153.927.222 1.46.222.492 0 .943-.073 1.352-.215l-.78-2.462c-.167.05-.341.075-.517.073z\" fill=\"url(#lobe-icons-meta-10-_R_0_-tk)\"></path><path d=\"M18.323 16.534l-.014-.017-1.836 1.914.016.017c.637.682 1.246 1.105 1.937 1.337l.797-2.52c-.291-.125-.573-.353-.9-.731z\" fill=\"url(#lobe-icons-meta-11-_R_0_-tk)\"></path><path d=\"M18.309 16.515c-.55-.642-1.232-1.712-2.303-3.44l-1.396-2.336-.011-.02-1.62 2.438.012.02.989 1.668c.959 1.61 1.74 2.774 2.493 3.585l.016.016 1.834-1.914a2.353 2.353 0 01-.014-.017z\" fill=\"url(#lobe-icons-meta-12-_R_0_-tk)\"></path><defs><linearGradient id=\"lobe-icons-meta-0-_R_0_-tk\" x1=\"75.897%\" x2=\"26.312%\" y1=\"89.199%\" y2=\"12.194%\"><stop offset=\".06%\" stop-color=\"#0867DF\"></stop><stop offset=\"45.39%\" stop-color=\"#0668E1\"></stop><stop offset=\"85.91%\" stop-color=\"#0064E0\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-1-_R_0_-tk\" x1=\"21.67%\" x2=\"97.068%\" y1=\"75.874%\" y2=\"23.985%\"><stop offset=\"13.23%\" stop-color=\"#0064DF\"></stop><stop offset=\"99.88%\" stop-color=\"#0064E0\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-2-_R_0_-tk\" x1=\"38.263%\" x2=\"60.895%\" y1=\"89.127%\" y2=\"16.131%\"><stop offset=\"1.47%\" stop-color=\"#0072EC\"></stop><stop offset=\"68.81%\" stop-color=\"#0064DF\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-3-_R_0_-tk\" x1=\"47.032%\" x2=\"52.15%\" y1=\"90.19%\" y2=\"15.745%\"><stop offset=\"7.31%\" stop-color=\"#007CF6\"></stop><stop offset=\"99.43%\" stop-color=\"#0072EC\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-4-_R_0_-tk\" x1=\"52.155%\" x2=\"47.591%\" y1=\"58.301%\" y2=\"37.004%\"><stop offset=\"7.31%\" stop-color=\"#007FF9\"></stop><stop offset=\"100%\" stop-color=\"#007CF6\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-5-_R_0_-tk\" x1=\"37.689%\" x2=\"61.961%\" y1=\"12.502%\" y2=\"63.624%\"><stop offset=\"7.31%\" stop-color=\"#007FF9\"></stop><stop offset=\"100%\" stop-color=\"#0082FB\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-6-_R_0_-tk\" x1=\"34.808%\" x2=\"62.313%\" y1=\"68.859%\" y2=\"23.174%\"><stop offset=\"27.99%\" stop-color=\"#007FF8\"></stop><stop offset=\"91.41%\" stop-color=\"#0082FB\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-7-_R_0_-tk\" x1=\"43.762%\" x2=\"57.602%\" y1=\"6.235%\" y2=\"98.514%\"><stop offset=\"0%\" stop-color=\"#0082FB\"></stop><stop offset=\"99.95%\" stop-color=\"#0081FA\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-8-_R_0_-tk\" x1=\"60.055%\" x2=\"39.88%\" y1=\"4.661%\" y2=\"69.077%\"><stop offset=\"6.19%\" stop-color=\"#0081FA\"></stop><stop offset=\"100%\" stop-color=\"#0080F9\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-9-_R_0_-tk\" x1=\"30.282%\" x2=\"61.081%\" y1=\"59.32%\" y2=\"33.244%\"><stop offset=\"0%\" stop-color=\"#027AF3\"></stop><stop offset=\"100%\" stop-color=\"#0080F9\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-10-_R_0_-tk\" x1=\"20.433%\" x2=\"82.112%\" y1=\"50.001%\" y2=\"50.001%\"><stop offset=\"0%\" stop-color=\"#0377EF\"></stop><stop offset=\"99.94%\" stop-color=\"#0279F1\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-11-_R_0_-tk\" x1=\"40.303%\" x2=\"72.394%\" y1=\"35.298%\" y2=\"57.811%\"><stop offset=\".19%\" stop-color=\"#0471E9\"></stop><stop offset=\"100%\" stop-color=\"#0377EF\"></stop></linearGradient><linearGradient id=\"lobe-icons-meta-12-_R_0_-tk\" x1=\"32.254%\" x2=\"68.003%\" y1=\"19.719%\" y2=\"84.908%\"><stop offset=\"27.65%\" stop-color=\"#0867DF\"></stop><stop offset=\"100%\" stop-color=\"#0471E9\"></stop></linearGradient></defs>", "rule": null}, "Google": {"icon": "google-color", "viewBox": "0 0 24 24", "inner": "<path d=\"M23 12.245c0-.905-.075-1.565-.236-2.25h-10.54v4.083h6.186c-.124 1.014-.797 2.542-2.294 3.569l-.021.136 3.332 2.53.23.022C21.779 18.417 23 15.593 23 12.245z\" fill=\"#4285F4\"></path><path d=\"M12.225 23c3.03 0 5.574-.978 7.433-2.665l-3.542-2.688c-.948.648-2.22 1.1-3.891 1.1a6.745 6.745 0 01-6.386-4.572l-.132.011-3.465 2.628-.045.124C4.043 20.531 7.835 23 12.225 23z\" fill=\"#34A853\"></path><path d=\"M5.84 14.175A6.65 6.65 0 015.463 12c0-.758.138-1.491.361-2.175l-.006-.147-3.508-2.67-.115.054A10.831 10.831 0 001 12c0 1.772.436 3.447 1.197 4.938l3.642-2.763z\" fill=\"#FBBC05\"></path><path d=\"M12.225 5.253c2.108 0 3.529.892 4.34 1.638l3.167-3.031C17.787 2.088 15.255 1 12.225 1 7.834 1 4.043 3.469 2.197 7.062l3.63 2.763a6.77 6.77 0 016.398-4.572z\" fill=\"#EB4335\"></path>", "rule": null}, "DeepSeek": {"icon": "deepseek-color", "viewBox": "0 0 24 24", "inner": "<path d=\"M23.748 4.482c-.254-.124-.364.113-.512.234-.051.039-.094.09-.137.136-.372.397-.806.657-1.373.626-.829-.046-1.537.214-2.163.848-.133-.782-.575-1.248-1.247-1.548-.352-.156-.708-.311-.955-.65-.172-.241-.219-.51-.305-.774-.055-.16-.11-.323-.293-.35-.2-.031-.278.136-.356.276-.313.572-.434 1.202-.422 1.84.027 1.436.633 2.58 1.838 3.393.137.093.172.187.129.323-.082.28-.18.552-.266.833-.055.179-.137.217-.329.14a5.526 5.526 0 01-1.736-1.18c-.857-.828-1.631-1.742-2.597-2.458a11.365 11.365 0 00-.689-.471c-.985-.957.13-1.743.388-1.836.27-.098.093-.432-.779-.428-.872.004-1.67.295-2.687.684a3.055 3.055 0 01-.465.137 9.597 9.597 0 00-2.883-.102c-1.885.21-3.39 1.102-4.497 2.623C.082 8.606-.231 10.684.152 12.85c.403 2.284 1.569 4.175 3.36 5.653 1.858 1.533 3.997 2.284 6.438 2.14 1.482-.085 3.133-.284 4.994-1.86.47.234.962.327 1.78.397.63.059 1.236-.03 1.705-.128.735-.156.684-.837.419-.961-2.155-1.004-1.682-.595-2.113-.926 1.096-1.296 2.746-2.642 3.392-7.003.05-.347.007-.565 0-.845-.004-.17.035-.237.23-.256a4.173 4.173 0 001.545-.475c1.396-.763 1.96-2.015 2.093-3.517.02-.23-.004-.467-.247-.588zM11.581 18c-2.089-1.642-3.102-2.183-3.52-2.16-.392.024-.321.471-.235.763.09.288.207.486.371.739.114.167.192.416-.113.603-.673.416-1.842-.14-1.897-.167-1.361-.802-2.5-1.86-3.301-3.307-.774-1.393-1.224-2.887-1.298-4.482-.02-.386.093-.522.477-.592a4.696 4.696 0 011.529-.039c2.132.312 3.946 1.265 5.468 2.774.868.86 1.525 1.887 2.202 2.891.72 1.066 1.494 2.082 2.48 2.914.348.292.625.514.891.677-.802.09-2.14.11-3.054-.614zm1-6.44a.306.306 0 01.415-.287.302.302 0 01.2.288.306.306 0 01-.31.307.303.303 0 01-.304-.308zm3.11 1.596c-.2.081-.399.151-.59.16a1.245 1.245 0 01-.798-.254c-.274-.23-.47-.358-.552-.758a1.73 1.73 0 01.016-.588c.07-.327-.008-.537-.239-.727-.187-.156-.426-.199-.688-.199a.559.559 0 01-.254-.078c-.11-.054-.2-.19-.114-.358.028-.054.16-.186.192-.21.356-.202.767-.136 1.146.016.352.144.618.408 1.001.782.391.451.462.576.685.914.176.265.336.537.445.848.067.195-.019.354-.25.452z\" fill=\"#4D6BFE\"></path>", "rule": null}, "xAI": {"icon": "xai", "tile": "#111111", "shape": "square", "viewBox": "0 0 24 24", "inner": "<path d=\"M6.469 8.776L16.512 23h-4.464L2.005 8.776H6.47zm-.004 7.9l2.233 3.164L6.467 23H2l4.465-6.324zM22 2.582V23h-3.659V7.764L22 2.582zM22 1l-9.952 14.095-2.233-3.163L17.533 1H22z\"></path>", "rule": "evenodd"}, "Z.ai": {"icon": "zai", "tile": "#111111", "shape": "square", "viewBox": "0 0 24 24", "inner": "<path d=\"M12.105 2L9.927 4.953H.653L2.83 2h9.276zM23.254 19.048L21.078 22h-9.242l2.174-2.952h9.244zM24 2L9.264 22H0L14.736 2H24z\"></path>", "rule": "evenodd"}, "Alibaba": {"icon": "qwen-color", "viewBox": "0 0 24 24", "inner": "<path d=\"M12.604 1.34c.393.69.784 1.382 1.174 2.075a.18.18 0 00.157.091h5.552c.174 0 .322.11.446.327l1.454 2.57c.19.337.24.478.024.837-.26.43-.513.864-.76 1.3l-.367.658c-.106.196-.223.28-.04.512l2.652 4.637c.172.301.111.494-.043.77-.437.785-.882 1.564-1.335 2.34-.159.272-.352.375-.68.37-.777-.016-1.552-.01-2.327.016a.099.099 0 00-.081.05 575.097 575.097 0 01-2.705 4.74c-.169.293-.38.363-.725.364-.997.003-2.002.004-3.017.002a.537.537 0 01-.465-.271l-1.335-2.323a.09.09 0 00-.083-.049H4.982c-.285.03-.553-.001-.805-.092l-1.603-2.77a.543.543 0 01-.002-.54l1.207-2.12a.198.198 0 000-.197 550.951 550.951 0 01-1.875-3.272l-.79-1.395c-.16-.31-.173-.496.095-.965.465-.813.927-1.625 1.387-2.436.132-.234.304-.334.584-.335a338.3 338.3 0 012.589-.001.124.124 0 00.107-.063l2.806-4.895a.488.488 0 01.422-.246c.524-.001 1.053 0 1.583-.006L11.704 1c.341-.003.724.032.9.34zm-3.432.403a.06.06 0 00-.052.03L6.254 6.788a.157.157 0 01-.135.078H3.253c-.056 0-.07.025-.041.074l5.81 10.156c.025.042.013.062-.034.063l-2.795.015a.218.218 0 00-.2.116l-1.32 2.31c-.044.078-.021.118.068.118l5.716.008c.046 0 .08.02.104.061l1.403 2.454c.046.081.092.082.139 0l5.006-8.76.783-1.382a.055.055 0 01.096 0l1.424 2.53a.122.122 0 00.107.062l2.763-.02a.04.04 0 00.035-.02.041.041 0 000-.04l-2.9-5.086a.108.108 0 010-.113l.293-.507 1.12-1.977c.024-.041.012-.062-.035-.062H9.2c-.059 0-.073-.026-.043-.077l1.434-2.505a.107.107 0 000-.114L9.225 1.774a.06.06 0 00-.053-.031zm6.29 8.02c.046 0 .058.02.034.06l-.832 1.465-2.613 4.585a.056.056 0 01-.05.029.058.058 0 01-.05-.029L8.498 9.841c-.02-.034-.01-.052.028-.054l.216-.012 6.722-.012z\" fill=\"url(#lobe-icons-qwen-_R_0_-tk)\" fill-rule=\"nonzero\"></path><defs><linearGradient id=\"lobe-icons-qwen-_R_0_-tk\" x1=\"0%\" x2=\"100%\" y1=\"0%\" y2=\"0%\"><stop offset=\"0%\" stop-color=\"#6336E7\" stop-opacity=\".84\"></stop><stop offset=\"100%\" stop-color=\"#6F69F7\" stop-opacity=\".84\"></stop></linearGradient></defs>", "rule": null}, "Thinking Machines": {"wordmark": ["THINKING", "MACHINES"]}};
|
| 7 |
+
await Promise.all(['400 13px Inter', '700 13px Inter', '500 12.5px "IBM Plex Mono"'].map(f => document.fonts.load(f)));
|
| 8 |
+
const svg = document.getElementById('tk-families'), W = 1200, H = 410, NS = 'http://www.w3.org/2000/svg';
|
| 9 |
+
const SANS = 'Inter, Arial, sans-serif', MONO = '"IBM Plex Mono", monospace';
|
| 10 |
+
const G = {left: 40, right: 24, top: 34, base: 262, groupFill: 0.80, gap: 2, radius: 3.5,
|
| 11 |
+
logo: 18, logoGap: 8, nameGap: 9, nameAngle: -60, nameSize: 13, nameLead: 14.5, nameOneLine: 97, nameLine: 93, nameTol: 12};
|
| 12 |
+
function el(tag, attrs, parent) { const e = document.createElementNS(NS, tag); for (const k in attrs) e.setAttribute(k, attrs[k]); (parent || svg).appendChild(e); return e; }
|
| 13 |
+
function text(x, y, s, attrs, parent) { const t = el('text', Object.assign({x, y}, attrs), parent); t.textContent = s; return t; }
|
| 14 |
+
const ctx = document.createElement('canvas').getContext('2d');
|
| 15 |
+
function width(t, font) { ctx.font = font; return ctx.measureText(t).width; }
|
| 16 |
+
function wrap(name, font) {
|
| 17 |
+
if (width(name, font) <= G.nameOneLine) return [name];
|
| 18 |
+
const w = name.split(' ');
|
| 19 |
+
const splits = k => { const out = [];
|
| 20 |
+
if (k === 2) for (let i = 1; i < w.length; i++) out.push([w.slice(0, i), w.slice(i)]);
|
| 21 |
+
if (k === 3) for (let i = 1; i < w.length; i++) for (let j = i + 1; j < w.length; j++) out.push([w.slice(0, i), w.slice(i, j), w.slice(j)]);
|
| 22 |
+
return out.map(p => p.map(x => x.join(' '))); };
|
| 23 |
+
for (const k of [2, 3]) {
|
| 24 |
+
const c = splits(k).map(lines => { const ws = lines.map(l => width(l, font)); return {lines, worst: Math.max(...ws), last: ws[ws.length - 1]}; })
|
| 25 |
+
.filter(o => o.worst <= G.nameLine);
|
| 26 |
+
if (!c.length) continue;
|
| 27 |
+
const floor = Math.min(...c.map(o => o.worst));
|
| 28 |
+
return c.filter(o => o.worst <= floor + G.nameTol).sort((a, b) => b.last - a.last)[0].lines;
|
| 29 |
+
}
|
| 30 |
+
return [name];
|
| 31 |
+
}
|
| 32 |
+
function logo(org, cx, ly, parent) {
|
| 33 |
+
const L = LOGOS[org], s = G.logo, lx = cx - s / 2;
|
| 34 |
+
if (L.wordmark) { L.wordmark.forEach((w, k) => text(cx, ly + 7.5 + k * 6.2, w, {'text-anchor': 'middle', 'font-family': SANS, 'font-weight': 500, 'font-size': 5.2, 'letter-spacing': 0.35, fill: '#4a4a4a'}, parent)); return; }
|
| 35 |
+
let pad = 0, fill = L.fill || null;
|
| 36 |
+
if (L.tile) { if (L.shape === 'circle') el('circle', {cx, cy: ly + s / 2, r: s / 2, fill: L.tile}, parent); else el('rect', {x: lx, y: ly, width: s, height: s, rx: 3.2, fill: L.tile}, parent); pad = 3.4; fill = '#ffffff'; }
|
| 37 |
+
const ic = el('svg', {x: lx + pad, y: ly + pad, width: s - 2 * pad, height: s - 2 * pad, viewBox: L.viewBox}, parent);
|
| 38 |
+
if (fill) { ic.setAttribute('fill', fill); ic.setAttribute('color', fill); }
|
| 39 |
+
if (L.rule) ic.setAttribute('fill-rule', L.rule);
|
| 40 |
+
ic.innerHTML = L.inner;
|
| 41 |
+
}
|
| 42 |
+
const y = v => G.base - v / 100 * (G.base - G.top);
|
| 43 |
+
for (const v of [25, 50, 75, 100])
|
| 44 |
+
el('line', {x1: G.left - 8, x2: W - G.right, y1: y(v), y2: y(v), stroke: '#cfcfcf', 'stroke-width': 1.4, 'stroke-dasharray': '0.1 4.2', 'stroke-linecap': 'round'});
|
| 45 |
+
el('line', {x1: G.left - 8, x2: W - G.right, y1: G.base, y2: G.base, stroke: '#e6e6e6', 'stroke-width': 1});
|
| 46 |
+
const n = D.rows.length, pitch = (W - G.left - G.right) / n, gw = pitch * G.groupFill, bw = (gw - 3 * G.gap) / 4;
|
| 47 |
+
D.rows.forEach((d, i) => {
|
| 48 |
+
const cx = G.left + (i + 0.5) * pitch, g0 = cx - gw / 2;
|
| 49 |
+
const node = el('g', {class: 'bar-node'});
|
| 50 |
+
el('rect', {x: G.left + i * pitch, y: G.top - 20, width: pitch, height: H - G.top + 20, fill: 'transparent', class: 'bar-hit'}, node);
|
| 51 |
+
d.fam.forEach((v, j) => {
|
| 52 |
+
const x0 = g0 + j * (bw + G.gap), top = y(v * 100), r = Math.min(G.radius, bw / 2);
|
| 53 |
+
el('path', {class: j === 3 ? 'bar' : 'bar fam', d: `M${x0},${G.base} V${top + r} Q${x0},${top} ${x0 + r},${top} H${x0 + bw - r} Q${x0 + bw},${top} ${x0 + bw},${top + r} V${G.base} Z`,
|
| 54 |
+
fill: j === 3 ? d.colour : D.greys[j]}, node);
|
| 55 |
+
if (j === 3) text(x0 + bw / 2, top - 6, String(Math.round(v * 100)), {'text-anchor': 'middle', 'font-family': MONO, 'font-weight': 500, 'font-size': 12.5, fill: '#262626'}, node);
|
| 56 |
+
});
|
| 57 |
+
logo(d.org, cx, G.base + G.logoGap, node);
|
| 58 |
+
const font = `400 ${G.nameSize}px ${SANS}`, lines = wrap(d.name, font);
|
| 59 |
+
const g = el('g', {transform: `translate(${cx + 3},${G.base + G.logoGap + G.logo + G.nameGap}) rotate(${G.nameAngle})`}, node);
|
| 60 |
+
const off = -(lines.length - 1) / 2 * G.nameLead;
|
| 61 |
+
lines.forEach((ln, k) => text(0, off + k * G.nameLead + 4.5, ln, {'text-anchor': 'end', 'font-family': SANS, 'font-size': G.nameSize, fill: '#262626'}, g));
|
| 62 |
+
bindRecordTooltip(node, svg, {name: d.name, provider: d.org,
|
| 63 |
+
metrics: D.families.map((f, j) => ({label: f, value: d.fam[j].toFixed(3)}))}, '.bar:not(.fam)');
|
| 64 |
+
});
|
| 65 |
+
svg.dataset.ready = 'true';
|
index.html
CHANGED
|
@@ -19,11 +19,11 @@ svg.ch .dim{opacity:.42}svg.ch g.row:hover{opacity:1}svg.ch .hot{fill:var(--c4s-
|
|
| 19 |
.chip.ok{color:var(--c4s-ok);border-color:currentColor}.chip.no{color:var(--c4s-hot);border-color:currentColor}.chip small{font-weight:500;color:var(--text-muted);margin-left:4px}
|
| 20 |
.bigstats{display:flex;flex-wrap:wrap;gap:12px 34px;margin:0 0 12px}.bigstats b{display:block;font-size:30px;letter-spacing:-.02em;font-variant-numeric:tabular-nums}
|
| 21 |
.bigstats span{font-size:13px;color:var(--text-secondary)}
|
| 22 |
-
</style><link href="assets/buildingbench.css?v=
|
| 23 |
<header class="site-header"><div class="brand"><a aria-label="Code4Scene home" class="brand-home" href="index.html#top"><svg aria-hidden="true" fill="none" viewbox="0 0 30 32"><path d="M15 2 27 9v14l-12 7L3 23V9L15 2Z M3 9l12 7 12-7 M15 16v14 M9 5.5l12 7v14 M21 5.5l-12 7v14" stroke="currentColor" stroke-linejoin="round" stroke-width="1.6"></path></svg><span>Code4Scene</span></a><a class="brand-by" href="https://simworld.org/">by SimWorld</a></div><nav aria-label="Main navigation"><a class="nav-active" href="index.html#top">Benchmark</a><a href="index.html#leaderboard">Leaderboard</a><a href="index.html#cases">Visualization</a><a href="https://arxiv.org/abs/2609.36777" rel="noopener" target="_blank">Paper ↗</a><a href="https://github.com/SimWorld-AI/Code4Scene" rel="noopener" target="_blank">Github ↗</a></nav></header><main class="page-main"><section class="intro" id="top"><div class="intro-copy"><h1>Constructing<br/><em>3D Scenes with Code</em></h1><p class="intro-text">Benchmarking coding agents that turn open-ended scene descriptions into engine-native 3D scenes in Unreal Engine. Measuring spatial reasoning, task fulfillment and physical validity.</p><div class="intro-links"><a href="https://arxiv.org/abs/2609.36777">Read the paper ↗</a><a href="https://github.com/SimWorld-AI/Code4Scene">Code & Data ↗</a><a href="#hero-video">Watch the teaser ↘</a></div></div><div class="featured featured-video" id="hero-video"><video aria-label="Code4Scene overview video" controls="" playsinline="" poster="media/teaser-poster.jpg" preload="metadata"><source src="media/teaser-v31.mp4" type="video/mp4"/>Your browser does not support embedded video. <a href="media/teaser-v31.mp4">Watch the Code4Scene video</a>.</video><div class="featured-video-caption"><strong>Code4Scene in 90 seconds</strong><span>Construction and evaluation</span></div></div></section><section class="task-section" id="task-format"><div class="container"><h2 class="section-title">Task Format</h2><div class="task-shell"><div aria-labelledby="task-tab-t2s" class="task-content" data-task="t2s" id="task-t2s" role="tabpanel"><h3>Nordic Harbour <span>· Text-to-Scene</span></h3><p class="task-subtitle">An open-ended scene description, an asset catalog and the Unreal Engine editor.</p><div class="task-columns"><article class="task-panel"><div class="panel-heading"><strong>Agent input</strong><span>Text + assets</span></div><div class="prompt-preview"><span class="eyebrow">Task instruction</span><p>Build me a compact, sunlit Copenhagen-style canal district with narrow cobblestone streets and rectangular waterways enclosing dense urban blocks.</p><p class="prompt-detail">Townhouses, canals, bridges and a waterfront promenade — with detailed requirements for materials, layout and street furniture.</p></div><p class="task-foot"><strong>Given:</strong> an empty level and the content pack’s asset catalog.</p></article><article class="task-panel output-panel"><div class="panel-heading"><strong>Agent output</strong><span>GPT-6 Astra</span></div><div class="live-scene" data-agent="astra" data-case="nordic-harbour"><canvas aria-label="Interactive 3D scene: Nordic Harbour. Drag to orbit, scroll or use plus and minus to zoom." tabindex="0"></canvas><span class="scene-loading" role="status">Loading 3D scene…</span><span class="scene-hint">Drag to orbit · Scroll to zoom</span><button aria-label="Pause scene rotation" class="scene-motion">Ⅱ</button></div><a class="task-foot text-button" href="cases.html#nordic-harbour">Compare agents and evaluator views ↗</a></article><article class="task-panel evaluation-panel"><div class="panel-heading"><strong>Evaluation</strong><span>Engine-native scene</span></div><div class="score-block"><span>CASE SCORE</span><strong>0.877</strong><dl><div><dt>Detailed Alignment</dt><dd>0.757</dd></div><div><dt>Overview Alignment</dt><dd>0.909</dd></div><div><dt>Physical Safety</dt><dd>0.900</dd></div></dl></div><p class="task-foot">Task fulfillment, artifact integrity and static physical validity.</p><p class="task-formula">0.2 × Detailed + 0.6 × Overview + 0.2 × Physical</p></article></div><details class="task-full"><summary><span>+ Full task instruction</span><span>TEXT-TO-SCENE</span></summary><p>Build me a compact, sunlit Copenhagen-style canal district with narrow cobblestone streets and rectangular waterways enclosing dense urban blocks. Arrange roughly thirty attached four- to six-storey townhouses in continuous street and canal-front rows, using slender façades, repetitive white-framed windows, arched doors and carriage entrances, modest cornices, and steep tiled roofs packed with dormers and brick chimneys. Include occasional exposed party walls between staggered building rows. Finish the façades in weathered muted red, mustard yellow, dusty blue, warm grey, beige, and exposed brick, with red, orange, and charcoal roof tiles. Run a principal waterfront street along a stone quay, crossed by smaller streets and linked across the canals by short, gently arched stone-and-metal bridges, including one opening onto a broad paved promenade with rounded landings. Form the canal edges from stepped pale stone, dark timber retaining walls, and iron railings around reflective rippling water. Add regularly spaced leafy trees, ornate black lamps, stone bollards, benches, European road signs, zebra crossings, puddles, grime, and sparse weeds.</p></details></div></div><div class="availability"><span><b>160</b> construction cases in the full benchmark target, including private cases</span><span><b>129</b> public construction cases</span></div></div></section><section id="leaderboard"><div class="container">
|
| 24 |
<div class="section-header centered"><span class="section-label">Leaderboard</span><h2 class="section-title">Leaderboard</h2>
|
| 25 |
<p class="section-desc">14 coding-agent configurations · paper results on the original 20 public construction cases.</p></div>
|
| 26 |
-
<div id="lbwrap"><div class="card bar-card"><div class="bar-heading"><h3>Code4Scene</h3><span>Paper evaluation · score out of 100</span></div><p class="bar-mobile-hint">Swipe to view all 14 configurations →</p><div aria-label="All 14 model configurations" class="bar-scroll" role="region" tabindex="0"><div data-view="t2s"><svg aria-label="Text-to-Scene scores for the 14 Code4Scene paper configurations" class="kit-bars" id="bars-t2s" role="group" viewbox="0 0 1200 420"></svg></div></div><p class="note">Sorted by score. Bar labels round scores ×100; the table preserves three decimal places. One color per model provider.</p></div>
|
| 27 |
<div class="card chart-card pareto-card"><h3 class="figure-title">Code4Scene <span>Pareto Frontier</span></h3><p class="figure-subtitle">Quality against the cost of one case · original 20-case public evaluation</p><p class="pareto-swipe">Swipe to explore all configurations →</p><div class="pareto-view" data-view="t2s"><div class="pareto-scroll"><svg aria-label="t2s score against average cost per case. More expensive runs on the left; logarithmic scale." class="pareto-svg" data-mode="t2s" role="group" viewbox="0 0 1200 560"></svg></div></div><p class="figure-caption">Solid: the frontier — nothing cheaper scores higher. One point per configuration, at its evaluated reasoning effort. Cost is the mean USD per case.</p></div>
|
| 28 |
<div class="card ranking-card"><div class="ranking-toolbar"><label for="model-search">Search configurations<input autocomplete="off" id="model-search" placeholder="Model or provider" type="search"/></label><span aria-live="polite" id="ranking-count">14 configurations</span></div><div class="sub-title">Ranking</div><div data-view="t2s"><p class="board-note" style="margin:0 .2rem .7rem">20 public cases. Case score = 0.2 × Detailed + 0.6 × Overview + 0.2 × Physical. Detailed and Overview are means of the per-case scores.</p><div class="board-wrap"><table class="board"><thead><tr><th>#</th><th>Agent configuration</th><th class="sorted" data-best="high" data-k="st">Score</th><th data-best="high" data-k="detailed">Detailed</th><th data-best="high" data-k="overview">Overview</th><th data-best="high" data-k="pt">Physical</th><th data-best="low" data-k="c_t2s">Cost / case</th></tr></thead><tbody><tr data-model-key="fable" data-search="claude fable 5.1 (max) anthropic"><td class="rk">1</td><td class="nm"><span class="logo"><svg fill="#D97757" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Claude</title><path d="M4.709 15.955l4.72-2.647.08-.23-.08-.128H9.2l-.79-.048-2.698-.073-2.339-.097-2.266-.122-.571-.121L0 11.784l.055-.352.48-.321.686.06 1.52.103 2.278.158 1.652.097 2.449.255h.389l.055-.157-.134-.098-.103-.097-2.358-1.596-2.552-1.688-1.336-.972-.724-.491-.364-.462-.158-1.008.656-.722.881.06.225.061.893.686 1.908 1.476 2.491 1.833.365.304.145-.103.019-.073-.164-.274-1.355-2.446-1.446-2.49-.644-1.032-.17-.619a2.97 2.97 0 01-.104-.729L6.283.134 6.696 0l.996.134.42.364.62 1.414 1.002 2.229 1.555 3.03.456.898.243.832.091.255h.158V9.01l.128-1.706.237-2.095.23-2.695.08-.76.376-.91.747-.492.584.28.48.685-.067.444-.286 1.851-.559 2.903-.364 1.942h.212l.243-.242.985-1.306 1.652-2.064.73-.82.85-.904.547-.431h1.033l.76 1.129-.34 1.166-1.064 1.347-.881 1.142-1.264 1.7-.79 1.36.073.11.188-.02 2.856-.606 1.543-.28 1.841-.315.833.388.091.395-.328.807-1.969.486-2.309.462-3.439.813-.042.03.049.061 1.549.146.662.036h1.622l3.02.225.79.522.474.638-.079.485-1.215.62-1.64-.389-3.829-.91-1.312-.329h-.182v.11l1.093 1.068 2.006 1.81 2.509 2.33.127.578-.322.455-.34-.049-2.205-1.657-.851-.747-1.926-1.62h-.128v.17l.444.649 2.345 3.521.122 1.08-.17.353-.608.213-.668-.122-1.374-1.925-1.415-2.167-1.143-1.943-.14.08-.674 7.254-.316.37-.729.28-.607-.461-.322-.747.322-1.476.389-1.924.315-1.53.286-1.9.17-.632-.012-.042-.14.018-1.434 1.967-2.18 2.945-1.726 1.845-.414.164-.717-.37.067-.662.401-.589 2.388-3.036 1.44-1.882.93-1.086-.006-.158h-.055L4.132 18.56l-1.13.146-.487-.456.061-.746.231-.243 1.908-1.312-.006.006z"></path></svg></span>Claude Fable 5.1 (max)<span class="tag front">Pareto</span><span class="org">Anthropic</span></td><td data-v="0.78775"><div class="cellbar"><i style="width:80.0px;--bar:var(--c4s-a)"></i><b>0.788</b></div></td><td class="dim" data-v="0.7253149999999999">0.725</td><td class="dim" data-v="0.7717050000000001">0.772</td><td class="dim" data-v="0.898">0.898</td><td class="dim" data-v="18.1576316">$18.16</td></tr><tr data-model-key="astra" data-search="gpt-6 astra (max) openai"><td class="rk">2</td><td class="nm"><span class="logo"><svg fill="currentColor" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>OpenAI</title><path d="M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z"></path></svg></span>GPT-6 Astra (max)<span class="org">OpenAI</span></td><td data-v="0.72382"><div class="cellbar"><i style="width:73.5px;--bar:var(--c4s-a)"></i><b>0.724</b></div></td><td class="dim" data-v="0.707165">0.707</td><td class="dim" data-v="0.733235">0.733</td><td class="dim" data-v="0.712">0.712</td><td class="dim" data-v="22.472260736842106">$22.47</td></tr><tr data-model-key="opus" data-search="claude opus 5 (max) anthropic"><td class="rk">3</td><td class="nm"><span class="logo"><svg fill="#D97757" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Claude</title><path d="M4.709 15.955l4.72-2.647.08-.23-.08-.128H9.2l-.79-.048-2.698-.073-2.339-.097-2.266-.122-.571-.121L0 11.784l.055-.352.48-.321.686.06 1.52.103 2.278.158 1.652.097 2.449.255h.389l.055-.157-.134-.098-.103-.097-2.358-1.596-2.552-1.688-1.336-.972-.724-.491-.364-.462-.158-1.008.656-.722.881.06.225.061.893.686 1.908 1.476 2.491 1.833.365.304.145-.103.019-.073-.164-.274-1.355-2.446-1.446-2.49-.644-1.032-.17-.619a2.97 2.97 0 01-.104-.729L6.283.134 6.696 0l.996.134.42.364.62 1.414 1.002 2.229 1.555 3.03.456.898.243.832.091.255h.158V9.01l.128-1.706.237-2.095.23-2.695.08-.76.376-.91.747-.492.584.28.48.685-.067.444-.286 1.851-.559 2.903-.364 1.942h.212l.243-.242.985-1.306 1.652-2.064.73-.82.85-.904.547-.431h1.033l.76 1.129-.34 1.166-1.064 1.347-.881 1.142-1.264 1.7-.79 1.36.073.11.188-.02 2.856-.606 1.543-.28 1.841-.315.833.388.091.395-.328.807-1.969.486-2.309.462-3.439.813-.042.03.049.061 1.549.146.662.036h1.622l3.02.225.79.522.474.638-.079.485-1.215.62-1.64-.389-3.829-.91-1.312-.329h-.182v.11l1.093 1.068 2.006 1.81 2.509 2.33.127.578-.322.455-.34-.049-2.205-1.657-.851-.747-1.926-1.62h-.128v.17l.444.649 2.345 3.521.122 1.08-.17.353-.608.213-.668-.122-1.374-1.925-1.415-2.167-1.143-1.943-.14.08-.674 7.254-.316.37-.729.28-.607-.461-.322-.747.322-1.476.389-1.924.315-1.53.286-1.9.17-.632-.012-.042-.14.018-1.434 1.967-2.18 2.945-1.726 1.845-.414.164-.717-.37.067-.662.401-.589 2.388-3.036 1.44-1.882.93-1.086-.006-.158h-.055L4.132 18.56l-1.13.146-.487-.456.061-.746.231-.243 1.908-1.312-.006.006z"></path></svg></span>Claude Opus 5 (max)<span class="org">Anthropic</span></td><td data-v="0.7181599999999999"><div class="cellbar"><i style="width:72.9px;--bar:var(--c4s-a)"></i><b>0.718</b></div></td><td class="dim" data-v="0.63106">0.631</td><td class="dim" data-v="0.71617">0.716</td><td class="dim" data-v="0.811">0.811</td><td class="dim" data-v="34.49611078947368">$34.50</td></tr><tr data-model-key="sol" data-search="gpt-5.6 sol (high) openai"><td class="rk">4</td><td class="nm"><span class="logo"><svg fill="currentColor" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>OpenAI</title><path d="M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z"></path></svg></span>GPT-5.6 Sol (high)<span class="tag front">Pareto</span><span class="org">OpenAI</span></td><td data-v="0.70704"><div class="cellbar"><i style="width:71.8px;--bar:var(--c4s-a)"></i><b>0.707</b></div></td><td class="dim" data-v="0.63186">0.632</td><td class="dim" data-v="0.710485">0.710</td><td class="dim" data-v="0.772">0.772</td><td class="dim" data-v="4.07260365">$4.07</td></tr><tr data-model-key="gemini" data-search="gemini 3.8 flash (high) google"><td class="rk">5</td><td class="nm"><span class="logo"><svg fill="url(#c4sGem-19)" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><defs><lineargradient id="c4sGem-19" x1="0" x2="1" y1="0" y2="1"><stop offset="0" stop-color="#4796E3"></stop><stop offset=".55" stop-color="#9168C0"></stop><stop offset="1" stop-color="#D96570"></stop></lineargradient></defs><title>Gemini</title><path d="M20.616 10.835a14.147 14.147 0 01-4.45-3.001 14.111 14.111 0 01-3.678-6.452.503.503 0 00-.975 0 14.134 14.134 0 01-3.679 6.452 14.155 14.155 0 01-4.45 3.001c-.65.28-1.318.505-2.002.678a.502.502 0 000 .975c.684.172 1.35.397 2.002.677a14.147 14.147 0 014.45 3.001 14.112 14.112 0 013.679 6.453.502.502 0 00.975 0c.172-.685.397-1.351.677-2.003a14.145 14.145 0 013.001-4.45 14.113 14.113 0 016.453-3.678.503.503 0 000-.975 13.245 13.245 0 01-2.003-.678z"></path></svg></span>Gemini 3.8 Flash (high)<span class="tag front">Pareto</span><span class="org">Google</span></td><td data-v="0.65689"><div class="cellbar"><i style="width:66.7px;--bar:var(--c4s-a)"></i><b>0.657</b></div></td><td class="dim" data-v="0.6369100000000001">0.637</td><td class="dim" data-v="0.66406">0.664</td><td class="dim" data-v="0.655">0.655</td><td class="dim" data-v="2.49471265">$2.49</td></tr><tr data-model-key="muse" data-search="muse spark 1.3 (medium) meta"><td class="rk">6</td><td class="nm"><span class="logo"><svg fill="#0081FB" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Meta</title><path d="M6.897 4c1.915 0 3.516.932 5.43 3.376l.282-.373c.19-.246.383-.484.58-.71l.313-.35C14.588 4.788 15.792 4 17.225 4c1.273 0 2.469.557 3.491 1.516l.218.213c1.73 1.765 2.917 4.71 3.053 8.026l.011.392.002.25c0 1.501-.28 2.759-.818 3.7l-.14.23-.108.153c-.301.42-.664.758-1.086 1.009l-.265.142-.087.04a3.493 3.493 0 01-.302.118 4.117 4.117 0 01-1.33.208c-.524 0-.996-.067-1.438-.215-.614-.204-1.163-.56-1.726-1.116l-.227-.235c-.753-.812-1.534-1.976-2.493-3.586l-1.43-2.41-.544-.895-1.766 3.13-.343.592C7.597 19.156 6.227 20 4.356 20c-1.21 0-2.205-.42-2.936-1.182l-.168-.184c-.484-.573-.837-1.311-1.043-2.189l-.067-.32a8.69 8.69 0 01-.136-1.288L0 14.468c.002-.745.06-1.49.174-2.23l.1-.573c.298-1.53.828-2.958 1.536-4.157l.209-.34c1.177-1.83 2.789-3.053 4.615-3.16L6.897 4zm-.033 2.615l-.201.01c-.83.083-1.606.673-2.252 1.577l-.138.199-.01.018c-.67 1.017-1.185 2.378-1.456 3.845l-.004.022a12.591 12.591 0 00-.207 2.254l.002.188c.004.18.017.36.04.54l.043.291c.092.503.257.908.486 1.208l.117.137c.303.323.698.492 1.17.492 1.1 0 1.796-.676 3.696-3.641l2.175-3.4.454-.701-.139-.198C9.11 7.3 8.084 6.616 6.864 6.616zm10.196-.552l-.176.007c-.635.048-1.223.359-1.82.933l-.196.198c-.439.462-.887 1.064-1.367 1.807l.266.398c.18.274.362.56.55.858l.293.475 1.396 2.335.695 1.114c.583.926 1.03 1.6 1.408 2.082l.213.262c.282.326.529.54.777.673l.102.05c.227.1.457.138.718.138.176.002.35-.023.518-.073.338-.104.61-.32.813-.637l.095-.163.077-.162c.194-.459.29-1.06.29-1.785l-.006-.449c-.08-2.871-.938-5.372-2.2-6.798l-.176-.189c-.67-.683-1.444-1.074-2.27-1.074z"></path></svg></span>Muse Spark 1.3 (medium)<span class="tag front">Pareto</span><span class="org">Meta</span></td><td data-v="0.646195"><div class="cellbar"><i style="width:65.6px;--bar:var(--c4s-a)"></i><b>0.646</b></div></td><td class="dim" data-v="0.5981099999999999">0.598</td><td class="dim" data-v="0.660675">0.661</td><td class="dim" data-v="0.651">0.651</td><td class="dim" data-v="0.10048539999999999">$0.10</td></tr><tr data-model-key="grok" data-search="grok 4.6 (high) xai"><td class="rk">7</td><td class="nm"><span class="logo"><svg fill="currentColor" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Grok</title><path d="M9.27 15.29l7.978-5.897c.391-.29.95-.177 1.137.272.98 2.369.542 5.215-1.41 7.169-1.951 1.954-4.667 2.382-7.149 1.406l-2.711 1.257c3.889 2.661 8.611 2.003 11.562-.953 2.341-2.344 3.066-5.539 2.388-8.42l.006.007c-.983-4.232.242-5.924 2.75-9.383.06-.082.12-.164.179-.248l-3.301 3.305v-.01L9.267 15.292M7.623 16.723c-2.792-2.67-2.31-6.801.071-9.184 1.761-1.763 4.647-2.483 7.166-1.425l2.705-1.25a7.808 7.808 0 00-1.829-1A8.975 8.975 0 005.984 5.83c-2.533 2.536-3.33 6.436-1.962 9.764 1.022 2.487-.653 4.246-2.34 6.022-.599.63-1.199 1.259-1.682 1.925l7.62-6.815"></path></svg></span>Grok 4.6 (high)<span class="org">xAI</span></td><td data-v="0.56721"><div class="cellbar"><i style="width:57.6px;--bar:var(--c4s-a)"></i><b>0.567</b></div></td><td class="dim" data-v="0.52211">0.522</td><td class="dim" data-v="0.5764199999999999">0.576</td><td class="dim" data-v="0.585">0.585</td><td class="dim" data-v="4.326270533333333">$4.33</td></tr><tr data-model-key="qwen-off" data-search="qwen 3.8 27b (thinking off) alibaba"><td class="rk">8</td><td class="nm"><span class="logo"><svg fill="#615CED" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Qwen</title><path d="M12.604 1.34c.393.69.784 1.382 1.174 2.075a.18.18 0 00.157.091h5.552c.174 0 .322.11.446.327l1.454 2.57c.19.337.24.478.024.837-.26.43-.513.864-.76 1.3l-.367.658c-.106.196-.223.28-.04.512l2.652 4.637c.172.301.111.494-.043.77-.437.785-.882 1.564-1.335 2.34-.159.272-.352.375-.68.37-.777-.016-1.552-.01-2.327.016a.099.099 0 00-.081.05 575.097 575.097 0 01-2.705 4.74c-.169.293-.38.363-.725.364-.997.003-2.002.004-3.017.002a.537.537 0 01-.465-.271l-1.335-2.323a.09.09 0 00-.083-.049H4.982c-.285.03-.553-.001-.805-.092l-1.603-2.77a.543.543 0 01-.002-.54l1.207-2.12a.198.198 0 000-.197 550.951 550.951 0 01-1.875-3.272l-.79-1.395c-.16-.31-.173-.496.095-.965.465-.813.927-1.625 1.387-2.436.132-.234.304-.334.584-.335a338.3 338.3 0 012.589-.001.124.124 0 00.107-.063l2.806-4.895a.488.488 0 01.422-.246c.524-.001 1.053 0 1.583-.006L11.704 1c.341-.003.724.032.9.34zm-3.432.403a.06.06 0 00-.052.03L6.254 6.788a.157.157 0 01-.135.078H3.253c-.056 0-.07.025-.041.074l5.81 10.156c.025.042.013.062-.034.063l-2.795.015a.218.218 0 00-.2.116l-1.32 2.31c-.044.078-.021.118.068.118l5.716.008c.046 0 .08.02.104.061l1.403 2.454c.046.081.092.082.139 0l5.006-8.76.783-1.382a.055.055 0 01.096 0l1.424 2.53a.122.122 0 00.107.062l2.763-.02a.04.04 0 00.035-.02.041.041 0 000-.04l-2.9-5.086a.108.108 0 010-.113l.293-.507 1.12-1.977c.024-.041.012-.062-.035-.062H9.2c-.059 0-.073-.026-.043-.077l1.434-2.505a.107.107 0 000-.114L9.225 1.774a.06.06 0 00-.053-.031zm6.29 8.02c.046 0 .058.02.034.06l-.832 1.465-2.613 4.585a.056.056 0 01-.05.029.058.058 0 01-.05-.029L8.498 9.841c-.02-.034-.01-.052.028-.054l.216-.012 6.722-.012z"></path></svg></span>Qwen 3.8 27B (thinking off)<span class="tag open">open weights</span><span class="org">Alibaba</span></td><td data-v="0.55666"><div class="cellbar"><i style="width:56.5px;--bar:var(--c4s-a)"></i><b>0.557</b></div></td><td class="dim" data-v="0.506755">0.507</td><td class="dim" data-v="0.529525">0.530</td><td class="dim" data-v="0.688">0.688</td><td class="dim" data-v="0.49154169999999997">$0.49</td></tr><tr data-model-key="qwen-on" data-search="qwen 3.8 27b (thinking on) alibaba"><td class="rk">9</td><td class="nm"><span class="logo"><svg fill="#615CED" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Qwen</title><path d="M12.604 1.34c.393.69.784 1.382 1.174 2.075a.18.18 0 00.157.091h5.552c.174 0 .322.11.446.327l1.454 2.57c.19.337.24.478.024.837-.26.43-.513.864-.76 1.3l-.367.658c-.106.196-.223.28-.04.512l2.652 4.637c.172.301.111.494-.043.77-.437.785-.882 1.564-1.335 2.34-.159.272-.352.375-.68.37-.777-.016-1.552-.01-2.327.016a.099.099 0 00-.081.05 575.097 575.097 0 01-2.705 4.74c-.169.293-.38.363-.725.364-.997.003-2.002.004-3.017.002a.537.537 0 01-.465-.271l-1.335-2.323a.09.09 0 00-.083-.049H4.982c-.285.03-.553-.001-.805-.092l-1.603-2.77a.543.543 0 01-.002-.54l1.207-2.12a.198.198 0 000-.197 550.951 550.951 0 01-1.875-3.272l-.79-1.395c-.16-.31-.173-.496.095-.965.465-.813.927-1.625 1.387-2.436.132-.234.304-.334.584-.335a338.3 338.3 0 012.589-.001.124.124 0 00.107-.063l2.806-4.895a.488.488 0 01.422-.246c.524-.001 1.053 0 1.583-.006L11.704 1c.341-.003.724.032.9.34zm-3.432.403a.06.06 0 00-.052.03L6.254 6.788a.157.157 0 01-.135.078H3.253c-.056 0-.07.025-.041.074l5.81 10.156c.025.042.013.062-.034.063l-2.795.015a.218.218 0 00-.2.116l-1.32 2.31c-.044.078-.021.118.068.118l5.716.008c.046 0 .08.02.104.061l1.403 2.454c.046.081.092.082.139 0l5.006-8.76.783-1.382a.055.055 0 01.096 0l1.424 2.53a.122.122 0 00.107.062l2.763-.02a.04.04 0 00.035-.02.041.041 0 000-.04l-2.9-5.086a.108.108 0 010-.113l.293-.507 1.12-1.977c.024-.041.012-.062-.035-.062H9.2c-.059 0-.073-.026-.043-.077l1.434-2.505a.107.107 0 000-.114L9.225 1.774a.06.06 0 00-.053-.031zm6.29 8.02c.046 0 .058.02.034.06l-.832 1.465-2.613 4.585a.056.056 0 01-.05.029.058.058 0 01-.05-.029L8.498 9.841c-.02-.034-.01-.052.028-.054l.216-.012 6.722-.012z"></path></svg></span>Qwen 3.8 27B (thinking on)<span class="tag open">open weights</span><span class="org">Alibaba</span></td><td data-v="0.51571"><div class="cellbar"><i style="width:52.4px;--bar:var(--c4s-a)"></i><b>0.516</b></div></td><td class="dim" data-v="0.50522">0.505</td><td class="dim" data-v="0.45978">0.460</td><td class="dim" data-v="0.694">0.694</td><td class="dim" data-v="0.3344125">$0.33</td></tr><tr data-model-key="glm" data-search="glm-5.3 flash (max) z.ai"><td class="rk">10</td><td class="nm"><span class="logo"><svg fill="currentColor" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Z.ai</title><path d="M12.105 2L9.927 4.953H.653L2.83 2h9.276zM23.254 19.048L21.078 22h-9.242l2.174-2.952h9.244zM24 2L9.264 22H0L14.736 2H24z"></path></svg></span>GLM-5.3 Flash (max)<span class="tag open">open weights</span><span class="org">Z.ai</span></td><td data-v="0.5094299999999999"><div class="cellbar"><i style="width:51.7px;--bar:var(--c4s-a)"></i><b>0.509</b></div></td><td class="dim" data-v="0.403365">0.403</td><td class="dim" data-v="0.537015">0.537</td><td class="dim" data-v="0.533">0.533</td><td class="dim" data-v="0.710521125">$0.71</td></tr><tr data-model-key="gemma-on" data-search="gemma 4 31b (thinking on) google"><td class="rk">11</td><td class="nm"><span class="logo"><svg fill="url(#c4sGem-25)" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><defs><lineargradient id="c4sGem-25" x1="0" x2="1" y1="0" y2="1"><stop offset="0" stop-color="#4796E3"></stop><stop offset=".55" stop-color="#9168C0"></stop><stop offset="1" stop-color="#D96570"></stop></lineargradient></defs><title>Gemini</title><path d="M20.616 10.835a14.147 14.147 0 01-4.45-3.001 14.111 14.111 0 01-3.678-6.452.503.503 0 00-.975 0 14.134 14.134 0 01-3.679 6.452 14.155 14.155 0 01-4.45 3.001c-.65.28-1.318.505-2.002.678a.502.502 0 000 .975c.684.172 1.35.397 2.002.677a14.147 14.147 0 014.45 3.001 14.112 14.112 0 013.679 6.453.502.502 0 00.975 0c.172-.685.397-1.351.677-2.003a14.145 14.145 0 013.001-4.45 14.113 14.113 0 016.453-3.678.503.503 0 000-.975 13.245 13.245 0 01-2.003-.678z"></path></svg></span>Gemma 4 31B (thinking on)<span class="tag open">open weights</span><span class="tag front">Pareto</span><span class="org">Google</span></td><td data-v="0.470705"><div class="cellbar"><i style="width:47.8px;--bar:var(--c4s-a)"></i><b>0.471</b></div></td><td class="dim" data-v="0.4117">0.412</td><td class="dim" data-v="0.41593">0.416</td><td class="dim" data-v="0.694">0.694</td><td class="dim" data-v="0.0253001">$0.03</td></tr><tr data-model-key="gemma-off" data-search="gemma 4 31b (thinking off) google"><td class="rk">12</td><td class="nm"><span class="logo"><svg fill="url(#c4sGem-26)" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><defs><lineargradient id="c4sGem-26" x1="0" x2="1" y1="0" y2="1"><stop offset="0" stop-color="#4796E3"></stop><stop offset=".55" stop-color="#9168C0"></stop><stop offset="1" stop-color="#D96570"></stop></lineargradient></defs><title>Gemini</title><path d="M20.616 10.835a14.147 14.147 0 01-4.45-3.001 14.111 14.111 0 01-3.678-6.452.503.503 0 00-.975 0 14.134 14.134 0 01-3.679 6.452 14.155 14.155 0 01-4.45 3.001c-.65.28-1.318.505-2.002.678a.502.502 0 000 .975c.684.172 1.35.397 2.002.677a14.147 14.147 0 014.45 3.001 14.112 14.112 0 013.679 6.453.502.502 0 00.975 0c.172-.685.397-1.351.677-2.003a14.145 14.145 0 013.001-4.45 14.113 14.113 0 016.453-3.678.503.503 0 000-.975 13.245 13.245 0 01-2.003-.678z"></path></svg></span>Gemma 4 31B (thinking off)<span class="tag open">open weights</span><span class="tag front">Pareto</span><span class="org">Google</span></td><td data-v="0.45494"><div class="cellbar"><i style="width:46.2px;--bar:var(--c4s-a)"></i><b>0.455</b></div></td><td class="dim" data-v="0.43810000000000004">0.438</td><td class="dim" data-v="0.370465">0.370</td><td class="dim" data-v="0.725">0.725</td><td class="dim" data-v="0.0151612">$0.02</td></tr><tr data-model-key="inkling" data-search="inkling (high) thinking machines"><td class="rk">13</td><td class="nm"><span class="logo"><svg fill="currentColor" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Thinking Machines</title><rect height="15.6" rx="3.6" width="15.6" x="4.2" y="4.2"></rect></svg>
|
| 29 |
</span>Inkling (high)<span class="org">Thinking Machines</span></td><td data-v="0.42498500000000006"><div class="cellbar"><i style="width:43.1px;--bar:var(--c4s-a)"></i><b>0.425</b></div></td><td class="dim" data-v="0.392605">0.393</td><td class="dim" data-v="0.336705">0.337</td><td class="dim" data-v="0.722">0.722</td><td class="dim" data-v="0.20252835">$0.20</td></tr><tr data-model-key="deepseek" data-search="deepseek v4.1 flash (high) deepseek"><td class="rk">14</td><td class="nm"><span class="logo"><svg fill="#4D6BFE" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>DeepSeek</title><path d="M23.748 4.482c-.254-.124-.364.113-.512.234-.051.039-.094.09-.137.136-.372.397-.806.657-1.373.626-.829-.046-1.537.214-2.163.848-.133-.782-.575-1.248-1.247-1.548-.352-.156-.708-.311-.955-.65-.172-.241-.219-.51-.305-.774-.055-.16-.11-.323-.293-.35-.2-.031-.278.136-.356.276-.313.572-.434 1.202-.422 1.84.027 1.436.633 2.58 1.838 3.393.137.093.172.187.129.323-.082.28-.18.552-.266.833-.055.179-.137.217-.329.14a5.526 5.526 0 01-1.736-1.18c-.857-.828-1.631-1.742-2.597-2.458a11.365 11.365 0 00-.689-.471c-.985-.957.13-1.743.388-1.836.27-.098.093-.432-.779-.428-.872.004-1.67.295-2.687.684a3.055 3.055 0 01-.465.137 9.597 9.597 0 00-2.883-.102c-1.885.21-3.39 1.102-4.497 2.623C.082 8.606-.231 10.684.152 12.85c.403 2.284 1.569 4.175 3.36 5.653 1.858 1.533 3.997 2.284 6.438 2.14 1.482-.085 3.133-.284 4.994-1.86.47.234.962.327 1.78.397.63.059 1.236-.03 1.705-.128.735-.156.684-.837.419-.961-2.155-1.004-1.682-.595-2.113-.926 1.096-1.296 2.746-2.642 3.392-7.003.05-.347.007-.565 0-.845-.004-.17.035-.237.23-.256a4.173 4.173 0 001.545-.475c1.396-.763 1.96-2.015 2.093-3.517.02-.23-.004-.467-.247-.588zM11.581 18c-2.089-1.642-3.102-2.183-3.52-2.16-.392.024-.321.471-.235.763.09.288.207.486.371.739.114.167.192.416-.113.603-.673.416-1.842-.14-1.897-.167-1.361-.802-2.5-1.86-3.301-3.307-.774-1.393-1.224-2.887-1.298-4.482-.02-.386.093-.522.477-.592a4.696 4.696 0 011.529-.039c2.132.312 3.946 1.265 5.468 2.774.868.86 1.525 1.887 2.202 2.891.72 1.066 1.494 2.082 2.48 2.914.348.292.625.514.891.677-.802.09-2.14.11-3.054-.614zm1-6.44a.306.306 0 01.415-.287.302.302 0 01.2.288.306.306 0 01-.31.307.303.303 0 01-.304-.308zm3.11 1.596c-.2.081-.399.151-.59.16a1.245 1.245 0 01-.798-.254c-.274-.23-.47-.358-.552-.758a1.73 1.73 0 01.016-.588c.07-.327-.008-.537-.239-.727-.187-.156-.426-.199-.688-.199a.559.559 0 01-.254-.078c-.11-.054-.2-.19-.114-.358.028-.054.16-.186.192-.21.356-.202.767-.136 1.146.016.352.144.618.408 1.001.782.391.451.462.576.685.914.176.265.336.537.445.848.067.195-.019.354-.25.452z"></path></svg></span>DeepSeek V4.1 Flash (high)<span class="tag open">open weights</span><span class="org">DeepSeek</span></td><td data-v="0.24269"><div class="cellbar"><i style="width:24.7px;--bar:var(--c4s-a)"></i><b>0.243</b></div></td><td class="dim" data-v="0.216595">0.217</td><td class="dim" data-v="0.16134500000000002">0.161</td><td class="dim" data-v="0.513">0.513</td><td class="dim" data-v="0.193865">$0.19</td></tr></tbody></table></div></div>
|
|
@@ -36,7 +36,7 @@ svg.ch .dim{opacity:.42}svg.ch g.row:hover{opacity:1}svg.ch .hot{fill:var(--c4s-
|
|
| 36 |
</div></section><section class="alt" id="takeaways"><div class="container">
|
| 37 |
<div class="section-header centered"><span class="section-label">Findings</span><h2 class="section-title">Takeaway</h2>
|
| 38 |
<p class="section-desc">What the evaluation shows across the 14 agents. Every chart below is drawn from the paper's numbers; hover for values.</p></div>
|
| 39 |
-
<div class="tk"><span class="n">Takeaway</span><h3>Spatial Composition remains the weakest requirement family for all 14 agents.</h3><p class="s">Errors persist even when the required objects are present: generating the right objects does not ensure that their relationships satisfy the specification.</p><div class="figs"><figure><div class="fh">Requirement-family score per agent</div><div class="ch-legend"><span><i style="background:var(--c4s-g1)"></i>Identity & Environment</span><span><i style="background:var(--c4s-g2)"></i>Content & Quantity</span><span><i style="background:var(--c4s-g3)"></i>Attributes & Materials</span><span><i style="background:var(--c4s-hot)"></i>Spatial Composition</span></div><svg aria-label="Requirement-family scores per agent" class="ch" role="img" viewbox="0 0 880 450"><line class="grid" x1="245.6" x2="245.6" y1="18" y2="416"></line><text class="m num" text-anchor="middle" x="245.6" y="432.0">0.2</text><line class="grid" x1="396.7" x2="396.7" y1="18" y2="416"></line><text class="m num" text-anchor="middle" x="396.7" y="432.0">0.4</text><line class="grid" x1="547.8" x2="547.8" y1="18" y2="416"></line><text class="m num" text-anchor="middle" x="547.8" y="432.0">0.6</text><line class="grid" x1="698.9" x2="698.9" y1="18" y2="416"></line><text class="m num" text-anchor="middle" x="698.9" y="432.0">0.8</text><line class="grid" x1="850.0" x2="850.0" y1="18" y2="416"></line><text class="m num" text-anchor="middle" x="850.0" y="432.0">1.0</text><g class="row" data-tip="GPT-6 Astra: Identity & Environment 0.875 · Content & Quantity 0.715 · Spatial Composition 0.543 · Attributes & Materials 0.643"><text class="" text-anchor="end" x="158.0" y="43.0">GPT-6 Astra</text><line stroke="var(--line-strong)" stroke-width="2" x1="504.7" x2="755.6" y1="39.0" y2="39.0"></line><circle cx="755.6" cy="39.0" fill="var(--c4s-g1)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="634.7" cy="39.0" fill="var(--c4s-g2)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="580.3" cy="39.0" fill="var(--c4s-g3)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="504.7" cy="39.0" fill="var(--c4s-hot)" r="7" stroke="var(--surface-1)" stroke-width="2"></circle></g><g class="row" data-tip="Gemini 3.8 Flash: Identity & Environment 0.814 · Content & Quantity 0.639 · Spatial Composition 0.473 · Attributes & Materials 0.572"><text class="" text-anchor="end" x="158.0" y="69.0">Gemini 3.8 Flash</text><line stroke="var(--line-strong)" stroke-width="2" x1="451.8" x2="709.5" y1="65.0" y2="65.0"></line><circle cx="709.5" cy="65.0" fill="var(--c4s-g1)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="577.2" cy="65.0" fill="var(--c4s-g2)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="526.6" cy="65.0" fill="var(--c4s-g3)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="451.8" cy="65.0" fill="var(--c4s-hot)" r="7" stroke="var(--surface-1)" stroke-width="2"></circle></g><g class="row" data-tip="Claude Fable 5.1: Identity & Environment 0.906 · Content & Quantity 0.738 · Spatial Composition 0.538 · Attributes & Materials 0.670"><text class="" text-anchor="end" x="158.0" y="95.0">Claude Fable 5.1</text><line stroke="var(--line-strong)" stroke-width="2" x1="500.9" x2="779.0" y1="91.0" y2="91.0"></line><circle cx="779.0" cy="91.0" fill="var(--c4s-g1)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="652.0" cy="91.0" fill="var(--c4s-g2)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="600.7" cy="91.0" fill="var(--c4s-g3)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="500.9" cy="91.0" fill="var(--c4s-hot)" r="7" stroke="var(--surface-1)" stroke-width="2"></circle></g><g class="row" data-tip="Claude Opus 5: Identity & Environment 0.803 · Content & Quantity 0.624 · Spatial Composition 0.479 · Attributes & Materials 0.573"><text class="" text-anchor="end" x="158.0" y="121.0">Claude Opus 5</text><line stroke="var(--line-strong)" stroke-width="2" x1="456.4" x2="701.2" y1="117.0" y2="117.0"></line><circle cx="701.2" cy="117.0" fill="var(--c4s-g1)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="565.9" cy="117.0" fill="var(--c4s-g2)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="527.4" cy="117.0" fill="var(--c4s-g3)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="456.4" cy="117.0" fill="var(--c4s-hot)" r="7" stroke="var(--surface-1)" stroke-width="2"></circle></g><g class="row" data-tip="GPT-5.6 Sol: Identity & Environment 0.750 · Content & Quantity 0.660 · Spatial Composition 0.469 · Attributes & Materials 0.602"><text class="" text-anchor="end" x="158.0" y="147.0">GPT-5.6 Sol</text><line stroke="var(--line-strong)" stroke-width="2" x1="448.8" x2="661.1" y1="143.0" y2="143.0"></line><circle cx="661.1" cy="143.0" fill="var(--c4s-g1)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="593.1" cy="143.0" fill="var(--c4s-g2)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="549.3" cy="143.0" fill="var(--c4s-g3)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="448.8" cy="143.0" fill="var(--c4s-hot)" r="7" stroke="var(--surface-1)" stroke-width="2"></circle></g><g class="row" data-tip="Muse Spark 1.3: Identity & Environment 0.730 · Content & Quantity 0.641 · Spatial Composition 0.393 · Attributes & Materials 0.546"><text class="" text-anchor="end" x="158.0" y="173.0">Muse Spark 1.3</text><line stroke="var(--line-strong)" stroke-width="2" x1="391.4" x2="646.0" y1="169.0" y2="169.0"></line><circle cx="646.0" cy="169.0" fill="var(--c4s-g1)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="578.8" cy="169.0" fill="var(--c4s-g2)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="507.0" cy="169.0" fill="var(--c4s-g3)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="391.4" cy="169.0" fill="var(--c4s-hot)" r="7" stroke="var(--surface-1)" stroke-width="2"></circle></g><g class="row" data-tip="GLM-5.3 Flash: Identity & Environment 0.467 · Content & Quantity 0.421 · Spatial Composition 0.299 · Attributes & Materials 0.377"><text class="" text-anchor="end" x="158.0" y="199.0">GLM-5.3 Flash</text><line stroke="var(--line-strong)" stroke-width="2" x1="320.4" x2="447.3" y1="195.0" y2="195.0"></line><circle cx="447.3" cy="195.0" fill="var(--c4s-g1)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="412.5" cy="195.0" fill="var(--c4s-g2)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="379.3" cy="195.0" fill="var(--c4s-g3)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="320.4" cy="195.0" fill="var(--c4s-hot)" r="7" stroke="var(--surface-1)" stroke-width="2"></circle></g><g class="row" data-tip="Qwen 3.8 27B · off: Identity & Environment 0.724 · Content & Quantity 0.520 · Spatial Composition 0.300 · Attributes & Materials 0.428"><text class="" text-anchor="end" x="158.0" y="225.0">Qwen 3.8 27B · off</text><line stroke="var(--line-strong)" stroke-width="2" x1="321.1" x2="641.5" y1="221.0" y2="221.0"></line><circle cx="641.5" cy="221.0" fill="var(--c4s-g1)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="487.3" cy="221.0" fill="var(--c4s-g2)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="417.8" cy="221.0" fill="var(--c4s-g3)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="321.1" cy="221.0" fill="var(--c4s-hot)" r="7" stroke="var(--surface-1)" stroke-width="2"></circle></g><g class="row" data-tip="Grok 4.6: Identity & Environment 0.647 · Content & Quantity 0.558 · Spatial Composition 0.334 · Attributes & Materials 0.457"><text class="" text-anchor="end" x="158.0" y="251.0">Grok 4.6</text><line stroke="var(--line-strong)" stroke-width="2" x1="346.8" x2="583.3" y1="247.0" y2="247.0"></line><circle cx="583.3" cy="247.0" fill="var(--c4s-g1)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="516.0" cy="247.0" fill="var(--c4s-g2)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="439.7" cy="247.0" fill="var(--c4s-g3)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="346.8" cy="247.0" fill="var(--c4s-hot)" r="7" stroke="var(--surface-1)" stroke-width="2"></circle></g><g class="row" data-tip="Qwen 3.8 27B · on: Identity & Environment 0.660 · Content & Quantity 0.525 · Spatial Composition 0.307 · Attributes & Materials 0.470"><text class="" text-anchor="end" x="158.0" y="277.0">Qwen 3.8 27B · on</text><line stroke="var(--line-strong)" stroke-width="2" x1="326.4" x2="593.1" y1="273.0" y2="273.0"></line><circle cx="593.1" cy="273.0" fill="var(--c4s-g1)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="491.1" cy="273.0" fill="var(--c4s-g2)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="449.6" cy="273.0" fill="var(--c4s-g3)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="326.4" cy="273.0" fill="var(--c4s-hot)" r="7" stroke="var(--surface-1)" stroke-width="2"></circle></g><g class="row" data-tip="Inkling: Identity & Environment 0.572 · Content & Quantity 0.394 · Spatial Composition 0.193 · Attributes & Materials 0.361"><text class="" text-anchor="end" x="158.0" y="303.0">Inkling</text><line stroke="var(--line-strong)" stroke-width="2" x1="240.3" x2="526.6" y1="299.0" y2="299.0"></line><circle cx="526.6" cy="299.0" fill="var(--c4s-g1)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="392.1" cy="299.0" fill="var(--c4s-g2)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="367.2" cy="299.0" fill="var(--c4s-g3)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="240.3" cy="299.0" fill="var(--c4s-hot)" r="7" stroke="var(--surface-1)" stroke-width="2"></circle></g><g class="row" data-tip="Gemma 4 31B · on: Identity & Environment 0.561 · Content & Quantity 0.407 · Spatial Composition 0.272 · Attributes & Materials 0.387"><text class="" text-anchor="end" x="158.0" y="329.0">Gemma 4 31B · on</text><line stroke="var(--line-strong)" stroke-width="2" x1="300.0" x2="518.3" y1="325.0" y2="325.0"></line><circle cx="518.3" cy="325.0" fill="var(--c4s-g1)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="402.0" cy="325.0" fill="var(--c4s-g2)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="386.8" cy="325.0" fill="var(--c4s-g3)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="300.0" cy="325.0" fill="var(--c4s-hot)" r="7" stroke="var(--surface-1)" stroke-width="2"></circle></g><g class="row" data-tip="Gemma 4 31B · off: Identity & Environment 0.612 · Content & Quantity 0.453 · Spatial Composition 0.252 · Attributes & Materials 0.387"><text class="" text-anchor="end" x="158.0" y="355.0">Gemma 4 31B · off</text><line stroke="var(--line-strong)" stroke-width="2" x1="284.8" x2="556.8" y1="351.0" y2="351.0"></line><circle cx="556.8" cy="351.0" fill="var(--c4s-g1)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="436.7" cy="351.0" fill="var(--c4s-g2)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="386.8" cy="351.0" fill="var(--c4s-g3)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="284.8" cy="351.0" fill="var(--c4s-hot)" r="7" stroke="var(--surface-1)" stroke-width="2"></circle></g><g class="row" data-tip="DeepSeek V4.1 Flash: Identity & Environment 0.294 · Content & Quantity 0.215 · Spatial Composition 0.120 · Attributes & Materials 0.259"><text class="" text-anchor="end" x="158.0" y="381.0">DeepSeek V4.1 Flash</text><line stroke="var(--line-strong)" stroke-width="2" x1="185.1" x2="316.6" y1="377.0" y2="377.0"></line><circle cx="316.6" cy="377.0" fill="var(--c4s-g1)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="256.9" cy="377.0" fill="var(--c4s-g2)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="290.1" cy="377.0" fill="var(--c4s-g3)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="185.1" cy="377.0" fill="var(--c4s-hot)" r="7" stroke="var(--surface-1)" stroke-width="2"></circle></g><line class="axis" x1="10" x2="850" y1="390.0" y2="390.0"></line><g class="row" data-tip="Mean of the 14 agents: Identity & Environment 0.672 · Content & Quantity 0.536 · Spatial Composition 0.355 · Attributes & Materials 0.481"><text class="b" text-anchor="end" x="158.0" y="407.0">Mean of 14</text><line stroke="var(--line-strong)" stroke-width="2" x1="362.8" x2="602.6" y1="403.0" y2="403.0"></line><circle cx="602.6" cy="403.0" fill="var(--c4s-g1)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="499.7" cy="403.0" fill="var(--c4s-g2)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="457.8" cy="403.0" fill="var(--c4s-g3)" r="6" stroke="var(--surface-1)" stroke-width="2"></circle><circle cx="362.8" cy="403.0" fill="var(--c4s-hot)" r="7" stroke="var(--surface-1)" stroke-width="2"></circle><text class="hot num" text-anchor="end" x="351.8" y="407.0">0.36</text><text class="m num" text-anchor="start" x="613.6" y="407.0">0.67</text></g></svg><figcaption>Each row is one agent's four family scores; the orange dot, Spatial Composition, is the lowest in every row (paper Figure 14b).</figcaption></figure><div class="figs two" style="margin-top:0"><figure><div class="fh">The objects are there; the relation is not</div><div class="live-scene" data-agent="sol" data-case="egyptian-temple"><canvas aria-label="Interactive 3D scene: Egyptian Temple by GPT-5.6 Sol. Drag to orbit; scroll to zoom." tabindex="0"></canvas><span class="scene-loading" role="status">Loading 3D scene…</span><span class="scene-hint">Drag to orbit · Scroll to zoom</span><button aria-label="Pause scene rotation" class="scene-motion">Ⅱ</button></div><div class="chips"><span class="chip ok">✓ slab roof <small>0.95</small></span><span class="chip ok">✓ stone coping <small>0.82</small></span><span class="chip no">✗ slab roof enclosed by the coping <small>0.78</small></span></div><figcaption><span>GPT-5.6 Sol’s saved Egyptian Temple scene. The judge finds the roof and stone coping, but flags their spatial relationship. <a href="img/judge_egyptian-temple_sol_s3f_000025.jpg?v=1790731787" rel="noopener" target="_blank">View the judge’s evidence ↗</a></span></figcaption></figure><figure><div class="fh">Mismatch rate when the required objects are present</div><svg aria-label="Mismatch rate when the required objects are present" class="ch" role="img" viewbox="0 0 440 144"><g data-tip="Spatial relation: 32.0% of checks fail although the objects are present"><text class="b" text-anchor="end" x="120.0" y="34.0">Spatial relation</text><rect fill="var(--c4s-hot)" height="28" rx="4" width="228.6" x="130" y="16"></rect><text class="num hot" text-anchor="start" x="366.6" y="35.0">32.0%</text></g><g data-tip="Distribution: 19.1% of checks fail although the objects are present"><text class="" text-anchor="end" x="120.0" y="78.0">Distribution</text><rect fill="var(--c4s-g1)" height="28" rx="4" width="136.4" x="130" y="60"></rect><text class="num b" text-anchor="start" x="274.4" y="79.0">19.1%</text></g><g data-tip="Composition: 4.3% of checks fail although the objects are present"><text class="" text-anchor="end" x="120.0" y="122.0">Composition</text><rect fill="var(--c4s-g1)" height="28" rx="4" width="30.7" x="130" y="104"></rect><text class="num b" text-anchor="start" x="168.7" y="123.0">4.3%</text></g></svg><figcaption>Across the benchmark (paper Figure 6a).</figcaption></figure></div></div></div></div></section><section class="dark" id="citation"><div class="container">
|
| 40 |
<div class="section-header centered"><span class="section-label">Citation</span><h2 class="section-title">Cite This Work</h2></div>
|
| 41 |
<div class="cite-box"><button class="cite-copy-btn" id="copy-btn">Copy</button><pre id="bibtex-text">@misc{ye2026code4scene,
|
| 42 |
title = {Code4Scene: Benchmarking Coding Agents for Constructing and Editing 3D Scenes},
|
|
@@ -56,4 +56,4 @@ svg.ch .dim{opacity:.42}svg.ch g.row:hover{opacity:1}svg.ch .hot{fill:var(--c4s-
|
|
| 56 |
<a href="https://simworld.org/" rel="noopener" target="_blank">SimWorld Home</a></p>
|
| 57 |
<p style="margin-top:.5rem;font-size:.75rem;color:rgba(255,255,255,.25)">Part of the SimWorld project.</p></footer>
|
| 58 |
<script src="assets/paper.js?v=5427505a"></script>
|
| 59 |
-
<script defer="" src="assets/leaderboard-bars.js?v=
|
|
|
|
| 19 |
.chip.ok{color:var(--c4s-ok);border-color:currentColor}.chip.no{color:var(--c4s-hot);border-color:currentColor}.chip small{font-weight:500;color:var(--text-muted);margin-left:4px}
|
| 20 |
.bigstats{display:flex;flex-wrap:wrap;gap:12px 34px;margin:0 0 12px}.bigstats b{display:block;font-size:30px;letter-spacing:-.02em;font-variant-numeric:tabular-nums}
|
| 21 |
.bigstats span{font-size:13px;color:var(--text-secondary)}
|
| 22 |
+
</style><link href="assets/buildingbench.css?v=20261004kit" rel="stylesheet"/><script type="importmap">{"imports":{"three":"./assets/vendor/three-r160/build/three.module.js","three/addons/":"./assets/vendor/three-r160/addons/"}}</script></head><body><a class="skip-link" href="#leaderboard">Skip to benchmark explorer</a>
|
| 23 |
<header class="site-header"><div class="brand"><a aria-label="Code4Scene home" class="brand-home" href="index.html#top"><svg aria-hidden="true" fill="none" viewbox="0 0 30 32"><path d="M15 2 27 9v14l-12 7L3 23V9L15 2Z M3 9l12 7 12-7 M15 16v14 M9 5.5l12 7v14 M21 5.5l-12 7v14" stroke="currentColor" stroke-linejoin="round" stroke-width="1.6"></path></svg><span>Code4Scene</span></a><a class="brand-by" href="https://simworld.org/">by SimWorld</a></div><nav aria-label="Main navigation"><a class="nav-active" href="index.html#top">Benchmark</a><a href="index.html#leaderboard">Leaderboard</a><a href="index.html#cases">Visualization</a><a href="https://arxiv.org/abs/2609.36777" rel="noopener" target="_blank">Paper ↗</a><a href="https://github.com/SimWorld-AI/Code4Scene" rel="noopener" target="_blank">Github ↗</a></nav></header><main class="page-main"><section class="intro" id="top"><div class="intro-copy"><h1>Constructing<br/><em>3D Scenes with Code</em></h1><p class="intro-text">Benchmarking coding agents that turn open-ended scene descriptions into engine-native 3D scenes in Unreal Engine. Measuring spatial reasoning, task fulfillment and physical validity.</p><div class="intro-links"><a href="https://arxiv.org/abs/2609.36777">Read the paper ↗</a><a href="https://github.com/SimWorld-AI/Code4Scene">Code & Data ↗</a><a href="#hero-video">Watch the teaser ↘</a></div></div><div class="featured featured-video" id="hero-video"><video aria-label="Code4Scene overview video" controls="" playsinline="" poster="media/teaser-poster.jpg" preload="metadata"><source src="media/teaser-v31.mp4" type="video/mp4"/>Your browser does not support embedded video. <a href="media/teaser-v31.mp4">Watch the Code4Scene video</a>.</video><div class="featured-video-caption"><strong>Code4Scene in 90 seconds</strong><span>Construction and evaluation</span></div></div></section><section class="task-section" id="task-format"><div class="container"><h2 class="section-title">Task Format</h2><div class="task-shell"><div aria-labelledby="task-tab-t2s" class="task-content" data-task="t2s" id="task-t2s" role="tabpanel"><h3>Nordic Harbour <span>· Text-to-Scene</span></h3><p class="task-subtitle">An open-ended scene description, an asset catalog and the Unreal Engine editor.</p><div class="task-columns"><article class="task-panel"><div class="panel-heading"><strong>Agent input</strong><span>Text + assets</span></div><div class="prompt-preview"><span class="eyebrow">Task instruction</span><p>Build me a compact, sunlit Copenhagen-style canal district with narrow cobblestone streets and rectangular waterways enclosing dense urban blocks.</p><p class="prompt-detail">Townhouses, canals, bridges and a waterfront promenade — with detailed requirements for materials, layout and street furniture.</p></div><p class="task-foot"><strong>Given:</strong> an empty level and the content pack’s asset catalog.</p></article><article class="task-panel output-panel"><div class="panel-heading"><strong>Agent output</strong><span>GPT-6 Astra</span></div><div class="live-scene" data-agent="astra" data-case="nordic-harbour"><canvas aria-label="Interactive 3D scene: Nordic Harbour. Drag to orbit, scroll or use plus and minus to zoom." tabindex="0"></canvas><span class="scene-loading" role="status">Loading 3D scene…</span><span class="scene-hint">Drag to orbit · Scroll to zoom</span><button aria-label="Pause scene rotation" class="scene-motion">Ⅱ</button></div><a class="task-foot text-button" href="cases.html#nordic-harbour">Compare agents and evaluator views ↗</a></article><article class="task-panel evaluation-panel"><div class="panel-heading"><strong>Evaluation</strong><span>Engine-native scene</span></div><div class="score-block"><span>CASE SCORE</span><strong>0.877</strong><dl><div><dt>Detailed Alignment</dt><dd>0.757</dd></div><div><dt>Overview Alignment</dt><dd>0.909</dd></div><div><dt>Physical Safety</dt><dd>0.900</dd></div></dl></div><p class="task-foot">Task fulfillment, artifact integrity and static physical validity.</p><p class="task-formula">0.2 × Detailed + 0.6 × Overview + 0.2 × Physical</p></article></div><details class="task-full"><summary><span>+ Full task instruction</span><span>TEXT-TO-SCENE</span></summary><p>Build me a compact, sunlit Copenhagen-style canal district with narrow cobblestone streets and rectangular waterways enclosing dense urban blocks. Arrange roughly thirty attached four- to six-storey townhouses in continuous street and canal-front rows, using slender façades, repetitive white-framed windows, arched doors and carriage entrances, modest cornices, and steep tiled roofs packed with dormers and brick chimneys. Include occasional exposed party walls between staggered building rows. Finish the façades in weathered muted red, mustard yellow, dusty blue, warm grey, beige, and exposed brick, with red, orange, and charcoal roof tiles. Run a principal waterfront street along a stone quay, crossed by smaller streets and linked across the canals by short, gently arched stone-and-metal bridges, including one opening onto a broad paved promenade with rounded landings. Form the canal edges from stepped pale stone, dark timber retaining walls, and iron railings around reflective rippling water. Add regularly spaced leafy trees, ornate black lamps, stone bollards, benches, European road signs, zebra crossings, puddles, grime, and sparse weeds.</p></details></div></div><div class="availability"><span><b>160</b> construction cases in the full benchmark target, including private cases</span><span><b>129</b> public construction cases</span></div></div></section><section id="leaderboard"><div class="container">
|
| 24 |
<div class="section-header centered"><span class="section-label">Leaderboard</span><h2 class="section-title">Leaderboard</h2>
|
| 25 |
<p class="section-desc">14 coding-agent configurations · paper results on the original 20 public construction cases.</p></div>
|
| 26 |
+
<div id="lbwrap"><div class="card bar-card"><div class="bar-heading"><h3>Code4Scene <span class="q">Text-to-Scene</span></h3><span>Paper evaluation · score out of 100</span></div><p class="bar-mobile-hint">Swipe to view all 14 configurations →</p><div aria-label="All 14 model configurations" class="bar-scroll" role="region" tabindex="0"><div data-view="t2s"><svg aria-label="Text-to-Scene scores for the 14 Code4Scene paper configurations" class="kit-bars" id="bars-t2s" role="group" viewbox="0 0 1200 420"></svg></div></div><p class="note">Sorted by score. Bar labels round scores ×100; the table preserves three decimal places. One color per model provider.</p></div>
|
| 27 |
<div class="card chart-card pareto-card"><h3 class="figure-title">Code4Scene <span>Pareto Frontier</span></h3><p class="figure-subtitle">Quality against the cost of one case · original 20-case public evaluation</p><p class="pareto-swipe">Swipe to explore all configurations →</p><div class="pareto-view" data-view="t2s"><div class="pareto-scroll"><svg aria-label="t2s score against average cost per case. More expensive runs on the left; logarithmic scale." class="pareto-svg" data-mode="t2s" role="group" viewbox="0 0 1200 560"></svg></div></div><p class="figure-caption">Solid: the frontier — nothing cheaper scores higher. One point per configuration, at its evaluated reasoning effort. Cost is the mean USD per case.</p></div>
|
| 28 |
<div class="card ranking-card"><div class="ranking-toolbar"><label for="model-search">Search configurations<input autocomplete="off" id="model-search" placeholder="Model or provider" type="search"/></label><span aria-live="polite" id="ranking-count">14 configurations</span></div><div class="sub-title">Ranking</div><div data-view="t2s"><p class="board-note" style="margin:0 .2rem .7rem">20 public cases. Case score = 0.2 × Detailed + 0.6 × Overview + 0.2 × Physical. Detailed and Overview are means of the per-case scores.</p><div class="board-wrap"><table class="board"><thead><tr><th>#</th><th>Agent configuration</th><th class="sorted" data-best="high" data-k="st">Score</th><th data-best="high" data-k="detailed">Detailed</th><th data-best="high" data-k="overview">Overview</th><th data-best="high" data-k="pt">Physical</th><th data-best="low" data-k="c_t2s">Cost / case</th></tr></thead><tbody><tr data-model-key="fable" data-search="claude fable 5.1 (max) anthropic"><td class="rk">1</td><td class="nm"><span class="logo"><svg fill="#D97757" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Claude</title><path d="M4.709 15.955l4.72-2.647.08-.23-.08-.128H9.2l-.79-.048-2.698-.073-2.339-.097-2.266-.122-.571-.121L0 11.784l.055-.352.48-.321.686.06 1.52.103 2.278.158 1.652.097 2.449.255h.389l.055-.157-.134-.098-.103-.097-2.358-1.596-2.552-1.688-1.336-.972-.724-.491-.364-.462-.158-1.008.656-.722.881.06.225.061.893.686 1.908 1.476 2.491 1.833.365.304.145-.103.019-.073-.164-.274-1.355-2.446-1.446-2.49-.644-1.032-.17-.619a2.97 2.97 0 01-.104-.729L6.283.134 6.696 0l.996.134.42.364.62 1.414 1.002 2.229 1.555 3.03.456.898.243.832.091.255h.158V9.01l.128-1.706.237-2.095.23-2.695.08-.76.376-.91.747-.492.584.28.48.685-.067.444-.286 1.851-.559 2.903-.364 1.942h.212l.243-.242.985-1.306 1.652-2.064.73-.82.85-.904.547-.431h1.033l.76 1.129-.34 1.166-1.064 1.347-.881 1.142-1.264 1.7-.79 1.36.073.11.188-.02 2.856-.606 1.543-.28 1.841-.315.833.388.091.395-.328.807-1.969.486-2.309.462-3.439.813-.042.03.049.061 1.549.146.662.036h1.622l3.02.225.79.522.474.638-.079.485-1.215.62-1.64-.389-3.829-.91-1.312-.329h-.182v.11l1.093 1.068 2.006 1.81 2.509 2.33.127.578-.322.455-.34-.049-2.205-1.657-.851-.747-1.926-1.62h-.128v.17l.444.649 2.345 3.521.122 1.08-.17.353-.608.213-.668-.122-1.374-1.925-1.415-2.167-1.143-1.943-.14.08-.674 7.254-.316.37-.729.28-.607-.461-.322-.747.322-1.476.389-1.924.315-1.53.286-1.9.17-.632-.012-.042-.14.018-1.434 1.967-2.18 2.945-1.726 1.845-.414.164-.717-.37.067-.662.401-.589 2.388-3.036 1.44-1.882.93-1.086-.006-.158h-.055L4.132 18.56l-1.13.146-.487-.456.061-.746.231-.243 1.908-1.312-.006.006z"></path></svg></span>Claude Fable 5.1 (max)<span class="tag front">Pareto</span><span class="org">Anthropic</span></td><td data-v="0.78775"><div class="cellbar"><i style="width:80.0px;--bar:var(--c4s-a)"></i><b>0.788</b></div></td><td class="dim" data-v="0.7253149999999999">0.725</td><td class="dim" data-v="0.7717050000000001">0.772</td><td class="dim" data-v="0.898">0.898</td><td class="dim" data-v="18.1576316">$18.16</td></tr><tr data-model-key="astra" data-search="gpt-6 astra (max) openai"><td class="rk">2</td><td class="nm"><span class="logo"><svg fill="currentColor" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>OpenAI</title><path d="M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z"></path></svg></span>GPT-6 Astra (max)<span class="org">OpenAI</span></td><td data-v="0.72382"><div class="cellbar"><i style="width:73.5px;--bar:var(--c4s-a)"></i><b>0.724</b></div></td><td class="dim" data-v="0.707165">0.707</td><td class="dim" data-v="0.733235">0.733</td><td class="dim" data-v="0.712">0.712</td><td class="dim" data-v="22.472260736842106">$22.47</td></tr><tr data-model-key="opus" data-search="claude opus 5 (max) anthropic"><td class="rk">3</td><td class="nm"><span class="logo"><svg fill="#D97757" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Claude</title><path d="M4.709 15.955l4.72-2.647.08-.23-.08-.128H9.2l-.79-.048-2.698-.073-2.339-.097-2.266-.122-.571-.121L0 11.784l.055-.352.48-.321.686.06 1.52.103 2.278.158 1.652.097 2.449.255h.389l.055-.157-.134-.098-.103-.097-2.358-1.596-2.552-1.688-1.336-.972-.724-.491-.364-.462-.158-1.008.656-.722.881.06.225.061.893.686 1.908 1.476 2.491 1.833.365.304.145-.103.019-.073-.164-.274-1.355-2.446-1.446-2.49-.644-1.032-.17-.619a2.97 2.97 0 01-.104-.729L6.283.134 6.696 0l.996.134.42.364.62 1.414 1.002 2.229 1.555 3.03.456.898.243.832.091.255h.158V9.01l.128-1.706.237-2.095.23-2.695.08-.76.376-.91.747-.492.584.28.48.685-.067.444-.286 1.851-.559 2.903-.364 1.942h.212l.243-.242.985-1.306 1.652-2.064.73-.82.85-.904.547-.431h1.033l.76 1.129-.34 1.166-1.064 1.347-.881 1.142-1.264 1.7-.79 1.36.073.11.188-.02 2.856-.606 1.543-.28 1.841-.315.833.388.091.395-.328.807-1.969.486-2.309.462-3.439.813-.042.03.049.061 1.549.146.662.036h1.622l3.02.225.79.522.474.638-.079.485-1.215.62-1.64-.389-3.829-.91-1.312-.329h-.182v.11l1.093 1.068 2.006 1.81 2.509 2.33.127.578-.322.455-.34-.049-2.205-1.657-.851-.747-1.926-1.62h-.128v.17l.444.649 2.345 3.521.122 1.08-.17.353-.608.213-.668-.122-1.374-1.925-1.415-2.167-1.143-1.943-.14.08-.674 7.254-.316.37-.729.28-.607-.461-.322-.747.322-1.476.389-1.924.315-1.53.286-1.9.17-.632-.012-.042-.14.018-1.434 1.967-2.18 2.945-1.726 1.845-.414.164-.717-.37.067-.662.401-.589 2.388-3.036 1.44-1.882.93-1.086-.006-.158h-.055L4.132 18.56l-1.13.146-.487-.456.061-.746.231-.243 1.908-1.312-.006.006z"></path></svg></span>Claude Opus 5 (max)<span class="org">Anthropic</span></td><td data-v="0.7181599999999999"><div class="cellbar"><i style="width:72.9px;--bar:var(--c4s-a)"></i><b>0.718</b></div></td><td class="dim" data-v="0.63106">0.631</td><td class="dim" data-v="0.71617">0.716</td><td class="dim" data-v="0.811">0.811</td><td class="dim" data-v="34.49611078947368">$34.50</td></tr><tr data-model-key="sol" data-search="gpt-5.6 sol (high) openai"><td class="rk">4</td><td class="nm"><span class="logo"><svg fill="currentColor" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>OpenAI</title><path d="M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z"></path></svg></span>GPT-5.6 Sol (high)<span class="tag front">Pareto</span><span class="org">OpenAI</span></td><td data-v="0.70704"><div class="cellbar"><i style="width:71.8px;--bar:var(--c4s-a)"></i><b>0.707</b></div></td><td class="dim" data-v="0.63186">0.632</td><td class="dim" data-v="0.710485">0.710</td><td class="dim" data-v="0.772">0.772</td><td class="dim" data-v="4.07260365">$4.07</td></tr><tr data-model-key="gemini" data-search="gemini 3.8 flash (high) google"><td class="rk">5</td><td class="nm"><span class="logo"><svg fill="url(#c4sGem-19)" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><defs><lineargradient id="c4sGem-19" x1="0" x2="1" y1="0" y2="1"><stop offset="0" stop-color="#4796E3"></stop><stop offset=".55" stop-color="#9168C0"></stop><stop offset="1" stop-color="#D96570"></stop></lineargradient></defs><title>Gemini</title><path d="M20.616 10.835a14.147 14.147 0 01-4.45-3.001 14.111 14.111 0 01-3.678-6.452.503.503 0 00-.975 0 14.134 14.134 0 01-3.679 6.452 14.155 14.155 0 01-4.45 3.001c-.65.28-1.318.505-2.002.678a.502.502 0 000 .975c.684.172 1.35.397 2.002.677a14.147 14.147 0 014.45 3.001 14.112 14.112 0 013.679 6.453.502.502 0 00.975 0c.172-.685.397-1.351.677-2.003a14.145 14.145 0 013.001-4.45 14.113 14.113 0 016.453-3.678.503.503 0 000-.975 13.245 13.245 0 01-2.003-.678z"></path></svg></span>Gemini 3.8 Flash (high)<span class="tag front">Pareto</span><span class="org">Google</span></td><td data-v="0.65689"><div class="cellbar"><i style="width:66.7px;--bar:var(--c4s-a)"></i><b>0.657</b></div></td><td class="dim" data-v="0.6369100000000001">0.637</td><td class="dim" data-v="0.66406">0.664</td><td class="dim" data-v="0.655">0.655</td><td class="dim" data-v="2.49471265">$2.49</td></tr><tr data-model-key="muse" data-search="muse spark 1.3 (medium) meta"><td class="rk">6</td><td class="nm"><span class="logo"><svg fill="#0081FB" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Meta</title><path d="M6.897 4c1.915 0 3.516.932 5.43 3.376l.282-.373c.19-.246.383-.484.58-.71l.313-.35C14.588 4.788 15.792 4 17.225 4c1.273 0 2.469.557 3.491 1.516l.218.213c1.73 1.765 2.917 4.71 3.053 8.026l.011.392.002.25c0 1.501-.28 2.759-.818 3.7l-.14.23-.108.153c-.301.42-.664.758-1.086 1.009l-.265.142-.087.04a3.493 3.493 0 01-.302.118 4.117 4.117 0 01-1.33.208c-.524 0-.996-.067-1.438-.215-.614-.204-1.163-.56-1.726-1.116l-.227-.235c-.753-.812-1.534-1.976-2.493-3.586l-1.43-2.41-.544-.895-1.766 3.13-.343.592C7.597 19.156 6.227 20 4.356 20c-1.21 0-2.205-.42-2.936-1.182l-.168-.184c-.484-.573-.837-1.311-1.043-2.189l-.067-.32a8.69 8.69 0 01-.136-1.288L0 14.468c.002-.745.06-1.49.174-2.23l.1-.573c.298-1.53.828-2.958 1.536-4.157l.209-.34c1.177-1.83 2.789-3.053 4.615-3.16L6.897 4zm-.033 2.615l-.201.01c-.83.083-1.606.673-2.252 1.577l-.138.199-.01.018c-.67 1.017-1.185 2.378-1.456 3.845l-.004.022a12.591 12.591 0 00-.207 2.254l.002.188c.004.18.017.36.04.54l.043.291c.092.503.257.908.486 1.208l.117.137c.303.323.698.492 1.17.492 1.1 0 1.796-.676 3.696-3.641l2.175-3.4.454-.701-.139-.198C9.11 7.3 8.084 6.616 6.864 6.616zm10.196-.552l-.176.007c-.635.048-1.223.359-1.82.933l-.196.198c-.439.462-.887 1.064-1.367 1.807l.266.398c.18.274.362.56.55.858l.293.475 1.396 2.335.695 1.114c.583.926 1.03 1.6 1.408 2.082l.213.262c.282.326.529.54.777.673l.102.05c.227.1.457.138.718.138.176.002.35-.023.518-.073.338-.104.61-.32.813-.637l.095-.163.077-.162c.194-.459.29-1.06.29-1.785l-.006-.449c-.08-2.871-.938-5.372-2.2-6.798l-.176-.189c-.67-.683-1.444-1.074-2.27-1.074z"></path></svg></span>Muse Spark 1.3 (medium)<span class="tag front">Pareto</span><span class="org">Meta</span></td><td data-v="0.646195"><div class="cellbar"><i style="width:65.6px;--bar:var(--c4s-a)"></i><b>0.646</b></div></td><td class="dim" data-v="0.5981099999999999">0.598</td><td class="dim" data-v="0.660675">0.661</td><td class="dim" data-v="0.651">0.651</td><td class="dim" data-v="0.10048539999999999">$0.10</td></tr><tr data-model-key="grok" data-search="grok 4.6 (high) xai"><td class="rk">7</td><td class="nm"><span class="logo"><svg fill="currentColor" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Grok</title><path d="M9.27 15.29l7.978-5.897c.391-.29.95-.177 1.137.272.98 2.369.542 5.215-1.41 7.169-1.951 1.954-4.667 2.382-7.149 1.406l-2.711 1.257c3.889 2.661 8.611 2.003 11.562-.953 2.341-2.344 3.066-5.539 2.388-8.42l.006.007c-.983-4.232.242-5.924 2.75-9.383.06-.082.12-.164.179-.248l-3.301 3.305v-.01L9.267 15.292M7.623 16.723c-2.792-2.67-2.31-6.801.071-9.184 1.761-1.763 4.647-2.483 7.166-1.425l2.705-1.25a7.808 7.808 0 00-1.829-1A8.975 8.975 0 005.984 5.83c-2.533 2.536-3.33 6.436-1.962 9.764 1.022 2.487-.653 4.246-2.34 6.022-.599.63-1.199 1.259-1.682 1.925l7.62-6.815"></path></svg></span>Grok 4.6 (high)<span class="org">xAI</span></td><td data-v="0.56721"><div class="cellbar"><i style="width:57.6px;--bar:var(--c4s-a)"></i><b>0.567</b></div></td><td class="dim" data-v="0.52211">0.522</td><td class="dim" data-v="0.5764199999999999">0.576</td><td class="dim" data-v="0.585">0.585</td><td class="dim" data-v="4.326270533333333">$4.33</td></tr><tr data-model-key="qwen-off" data-search="qwen 3.8 27b (thinking off) alibaba"><td class="rk">8</td><td class="nm"><span class="logo"><svg fill="#615CED" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Qwen</title><path d="M12.604 1.34c.393.69.784 1.382 1.174 2.075a.18.18 0 00.157.091h5.552c.174 0 .322.11.446.327l1.454 2.57c.19.337.24.478.024.837-.26.43-.513.864-.76 1.3l-.367.658c-.106.196-.223.28-.04.512l2.652 4.637c.172.301.111.494-.043.77-.437.785-.882 1.564-1.335 2.34-.159.272-.352.375-.68.37-.777-.016-1.552-.01-2.327.016a.099.099 0 00-.081.05 575.097 575.097 0 01-2.705 4.74c-.169.293-.38.363-.725.364-.997.003-2.002.004-3.017.002a.537.537 0 01-.465-.271l-1.335-2.323a.09.09 0 00-.083-.049H4.982c-.285.03-.553-.001-.805-.092l-1.603-2.77a.543.543 0 01-.002-.54l1.207-2.12a.198.198 0 000-.197 550.951 550.951 0 01-1.875-3.272l-.79-1.395c-.16-.31-.173-.496.095-.965.465-.813.927-1.625 1.387-2.436.132-.234.304-.334.584-.335a338.3 338.3 0 012.589-.001.124.124 0 00.107-.063l2.806-4.895a.488.488 0 01.422-.246c.524-.001 1.053 0 1.583-.006L11.704 1c.341-.003.724.032.9.34zm-3.432.403a.06.06 0 00-.052.03L6.254 6.788a.157.157 0 01-.135.078H3.253c-.056 0-.07.025-.041.074l5.81 10.156c.025.042.013.062-.034.063l-2.795.015a.218.218 0 00-.2.116l-1.32 2.31c-.044.078-.021.118.068.118l5.716.008c.046 0 .08.02.104.061l1.403 2.454c.046.081.092.082.139 0l5.006-8.76.783-1.382a.055.055 0 01.096 0l1.424 2.53a.122.122 0 00.107.062l2.763-.02a.04.04 0 00.035-.02.041.041 0 000-.04l-2.9-5.086a.108.108 0 010-.113l.293-.507 1.12-1.977c.024-.041.012-.062-.035-.062H9.2c-.059 0-.073-.026-.043-.077l1.434-2.505a.107.107 0 000-.114L9.225 1.774a.06.06 0 00-.053-.031zm6.29 8.02c.046 0 .058.02.034.06l-.832 1.465-2.613 4.585a.056.056 0 01-.05.029.058.058 0 01-.05-.029L8.498 9.841c-.02-.034-.01-.052.028-.054l.216-.012 6.722-.012z"></path></svg></span>Qwen 3.8 27B (thinking off)<span class="tag open">open weights</span><span class="org">Alibaba</span></td><td data-v="0.55666"><div class="cellbar"><i style="width:56.5px;--bar:var(--c4s-a)"></i><b>0.557</b></div></td><td class="dim" data-v="0.506755">0.507</td><td class="dim" data-v="0.529525">0.530</td><td class="dim" data-v="0.688">0.688</td><td class="dim" data-v="0.49154169999999997">$0.49</td></tr><tr data-model-key="qwen-on" data-search="qwen 3.8 27b (thinking on) alibaba"><td class="rk">9</td><td class="nm"><span class="logo"><svg fill="#615CED" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Qwen</title><path d="M12.604 1.34c.393.69.784 1.382 1.174 2.075a.18.18 0 00.157.091h5.552c.174 0 .322.11.446.327l1.454 2.57c.19.337.24.478.024.837-.26.43-.513.864-.76 1.3l-.367.658c-.106.196-.223.28-.04.512l2.652 4.637c.172.301.111.494-.043.77-.437.785-.882 1.564-1.335 2.34-.159.272-.352.375-.68.37-.777-.016-1.552-.01-2.327.016a.099.099 0 00-.081.05 575.097 575.097 0 01-2.705 4.74c-.169.293-.38.363-.725.364-.997.003-2.002.004-3.017.002a.537.537 0 01-.465-.271l-1.335-2.323a.09.09 0 00-.083-.049H4.982c-.285.03-.553-.001-.805-.092l-1.603-2.77a.543.543 0 01-.002-.54l1.207-2.12a.198.198 0 000-.197 550.951 550.951 0 01-1.875-3.272l-.79-1.395c-.16-.31-.173-.496.095-.965.465-.813.927-1.625 1.387-2.436.132-.234.304-.334.584-.335a338.3 338.3 0 012.589-.001.124.124 0 00.107-.063l2.806-4.895a.488.488 0 01.422-.246c.524-.001 1.053 0 1.583-.006L11.704 1c.341-.003.724.032.9.34zm-3.432.403a.06.06 0 00-.052.03L6.254 6.788a.157.157 0 01-.135.078H3.253c-.056 0-.07.025-.041.074l5.81 10.156c.025.042.013.062-.034.063l-2.795.015a.218.218 0 00-.2.116l-1.32 2.31c-.044.078-.021.118.068.118l5.716.008c.046 0 .08.02.104.061l1.403 2.454c.046.081.092.082.139 0l5.006-8.76.783-1.382a.055.055 0 01.096 0l1.424 2.53a.122.122 0 00.107.062l2.763-.02a.04.04 0 00.035-.02.041.041 0 000-.04l-2.9-5.086a.108.108 0 010-.113l.293-.507 1.12-1.977c.024-.041.012-.062-.035-.062H9.2c-.059 0-.073-.026-.043-.077l1.434-2.505a.107.107 0 000-.114L9.225 1.774a.06.06 0 00-.053-.031zm6.29 8.02c.046 0 .058.02.034.06l-.832 1.465-2.613 4.585a.056.056 0 01-.05.029.058.058 0 01-.05-.029L8.498 9.841c-.02-.034-.01-.052.028-.054l.216-.012 6.722-.012z"></path></svg></span>Qwen 3.8 27B (thinking on)<span class="tag open">open weights</span><span class="org">Alibaba</span></td><td data-v="0.51571"><div class="cellbar"><i style="width:52.4px;--bar:var(--c4s-a)"></i><b>0.516</b></div></td><td class="dim" data-v="0.50522">0.505</td><td class="dim" data-v="0.45978">0.460</td><td class="dim" data-v="0.694">0.694</td><td class="dim" data-v="0.3344125">$0.33</td></tr><tr data-model-key="glm" data-search="glm-5.3 flash (max) z.ai"><td class="rk">10</td><td class="nm"><span class="logo"><svg fill="currentColor" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Z.ai</title><path d="M12.105 2L9.927 4.953H.653L2.83 2h9.276zM23.254 19.048L21.078 22h-9.242l2.174-2.952h9.244zM24 2L9.264 22H0L14.736 2H24z"></path></svg></span>GLM-5.3 Flash (max)<span class="tag open">open weights</span><span class="org">Z.ai</span></td><td data-v="0.5094299999999999"><div class="cellbar"><i style="width:51.7px;--bar:var(--c4s-a)"></i><b>0.509</b></div></td><td class="dim" data-v="0.403365">0.403</td><td class="dim" data-v="0.537015">0.537</td><td class="dim" data-v="0.533">0.533</td><td class="dim" data-v="0.710521125">$0.71</td></tr><tr data-model-key="gemma-on" data-search="gemma 4 31b (thinking on) google"><td class="rk">11</td><td class="nm"><span class="logo"><svg fill="url(#c4sGem-25)" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><defs><lineargradient id="c4sGem-25" x1="0" x2="1" y1="0" y2="1"><stop offset="0" stop-color="#4796E3"></stop><stop offset=".55" stop-color="#9168C0"></stop><stop offset="1" stop-color="#D96570"></stop></lineargradient></defs><title>Gemini</title><path d="M20.616 10.835a14.147 14.147 0 01-4.45-3.001 14.111 14.111 0 01-3.678-6.452.503.503 0 00-.975 0 14.134 14.134 0 01-3.679 6.452 14.155 14.155 0 01-4.45 3.001c-.65.28-1.318.505-2.002.678a.502.502 0 000 .975c.684.172 1.35.397 2.002.677a14.147 14.147 0 014.45 3.001 14.112 14.112 0 013.679 6.453.502.502 0 00.975 0c.172-.685.397-1.351.677-2.003a14.145 14.145 0 013.001-4.45 14.113 14.113 0 016.453-3.678.503.503 0 000-.975 13.245 13.245 0 01-2.003-.678z"></path></svg></span>Gemma 4 31B (thinking on)<span class="tag open">open weights</span><span class="tag front">Pareto</span><span class="org">Google</span></td><td data-v="0.470705"><div class="cellbar"><i style="width:47.8px;--bar:var(--c4s-a)"></i><b>0.471</b></div></td><td class="dim" data-v="0.4117">0.412</td><td class="dim" data-v="0.41593">0.416</td><td class="dim" data-v="0.694">0.694</td><td class="dim" data-v="0.0253001">$0.03</td></tr><tr data-model-key="gemma-off" data-search="gemma 4 31b (thinking off) google"><td class="rk">12</td><td class="nm"><span class="logo"><svg fill="url(#c4sGem-26)" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><defs><lineargradient id="c4sGem-26" x1="0" x2="1" y1="0" y2="1"><stop offset="0" stop-color="#4796E3"></stop><stop offset=".55" stop-color="#9168C0"></stop><stop offset="1" stop-color="#D96570"></stop></lineargradient></defs><title>Gemini</title><path d="M20.616 10.835a14.147 14.147 0 01-4.45-3.001 14.111 14.111 0 01-3.678-6.452.503.503 0 00-.975 0 14.134 14.134 0 01-3.679 6.452 14.155 14.155 0 01-4.45 3.001c-.65.28-1.318.505-2.002.678a.502.502 0 000 .975c.684.172 1.35.397 2.002.677a14.147 14.147 0 014.45 3.001 14.112 14.112 0 013.679 6.453.502.502 0 00.975 0c.172-.685.397-1.351.677-2.003a14.145 14.145 0 013.001-4.45 14.113 14.113 0 016.453-3.678.503.503 0 000-.975 13.245 13.245 0 01-2.003-.678z"></path></svg></span>Gemma 4 31B (thinking off)<span class="tag open">open weights</span><span class="tag front">Pareto</span><span class="org">Google</span></td><td data-v="0.45494"><div class="cellbar"><i style="width:46.2px;--bar:var(--c4s-a)"></i><b>0.455</b></div></td><td class="dim" data-v="0.43810000000000004">0.438</td><td class="dim" data-v="0.370465">0.370</td><td class="dim" data-v="0.725">0.725</td><td class="dim" data-v="0.0151612">$0.02</td></tr><tr data-model-key="inkling" data-search="inkling (high) thinking machines"><td class="rk">13</td><td class="nm"><span class="logo"><svg fill="currentColor" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Thinking Machines</title><rect height="15.6" rx="3.6" width="15.6" x="4.2" y="4.2"></rect></svg>
|
| 29 |
</span>Inkling (high)<span class="org">Thinking Machines</span></td><td data-v="0.42498500000000006"><div class="cellbar"><i style="width:43.1px;--bar:var(--c4s-a)"></i><b>0.425</b></div></td><td class="dim" data-v="0.392605">0.393</td><td class="dim" data-v="0.336705">0.337</td><td class="dim" data-v="0.722">0.722</td><td class="dim" data-v="0.20252835">$0.20</td></tr><tr data-model-key="deepseek" data-search="deepseek v4.1 flash (high) deepseek"><td class="rk">14</td><td class="nm"><span class="logo"><svg fill="#4D6BFE" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>DeepSeek</title><path d="M23.748 4.482c-.254-.124-.364.113-.512.234-.051.039-.094.09-.137.136-.372.397-.806.657-1.373.626-.829-.046-1.537.214-2.163.848-.133-.782-.575-1.248-1.247-1.548-.352-.156-.708-.311-.955-.65-.172-.241-.219-.51-.305-.774-.055-.16-.11-.323-.293-.35-.2-.031-.278.136-.356.276-.313.572-.434 1.202-.422 1.84.027 1.436.633 2.58 1.838 3.393.137.093.172.187.129.323-.082.28-.18.552-.266.833-.055.179-.137.217-.329.14a5.526 5.526 0 01-1.736-1.18c-.857-.828-1.631-1.742-2.597-2.458a11.365 11.365 0 00-.689-.471c-.985-.957.13-1.743.388-1.836.27-.098.093-.432-.779-.428-.872.004-1.67.295-2.687.684a3.055 3.055 0 01-.465.137 9.597 9.597 0 00-2.883-.102c-1.885.21-3.39 1.102-4.497 2.623C.082 8.606-.231 10.684.152 12.85c.403 2.284 1.569 4.175 3.36 5.653 1.858 1.533 3.997 2.284 6.438 2.14 1.482-.085 3.133-.284 4.994-1.86.47.234.962.327 1.78.397.63.059 1.236-.03 1.705-.128.735-.156.684-.837.419-.961-2.155-1.004-1.682-.595-2.113-.926 1.096-1.296 2.746-2.642 3.392-7.003.05-.347.007-.565 0-.845-.004-.17.035-.237.23-.256a4.173 4.173 0 001.545-.475c1.396-.763 1.96-2.015 2.093-3.517.02-.23-.004-.467-.247-.588zM11.581 18c-2.089-1.642-3.102-2.183-3.52-2.16-.392.024-.321.471-.235.763.09.288.207.486.371.739.114.167.192.416-.113.603-.673.416-1.842-.14-1.897-.167-1.361-.802-2.5-1.86-3.301-3.307-.774-1.393-1.224-2.887-1.298-4.482-.02-.386.093-.522.477-.592a4.696 4.696 0 011.529-.039c2.132.312 3.946 1.265 5.468 2.774.868.86 1.525 1.887 2.202 2.891.72 1.066 1.494 2.082 2.48 2.914.348.292.625.514.891.677-.802.09-2.14.11-3.054-.614zm1-6.44a.306.306 0 01.415-.287.302.302 0 01.2.288.306.306 0 01-.31.307.303.303 0 01-.304-.308zm3.11 1.596c-.2.081-.399.151-.59.16a1.245 1.245 0 01-.798-.254c-.274-.23-.47-.358-.552-.758a1.73 1.73 0 01.016-.588c.07-.327-.008-.537-.239-.727-.187-.156-.426-.199-.688-.199a.559.559 0 01-.254-.078c-.11-.054-.2-.19-.114-.358.028-.054.16-.186.192-.21.356-.202.767-.136 1.146.016.352.144.618.408 1.001.782.391.451.462.576.685.914.176.265.336.537.445.848.067.195-.019.354-.25.452z"></path></svg></span>DeepSeek V4.1 Flash (high)<span class="tag open">open weights</span><span class="org">DeepSeek</span></td><td data-v="0.24269"><div class="cellbar"><i style="width:24.7px;--bar:var(--c4s-a)"></i><b>0.243</b></div></td><td class="dim" data-v="0.216595">0.217</td><td class="dim" data-v="0.16134500000000002">0.161</td><td class="dim" data-v="0.513">0.513</td><td class="dim" data-v="0.193865">$0.19</td></tr></tbody></table></div></div>
|
|
|
|
| 36 |
</div></section><section class="alt" id="takeaways"><div class="container">
|
| 37 |
<div class="section-header centered"><span class="section-label">Findings</span><h2 class="section-title">Takeaway</h2>
|
| 38 |
<p class="section-desc">What the evaluation shows across the 14 agents. Every chart below is drawn from the paper's numbers; hover for values.</p></div>
|
| 39 |
+
<div class="tk"><span class="n">Takeaway</span><h3>Spatial Composition remains the weakest requirement family for all 14 agents.</h3><p class="s">Errors persist even when the required objects are present: generating the right objects does not ensure that their relationships satisfy the specification.</p><div class="figs"><figure><div class="fh">Requirement-family score per agent</div><div class="kit-legend"><span><i style="background:#dedede"></i><span>Identity & Environment</span><em>mean 67</em></span><span><i style="background:#c6c6c6"></i><span>Content & Quantity</span><em>mean 54</em></span><span><i style="background:#a9a9a9"></i><span>Attributes & Materials</span><em>mean 48</em></span><span><i style="background:linear-gradient(90deg,#D97757 0% 25%,#10A37F 25% 50%,#7b1fa2 50% 75%,#42a5f5 75% 100%)"></i><span><b>Spatial Composition</b>, in the agent’s colour</span><em>mean 36</em></span></div><div aria-label="Requirement-family scores, all 14 configurations" class="bar-scroll" role="region" tabindex="0"><svg aria-label="Requirement-family scores per agent, in Text-to-Scene rank order" class="kit-bars" id="tk-families" role="group" viewbox="0 0 1200 410"></svg></div><figcaption>One group per agent, in Text-to-Scene rank order: Identity & Environment, Content & Quantity, Attributes & Materials, then Spatial Composition, the lowest bar in every group. Scores ×100 (paper Figure 14b).</figcaption></figure><div class="figs two" style="margin-top:0"><figure><div class="fh">The objects are there; the relation is not</div><div class="live-scene" data-agent="sol" data-case="egyptian-temple"><span aria-hidden="true" class="agent-chip" style="background:#10A37F"><svg fill="#fff" viewbox="0 0 24 24"><path d="M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z"></path></svg>GPT-5.6 Sol (high)</span><canvas aria-label="Interactive 3D scene: Egyptian Temple by GPT-5.6 Sol. Drag to orbit; scroll to zoom." tabindex="0"></canvas><span class="scene-loading" role="status">Loading 3D scene…</span><span class="scene-hint">Drag to orbit · Scroll to zoom</span><button aria-label="Pause scene rotation" class="scene-motion">Ⅱ</button></div><div class="chips"><span class="chip ok">✓ slab roof <small>0.95</small></span><span class="chip ok">✓ stone coping <small>0.82</small></span><span class="chip no">✗ slab roof enclosed by the coping <small>0.78</small></span></div><figcaption><span>GPT-5.6 Sol’s saved Egyptian Temple scene. The judge finds the roof and stone coping, but flags their spatial relationship. <a href="img/judge_egyptian-temple_sol_s3f_000025.jpg?v=1790731787" rel="noopener" target="_blank">View the judge’s evidence ↗</a></span></figcaption></figure><figure><div class="fh">Mismatch when the objects are present</div><svg aria-label="Mismatch rate when the required objects are present" class="kit-ch" role="img" viewbox="0 0 520 196"><line stroke="#cfcfcf" stroke-dasharray="0.1 4.2" stroke-linecap="round" stroke-width="1.4" x1="212.0" x2="212.0" y1="10" y2="166"></line><line stroke="#cfcfcf" stroke-dasharray="0.1 4.2" stroke-linecap="round" stroke-width="1.4" x1="288.0" x2="288.0" y1="10" y2="166"></line><line stroke="#cfcfcf" stroke-dasharray="0.1 4.2" stroke-linecap="round" stroke-width="1.4" x1="364.0" x2="364.0" y1="10" y2="166"></line><line stroke="#cfcfcf" stroke-dasharray="0.1 4.2" stroke-linecap="round" stroke-width="1.4" x1="440.0" x2="440.0" y1="10" y2="166"></line><line stroke="#e6e6e6" stroke-width="1" x1="136" x2="136" y1="10" y2="166"></line><text fill="#9a9a9a" font-family="IBM Plex Mono, monospace" font-size="12" font-weight="500" text-anchor="middle" x="136.0" y="186">0%</text><text fill="#9a9a9a" font-family="IBM Plex Mono, monospace" font-size="12" font-weight="500" text-anchor="middle" x="212.0" y="186">10%</text><text fill="#9a9a9a" font-family="IBM Plex Mono, monospace" font-size="12" font-weight="500" text-anchor="middle" x="288.0" y="186">20%</text><text fill="#9a9a9a" font-family="IBM Plex Mono, monospace" font-size="12" font-weight="500" text-anchor="middle" x="364.0" y="186">30%</text><text fill="#9a9a9a" font-family="IBM Plex Mono, monospace" font-size="12" font-weight="500" text-anchor="middle" x="440.0" y="186">40%</text><g data-tip="Spatial relation: 32.0% of checks fail although the objects are present"><text fill="#262626" font-family="Inter, sans-serif" font-size="13" font-weight="700" text-anchor="end" x="122" y="40.6">Spatial relation</text><path d="M136,19.0 H374.7 Q379.2,19.0 379.2,23.5 V48.5 Q379.2,53.0 374.7,53.0 H136 Z" fill="#2b3531"></path><text fill="#262626" font-family="IBM Plex Mono, monospace" font-size="15.5" font-weight="500" x="388.2" y="41.4">32.0%</text></g><g data-tip="Distribution: 19.1% of checks fail although the objects are present"><text fill="#262626" font-family="Inter, sans-serif" font-size="13" font-weight="400" text-anchor="end" x="122" y="92.6">Distribution</text><path d="M136,71.0 H276.7 Q281.2,71.0 281.2,75.5 V100.5 Q281.2,105.0 276.7,105.0 H136 Z" fill="#c6c6c6"></path><text fill="#262626" font-family="IBM Plex Mono, monospace" font-size="15.5" font-weight="500" x="290.2" y="93.4">19.1%</text></g><g data-tip="Composition: 4.3% of checks fail although the objects are present"><text fill="#262626" font-family="Inter, sans-serif" font-size="13" font-weight="400" text-anchor="end" x="122" y="144.6">Composition</text><path d="M136,123.0 H164.2 Q168.7,123.0 168.7,127.5 V152.5 Q168.7,157.0 164.2,157.0 H136 Z" fill="#c6c6c6"></path><text fill="#262626" font-family="IBM Plex Mono, monospace" font-size="15.5" font-weight="500" x="177.7" y="145.4">4.3%</text></g></svg><figcaption>Share of checks that fail although every object they name is in the scene, across the benchmark (paper Figure 6a).</figcaption></figure></div></div></div></div></section><section class="dark" id="citation"><div class="container">
|
| 40 |
<div class="section-header centered"><span class="section-label">Citation</span><h2 class="section-title">Cite This Work</h2></div>
|
| 41 |
<div class="cite-box"><button class="cite-copy-btn" id="copy-btn">Copy</button><pre id="bibtex-text">@misc{ye2026code4scene,
|
| 42 |
title = {Code4Scene: Benchmarking Coding Agents for Constructing and Editing 3D Scenes},
|
|
|
|
| 56 |
<a href="https://simworld.org/" rel="noopener" target="_blank">SimWorld Home</a></p>
|
| 57 |
<p style="margin-top:.5rem;font-size:.75rem;color:rgba(255,255,255,.25)">Part of the SimWorld project.</p></footer>
|
| 58 |
<script src="assets/paper.js?v=5427505a"></script>
|
| 59 |
+
<script defer="" src="assets/leaderboard-bars.js?v=20261004kit"></script><script src="data/site.js?v=20261003t2s"></script><script src="assets/pareto.js?v=20261004kit" type="module"></script><script src="assets/scene-cards.js?v=20261002a" type="module"></script><script src="assets/task-tabs.js?v=20261001c" type="module"></script><script src="assets/leaderboard-controls.js?v=20261001c" type="module"></script><script src="assets/takeaway-bars.js?v=20261004kit" type="module"></script></body></html>
|
scripts/kit_charts.py
ADDED
|
@@ -0,0 +1,292 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Draw the page's charts in the Figure Kit style of the paper figures (user, 2026-10-01: "网站上的图表也换成这个风格").
|
| 2 |
+
|
| 3 |
+
- Takeaway: the requirement-family dot plot becomes the kit's grouped bars (company colour for Spatial Composition,
|
| 4 |
+
greys for the other three families, logos, rotated names), drawn by assets/takeaway-bars.js; the mismatch chart and
|
| 5 |
+
the judge's checks take the kit's neutrals; the evidence scene gets the kit's company-colour chip.
|
| 6 |
+
- Leaderboard bars: the "Text-to-Scene" qualifier beside the title, and the Qwen glyph in place of its wordmark.
|
| 7 |
+
- Pareto: the y axis is the Text-to-Scene score (it said "Simulation Quality"); effort in title case, as in the kit.
|
| 8 |
+
|
| 9 |
+
Each step asserts it found what it edits, so a re-run stops instead of half-applying.
|
| 10 |
+
python3 scripts/kit_charts.py
|
| 11 |
+
"""
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
import json, re, sys
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
from bs4 import BeautifulSoup
|
| 16 |
+
|
| 17 |
+
ROOT = Path(__file__).resolve().parent.parent
|
| 18 |
+
KIT = ROOT.parent / "figure-kit" # ~/Downloads/code4scene-video/figure-kit (the paper figures' build)
|
| 19 |
+
V = "20261004kit"
|
| 20 |
+
sys.path.insert(0, str(KIT))
|
| 21 |
+
import build as kit # noqa: E402 (logos, colours)
|
| 22 |
+
|
| 23 |
+
# paper Figure 14b: Identity & Environment, Content & Quantity, Attributes & Materials, Spatial Composition
|
| 24 |
+
FAMILY = {
|
| 25 |
+
"Claude Fable 5.1 (max)": (.906, .738, .670, .538),
|
| 26 |
+
"GPT-6 Astra (max)": (.875, .715, .643, .543),
|
| 27 |
+
"Claude Opus 5 (max)": (.803, .624, .573, .479),
|
| 28 |
+
"GPT-5.6 Sol (high)": (.750, .660, .602, .469),
|
| 29 |
+
"Gemini 3.8 Flash (high)": (.814, .639, .572, .473),
|
| 30 |
+
"Muse Spark 1.3 (medium)": (.730, .641, .546, .393),
|
| 31 |
+
"Grok 4.6 (high)": (.647, .558, .457, .334),
|
| 32 |
+
"Qwen 3.8 27B (thinking off)": (.724, .520, .428, .300),
|
| 33 |
+
"Qwen 3.8 27B (thinking on)": (.660, .525, .470, .307),
|
| 34 |
+
"GLM-5.3 Flash (max)": (.467, .421, .377, .299),
|
| 35 |
+
"Gemma 4 31B (thinking on)": (.561, .407, .387, .272),
|
| 36 |
+
"Gemma 4 31B (thinking off)": (.612, .453, .387, .252),
|
| 37 |
+
"Inkling (high)": (.572, .394, .361, .193),
|
| 38 |
+
"DeepSeek V4.1 Flash (high)": (.294, .215, .259, .120),
|
| 39 |
+
}
|
| 40 |
+
FAMILIES = ["Identity & Environment", "Content & Quantity", "Attributes & Materials", "Spatial Composition"]
|
| 41 |
+
GREYS = ["#dedede", "#c6c6c6", "#a9a9a9"]
|
| 42 |
+
SC_SWATCH = ["#D97757", "#10A37F", "#7b1fa2", "#42a5f5"]
|
| 43 |
+
MISMATCH = [("Spatial relation", 32.0), ("Distribution", 19.1), ("Composition", 4.3)] # paper Figure 6a, %
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def one(found, what):
|
| 47 |
+
assert found, f"not found: {what}"
|
| 48 |
+
return found
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def uniq_ids(spec, suffix):
|
| 52 |
+
"""A logo's gradient ids, suffixed, so a second copy on the page cannot point at the first one's defs."""
|
| 53 |
+
if "inner" not in spec:
|
| 54 |
+
return spec
|
| 55 |
+
s = spec["inner"]
|
| 56 |
+
for i in set(re.findall(r'id="([^"]+)"', s)):
|
| 57 |
+
s = s.replace(f'id="{i}"', f'id="{i}{suffix}"').replace(f"url(#{i})", f"url(#{i}{suffix})")
|
| 58 |
+
return dict(spec, inner=s)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def rows():
|
| 62 |
+
board = json.loads(re.search(r"const ALL = (\{.*?\});\n", (ROOT / "assets/leaderboard-bars.js").read_text(), re.S).group(1))["t2s"]
|
| 63 |
+
out = []
|
| 64 |
+
for b in sorted(board, key=lambda b: -b["score"]):
|
| 65 |
+
fam = FAMILY[b["name"]]
|
| 66 |
+
assert min(fam) == fam[3], b["name"] # the claim the chart makes, per agent
|
| 67 |
+
out.append({"name": b["name"], "org": b["org"], "colour": b["colour"], "fam": fam})
|
| 68 |
+
assert len(out) == len(FAMILY) == 14
|
| 69 |
+
return out
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
TAKEAWAY_JS = r"""// Code4Scene takeaway: requirement-family scores per agent (paper Figure 14b), in the paper figures' kit style --
|
| 73 |
+
// grouped bars in Text-to-Scene rank order, Spatial Composition in the agent's company colour, the other families grey.
|
| 74 |
+
// Generated by scripts/kit_charts.py; the layout follows figure-kit/takeaway.py and the leaderboard bars.
|
| 75 |
+
import {bindRecordTooltip} from "./chart-interactions.js?v=%(v)s";
|
| 76 |
+
const D = %(data)s;
|
| 77 |
+
const LOGOS = %(logos)s;
|
| 78 |
+
await Promise.all(['400 13px Inter', '700 13px Inter', '500 12.5px "IBM Plex Mono"'].map(f => document.fonts.load(f)));
|
| 79 |
+
const svg = document.getElementById('tk-families'), W = 1200, H = 410, NS = 'http://www.w3.org/2000/svg';
|
| 80 |
+
const SANS = 'Inter, Arial, sans-serif', MONO = '"IBM Plex Mono", monospace';
|
| 81 |
+
const G = {left: 40, right: 24, top: 34, base: 262, groupFill: 0.80, gap: 2, radius: 3.5,
|
| 82 |
+
logo: 18, logoGap: 8, nameGap: 9, nameAngle: -60, nameSize: 13, nameLead: 14.5, nameOneLine: 97, nameLine: 93, nameTol: 12};
|
| 83 |
+
function el(tag, attrs, parent) { const e = document.createElementNS(NS, tag); for (const k in attrs) e.setAttribute(k, attrs[k]); (parent || svg).appendChild(e); return e; }
|
| 84 |
+
function text(x, y, s, attrs, parent) { const t = el('text', Object.assign({x, y}, attrs), parent); t.textContent = s; return t; }
|
| 85 |
+
const ctx = document.createElement('canvas').getContext('2d');
|
| 86 |
+
function width(t, font) { ctx.font = font; return ctx.measureText(t).width; }
|
| 87 |
+
function wrap(name, font) {
|
| 88 |
+
if (width(name, font) <= G.nameOneLine) return [name];
|
| 89 |
+
const w = name.split(' ');
|
| 90 |
+
const splits = k => { const out = [];
|
| 91 |
+
if (k === 2) for (let i = 1; i < w.length; i++) out.push([w.slice(0, i), w.slice(i)]);
|
| 92 |
+
if (k === 3) for (let i = 1; i < w.length; i++) for (let j = i + 1; j < w.length; j++) out.push([w.slice(0, i), w.slice(i, j), w.slice(j)]);
|
| 93 |
+
return out.map(p => p.map(x => x.join(' '))); };
|
| 94 |
+
for (const k of [2, 3]) {
|
| 95 |
+
const c = splits(k).map(lines => { const ws = lines.map(l => width(l, font)); return {lines, worst: Math.max(...ws), last: ws[ws.length - 1]}; })
|
| 96 |
+
.filter(o => o.worst <= G.nameLine);
|
| 97 |
+
if (!c.length) continue;
|
| 98 |
+
const floor = Math.min(...c.map(o => o.worst));
|
| 99 |
+
return c.filter(o => o.worst <= floor + G.nameTol).sort((a, b) => b.last - a.last)[0].lines;
|
| 100 |
+
}
|
| 101 |
+
return [name];
|
| 102 |
+
}
|
| 103 |
+
function logo(org, cx, ly, parent) {
|
| 104 |
+
const L = LOGOS[org], s = G.logo, lx = cx - s / 2;
|
| 105 |
+
if (L.wordmark) { L.wordmark.forEach((w, k) => text(cx, ly + 7.5 + k * 6.2, w, {'text-anchor': 'middle', 'font-family': SANS, 'font-weight': 500, 'font-size': 5.2, 'letter-spacing': 0.35, fill: '#4a4a4a'}, parent)); return; }
|
| 106 |
+
let pad = 0, fill = L.fill || null;
|
| 107 |
+
if (L.tile) { if (L.shape === 'circle') el('circle', {cx, cy: ly + s / 2, r: s / 2, fill: L.tile}, parent); else el('rect', {x: lx, y: ly, width: s, height: s, rx: 3.2, fill: L.tile}, parent); pad = 3.4; fill = '#ffffff'; }
|
| 108 |
+
const ic = el('svg', {x: lx + pad, y: ly + pad, width: s - 2 * pad, height: s - 2 * pad, viewBox: L.viewBox}, parent);
|
| 109 |
+
if (fill) { ic.setAttribute('fill', fill); ic.setAttribute('color', fill); }
|
| 110 |
+
if (L.rule) ic.setAttribute('fill-rule', L.rule);
|
| 111 |
+
ic.innerHTML = L.inner;
|
| 112 |
+
}
|
| 113 |
+
const y = v => G.base - v / 100 * (G.base - G.top);
|
| 114 |
+
for (const v of [25, 50, 75, 100])
|
| 115 |
+
el('line', {x1: G.left - 8, x2: W - G.right, y1: y(v), y2: y(v), stroke: '#cfcfcf', 'stroke-width': 1.4, 'stroke-dasharray': '0.1 4.2', 'stroke-linecap': 'round'});
|
| 116 |
+
el('line', {x1: G.left - 8, x2: W - G.right, y1: G.base, y2: G.base, stroke: '#e6e6e6', 'stroke-width': 1});
|
| 117 |
+
const n = D.rows.length, pitch = (W - G.left - G.right) / n, gw = pitch * G.groupFill, bw = (gw - 3 * G.gap) / 4;
|
| 118 |
+
D.rows.forEach((d, i) => {
|
| 119 |
+
const cx = G.left + (i + 0.5) * pitch, g0 = cx - gw / 2;
|
| 120 |
+
const node = el('g', {class: 'bar-node'});
|
| 121 |
+
el('rect', {x: G.left + i * pitch, y: G.top - 20, width: pitch, height: H - G.top + 20, fill: 'transparent', class: 'bar-hit'}, node);
|
| 122 |
+
d.fam.forEach((v, j) => {
|
| 123 |
+
const x0 = g0 + j * (bw + G.gap), top = y(v * 100), r = Math.min(G.radius, bw / 2);
|
| 124 |
+
el('path', {class: j === 3 ? 'bar' : 'bar fam', d: `M${x0},${G.base} V${top + r} Q${x0},${top} ${x0 + r},${top} H${x0 + bw - r} Q${x0 + bw},${top} ${x0 + bw},${top + r} V${G.base} Z`,
|
| 125 |
+
fill: j === 3 ? d.colour : D.greys[j]}, node);
|
| 126 |
+
if (j === 3) text(x0 + bw / 2, top - 6, String(Math.round(v * 100)), {'text-anchor': 'middle', 'font-family': MONO, 'font-weight': 500, 'font-size': 12.5, fill: '#262626'}, node);
|
| 127 |
+
});
|
| 128 |
+
logo(d.org, cx, G.base + G.logoGap, node);
|
| 129 |
+
const font = `400 ${G.nameSize}px ${SANS}`, lines = wrap(d.name, font);
|
| 130 |
+
const g = el('g', {transform: `translate(${cx + 3},${G.base + G.logoGap + G.logo + G.nameGap}) rotate(${G.nameAngle})`}, node);
|
| 131 |
+
const off = -(lines.length - 1) / 2 * G.nameLead;
|
| 132 |
+
lines.forEach((ln, k) => text(0, off + k * G.nameLead + 4.5, ln, {'text-anchor': 'end', 'font-family': SANS, 'font-size': G.nameSize, fill: '#262626'}, g));
|
| 133 |
+
bindRecordTooltip(node, svg, {name: d.name, provider: d.org,
|
| 134 |
+
metrics: D.families.map((f, j) => ({label: f, value: d.fam[j].toFixed(3)}))}, '.bar:not(.fam)');
|
| 135 |
+
});
|
| 136 |
+
svg.dataset.ready = 'true';
|
| 137 |
+
"""
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def write_takeaway_js():
|
| 141 |
+
rs = rows()
|
| 142 |
+
logos = {org: uniq_ids(spec, "-tk") for org, spec in kit.logos().items() if org in {r["org"] for r in rs}}
|
| 143 |
+
data = {"rows": rs, "families": FAMILIES, "greys": GREYS}
|
| 144 |
+
(ROOT / "assets/takeaway-bars.js").write_text(TAKEAWAY_JS % {"v": V, "data": json.dumps(data), "logos": json.dumps(logos)})
|
| 145 |
+
return rs
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def mismatch_svg():
|
| 149 |
+
"""Paper Figure 6a in the kit's neutrals: the spatial-relation bar dark, the others grey, values in Plex Mono."""
|
| 150 |
+
Wm, x0, x1, top, rh, bh = 520, 136, 440, 10, 52, 34
|
| 151 |
+
X = lambda v: x0 + v / 40 * (x1 - x0)
|
| 152 |
+
H = top + rh * len(MISMATCH) + 30
|
| 153 |
+
o = [f'<svg class="kit-ch" viewBox="0 0 {Wm} {H}" role="img" aria-label="Mismatch rate when the required objects are present">']
|
| 154 |
+
for v in (10, 20, 30, 40):
|
| 155 |
+
o.append(f'<line x1="{X(v):.1f}" x2="{X(v):.1f}" y1="{top}" y2="{top + rh * 3}" stroke="#cfcfcf" stroke-width="1.4" stroke-dasharray="0.1 4.2" stroke-linecap="round"/>')
|
| 156 |
+
o.append(f'<line x1="{x0}" x2="{x0}" y1="{top}" y2="{top + rh * 3}" stroke="#e6e6e6" stroke-width="1"/>')
|
| 157 |
+
for v in (0, 10, 20, 30, 40):
|
| 158 |
+
o.append(f'<text x="{X(v):.1f}" y="{top + rh * 3 + 20}" text-anchor="middle" font-family="IBM Plex Mono, monospace" font-weight="500" font-size="12" fill="#9a9a9a">{v}%</text>')
|
| 159 |
+
for i, (lab, v) in enumerate(MISMATCH):
|
| 160 |
+
yc = top + (i + .5) * rh; hot = i == 0; xe = X(v); r = min(4.5, (xe - x0) / 2)
|
| 161 |
+
o.append(f'<g data-tip="{lab}: {v:.1f}% of checks fail although the objects are present">')
|
| 162 |
+
o.append(f'<text x="{x0 - 14}" y="{yc + 4.6:.1f}" text-anchor="end" font-family="Inter, sans-serif" font-size="13" font-weight="{700 if hot else 400}" fill="#262626">{lab}</text>')
|
| 163 |
+
o.append(f'<path d="M{x0},{yc - bh / 2:.1f} H{xe - r:.1f} Q{xe:.1f},{yc - bh / 2:.1f} {xe:.1f},{yc - bh / 2 + r:.1f} V{yc + bh / 2 - r:.1f} '
|
| 164 |
+
f'Q{xe:.1f},{yc + bh / 2:.1f} {xe - r:.1f},{yc + bh / 2:.1f} H{x0} Z" fill="{"#2b3531" if hot else "#c6c6c6"}"/>')
|
| 165 |
+
o.append(f'<text x="{xe + 9:.1f}" y="{yc + 5.4:.1f}" font-family="IBM Plex Mono, monospace" font-weight="500" font-size="15.5" fill="#262626">{v:.1f}%</text></g>')
|
| 166 |
+
o.append("</svg>")
|
| 167 |
+
return "".join(o)
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def legend_html(rs):
|
| 171 |
+
mean = [sum(r["fam"][j] for r in rs) / len(rs) for j in range(4)]
|
| 172 |
+
sw = "linear-gradient(90deg," + ",".join(f"{c} {k * 25}% {(k + 1) * 25}%" for k, c in enumerate(SC_SWATCH)) + ")"
|
| 173 |
+
items = []
|
| 174 |
+
for j, f in enumerate(FAMILIES):
|
| 175 |
+
bg = GREYS[j] if j < 3 else sw
|
| 176 |
+
lab = f if j < 3 else f"<b>{f}</b>, in the agent’s colour"
|
| 177 |
+
items.append(f'<span><i style="background:{bg}"></i><span>{lab.replace("&", "&") if j < 3 else lab.replace(" & ", " & ")}</span><em>mean {round(mean[j] * 100)}</em></span>')
|
| 178 |
+
return '<div class="kit-legend">' + "".join(items) + "</div>"
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
def index(rs):
|
| 182 |
+
p = ROOT / "index.html"
|
| 183 |
+
soup = BeautifulSoup(p.read_text(), "html.parser")
|
| 184 |
+
frag = lambda html: BeautifulSoup(html, "html.parser")
|
| 185 |
+
# takeaway: the family chart
|
| 186 |
+
tk = one(soup.find(id="takeaways"), "takeaways")
|
| 187 |
+
figs = one(tk.find(class_="figs"), "takeaway figs")
|
| 188 |
+
fam_fig = one(figs.find("figure", recursive=False), "family figure")
|
| 189 |
+
fam_fig.clear()
|
| 190 |
+
fam_fig.append(frag('<div class="fh">Requirement-family score per agent</div>' + legend_html(rs) +
|
| 191 |
+
'<div class="bar-scroll" role="region" tabindex="0" aria-label="Requirement-family scores, all 14 configurations">'
|
| 192 |
+
'<svg id="tk-families" class="kit-bars" viewBox="0 0 1200 410" role="group" '
|
| 193 |
+
'aria-label="Requirement-family scores per agent, in Text-to-Scene rank order"></svg></div>'
|
| 194 |
+
'<figcaption>One group per agent, in Text-to-Scene rank order: Identity & Environment, Content & Quantity, '
|
| 195 |
+
'Attributes & Materials, then Spatial Composition, the lowest bar in every group. Scores ×100 (paper Figure 14b).</figcaption>'))
|
| 196 |
+
# the mismatch chart
|
| 197 |
+
mm = None
|
| 198 |
+
for f in tk.find_all("figure"):
|
| 199 |
+
h = f.find(class_="fh")
|
| 200 |
+
if h and h.get_text(strip=True).startswith("Mismatch"):
|
| 201 |
+
mm = f
|
| 202 |
+
one(mm, "mismatch figure")
|
| 203 |
+
one(mm.find("svg"), "mismatch svg").replace_with(frag(mismatch_svg()))
|
| 204 |
+
mm.find(class_="fh").string = "Mismatch when the objects are present"
|
| 205 |
+
mm.find("figcaption").string = "Share of checks that fail although every object they name is in the scene, across the benchmark (paper Figure 6a)."
|
| 206 |
+
# the evidence scene: the kit's company-colour chip over the panel
|
| 207 |
+
scene = one(tk.find(class_="live-scene", attrs={"data-agent": "sol"}), "evidence scene")
|
| 208 |
+
if not scene.find(class_="agent-chip"):
|
| 209 |
+
ic = kit.logos()["OpenAI"]
|
| 210 |
+
scene.insert(0, frag(f'<span class="agent-chip" style="background:{kit.COLOUR["OpenAI"]}" aria-hidden="true">'
|
| 211 |
+
f'<svg viewBox="{ic["viewBox"]}" fill="#fff">{ic["inner"]}</svg>GPT-5.6 Sol (high)</span>'))
|
| 212 |
+
# leaderboard bars: the qualifier beside the title
|
| 213 |
+
h3 = one(soup.select_one(".bar-heading h3"), "bar heading")
|
| 214 |
+
h3.clear(); h3.append("Code4Scene "); q = soup.new_tag("span", attrs={"class": "q"}); q.string = "Text-to-Scene"; h3.append(q)
|
| 215 |
+
# scripts and cache keys
|
| 216 |
+
if not soup.find("script", src=re.compile(r"takeaway-bars\.js")):
|
| 217 |
+
soup.body.append(soup.new_tag("script", attrs={"type": "module", "src": f"assets/takeaway-bars.js?v={V}"}))
|
| 218 |
+
for t in soup.find_all(["script", "link"]):
|
| 219 |
+
a = "src" if t.name == "script" else "href"
|
| 220 |
+
u = t.get(a) or ""
|
| 221 |
+
if re.match(r"assets/(buildingbench\.css|leaderboard-bars\.js|pareto\.js|takeaway-bars\.js)", u):
|
| 222 |
+
t[a] = re.sub(r"\?v=.*$", "", u) + f"?v={V}"
|
| 223 |
+
p.write_text(str(soup))
|
| 224 |
+
print("index.html: takeaway, mismatch, chip, bar heading, scripts")
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
def bars_js():
|
| 228 |
+
p = ROOT / "assets/leaderboard-bars.js"; s = p.read_text()
|
| 229 |
+
m = one(re.search(r"const LOGOS = (\{.*?\});\n", s, re.S), "LOGOS")
|
| 230 |
+
logos = json.loads(m.group(1))
|
| 231 |
+
logos["Alibaba"] = kit.logos()["Alibaba"] # the Qwen glyph, as in the paper figure
|
| 232 |
+
s = s[:m.start(1)] + json.dumps(logos) + s[m.end(1):]
|
| 233 |
+
s, n = re.subn(r"chart-interactions\.js\?v=\w+", f"chart-interactions.js?v={V}", s); assert n == 1
|
| 234 |
+
p.write_text(s); print("leaderboard-bars.js: Qwen glyph")
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def pareto_js():
|
| 238 |
+
p = ROOT / "assets/pareto.js"; s = p.read_text()
|
| 239 |
+
a = "'Simulation Quality'"; assert a in s or "'Text-to-Scene Score'" in s; s = s.replace(a, "'Text-to-Scene Score'")
|
| 240 |
+
a = ".replace(/\\((.*?)\\)/,(_,s)=>'('+s[0].toUpperCase()+s.slice(1)+')')"
|
| 241 |
+
assert a in s or "s.replace(/\\b\\w/g" in s, "pareto name casing"
|
| 242 |
+
s = s.replace(a, ".replace(/\\((.*?)\\)/,(_,s)=>'('+s.replace(/\\b\\w/g,c=>c.toUpperCase())+')')")
|
| 243 |
+
s, n = re.subn(r"chart-interactions\.js\?v=\w+", f"chart-interactions.js?v={V}", s); assert n == 1
|
| 244 |
+
p.write_text(s); print("pareto.js: axis title, effort casing")
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def interactions_js():
|
| 248 |
+
p = ROOT / "assets/chart-interactions.js"; s = p.read_text()
|
| 249 |
+
if "bindRecordTooltip" in s:
|
| 250 |
+
return
|
| 251 |
+
a = ("export async function bindChartTooltip(node,plot,key,mode,anchorSelector='.bar,.chip'){\n"
|
| 252 |
+
" const data=await detailPromise,record=data[mode].find(d=>d.key===key||d.name===key);if(!record)return;\n")
|
| 253 |
+
assert a in s, "bindChartTooltip head"
|
| 254 |
+
s = s.replace(a, "export async function bindChartTooltip(node,plot,key,mode,anchorSelector='.bar,.chip'){\n"
|
| 255 |
+
" const data=await detailPromise,record=data[mode].find(d=>d.key===key||d.name===key);if(!record)return;\n"
|
| 256 |
+
" bindRecordTooltip(node,plot,record,anchorSelector);\n}\n"
|
| 257 |
+
"// the same behaviour for a chart whose record is not a leaderboard row (the takeaway's family scores)\n"
|
| 258 |
+
"export function bindRecordTooltip(node,plot,record,anchorSelector='.bar,.chip'){\n")
|
| 259 |
+
p.write_text(s); print("chart-interactions.js: bindRecordTooltip")
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
CSS = """
|
| 263 |
+
/* kit-style charts (scripts/kit_charts.py) */
|
| 264 |
+
.bar-heading h3 span.q{font:inherit!important;color:#8d8d8d!important;max-width:none!important;text-align:left!important}
|
| 265 |
+
.tk figure{background:#fff;min-width:0}
|
| 266 |
+
.tk .figs{grid-template-columns:minmax(0,1fr)}.tk .figs.two{grid-template-columns:minmax(0,1.15fr) minmax(0,1fr)}
|
| 267 |
+
@media(max-width:900px){.tk .figs.two{grid-template-columns:minmax(0,1fr)}}
|
| 268 |
+
.tk figure .fh{font:400 22px/1.25 'Source Serif 4',Georgia,serif;color:#8d8d8d;margin-bottom:12px;letter-spacing:-.3px}
|
| 269 |
+
.kit-legend{display:flex;flex-wrap:wrap;gap:6px 26px;margin:0 0 6px;font:400 13px/1.5 Inter,sans-serif;color:#262626}
|
| 270 |
+
.kit-legend>span{display:inline-flex;align-items:center;gap:7px}.kit-legend b{font-weight:700}
|
| 271 |
+
.kit-legend i{width:12px;height:12px;border-radius:3px;display:inline-block;flex:none}
|
| 272 |
+
.kit-legend em{font:500 12.5px 'IBM Plex Mono',monospace;color:#8d8d8d;font-style:normal;margin-left:2px}
|
| 273 |
+
svg.kit-ch{display:block;width:100%;height:auto;overflow:visible}
|
| 274 |
+
.tk .chip{font:500 12.5px Inter,sans-serif;padding:5px 12px;border-radius:999px}
|
| 275 |
+
.tk .chip.ok{color:#262626;background:#f1f1ee;border-color:#e4e4e2}
|
| 276 |
+
.tk .chip.no{color:#fff;background:#111;border-color:#111}
|
| 277 |
+
.tk .chip small{font:500 12px 'IBM Plex Mono',monospace;color:#8d8d8d}.tk .chip.no small{color:#bdbdbd}
|
| 278 |
+
.agent-chip{position:absolute;z-index:4;top:12px;left:12px;display:inline-flex;align-items:center;gap:6px;padding:5px 10px;border-radius:6px;color:#fff;font:700 12.5px Inter,sans-serif;pointer-events:none}
|
| 279 |
+
.agent-chip svg{width:14px;height:14px}
|
| 280 |
+
@media(max-width:650px){.kit-legend{font-size:11px;gap:4px 14px}.kit-legend em{font-size:10.5px}.tk figure .fh{font-size:18px}.agent-chip{font-size:11px;top:8px;left:8px}}
|
| 281 |
+
"""
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
def css():
|
| 285 |
+
p = ROOT / "assets/buildingbench.css"; s = p.read_text()
|
| 286 |
+
if "kit-style charts (scripts/kit_charts.py)" in s:
|
| 287 |
+
s = s[:s.index("\n/* kit-style charts (scripts/kit_charts.py) */")]
|
| 288 |
+
p.write_text(s.rstrip("\n") + "\n" + CSS); print("buildingbench.css: kit chart rules")
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
if __name__ == "__main__":
|
| 292 |
+
rs = write_takeaway_js(); interactions_js(); bars_js(); pareto_js(); css(); index(rs)
|