Spaces:
Running
Running
Name the evaluation tracks: Research Track (20 scenes) and Full Benchmark (160 scenes)
Browse filesThe leaderboard's Text-to-Scene results are the Research Track's; the case-count strip states the two tracks, and the leaderboard section says why the smaller track exists.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
- assets/buildingbench.css +3 -0
- index.html +6 -6
- scripts/tracks.py +69 -0
assets/buildingbench.css
CHANGED
|
@@ -60,3 +60,6 @@ svg.kit-ch{display:block;width:100%;height:auto;overflow:visible}
|
|
| 60 |
.agent-chip{position:absolute;z-index:4;top:12px;left:12px;display:inline-flex;align-items:center;gap:6px;padding:5px 10px;border-radius:6px;color:#fff;font:700 12.5px Inter,sans-serif;pointer-events:none}
|
| 61 |
.agent-chip svg{width:14px;height:14px}
|
| 62 |
@media(max-width:650px){.kit-legend{font-size:11px;gap:4px 14px}.kit-legend em{font-size:10.5px}.tk figure .fh{font-size:18px}.agent-chip{font-size:11px;top:8px;left:8px}}
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
.agent-chip{position:absolute;z-index:4;top:12px;left:12px;display:inline-flex;align-items:center;gap:6px;padding:5px 10px;border-radius:6px;color:#fff;font:700 12.5px Inter,sans-serif;pointer-events:none}
|
| 61 |
.agent-chip svg{width:14px;height:14px}
|
| 62 |
@media(max-width:650px){.kit-legend{font-size:11px;gap:4px 14px}.kit-legend em{font-size:10.5px}.tk figure .fh{font-size:18px}.agent-chip{font-size:11px;top:8px;left:8px}}
|
| 63 |
+
/* evaluation tracks (scripts/tracks.py) */
|
| 64 |
+
.tracks-why{max-width:860px;margin:8px 0 0;font-size:13px;line-height:1.65;color:var(--text-muted);text-align:left}
|
| 65 |
+
@media(max-width:650px){.tracks-why{font-size:11px}}
|
index.html
CHANGED
|
@@ -19,13 +19,13 @@ svg.ch .dim{opacity:.42}svg.ch g.row:hover{opacity:1}svg.ch .hot{fill:var(--c4s-
|
|
| 19 |
.chip.ok{color:var(--c4s-ok);border-color:currentColor}.chip.no{color:var(--c4s-hot);border-color:currentColor}.chip small{font-weight:500;color:var(--text-muted);margin-left:4px}
|
| 20 |
.bigstats{display:flex;flex-wrap:wrap;gap:12px 34px;margin:0 0 12px}.bigstats b{display:block;font-size:30px;letter-spacing:-.02em;font-variant-numeric:tabular-nums}
|
| 21 |
.bigstats span{font-size:13px;color:var(--text-secondary)}
|
| 22 |
-
</style><link href="assets/buildingbench.css?v=
|
| 23 |
-
<header class="site-header"><div class="brand"><a aria-label="Code4Scene home" class="brand-home" href="index.html#top"><svg aria-hidden="true" fill="none" viewbox="0 0 30 32"><path d="M15 2 27 9v14l-12 7L3 23V9L15 2Z M3 9l12 7 12-7 M15 16v14 M9 5.5l12 7v14 M21 5.5l-12 7v14" stroke="currentColor" stroke-linejoin="round" stroke-width="1.6"></path></svg><span>Code4Scene</span></a><a class="brand-by" href="https://simworld.org/">by SimWorld</a></div><nav aria-label="Main navigation"><a class="nav-active" href="index.html#top">Benchmark</a><a href="index.html#leaderboard">Leaderboard</a><a href="index.html#cases">Visualization</a><a href="https://arxiv.org/abs/2609.36777" rel="noopener" target="_blank">Paper ↗</a><a href="https://github.com/SimWorld-AI/Code4Scene" rel="noopener" target="_blank">Github ↗</a></nav></header><main class="page-main"><section class="intro" id="top"><div class="intro-copy"><h1>Constructing<br/><em>3D Scenes with Code</em></h1><p class="intro-text">Benchmarking coding agents that turn open-ended scene descriptions into engine-native 3D scenes in Unreal Engine. Measuring spatial reasoning, task fulfillment and physical validity.</p><div class="intro-links"><a href="https://arxiv.org/abs/2609.36777">Read the paper ↗</a><a href="https://github.com/SimWorld-AI/Code4Scene">Code & Data ↗</a><a href="#hero-video">Watch the teaser ↘</a></div></div><div class="featured featured-video" id="hero-video"><video aria-label="Code4Scene overview video" controls="" playsinline="" poster="media/teaser-poster.jpg" preload="metadata"><source src="media/teaser-v31.mp4" type="video/mp4"/>Your browser does not support embedded video. <a href="media/teaser-v31.mp4">Watch the Code4Scene video</a>.</video><div class="featured-video-caption"><strong>Code4Scene in 90 seconds</strong><span>Construction and evaluation</span></div></div></section><section class="task-section" id="task-format"><div class="container"><h2 class="section-title">Task Format</h2><div class="task-shell"><div aria-labelledby="task-tab-t2s" class="task-content" data-task="t2s" id="task-t2s" role="tabpanel"><h3>Nordic Harbour <span>· Text-to-Scene</span></h3><p class="task-subtitle">An open-ended scene description, an asset catalog and the Unreal Engine editor.</p><div class="task-columns"><article class="task-panel"><div class="panel-heading"><strong>Agent input</strong><span>Text + assets</span></div><div class="prompt-preview"><span class="eyebrow">Task instruction</span><p>Build me a compact, sunlit Copenhagen-style canal district with narrow cobblestone streets and rectangular waterways enclosing dense urban blocks.</p><p class="prompt-detail">Townhouses, canals, bridges and a waterfront promenade — with detailed requirements for materials, layout and street furniture.</p></div><p class="task-foot"><strong>Given:</strong> an empty level and the content pack’s asset catalog.</p></article><article class="task-panel output-panel"><div class="panel-heading"><strong>Agent output</strong><span>GPT-6 Astra</span></div><div class="live-scene" data-agent="astra" data-case="nordic-harbour"><canvas aria-label="Interactive 3D scene: Nordic Harbour. Drag to orbit, scroll or use plus and minus to zoom." tabindex="0"></canvas><span class="scene-loading" role="status">Loading 3D scene…</span><span class="scene-hint">Drag to orbit · Scroll to zoom</span><button aria-label="Pause scene rotation" class="scene-motion">Ⅱ</button></div><a class="task-foot text-button" href="cases.html#nordic-harbour">Compare agents and evaluator views ↗</a></article><article class="task-panel evaluation-panel"><div class="panel-heading"><strong>Evaluation</strong><span>Engine-native scene</span></div><div class="score-block"><span>CASE SCORE</span><strong>0.877</strong><dl><div><dt>Detailed Alignment</dt><dd>0.757</dd></div><div><dt>Overview Alignment</dt><dd>0.909</dd></div><div><dt>Physical Safety</dt><dd>0.900</dd></div></dl></div><p class="task-foot">Task fulfillment, artifact integrity and static physical validity.</p><p class="task-formula">0.2 × Detailed + 0.6 × Overview + 0.2 × Physical</p></article></div><details class="task-full"><summary><span>+ Full task instruction</span><span>TEXT-TO-SCENE</span></summary><p>Build me a compact, sunlit Copenhagen-style canal district with narrow cobblestone streets and rectangular waterways enclosing dense urban blocks. Arrange roughly thirty attached four- to six-storey townhouses in continuous street and canal-front rows, using slender façades, repetitive white-framed windows, arched doors and carriage entrances, modest cornices, and steep tiled roofs packed with dormers and brick chimneys. Include occasional exposed party walls between staggered building rows. Finish the façades in weathered muted red, mustard yellow, dusty blue, warm grey, beige, and exposed brick, with red, orange, and charcoal roof tiles. Run a principal waterfront street along a stone quay, crossed by smaller streets and linked across the canals by short, gently arched stone-and-metal bridges, including one opening onto a broad paved promenade with rounded landings. Form the canal edges from stepped pale stone, dark timber retaining walls, and iron railings around reflective rippling water. Add regularly spaced leafy trees, ornate black lamps, stone bollards, benches, European road signs, zebra crossings, puddles, grime, and sparse weeds.</p></details></div></div><div class="availability"><span><b>
|
| 24 |
<div class="section-header centered"><span class="section-label">Leaderboard</span><h2 class="section-title">Leaderboard</h2>
|
| 25 |
-
<p class="section-desc">14 coding-agent configurations
|
| 26 |
-
<div id="lbwrap"><div class="card bar-card"><div class="bar-heading"><h3>Code4Scene <span class="q">
|
| 27 |
-
<div class="card chart-card pareto-card"><h3 class="figure-title">Code4Scene <span>Pareto Frontier</span></h3><p class="figure-subtitle">Quality against the cost of one case ·
|
| 28 |
-
<div class="card ranking-card"><div class="ranking-toolbar"><label for="model-search">Search configurations<input autocomplete="off" id="model-search" placeholder="Model or provider" type="search"/></label><span aria-live="polite" id="ranking-count">14 configurations</span></div><div class="sub-title">Ranking</div><div data-view="t2s"><p class="board-note" style="margin:0 .2rem .7rem">20 public cases. Case score = 0.2 × Detailed + 0.6 × Overview + 0.2 × Physical. Detailed and Overview are means of the per-case scores.</p><div class="board-wrap"><table class="board"><thead><tr><th>#</th><th>Agent configuration</th><th class="sorted" data-best="high" data-k="st">Score</th><th data-best="high" data-k="detailed">Detailed</th><th data-best="high" data-k="overview">Overview</th><th data-best="high" data-k="pt">Physical</th><th data-best="low" data-k="c_t2s">Cost / case</th></tr></thead><tbody><tr data-model-key="fable" data-search="claude fable 5.1 (max) anthropic"><td class="rk">1</td><td class="nm"><span class="logo"><svg fill="#D97757" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Claude</title><path d="M4.709 15.955l4.72-2.647.08-.23-.08-.128H9.2l-.79-.048-2.698-.073-2.339-.097-2.266-.122-.571-.121L0 11.784l.055-.352.48-.321.686.06 1.52.103 2.278.158 1.652.097 2.449.255h.389l.055-.157-.134-.098-.103-.097-2.358-1.596-2.552-1.688-1.336-.972-.724-.491-.364-.462-.158-1.008.656-.722.881.06.225.061.893.686 1.908 1.476 2.491 1.833.365.304.145-.103.019-.073-.164-.274-1.355-2.446-1.446-2.49-.644-1.032-.17-.619a2.97 2.97 0 01-.104-.729L6.283.134 6.696 0l.996.134.42.364.62 1.414 1.002 2.229 1.555 3.03.456.898.243.832.091.255h.158V9.01l.128-1.706.237-2.095.23-2.695.08-.76.376-.91.747-.492.584.28.48.685-.067.444-.286 1.851-.559 2.903-.364 1.942h.212l.243-.242.985-1.306 1.652-2.064.73-.82.85-.904.547-.431h1.033l.76 1.129-.34 1.166-1.064 1.347-.881 1.142-1.264 1.7-.79 1.36.073.11.188-.02 2.856-.606 1.543-.28 1.841-.315.833.388.091.395-.328.807-1.969.486-2.309.462-3.439.813-.042.03.049.061 1.549.146.662.036h1.622l3.02.225.79.522.474.638-.079.485-1.215.62-1.64-.389-3.829-.91-1.312-.329h-.182v.11l1.093 1.068 2.006 1.81 2.509 2.33.127.578-.322.455-.34-.049-2.205-1.657-.851-.747-1.926-1.62h-.128v.17l.444.649 2.345 3.521.122 1.08-.17.353-.608.213-.668-.122-1.374-1.925-1.415-2.167-1.143-1.943-.14.08-.674 7.254-.316.37-.729.28-.607-.461-.322-.747.322-1.476.389-1.924.315-1.53.286-1.9.17-.632-.012-.042-.14.018-1.434 1.967-2.18 2.945-1.726 1.845-.414.164-.717-.37.067-.662.401-.589 2.388-3.036 1.44-1.882.93-1.086-.006-.158h-.055L4.132 18.56l-1.13.146-.487-.456.061-.746.231-.243 1.908-1.312-.006.006z"></path></svg></span>Claude Fable 5.1 (max)<span class="tag front">Pareto</span><span class="org">Anthropic</span></td><td data-v="0.78775"><div class="cellbar"><i style="width:80.0px;--bar:var(--c4s-a)"></i><b>0.788</b></div></td><td class="dim" data-v="0.7253149999999999">0.725</td><td class="dim" data-v="0.7717050000000001">0.772</td><td class="dim" data-v="0.898">0.898</td><td class="dim" data-v="18.1576316">$18.16</td></tr><tr data-model-key="astra" data-search="gpt-6 astra (max) openai"><td class="rk">2</td><td class="nm"><span class="logo"><svg fill="currentColor" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>OpenAI</title><path d="M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z"></path></svg></span>GPT-6 Astra (max)<span class="org">OpenAI</span></td><td data-v="0.72382"><div class="cellbar"><i style="width:73.5px;--bar:var(--c4s-a)"></i><b>0.724</b></div></td><td class="dim" data-v="0.707165">0.707</td><td class="dim" data-v="0.733235">0.733</td><td class="dim" data-v="0.712">0.712</td><td class="dim" data-v="22.472260736842106">$22.47</td></tr><tr data-model-key="opus" data-search="claude opus 5 (max) anthropic"><td class="rk">3</td><td class="nm"><span class="logo"><svg fill="#D97757" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Claude</title><path d="M4.709 15.955l4.72-2.647.08-.23-.08-.128H9.2l-.79-.048-2.698-.073-2.339-.097-2.266-.122-.571-.121L0 11.784l.055-.352.48-.321.686.06 1.52.103 2.278.158 1.652.097 2.449.255h.389l.055-.157-.134-.098-.103-.097-2.358-1.596-2.552-1.688-1.336-.972-.724-.491-.364-.462-.158-1.008.656-.722.881.06.225.061.893.686 1.908 1.476 2.491 1.833.365.304.145-.103.019-.073-.164-.274-1.355-2.446-1.446-2.49-.644-1.032-.17-.619a2.97 2.97 0 01-.104-.729L6.283.134 6.696 0l.996.134.42.364.62 1.414 1.002 2.229 1.555 3.03.456.898.243.832.091.255h.158V9.01l.128-1.706.237-2.095.23-2.695.08-.76.376-.91.747-.492.584.28.48.685-.067.444-.286 1.851-.559 2.903-.364 1.942h.212l.243-.242.985-1.306 1.652-2.064.73-.82.85-.904.547-.431h1.033l.76 1.129-.34 1.166-1.064 1.347-.881 1.142-1.264 1.7-.79 1.36.073.11.188-.02 2.856-.606 1.543-.28 1.841-.315.833.388.091.395-.328.807-1.969.486-2.309.462-3.439.813-.042.03.049.061 1.549.146.662.036h1.622l3.02.225.79.522.474.638-.079.485-1.215.62-1.64-.389-3.829-.91-1.312-.329h-.182v.11l1.093 1.068 2.006 1.81 2.509 2.33.127.578-.322.455-.34-.049-2.205-1.657-.851-.747-1.926-1.62h-.128v.17l.444.649 2.345 3.521.122 1.08-.17.353-.608.213-.668-.122-1.374-1.925-1.415-2.167-1.143-1.943-.14.08-.674 7.254-.316.37-.729.28-.607-.461-.322-.747.322-1.476.389-1.924.315-1.53.286-1.9.17-.632-.012-.042-.14.018-1.434 1.967-2.18 2.945-1.726 1.845-.414.164-.717-.37.067-.662.401-.589 2.388-3.036 1.44-1.882.93-1.086-.006-.158h-.055L4.132 18.56l-1.13.146-.487-.456.061-.746.231-.243 1.908-1.312-.006.006z"></path></svg></span>Claude Opus 5 (max)<span class="org">Anthropic</span></td><td data-v="0.7181599999999999"><div class="cellbar"><i style="width:72.9px;--bar:var(--c4s-a)"></i><b>0.718</b></div></td><td class="dim" data-v="0.63106">0.631</td><td class="dim" data-v="0.71617">0.716</td><td class="dim" data-v="0.811">0.811</td><td class="dim" data-v="34.49611078947368">$34.50</td></tr><tr data-model-key="sol" data-search="gpt-5.6 sol (high) openai"><td class="rk">4</td><td class="nm"><span class="logo"><svg fill="currentColor" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>OpenAI</title><path d="M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z"></path></svg></span>GPT-5.6 Sol (high)<span class="tag front">Pareto</span><span class="org">OpenAI</span></td><td data-v="0.70704"><div class="cellbar"><i style="width:71.8px;--bar:var(--c4s-a)"></i><b>0.707</b></div></td><td class="dim" data-v="0.63186">0.632</td><td class="dim" data-v="0.710485">0.710</td><td class="dim" data-v="0.772">0.772</td><td class="dim" data-v="4.07260365">$4.07</td></tr><tr data-model-key="gemini" data-search="gemini 3.8 flash (high) google"><td class="rk">5</td><td class="nm"><span class="logo"><svg fill="url(#c4sGem-19)" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><defs><lineargradient id="c4sGem-19" x1="0" x2="1" y1="0" y2="1"><stop offset="0" stop-color="#4796E3"></stop><stop offset=".55" stop-color="#9168C0"></stop><stop offset="1" stop-color="#D96570"></stop></lineargradient></defs><title>Gemini</title><path d="M20.616 10.835a14.147 14.147 0 01-4.45-3.001 14.111 14.111 0 01-3.678-6.452.503.503 0 00-.975 0 14.134 14.134 0 01-3.679 6.452 14.155 14.155 0 01-4.45 3.001c-.65.28-1.318.505-2.002.678a.502.502 0 000 .975c.684.172 1.35.397 2.002.677a14.147 14.147 0 014.45 3.001 14.112 14.112 0 013.679 6.453.502.502 0 00.975 0c.172-.685.397-1.351.677-2.003a14.145 14.145 0 013.001-4.45 14.113 14.113 0 016.453-3.678.503.503 0 000-.975 13.245 13.245 0 01-2.003-.678z"></path></svg></span>Gemini 3.8 Flash (high)<span class="tag front">Pareto</span><span class="org">Google</span></td><td data-v="0.65689"><div class="cellbar"><i style="width:66.7px;--bar:var(--c4s-a)"></i><b>0.657</b></div></td><td class="dim" data-v="0.6369100000000001">0.637</td><td class="dim" data-v="0.66406">0.664</td><td class="dim" data-v="0.655">0.655</td><td class="dim" data-v="2.49471265">$2.49</td></tr><tr data-model-key="muse" data-search="muse spark 1.3 (medium) meta"><td class="rk">6</td><td class="nm"><span class="logo"><svg fill="#0081FB" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Meta</title><path d="M6.897 4c1.915 0 3.516.932 5.43 3.376l.282-.373c.19-.246.383-.484.58-.71l.313-.35C14.588 4.788 15.792 4 17.225 4c1.273 0 2.469.557 3.491 1.516l.218.213c1.73 1.765 2.917 4.71 3.053 8.026l.011.392.002.25c0 1.501-.28 2.759-.818 3.7l-.14.23-.108.153c-.301.42-.664.758-1.086 1.009l-.265.142-.087.04a3.493 3.493 0 01-.302.118 4.117 4.117 0 01-1.33.208c-.524 0-.996-.067-1.438-.215-.614-.204-1.163-.56-1.726-1.116l-.227-.235c-.753-.812-1.534-1.976-2.493-3.586l-1.43-2.41-.544-.895-1.766 3.13-.343.592C7.597 19.156 6.227 20 4.356 20c-1.21 0-2.205-.42-2.936-1.182l-.168-.184c-.484-.573-.837-1.311-1.043-2.189l-.067-.32a8.69 8.69 0 01-.136-1.288L0 14.468c.002-.745.06-1.49.174-2.23l.1-.573c.298-1.53.828-2.958 1.536-4.157l.209-.34c1.177-1.83 2.789-3.053 4.615-3.16L6.897 4zm-.033 2.615l-.201.01c-.83.083-1.606.673-2.252 1.577l-.138.199-.01.018c-.67 1.017-1.185 2.378-1.456 3.845l-.004.022a12.591 12.591 0 00-.207 2.254l.002.188c.004.18.017.36.04.54l.043.291c.092.503.257.908.486 1.208l.117.137c.303.323.698.492 1.17.492 1.1 0 1.796-.676 3.696-3.641l2.175-3.4.454-.701-.139-.198C9.11 7.3 8.084 6.616 6.864 6.616zm10.196-.552l-.176.007c-.635.048-1.223.359-1.82.933l-.196.198c-.439.462-.887 1.064-1.367 1.807l.266.398c.18.274.362.56.55.858l.293.475 1.396 2.335.695 1.114c.583.926 1.03 1.6 1.408 2.082l.213.262c.282.326.529.54.777.673l.102.05c.227.1.457.138.718.138.176.002.35-.023.518-.073.338-.104.61-.32.813-.637l.095-.163.077-.162c.194-.459.29-1.06.29-1.785l-.006-.449c-.08-2.871-.938-5.372-2.2-6.798l-.176-.189c-.67-.683-1.444-1.074-2.27-1.074z"></path></svg></span>Muse Spark 1.3 (medium)<span class="tag front">Pareto</span><span class="org">Meta</span></td><td data-v="0.646195"><div class="cellbar"><i style="width:65.6px;--bar:var(--c4s-a)"></i><b>0.646</b></div></td><td class="dim" data-v="0.5981099999999999">0.598</td><td class="dim" data-v="0.660675">0.661</td><td class="dim" data-v="0.651">0.651</td><td class="dim" data-v="0.10048539999999999">$0.10</td></tr><tr data-model-key="grok" data-search="grok 4.6 (high) xai"><td class="rk">7</td><td class="nm"><span class="logo"><svg fill="currentColor" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Grok</title><path d="M9.27 15.29l7.978-5.897c.391-.29.95-.177 1.137.272.98 2.369.542 5.215-1.41 7.169-1.951 1.954-4.667 2.382-7.149 1.406l-2.711 1.257c3.889 2.661 8.611 2.003 11.562-.953 2.341-2.344 3.066-5.539 2.388-8.42l.006.007c-.983-4.232.242-5.924 2.75-9.383.06-.082.12-.164.179-.248l-3.301 3.305v-.01L9.267 15.292M7.623 16.723c-2.792-2.67-2.31-6.801.071-9.184 1.761-1.763 4.647-2.483 7.166-1.425l2.705-1.25a7.808 7.808 0 00-1.829-1A8.975 8.975 0 005.984 5.83c-2.533 2.536-3.33 6.436-1.962 9.764 1.022 2.487-.653 4.246-2.34 6.022-.599.63-1.199 1.259-1.682 1.925l7.62-6.815"></path></svg></span>Grok 4.6 (high)<span class="org">xAI</span></td><td data-v="0.56721"><div class="cellbar"><i style="width:57.6px;--bar:var(--c4s-a)"></i><b>0.567</b></div></td><td class="dim" data-v="0.52211">0.522</td><td class="dim" data-v="0.5764199999999999">0.576</td><td class="dim" data-v="0.585">0.585</td><td class="dim" data-v="4.326270533333333">$4.33</td></tr><tr data-model-key="qwen-off" data-search="qwen 3.8 27b (thinking off) alibaba"><td class="rk">8</td><td class="nm"><span class="logo"><svg fill="#615CED" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Qwen</title><path d="M12.604 1.34c.393.69.784 1.382 1.174 2.075a.18.18 0 00.157.091h5.552c.174 0 .322.11.446.327l1.454 2.57c.19.337.24.478.024.837-.26.43-.513.864-.76 1.3l-.367.658c-.106.196-.223.28-.04.512l2.652 4.637c.172.301.111.494-.043.77-.437.785-.882 1.564-1.335 2.34-.159.272-.352.375-.68.37-.777-.016-1.552-.01-2.327.016a.099.099 0 00-.081.05 575.097 575.097 0 01-2.705 4.74c-.169.293-.38.363-.725.364-.997.003-2.002.004-3.017.002a.537.537 0 01-.465-.271l-1.335-2.323a.09.09 0 00-.083-.049H4.982c-.285.03-.553-.001-.805-.092l-1.603-2.77a.543.543 0 01-.002-.54l1.207-2.12a.198.198 0 000-.197 550.951 550.951 0 01-1.875-3.272l-.79-1.395c-.16-.31-.173-.496.095-.965.465-.813.927-1.625 1.387-2.436.132-.234.304-.334.584-.335a338.3 338.3 0 012.589-.001.124.124 0 00.107-.063l2.806-4.895a.488.488 0 01.422-.246c.524-.001 1.053 0 1.583-.006L11.704 1c.341-.003.724.032.9.34zm-3.432.403a.06.06 0 00-.052.03L6.254 6.788a.157.157 0 01-.135.078H3.253c-.056 0-.07.025-.041.074l5.81 10.156c.025.042.013.062-.034.063l-2.795.015a.218.218 0 00-.2.116l-1.32 2.31c-.044.078-.021.118.068.118l5.716.008c.046 0 .08.02.104.061l1.403 2.454c.046.081.092.082.139 0l5.006-8.76.783-1.382a.055.055 0 01.096 0l1.424 2.53a.122.122 0 00.107.062l2.763-.02a.04.04 0 00.035-.02.041.041 0 000-.04l-2.9-5.086a.108.108 0 010-.113l.293-.507 1.12-1.977c.024-.041.012-.062-.035-.062H9.2c-.059 0-.073-.026-.043-.077l1.434-2.505a.107.107 0 000-.114L9.225 1.774a.06.06 0 00-.053-.031zm6.29 8.02c.046 0 .058.02.034.06l-.832 1.465-2.613 4.585a.056.056 0 01-.05.029.058.058 0 01-.05-.029L8.498 9.841c-.02-.034-.01-.052.028-.054l.216-.012 6.722-.012z"></path></svg></span>Qwen 3.8 27B (thinking off)<span class="tag open">open weights</span><span class="org">Alibaba</span></td><td data-v="0.55666"><div class="cellbar"><i style="width:56.5px;--bar:var(--c4s-a)"></i><b>0.557</b></div></td><td class="dim" data-v="0.506755">0.507</td><td class="dim" data-v="0.529525">0.530</td><td class="dim" data-v="0.688">0.688</td><td class="dim" data-v="0.49154169999999997">$0.49</td></tr><tr data-model-key="qwen-on" data-search="qwen 3.8 27b (thinking on) alibaba"><td class="rk">9</td><td class="nm"><span class="logo"><svg fill="#615CED" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Qwen</title><path d="M12.604 1.34c.393.69.784 1.382 1.174 2.075a.18.18 0 00.157.091h5.552c.174 0 .322.11.446.327l1.454 2.57c.19.337.24.478.024.837-.26.43-.513.864-.76 1.3l-.367.658c-.106.196-.223.28-.04.512l2.652 4.637c.172.301.111.494-.043.77-.437.785-.882 1.564-1.335 2.34-.159.272-.352.375-.68.37-.777-.016-1.552-.01-2.327.016a.099.099 0 00-.081.05 575.097 575.097 0 01-2.705 4.74c-.169.293-.38.363-.725.364-.997.003-2.002.004-3.017.002a.537.537 0 01-.465-.271l-1.335-2.323a.09.09 0 00-.083-.049H4.982c-.285.03-.553-.001-.805-.092l-1.603-2.77a.543.543 0 01-.002-.54l1.207-2.12a.198.198 0 000-.197 550.951 550.951 0 01-1.875-3.272l-.79-1.395c-.16-.31-.173-.496.095-.965.465-.813.927-1.625 1.387-2.436.132-.234.304-.334.584-.335a338.3 338.3 0 012.589-.001.124.124 0 00.107-.063l2.806-4.895a.488.488 0 01.422-.246c.524-.001 1.053 0 1.583-.006L11.704 1c.341-.003.724.032.9.34zm-3.432.403a.06.06 0 00-.052.03L6.254 6.788a.157.157 0 01-.135.078H3.253c-.056 0-.07.025-.041.074l5.81 10.156c.025.042.013.062-.034.063l-2.795.015a.218.218 0 00-.2.116l-1.32 2.31c-.044.078-.021.118.068.118l5.716.008c.046 0 .08.02.104.061l1.403 2.454c.046.081.092.082.139 0l5.006-8.76.783-1.382a.055.055 0 01.096 0l1.424 2.53a.122.122 0 00.107.062l2.763-.02a.04.04 0 00.035-.02.041.041 0 000-.04l-2.9-5.086a.108.108 0 010-.113l.293-.507 1.12-1.977c.024-.041.012-.062-.035-.062H9.2c-.059 0-.073-.026-.043-.077l1.434-2.505a.107.107 0 000-.114L9.225 1.774a.06.06 0 00-.053-.031zm6.29 8.02c.046 0 .058.02.034.06l-.832 1.465-2.613 4.585a.056.056 0 01-.05.029.058.058 0 01-.05-.029L8.498 9.841c-.02-.034-.01-.052.028-.054l.216-.012 6.722-.012z"></path></svg></span>Qwen 3.8 27B (thinking on)<span class="tag open">open weights</span><span class="org">Alibaba</span></td><td data-v="0.51571"><div class="cellbar"><i style="width:52.4px;--bar:var(--c4s-a)"></i><b>0.516</b></div></td><td class="dim" data-v="0.50522">0.505</td><td class="dim" data-v="0.45978">0.460</td><td class="dim" data-v="0.694">0.694</td><td class="dim" data-v="0.3344125">$0.33</td></tr><tr data-model-key="glm" data-search="glm-5.3 flash (max) z.ai"><td class="rk">10</td><td class="nm"><span class="logo"><svg fill="currentColor" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Z.ai</title><path d="M12.105 2L9.927 4.953H.653L2.83 2h9.276zM23.254 19.048L21.078 22h-9.242l2.174-2.952h9.244zM24 2L9.264 22H0L14.736 2H24z"></path></svg></span>GLM-5.3 Flash (max)<span class="tag open">open weights</span><span class="org">Z.ai</span></td><td data-v="0.5094299999999999"><div class="cellbar"><i style="width:51.7px;--bar:var(--c4s-a)"></i><b>0.509</b></div></td><td class="dim" data-v="0.403365">0.403</td><td class="dim" data-v="0.537015">0.537</td><td class="dim" data-v="0.533">0.533</td><td class="dim" data-v="0.710521125">$0.71</td></tr><tr data-model-key="gemma-on" data-search="gemma 4 31b (thinking on) google"><td class="rk">11</td><td class="nm"><span class="logo"><svg fill="url(#c4sGem-25)" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><defs><lineargradient id="c4sGem-25" x1="0" x2="1" y1="0" y2="1"><stop offset="0" stop-color="#4796E3"></stop><stop offset=".55" stop-color="#9168C0"></stop><stop offset="1" stop-color="#D96570"></stop></lineargradient></defs><title>Gemini</title><path d="M20.616 10.835a14.147 14.147 0 01-4.45-3.001 14.111 14.111 0 01-3.678-6.452.503.503 0 00-.975 0 14.134 14.134 0 01-3.679 6.452 14.155 14.155 0 01-4.45 3.001c-.65.28-1.318.505-2.002.678a.502.502 0 000 .975c.684.172 1.35.397 2.002.677a14.147 14.147 0 014.45 3.001 14.112 14.112 0 013.679 6.453.502.502 0 00.975 0c.172-.685.397-1.351.677-2.003a14.145 14.145 0 013.001-4.45 14.113 14.113 0 016.453-3.678.503.503 0 000-.975 13.245 13.245 0 01-2.003-.678z"></path></svg></span>Gemma 4 31B (thinking on)<span class="tag open">open weights</span><span class="tag front">Pareto</span><span class="org">Google</span></td><td data-v="0.470705"><div class="cellbar"><i style="width:47.8px;--bar:var(--c4s-a)"></i><b>0.471</b></div></td><td class="dim" data-v="0.4117">0.412</td><td class="dim" data-v="0.41593">0.416</td><td class="dim" data-v="0.694">0.694</td><td class="dim" data-v="0.0253001">$0.03</td></tr><tr data-model-key="gemma-off" data-search="gemma 4 31b (thinking off) google"><td class="rk">12</td><td class="nm"><span class="logo"><svg fill="url(#c4sGem-26)" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><defs><lineargradient id="c4sGem-26" x1="0" x2="1" y1="0" y2="1"><stop offset="0" stop-color="#4796E3"></stop><stop offset=".55" stop-color="#9168C0"></stop><stop offset="1" stop-color="#D96570"></stop></lineargradient></defs><title>Gemini</title><path d="M20.616 10.835a14.147 14.147 0 01-4.45-3.001 14.111 14.111 0 01-3.678-6.452.503.503 0 00-.975 0 14.134 14.134 0 01-3.679 6.452 14.155 14.155 0 01-4.45 3.001c-.65.28-1.318.505-2.002.678a.502.502 0 000 .975c.684.172 1.35.397 2.002.677a14.147 14.147 0 014.45 3.001 14.112 14.112 0 013.679 6.453.502.502 0 00.975 0c.172-.685.397-1.351.677-2.003a14.145 14.145 0 013.001-4.45 14.113 14.113 0 016.453-3.678.503.503 0 000-.975 13.245 13.245 0 01-2.003-.678z"></path></svg></span>Gemma 4 31B (thinking off)<span class="tag open">open weights</span><span class="tag front">Pareto</span><span class="org">Google</span></td><td data-v="0.45494"><div class="cellbar"><i style="width:46.2px;--bar:var(--c4s-a)"></i><b>0.455</b></div></td><td class="dim" data-v="0.43810000000000004">0.438</td><td class="dim" data-v="0.370465">0.370</td><td class="dim" data-v="0.725">0.725</td><td class="dim" data-v="0.0151612">$0.02</td></tr><tr data-model-key="inkling" data-search="inkling (high) thinking machines"><td class="rk">13</td><td class="nm"><span class="logo"><svg fill="currentColor" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Thinking Machines</title><rect height="15.6" rx="3.6" width="15.6" x="4.2" y="4.2"></rect></svg>
|
| 29 |
</span>Inkling (high)<span class="org">Thinking Machines</span></td><td data-v="0.42498500000000006"><div class="cellbar"><i style="width:43.1px;--bar:var(--c4s-a)"></i><b>0.425</b></div></td><td class="dim" data-v="0.392605">0.393</td><td class="dim" data-v="0.336705">0.337</td><td class="dim" data-v="0.722">0.722</td><td class="dim" data-v="0.20252835">$0.20</td></tr><tr data-model-key="deepseek" data-search="deepseek v4.1 flash (high) deepseek"><td class="rk">14</td><td class="nm"><span class="logo"><svg fill="#4D6BFE" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>DeepSeek</title><path d="M23.748 4.482c-.254-.124-.364.113-.512.234-.051.039-.094.09-.137.136-.372.397-.806.657-1.373.626-.829-.046-1.537.214-2.163.848-.133-.782-.575-1.248-1.247-1.548-.352-.156-.708-.311-.955-.65-.172-.241-.219-.51-.305-.774-.055-.16-.11-.323-.293-.35-.2-.031-.278.136-.356.276-.313.572-.434 1.202-.422 1.84.027 1.436.633 2.58 1.838 3.393.137.093.172.187.129.323-.082.28-.18.552-.266.833-.055.179-.137.217-.329.14a5.526 5.526 0 01-1.736-1.18c-.857-.828-1.631-1.742-2.597-2.458a11.365 11.365 0 00-.689-.471c-.985-.957.13-1.743.388-1.836.27-.098.093-.432-.779-.428-.872.004-1.67.295-2.687.684a3.055 3.055 0 01-.465.137 9.597 9.597 0 00-2.883-.102c-1.885.21-3.39 1.102-4.497 2.623C.082 8.606-.231 10.684.152 12.85c.403 2.284 1.569 4.175 3.36 5.653 1.858 1.533 3.997 2.284 6.438 2.14 1.482-.085 3.133-.284 4.994-1.86.47.234.962.327 1.78.397.63.059 1.236-.03 1.705-.128.735-.156.684-.837.419-.961-2.155-1.004-1.682-.595-2.113-.926 1.096-1.296 2.746-2.642 3.392-7.003.05-.347.007-.565 0-.845-.004-.17.035-.237.23-.256a4.173 4.173 0 001.545-.475c1.396-.763 1.96-2.015 2.093-3.517.02-.23-.004-.467-.247-.588zM11.581 18c-2.089-1.642-3.102-2.183-3.52-2.16-.392.024-.321.471-.235.763.09.288.207.486.371.739.114.167.192.416-.113.603-.673.416-1.842-.14-1.897-.167-1.361-.802-2.5-1.86-3.301-3.307-.774-1.393-1.224-2.887-1.298-4.482-.02-.386.093-.522.477-.592a4.696 4.696 0 011.529-.039c2.132.312 3.946 1.265 5.468 2.774.868.86 1.525 1.887 2.202 2.891.72 1.066 1.494 2.082 2.48 2.914.348.292.625.514.891.677-.802.09-2.14.11-3.054-.614zm1-6.44a.306.306 0 01.415-.287.302.302 0 01.2.288.306.306 0 01-.31.307.303.303 0 01-.304-.308zm3.11 1.596c-.2.081-.399.151-.59.16a1.245 1.245 0 01-.798-.254c-.274-.23-.47-.358-.552-.758a1.73 1.73 0 01.016-.588c.07-.327-.008-.537-.239-.727-.187-.156-.426-.199-.688-.199a.559.559 0 01-.254-.078c-.11-.054-.2-.19-.114-.358.028-.054.16-.186.192-.21.356-.202.767-.136 1.146.016.352.144.618.408 1.001.782.391.451.462.576.685.914.176.265.336.537.445.848.067.195-.019.354-.25.452z"></path></svg></span>DeepSeek V4.1 Flash (high)<span class="tag open">open weights</span><span class="org">DeepSeek</span></td><td data-v="0.24269"><div class="cellbar"><i style="width:24.7px;--bar:var(--c4s-a)"></i><b>0.243</b></div></td><td class="dim" data-v="0.216595">0.217</td><td class="dim" data-v="0.16134500000000002">0.161</td><td class="dim" data-v="0.513">0.513</td><td class="dim" data-v="0.193865">$0.19</td></tr></tbody></table></div></div>
|
| 30 |
<p class="board-note">Click a column to sort. Pareto: on the score-against-cost frontier.</p></div></div>
|
| 31 |
</div></section><section id="cases"><div class="container">
|
|
|
|
| 19 |
.chip.ok{color:var(--c4s-ok);border-color:currentColor}.chip.no{color:var(--c4s-hot);border-color:currentColor}.chip small{font-weight:500;color:var(--text-muted);margin-left:4px}
|
| 20 |
.bigstats{display:flex;flex-wrap:wrap;gap:12px 34px;margin:0 0 12px}.bigstats b{display:block;font-size:30px;letter-spacing:-.02em;font-variant-numeric:tabular-nums}
|
| 21 |
.bigstats span{font-size:13px;color:var(--text-secondary)}
|
| 22 |
+
</style><link href="assets/buildingbench.css?v=20261005tracks" rel="stylesheet"/><script type="importmap">{"imports":{"three":"./assets/vendor/three-r160/build/three.module.js","three/addons/":"./assets/vendor/three-r160/addons/"}}</script></head><body><a class="skip-link" href="#leaderboard">Skip to benchmark explorer</a>
|
| 23 |
+
<header class="site-header"><div class="brand"><a aria-label="Code4Scene home" class="brand-home" href="index.html#top"><svg aria-hidden="true" fill="none" viewbox="0 0 30 32"><path d="M15 2 27 9v14l-12 7L3 23V9L15 2Z M3 9l12 7 12-7 M15 16v14 M9 5.5l12 7v14 M21 5.5l-12 7v14" stroke="currentColor" stroke-linejoin="round" stroke-width="1.6"></path></svg><span>Code4Scene</span></a><a class="brand-by" href="https://simworld.org/">by SimWorld</a></div><nav aria-label="Main navigation"><a class="nav-active" href="index.html#top">Benchmark</a><a href="index.html#leaderboard">Leaderboard</a><a href="index.html#cases">Visualization</a><a href="https://arxiv.org/abs/2609.36777" rel="noopener" target="_blank">Paper ↗</a><a href="https://github.com/SimWorld-AI/Code4Scene" rel="noopener" target="_blank">Github ↗</a></nav></header><main class="page-main"><section class="intro" id="top"><div class="intro-copy"><h1>Constructing<br/><em>3D Scenes with Code</em></h1><p class="intro-text">Benchmarking coding agents that turn open-ended scene descriptions into engine-native 3D scenes in Unreal Engine. Measuring spatial reasoning, task fulfillment and physical validity.</p><div class="intro-links"><a href="https://arxiv.org/abs/2609.36777">Read the paper ↗</a><a href="https://github.com/SimWorld-AI/Code4Scene">Code & Data ↗</a><a href="#hero-video">Watch the teaser ↘</a></div></div><div class="featured featured-video" id="hero-video"><video aria-label="Code4Scene overview video" controls="" playsinline="" poster="media/teaser-poster.jpg" preload="metadata"><source src="media/teaser-v31.mp4" type="video/mp4"/>Your browser does not support embedded video. <a href="media/teaser-v31.mp4">Watch the Code4Scene video</a>.</video><div class="featured-video-caption"><strong>Code4Scene in 90 seconds</strong><span>Construction and evaluation</span></div></div></section><section class="task-section" id="task-format"><div class="container"><h2 class="section-title">Task Format</h2><div class="task-shell"><div aria-labelledby="task-tab-t2s" class="task-content" data-task="t2s" id="task-t2s" role="tabpanel"><h3>Nordic Harbour <span>· Text-to-Scene</span></h3><p class="task-subtitle">An open-ended scene description, an asset catalog and the Unreal Engine editor.</p><div class="task-columns"><article class="task-panel"><div class="panel-heading"><strong>Agent input</strong><span>Text + assets</span></div><div class="prompt-preview"><span class="eyebrow">Task instruction</span><p>Build me a compact, sunlit Copenhagen-style canal district with narrow cobblestone streets and rectangular waterways enclosing dense urban blocks.</p><p class="prompt-detail">Townhouses, canals, bridges and a waterfront promenade — with detailed requirements for materials, layout and street furniture.</p></div><p class="task-foot"><strong>Given:</strong> an empty level and the content pack’s asset catalog.</p></article><article class="task-panel output-panel"><div class="panel-heading"><strong>Agent output</strong><span>GPT-6 Astra</span></div><div class="live-scene" data-agent="astra" data-case="nordic-harbour"><canvas aria-label="Interactive 3D scene: Nordic Harbour. Drag to orbit, scroll or use plus and minus to zoom." tabindex="0"></canvas><span class="scene-loading" role="status">Loading 3D scene…</span><span class="scene-hint">Drag to orbit · Scroll to zoom</span><button aria-label="Pause scene rotation" class="scene-motion">Ⅱ</button></div><a class="task-foot text-button" href="cases.html#nordic-harbour">Compare agents and evaluator views ↗</a></article><article class="task-panel evaluation-panel"><div class="panel-heading"><strong>Evaluation</strong><span>Engine-native scene</span></div><div class="score-block"><span>CASE SCORE</span><strong>0.877</strong><dl><div><dt>Detailed Alignment</dt><dd>0.757</dd></div><div><dt>Overview Alignment</dt><dd>0.909</dd></div><div><dt>Physical Safety</dt><dd>0.900</dd></div></dl></div><p class="task-foot">Task fulfillment, artifact integrity and static physical validity.</p><p class="task-formula">0.2 × Detailed + 0.6 × Overview + 0.2 × Physical</p></article></div><details class="task-full"><summary><span>+ Full task instruction</span><span>TEXT-TO-SCENE</span></summary><p>Build me a compact, sunlit Copenhagen-style canal district with narrow cobblestone streets and rectangular waterways enclosing dense urban blocks. Arrange roughly thirty attached four- to six-storey townhouses in continuous street and canal-front rows, using slender façades, repetitive white-framed windows, arched doors and carriage entrances, modest cornices, and steep tiled roofs packed with dormers and brick chimneys. Include occasional exposed party walls between staggered building rows. Finish the façades in weathered muted red, mustard yellow, dusty blue, warm grey, beige, and exposed brick, with red, orange, and charcoal roof tiles. Run a principal waterfront street along a stone quay, crossed by smaller streets and linked across the canals by short, gently arched stone-and-metal bridges, including one opening onto a broad paved promenade with rounded landings. Form the canal edges from stepped pale stone, dark timber retaining walls, and iron railings around reflective rippling water. Add regularly spaced leafy trees, ornate black lamps, stone bollards, benches, European road signs, zebra crossings, puddles, grime, and sparse weeds.</p></details></div></div><div class="availability"><span><b>20</b> scenes in the <strong>Research Track</strong>: affordable, reproducible evaluation for academic and model-development use</span><span><b>160</b> scenes in the <strong>Full Benchmark</strong>: comprehensive evaluation for leaderboard and final reporting</span></div></div></section><section id="leaderboard"><div class="container">
|
| 24 |
<div class="section-header centered"><span class="section-label">Leaderboard</span><h2 class="section-title">Leaderboard</h2>
|
| 25 |
+
<p class="section-desc">Research Track · 20 representative scenes · 14 coding-agent configurations.</p><p class="tracks-why">Text-to-scene generation with frontier models is a long-horizon agentic task with substantial inference cost and runtime, which makes repeated full-scale evaluation difficult for many research groups. The Research Track supports affordable, reproducible experimentation; the Full Benchmark provides the more comprehensive evaluation for final model comparison and leaderboard reporting.</p></div>
|
| 26 |
+
<div id="lbwrap"><div class="card bar-card"><div class="bar-heading"><h3>Code4Scene <span class="q">Research Track</span></h3><span>20 scenes · score out of 100</span></div><p class="bar-mobile-hint">Swipe to view all 14 configurations →</p><div aria-label="All 14 model configurations" class="bar-scroll" role="region" tabindex="0"><div data-view="t2s"><svg aria-label="Research Track scores for the 14 Code4Scene configurations" class="kit-bars" id="bars-t2s" role="group" viewbox="0 0 1200 420"></svg></div></div><p class="note">Sorted by score. Bar labels round scores ×100; the table preserves three decimal places. One color per model provider.</p></div>
|
| 27 |
+
<div class="card chart-card pareto-card"><h3 class="figure-title">Code4Scene <span>Pareto Frontier</span></h3><p class="figure-subtitle">Quality against the cost of one case · Research Track, 20 scenes</p><p class="pareto-swipe">Swipe to explore all configurations →</p><div class="pareto-view" data-view="t2s"><div class="pareto-scroll"><svg aria-label="t2s score against average cost per case. More expensive runs on the left; logarithmic scale." class="pareto-svg" data-mode="t2s" role="group" viewbox="0 0 1200 560"></svg></div></div><p class="figure-caption">Solid: the frontier — nothing cheaper scores higher. One point per configuration, at its evaluated reasoning effort. Cost is the mean USD per case.</p></div>
|
| 28 |
+
<div class="card ranking-card"><div class="ranking-toolbar"><label for="model-search">Search configurations<input autocomplete="off" id="model-search" placeholder="Model or provider" type="search"/></label><span aria-live="polite" id="ranking-count">14 configurations</span></div><div class="sub-title">Ranking</div><div data-view="t2s"><p class="board-note" style="margin:0 .2rem .7rem">Research Track, 20 scenes. Case score = 0.2 × Detailed + 0.6 × Overview + 0.2 × Physical. Detailed and Overview are means of the per-case scores.</p><div class="board-wrap"><table class="board"><thead><tr><th>#</th><th>Agent configuration</th><th class="sorted" data-best="high" data-k="st">Score</th><th data-best="high" data-k="detailed">Detailed</th><th data-best="high" data-k="overview">Overview</th><th data-best="high" data-k="pt">Physical</th><th data-best="low" data-k="c_t2s">Cost / case</th></tr></thead><tbody><tr data-model-key="fable" data-search="claude fable 5.1 (max) anthropic"><td class="rk">1</td><td class="nm"><span class="logo"><svg fill="#D97757" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Claude</title><path d="M4.709 15.955l4.72-2.647.08-.23-.08-.128H9.2l-.79-.048-2.698-.073-2.339-.097-2.266-.122-.571-.121L0 11.784l.055-.352.48-.321.686.06 1.52.103 2.278.158 1.652.097 2.449.255h.389l.055-.157-.134-.098-.103-.097-2.358-1.596-2.552-1.688-1.336-.972-.724-.491-.364-.462-.158-1.008.656-.722.881.06.225.061.893.686 1.908 1.476 2.491 1.833.365.304.145-.103.019-.073-.164-.274-1.355-2.446-1.446-2.49-.644-1.032-.17-.619a2.97 2.97 0 01-.104-.729L6.283.134 6.696 0l.996.134.42.364.62 1.414 1.002 2.229 1.555 3.03.456.898.243.832.091.255h.158V9.01l.128-1.706.237-2.095.23-2.695.08-.76.376-.91.747-.492.584.28.48.685-.067.444-.286 1.851-.559 2.903-.364 1.942h.212l.243-.242.985-1.306 1.652-2.064.73-.82.85-.904.547-.431h1.033l.76 1.129-.34 1.166-1.064 1.347-.881 1.142-1.264 1.7-.79 1.36.073.11.188-.02 2.856-.606 1.543-.28 1.841-.315.833.388.091.395-.328.807-1.969.486-2.309.462-3.439.813-.042.03.049.061 1.549.146.662.036h1.622l3.02.225.79.522.474.638-.079.485-1.215.62-1.64-.389-3.829-.91-1.312-.329h-.182v.11l1.093 1.068 2.006 1.81 2.509 2.33.127.578-.322.455-.34-.049-2.205-1.657-.851-.747-1.926-1.62h-.128v.17l.444.649 2.345 3.521.122 1.08-.17.353-.608.213-.668-.122-1.374-1.925-1.415-2.167-1.143-1.943-.14.08-.674 7.254-.316.37-.729.28-.607-.461-.322-.747.322-1.476.389-1.924.315-1.53.286-1.9.17-.632-.012-.042-.14.018-1.434 1.967-2.18 2.945-1.726 1.845-.414.164-.717-.37.067-.662.401-.589 2.388-3.036 1.44-1.882.93-1.086-.006-.158h-.055L4.132 18.56l-1.13.146-.487-.456.061-.746.231-.243 1.908-1.312-.006.006z"></path></svg></span>Claude Fable 5.1 (max)<span class="tag front">Pareto</span><span class="org">Anthropic</span></td><td data-v="0.78775"><div class="cellbar"><i style="width:80.0px;--bar:var(--c4s-a)"></i><b>0.788</b></div></td><td class="dim" data-v="0.7253149999999999">0.725</td><td class="dim" data-v="0.7717050000000001">0.772</td><td class="dim" data-v="0.898">0.898</td><td class="dim" data-v="18.1576316">$18.16</td></tr><tr data-model-key="astra" data-search="gpt-6 astra (max) openai"><td class="rk">2</td><td class="nm"><span class="logo"><svg fill="currentColor" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>OpenAI</title><path d="M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z"></path></svg></span>GPT-6 Astra (max)<span class="org">OpenAI</span></td><td data-v="0.72382"><div class="cellbar"><i style="width:73.5px;--bar:var(--c4s-a)"></i><b>0.724</b></div></td><td class="dim" data-v="0.707165">0.707</td><td class="dim" data-v="0.733235">0.733</td><td class="dim" data-v="0.712">0.712</td><td class="dim" data-v="22.472260736842106">$22.47</td></tr><tr data-model-key="opus" data-search="claude opus 5 (max) anthropic"><td class="rk">3</td><td class="nm"><span class="logo"><svg fill="#D97757" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Claude</title><path d="M4.709 15.955l4.72-2.647.08-.23-.08-.128H9.2l-.79-.048-2.698-.073-2.339-.097-2.266-.122-.571-.121L0 11.784l.055-.352.48-.321.686.06 1.52.103 2.278.158 1.652.097 2.449.255h.389l.055-.157-.134-.098-.103-.097-2.358-1.596-2.552-1.688-1.336-.972-.724-.491-.364-.462-.158-1.008.656-.722.881.06.225.061.893.686 1.908 1.476 2.491 1.833.365.304.145-.103.019-.073-.164-.274-1.355-2.446-1.446-2.49-.644-1.032-.17-.619a2.97 2.97 0 01-.104-.729L6.283.134 6.696 0l.996.134.42.364.62 1.414 1.002 2.229 1.555 3.03.456.898.243.832.091.255h.158V9.01l.128-1.706.237-2.095.23-2.695.08-.76.376-.91.747-.492.584.28.48.685-.067.444-.286 1.851-.559 2.903-.364 1.942h.212l.243-.242.985-1.306 1.652-2.064.73-.82.85-.904.547-.431h1.033l.76 1.129-.34 1.166-1.064 1.347-.881 1.142-1.264 1.7-.79 1.36.073.11.188-.02 2.856-.606 1.543-.28 1.841-.315.833.388.091.395-.328.807-1.969.486-2.309.462-3.439.813-.042.03.049.061 1.549.146.662.036h1.622l3.02.225.79.522.474.638-.079.485-1.215.62-1.64-.389-3.829-.91-1.312-.329h-.182v.11l1.093 1.068 2.006 1.81 2.509 2.33.127.578-.322.455-.34-.049-2.205-1.657-.851-.747-1.926-1.62h-.128v.17l.444.649 2.345 3.521.122 1.08-.17.353-.608.213-.668-.122-1.374-1.925-1.415-2.167-1.143-1.943-.14.08-.674 7.254-.316.37-.729.28-.607-.461-.322-.747.322-1.476.389-1.924.315-1.53.286-1.9.17-.632-.012-.042-.14.018-1.434 1.967-2.18 2.945-1.726 1.845-.414.164-.717-.37.067-.662.401-.589 2.388-3.036 1.44-1.882.93-1.086-.006-.158h-.055L4.132 18.56l-1.13.146-.487-.456.061-.746.231-.243 1.908-1.312-.006.006z"></path></svg></span>Claude Opus 5 (max)<span class="org">Anthropic</span></td><td data-v="0.7181599999999999"><div class="cellbar"><i style="width:72.9px;--bar:var(--c4s-a)"></i><b>0.718</b></div></td><td class="dim" data-v="0.63106">0.631</td><td class="dim" data-v="0.71617">0.716</td><td class="dim" data-v="0.811">0.811</td><td class="dim" data-v="34.49611078947368">$34.50</td></tr><tr data-model-key="sol" data-search="gpt-5.6 sol (high) openai"><td class="rk">4</td><td class="nm"><span class="logo"><svg fill="currentColor" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>OpenAI</title><path d="M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z"></path></svg></span>GPT-5.6 Sol (high)<span class="tag front">Pareto</span><span class="org">OpenAI</span></td><td data-v="0.70704"><div class="cellbar"><i style="width:71.8px;--bar:var(--c4s-a)"></i><b>0.707</b></div></td><td class="dim" data-v="0.63186">0.632</td><td class="dim" data-v="0.710485">0.710</td><td class="dim" data-v="0.772">0.772</td><td class="dim" data-v="4.07260365">$4.07</td></tr><tr data-model-key="gemini" data-search="gemini 3.8 flash (high) google"><td class="rk">5</td><td class="nm"><span class="logo"><svg fill="url(#c4sGem-19)" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><defs><lineargradient id="c4sGem-19" x1="0" x2="1" y1="0" y2="1"><stop offset="0" stop-color="#4796E3"></stop><stop offset=".55" stop-color="#9168C0"></stop><stop offset="1" stop-color="#D96570"></stop></lineargradient></defs><title>Gemini</title><path d="M20.616 10.835a14.147 14.147 0 01-4.45-3.001 14.111 14.111 0 01-3.678-6.452.503.503 0 00-.975 0 14.134 14.134 0 01-3.679 6.452 14.155 14.155 0 01-4.45 3.001c-.65.28-1.318.505-2.002.678a.502.502 0 000 .975c.684.172 1.35.397 2.002.677a14.147 14.147 0 014.45 3.001 14.112 14.112 0 013.679 6.453.502.502 0 00.975 0c.172-.685.397-1.351.677-2.003a14.145 14.145 0 013.001-4.45 14.113 14.113 0 016.453-3.678.503.503 0 000-.975 13.245 13.245 0 01-2.003-.678z"></path></svg></span>Gemini 3.8 Flash (high)<span class="tag front">Pareto</span><span class="org">Google</span></td><td data-v="0.65689"><div class="cellbar"><i style="width:66.7px;--bar:var(--c4s-a)"></i><b>0.657</b></div></td><td class="dim" data-v="0.6369100000000001">0.637</td><td class="dim" data-v="0.66406">0.664</td><td class="dim" data-v="0.655">0.655</td><td class="dim" data-v="2.49471265">$2.49</td></tr><tr data-model-key="muse" data-search="muse spark 1.3 (medium) meta"><td class="rk">6</td><td class="nm"><span class="logo"><svg fill="#0081FB" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Meta</title><path d="M6.897 4c1.915 0 3.516.932 5.43 3.376l.282-.373c.19-.246.383-.484.58-.71l.313-.35C14.588 4.788 15.792 4 17.225 4c1.273 0 2.469.557 3.491 1.516l.218.213c1.73 1.765 2.917 4.71 3.053 8.026l.011.392.002.25c0 1.501-.28 2.759-.818 3.7l-.14.23-.108.153c-.301.42-.664.758-1.086 1.009l-.265.142-.087.04a3.493 3.493 0 01-.302.118 4.117 4.117 0 01-1.33.208c-.524 0-.996-.067-1.438-.215-.614-.204-1.163-.56-1.726-1.116l-.227-.235c-.753-.812-1.534-1.976-2.493-3.586l-1.43-2.41-.544-.895-1.766 3.13-.343.592C7.597 19.156 6.227 20 4.356 20c-1.21 0-2.205-.42-2.936-1.182l-.168-.184c-.484-.573-.837-1.311-1.043-2.189l-.067-.32a8.69 8.69 0 01-.136-1.288L0 14.468c.002-.745.06-1.49.174-2.23l.1-.573c.298-1.53.828-2.958 1.536-4.157l.209-.34c1.177-1.83 2.789-3.053 4.615-3.16L6.897 4zm-.033 2.615l-.201.01c-.83.083-1.606.673-2.252 1.577l-.138.199-.01.018c-.67 1.017-1.185 2.378-1.456 3.845l-.004.022a12.591 12.591 0 00-.207 2.254l.002.188c.004.18.017.36.04.54l.043.291c.092.503.257.908.486 1.208l.117.137c.303.323.698.492 1.17.492 1.1 0 1.796-.676 3.696-3.641l2.175-3.4.454-.701-.139-.198C9.11 7.3 8.084 6.616 6.864 6.616zm10.196-.552l-.176.007c-.635.048-1.223.359-1.82.933l-.196.198c-.439.462-.887 1.064-1.367 1.807l.266.398c.18.274.362.56.55.858l.293.475 1.396 2.335.695 1.114c.583.926 1.03 1.6 1.408 2.082l.213.262c.282.326.529.54.777.673l.102.05c.227.1.457.138.718.138.176.002.35-.023.518-.073.338-.104.61-.32.813-.637l.095-.163.077-.162c.194-.459.29-1.06.29-1.785l-.006-.449c-.08-2.871-.938-5.372-2.2-6.798l-.176-.189c-.67-.683-1.444-1.074-2.27-1.074z"></path></svg></span>Muse Spark 1.3 (medium)<span class="tag front">Pareto</span><span class="org">Meta</span></td><td data-v="0.646195"><div class="cellbar"><i style="width:65.6px;--bar:var(--c4s-a)"></i><b>0.646</b></div></td><td class="dim" data-v="0.5981099999999999">0.598</td><td class="dim" data-v="0.660675">0.661</td><td class="dim" data-v="0.651">0.651</td><td class="dim" data-v="0.10048539999999999">$0.10</td></tr><tr data-model-key="grok" data-search="grok 4.6 (high) xai"><td class="rk">7</td><td class="nm"><span class="logo"><svg fill="currentColor" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Grok</title><path d="M9.27 15.29l7.978-5.897c.391-.29.95-.177 1.137.272.98 2.369.542 5.215-1.41 7.169-1.951 1.954-4.667 2.382-7.149 1.406l-2.711 1.257c3.889 2.661 8.611 2.003 11.562-.953 2.341-2.344 3.066-5.539 2.388-8.42l.006.007c-.983-4.232.242-5.924 2.75-9.383.06-.082.12-.164.179-.248l-3.301 3.305v-.01L9.267 15.292M7.623 16.723c-2.792-2.67-2.31-6.801.071-9.184 1.761-1.763 4.647-2.483 7.166-1.425l2.705-1.25a7.808 7.808 0 00-1.829-1A8.975 8.975 0 005.984 5.83c-2.533 2.536-3.33 6.436-1.962 9.764 1.022 2.487-.653 4.246-2.34 6.022-.599.63-1.199 1.259-1.682 1.925l7.62-6.815"></path></svg></span>Grok 4.6 (high)<span class="org">xAI</span></td><td data-v="0.56721"><div class="cellbar"><i style="width:57.6px;--bar:var(--c4s-a)"></i><b>0.567</b></div></td><td class="dim" data-v="0.52211">0.522</td><td class="dim" data-v="0.5764199999999999">0.576</td><td class="dim" data-v="0.585">0.585</td><td class="dim" data-v="4.326270533333333">$4.33</td></tr><tr data-model-key="qwen-off" data-search="qwen 3.8 27b (thinking off) alibaba"><td class="rk">8</td><td class="nm"><span class="logo"><svg fill="#615CED" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Qwen</title><path d="M12.604 1.34c.393.69.784 1.382 1.174 2.075a.18.18 0 00.157.091h5.552c.174 0 .322.11.446.327l1.454 2.57c.19.337.24.478.024.837-.26.43-.513.864-.76 1.3l-.367.658c-.106.196-.223.28-.04.512l2.652 4.637c.172.301.111.494-.043.77-.437.785-.882 1.564-1.335 2.34-.159.272-.352.375-.68.37-.777-.016-1.552-.01-2.327.016a.099.099 0 00-.081.05 575.097 575.097 0 01-2.705 4.74c-.169.293-.38.363-.725.364-.997.003-2.002.004-3.017.002a.537.537 0 01-.465-.271l-1.335-2.323a.09.09 0 00-.083-.049H4.982c-.285.03-.553-.001-.805-.092l-1.603-2.77a.543.543 0 01-.002-.54l1.207-2.12a.198.198 0 000-.197 550.951 550.951 0 01-1.875-3.272l-.79-1.395c-.16-.31-.173-.496.095-.965.465-.813.927-1.625 1.387-2.436.132-.234.304-.334.584-.335a338.3 338.3 0 012.589-.001.124.124 0 00.107-.063l2.806-4.895a.488.488 0 01.422-.246c.524-.001 1.053 0 1.583-.006L11.704 1c.341-.003.724.032.9.34zm-3.432.403a.06.06 0 00-.052.03L6.254 6.788a.157.157 0 01-.135.078H3.253c-.056 0-.07.025-.041.074l5.81 10.156c.025.042.013.062-.034.063l-2.795.015a.218.218 0 00-.2.116l-1.32 2.31c-.044.078-.021.118.068.118l5.716.008c.046 0 .08.02.104.061l1.403 2.454c.046.081.092.082.139 0l5.006-8.76.783-1.382a.055.055 0 01.096 0l1.424 2.53a.122.122 0 00.107.062l2.763-.02a.04.04 0 00.035-.02.041.041 0 000-.04l-2.9-5.086a.108.108 0 010-.113l.293-.507 1.12-1.977c.024-.041.012-.062-.035-.062H9.2c-.059 0-.073-.026-.043-.077l1.434-2.505a.107.107 0 000-.114L9.225 1.774a.06.06 0 00-.053-.031zm6.29 8.02c.046 0 .058.02.034.06l-.832 1.465-2.613 4.585a.056.056 0 01-.05.029.058.058 0 01-.05-.029L8.498 9.841c-.02-.034-.01-.052.028-.054l.216-.012 6.722-.012z"></path></svg></span>Qwen 3.8 27B (thinking off)<span class="tag open">open weights</span><span class="org">Alibaba</span></td><td data-v="0.55666"><div class="cellbar"><i style="width:56.5px;--bar:var(--c4s-a)"></i><b>0.557</b></div></td><td class="dim" data-v="0.506755">0.507</td><td class="dim" data-v="0.529525">0.530</td><td class="dim" data-v="0.688">0.688</td><td class="dim" data-v="0.49154169999999997">$0.49</td></tr><tr data-model-key="qwen-on" data-search="qwen 3.8 27b (thinking on) alibaba"><td class="rk">9</td><td class="nm"><span class="logo"><svg fill="#615CED" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Qwen</title><path d="M12.604 1.34c.393.69.784 1.382 1.174 2.075a.18.18 0 00.157.091h5.552c.174 0 .322.11.446.327l1.454 2.57c.19.337.24.478.024.837-.26.43-.513.864-.76 1.3l-.367.658c-.106.196-.223.28-.04.512l2.652 4.637c.172.301.111.494-.043.77-.437.785-.882 1.564-1.335 2.34-.159.272-.352.375-.68.37-.777-.016-1.552-.01-2.327.016a.099.099 0 00-.081.05 575.097 575.097 0 01-2.705 4.74c-.169.293-.38.363-.725.364-.997.003-2.002.004-3.017.002a.537.537 0 01-.465-.271l-1.335-2.323a.09.09 0 00-.083-.049H4.982c-.285.03-.553-.001-.805-.092l-1.603-2.77a.543.543 0 01-.002-.54l1.207-2.12a.198.198 0 000-.197 550.951 550.951 0 01-1.875-3.272l-.79-1.395c-.16-.31-.173-.496.095-.965.465-.813.927-1.625 1.387-2.436.132-.234.304-.334.584-.335a338.3 338.3 0 012.589-.001.124.124 0 00.107-.063l2.806-4.895a.488.488 0 01.422-.246c.524-.001 1.053 0 1.583-.006L11.704 1c.341-.003.724.032.9.34zm-3.432.403a.06.06 0 00-.052.03L6.254 6.788a.157.157 0 01-.135.078H3.253c-.056 0-.07.025-.041.074l5.81 10.156c.025.042.013.062-.034.063l-2.795.015a.218.218 0 00-.2.116l-1.32 2.31c-.044.078-.021.118.068.118l5.716.008c.046 0 .08.02.104.061l1.403 2.454c.046.081.092.082.139 0l5.006-8.76.783-1.382a.055.055 0 01.096 0l1.424 2.53a.122.122 0 00.107.062l2.763-.02a.04.04 0 00.035-.02.041.041 0 000-.04l-2.9-5.086a.108.108 0 010-.113l.293-.507 1.12-1.977c.024-.041.012-.062-.035-.062H9.2c-.059 0-.073-.026-.043-.077l1.434-2.505a.107.107 0 000-.114L9.225 1.774a.06.06 0 00-.053-.031zm6.29 8.02c.046 0 .058.02.034.06l-.832 1.465-2.613 4.585a.056.056 0 01-.05.029.058.058 0 01-.05-.029L8.498 9.841c-.02-.034-.01-.052.028-.054l.216-.012 6.722-.012z"></path></svg></span>Qwen 3.8 27B (thinking on)<span class="tag open">open weights</span><span class="org">Alibaba</span></td><td data-v="0.51571"><div class="cellbar"><i style="width:52.4px;--bar:var(--c4s-a)"></i><b>0.516</b></div></td><td class="dim" data-v="0.50522">0.505</td><td class="dim" data-v="0.45978">0.460</td><td class="dim" data-v="0.694">0.694</td><td class="dim" data-v="0.3344125">$0.33</td></tr><tr data-model-key="glm" data-search="glm-5.3 flash (max) z.ai"><td class="rk">10</td><td class="nm"><span class="logo"><svg fill="currentColor" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Z.ai</title><path d="M12.105 2L9.927 4.953H.653L2.83 2h9.276zM23.254 19.048L21.078 22h-9.242l2.174-2.952h9.244zM24 2L9.264 22H0L14.736 2H24z"></path></svg></span>GLM-5.3 Flash (max)<span class="tag open">open weights</span><span class="org">Z.ai</span></td><td data-v="0.5094299999999999"><div class="cellbar"><i style="width:51.7px;--bar:var(--c4s-a)"></i><b>0.509</b></div></td><td class="dim" data-v="0.403365">0.403</td><td class="dim" data-v="0.537015">0.537</td><td class="dim" data-v="0.533">0.533</td><td class="dim" data-v="0.710521125">$0.71</td></tr><tr data-model-key="gemma-on" data-search="gemma 4 31b (thinking on) google"><td class="rk">11</td><td class="nm"><span class="logo"><svg fill="url(#c4sGem-25)" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><defs><lineargradient id="c4sGem-25" x1="0" x2="1" y1="0" y2="1"><stop offset="0" stop-color="#4796E3"></stop><stop offset=".55" stop-color="#9168C0"></stop><stop offset="1" stop-color="#D96570"></stop></lineargradient></defs><title>Gemini</title><path d="M20.616 10.835a14.147 14.147 0 01-4.45-3.001 14.111 14.111 0 01-3.678-6.452.503.503 0 00-.975 0 14.134 14.134 0 01-3.679 6.452 14.155 14.155 0 01-4.45 3.001c-.65.28-1.318.505-2.002.678a.502.502 0 000 .975c.684.172 1.35.397 2.002.677a14.147 14.147 0 014.45 3.001 14.112 14.112 0 013.679 6.453.502.502 0 00.975 0c.172-.685.397-1.351.677-2.003a14.145 14.145 0 013.001-4.45 14.113 14.113 0 016.453-3.678.503.503 0 000-.975 13.245 13.245 0 01-2.003-.678z"></path></svg></span>Gemma 4 31B (thinking on)<span class="tag open">open weights</span><span class="tag front">Pareto</span><span class="org">Google</span></td><td data-v="0.470705"><div class="cellbar"><i style="width:47.8px;--bar:var(--c4s-a)"></i><b>0.471</b></div></td><td class="dim" data-v="0.4117">0.412</td><td class="dim" data-v="0.41593">0.416</td><td class="dim" data-v="0.694">0.694</td><td class="dim" data-v="0.0253001">$0.03</td></tr><tr data-model-key="gemma-off" data-search="gemma 4 31b (thinking off) google"><td class="rk">12</td><td class="nm"><span class="logo"><svg fill="url(#c4sGem-26)" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><defs><lineargradient id="c4sGem-26" x1="0" x2="1" y1="0" y2="1"><stop offset="0" stop-color="#4796E3"></stop><stop offset=".55" stop-color="#9168C0"></stop><stop offset="1" stop-color="#D96570"></stop></lineargradient></defs><title>Gemini</title><path d="M20.616 10.835a14.147 14.147 0 01-4.45-3.001 14.111 14.111 0 01-3.678-6.452.503.503 0 00-.975 0 14.134 14.134 0 01-3.679 6.452 14.155 14.155 0 01-4.45 3.001c-.65.28-1.318.505-2.002.678a.502.502 0 000 .975c.684.172 1.35.397 2.002.677a14.147 14.147 0 014.45 3.001 14.112 14.112 0 013.679 6.453.502.502 0 00.975 0c.172-.685.397-1.351.677-2.003a14.145 14.145 0 013.001-4.45 14.113 14.113 0 016.453-3.678.503.503 0 000-.975 13.245 13.245 0 01-2.003-.678z"></path></svg></span>Gemma 4 31B (thinking off)<span class="tag open">open weights</span><span class="tag front">Pareto</span><span class="org">Google</span></td><td data-v="0.45494"><div class="cellbar"><i style="width:46.2px;--bar:var(--c4s-a)"></i><b>0.455</b></div></td><td class="dim" data-v="0.43810000000000004">0.438</td><td class="dim" data-v="0.370465">0.370</td><td class="dim" data-v="0.725">0.725</td><td class="dim" data-v="0.0151612">$0.02</td></tr><tr data-model-key="inkling" data-search="inkling (high) thinking machines"><td class="rk">13</td><td class="nm"><span class="logo"><svg fill="currentColor" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Thinking Machines</title><rect height="15.6" rx="3.6" width="15.6" x="4.2" y="4.2"></rect></svg>
|
| 29 |
</span>Inkling (high)<span class="org">Thinking Machines</span></td><td data-v="0.42498500000000006"><div class="cellbar"><i style="width:43.1px;--bar:var(--c4s-a)"></i><b>0.425</b></div></td><td class="dim" data-v="0.392605">0.393</td><td class="dim" data-v="0.336705">0.337</td><td class="dim" data-v="0.722">0.722</td><td class="dim" data-v="0.20252835">$0.20</td></tr><tr data-model-key="deepseek" data-search="deepseek v4.1 flash (high) deepseek"><td class="rk">14</td><td class="nm"><span class="logo"><svg fill="#4D6BFE" fill-rule="evenodd" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>DeepSeek</title><path d="M23.748 4.482c-.254-.124-.364.113-.512.234-.051.039-.094.09-.137.136-.372.397-.806.657-1.373.626-.829-.046-1.537.214-2.163.848-.133-.782-.575-1.248-1.247-1.548-.352-.156-.708-.311-.955-.65-.172-.241-.219-.51-.305-.774-.055-.16-.11-.323-.293-.35-.2-.031-.278.136-.356.276-.313.572-.434 1.202-.422 1.84.027 1.436.633 2.58 1.838 3.393.137.093.172.187.129.323-.082.28-.18.552-.266.833-.055.179-.137.217-.329.14a5.526 5.526 0 01-1.736-1.18c-.857-.828-1.631-1.742-2.597-2.458a11.365 11.365 0 00-.689-.471c-.985-.957.13-1.743.388-1.836.27-.098.093-.432-.779-.428-.872.004-1.67.295-2.687.684a3.055 3.055 0 01-.465.137 9.597 9.597 0 00-2.883-.102c-1.885.21-3.39 1.102-4.497 2.623C.082 8.606-.231 10.684.152 12.85c.403 2.284 1.569 4.175 3.36 5.653 1.858 1.533 3.997 2.284 6.438 2.14 1.482-.085 3.133-.284 4.994-1.86.47.234.962.327 1.78.397.63.059 1.236-.03 1.705-.128.735-.156.684-.837.419-.961-2.155-1.004-1.682-.595-2.113-.926 1.096-1.296 2.746-2.642 3.392-7.003.05-.347.007-.565 0-.845-.004-.17.035-.237.23-.256a4.173 4.173 0 001.545-.475c1.396-.763 1.96-2.015 2.093-3.517.02-.23-.004-.467-.247-.588zM11.581 18c-2.089-1.642-3.102-2.183-3.52-2.16-.392.024-.321.471-.235.763.09.288.207.486.371.739.114.167.192.416-.113.603-.673.416-1.842-.14-1.897-.167-1.361-.802-2.5-1.86-3.301-3.307-.774-1.393-1.224-2.887-1.298-4.482-.02-.386.093-.522.477-.592a4.696 4.696 0 011.529-.039c2.132.312 3.946 1.265 5.468 2.774.868.86 1.525 1.887 2.202 2.891.72 1.066 1.494 2.082 2.48 2.914.348.292.625.514.891.677-.802.09-2.14.11-3.054-.614zm1-6.44a.306.306 0 01.415-.287.302.302 0 01.2.288.306.306 0 01-.31.307.303.303 0 01-.304-.308zm3.11 1.596c-.2.081-.399.151-.59.16a1.245 1.245 0 01-.798-.254c-.274-.23-.47-.358-.552-.758a1.73 1.73 0 01.016-.588c.07-.327-.008-.537-.239-.727-.187-.156-.426-.199-.688-.199a.559.559 0 01-.254-.078c-.11-.054-.2-.19-.114-.358.028-.054.16-.186.192-.21.356-.202.767-.136 1.146.016.352.144.618.408 1.001.782.391.451.462.576.685.914.176.265.336.537.445.848.067.195-.019.354-.25.452z"></path></svg></span>DeepSeek V4.1 Flash (high)<span class="tag open">open weights</span><span class="org">DeepSeek</span></td><td data-v="0.24269"><div class="cellbar"><i style="width:24.7px;--bar:var(--c4s-a)"></i><b>0.243</b></div></td><td class="dim" data-v="0.216595">0.217</td><td class="dim" data-v="0.16134500000000002">0.161</td><td class="dim" data-v="0.513">0.513</td><td class="dim" data-v="0.193865">$0.19</td></tr></tbody></table></div></div>
|
| 30 |
<p class="board-note">Click a column to sort. Pareto: on the score-against-cost frontier.</p></div></div>
|
| 31 |
</div></section><section id="cases"><div class="container">
|
scripts/tracks.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Name the two evaluation tracks (user, 2026-10-01): the Research Track (20 scenes) and the Full Benchmark (160 scenes).
|
| 2 |
+
The page's Text-to-Scene results are the Research Track's; the case-count strip states the two tracks.
|
| 3 |
+
|
| 4 |
+
Each step asserts it found what it edits.
|
| 5 |
+
python3 scripts/tracks.py
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
import re
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from bs4 import BeautifulSoup
|
| 11 |
+
|
| 12 |
+
ROOT = Path(__file__).resolve().parent.parent
|
| 13 |
+
V = "20261005tracks"
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def one(found, what):
|
| 17 |
+
assert found, f"not found: {what}"
|
| 18 |
+
return found
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def main():
|
| 22 |
+
p = ROOT / "index.html"
|
| 23 |
+
soup = BeautifulSoup(p.read_text(), "html.parser")
|
| 24 |
+
frag = lambda html: BeautifulSoup(html, "html.parser")
|
| 25 |
+
# the case-count strip: the two tracks
|
| 26 |
+
av = one(soup.find(class_="availability"), "availability strip")
|
| 27 |
+
av.clear()
|
| 28 |
+
av.append(frag('<span><b>20</b> scenes in the <strong>Research Track</strong>: affordable, reproducible evaluation '
|
| 29 |
+
'for academic and model-development use</span>'
|
| 30 |
+
'<span><b>160</b> scenes in the <strong>Full Benchmark</strong>: comprehensive evaluation '
|
| 31 |
+
'for leaderboard and final reporting</span>'))
|
| 32 |
+
# leaderboard: the Research Track
|
| 33 |
+
lb = one(soup.find(id="leaderboard"), "leaderboard")
|
| 34 |
+
one(lb.find(class_="section-desc"), "leaderboard desc").string = \
|
| 35 |
+
"Research Track · 20 representative scenes · 14 coding-agent configurations."
|
| 36 |
+
lb.find(class_="section-desc").insert_after(frag(
|
| 37 |
+
'<p class="tracks-why">Text-to-scene generation with frontier models is a long-horizon agentic task with substantial '
|
| 38 |
+
'inference cost and runtime, which makes repeated full-scale evaluation difficult for many research groups. The '
|
| 39 |
+
'Research Track supports affordable, reproducible experimentation; the Full Benchmark provides the more comprehensive '
|
| 40 |
+
'evaluation for final model comparison and leaderboard reporting.</p>'))
|
| 41 |
+
h3 = one(lb.select_one(".bar-heading h3"), "bar heading")
|
| 42 |
+
one(h3.find("span", class_="q"), "bar qualifier").string = "Research Track"
|
| 43 |
+
side = [s for s in lb.select(".bar-heading > span")]
|
| 44 |
+
assert len(side) == 1, side
|
| 45 |
+
side[0].string = "20 scenes · score out of 100"
|
| 46 |
+
svg = one(lb.find(id="bars-t2s"), "bar svg")
|
| 47 |
+
svg["aria-label"] = "Research Track scores for the 14 Code4Scene configurations"
|
| 48 |
+
one(lb.find(class_="figure-subtitle"), "pareto subtitle").string = \
|
| 49 |
+
"Quality against the cost of one case · Research Track, 20 scenes"
|
| 50 |
+
note = one(lb.find(class_="board-note"), "board note")
|
| 51 |
+
t = note.get_text()
|
| 52 |
+
assert t.startswith("20 public cases."), t
|
| 53 |
+
note.string = "Research Track, 20 scenes." + t[len("20 public cases."):]
|
| 54 |
+
# cache key for the stylesheet (one new rule)
|
| 55 |
+
for link in soup.find_all("link", href=re.compile(r"^assets/buildingbench\.css")):
|
| 56 |
+
link["href"] = f"assets/buildingbench.css?v={V}"
|
| 57 |
+
p.write_text(str(soup))
|
| 58 |
+
left = re.findall(r"(?i)20 public|original 20|paper evaluation|129 public|full benchmark target", p.read_text())
|
| 59 |
+
assert not left, left
|
| 60 |
+
css = ROOT / "assets/buildingbench.css"; s = css.read_text()
|
| 61 |
+
rule = ".tracks-why{max-width:860px;margin:8px 0 0;font-size:13px;line-height:1.65;color:var(--text-muted);text-align:left}"
|
| 62 |
+
if ".tracks-why{" not in s:
|
| 63 |
+
css.write_text(s.rstrip("\n") + "\n/* evaluation tracks (scripts/tracks.py) */\n" + rule +
|
| 64 |
+
"\n@media(max-width:650px){.tracks-why{font-size:11px}}\n")
|
| 65 |
+
print("index.html: tracks strip, Research Track leaderboard")
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
if __name__ == "__main__":
|
| 69 |
+
main()
|