AgentArena / docs /architecture.html
smlflg's picture
Initial public upload from Projekte/AgentArena
3b0d6b3 verified
Raw History Blame Contribute Delete
64.7 kB
<!DOCTYPE html>
<html lang="de">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>AgentArena — Technische Architektur</title>
<style>
:root {
--bg: #0d1117;
--surface: #161b22;
--surface2: #21262d;
--border: #30363d;
--text: #c9d1d9;
--muted: #8b949e;
--green: #2ea043;
--green-dim: #1a3a20;
--green-bright: #56d364;
--blue: #1f6feb;
--blue-dim: #0d2a4a;
--blue-bright: #58a6ff;
--orange: #d29922;
--orange-dim:#3a2700;
--orange-bright: #e3b341;
--red: #da3633;
--red-dim: #3d0c09;
--red-bright:#f85149;
--purple: #8957e5;
--purple-dim:#2a1a5e;
--teal: #1b6e6e;
--teal-bright:#4dd0d0;
}
* { box-sizing: border-box; margin: 0; padding: 0; }
body {
background: var(--bg);
color: var(--text);
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, monospace;
font-size: 14px;
line-height: 1.6;
}
/* ---- NAV ---- */
nav {
position: sticky;
top: 0;
z-index: 100;
background: rgba(13,17,23,0.96);
border-bottom: 1px solid var(--border);
display: flex;
align-items: center;
gap: 2px;
padding: 0 24px;
height: 52px;
backdrop-filter: blur(8px);
}
nav .brand {
font-weight: 700;
font-size: 15px;
color: var(--green-bright);
margin-right: 20px;
white-space: nowrap;
}
nav a {
color: var(--muted);
text-decoration: none;
padding: 6px 10px;
border-radius: 6px;
font-size: 12px;
transition: all 0.15s;
white-space: nowrap;
}
nav a:hover { background: var(--surface2); color: var(--text); }
/* ---- LAYOUT ---- */
.page { max-width: 1200px; margin: 0 auto; padding: 40px 24px 80px; }
h1 { font-size: 32px; font-weight: 700; color: #fff; margin-bottom: 8px; }
h2 { font-size: 22px; font-weight: 600; color: #fff; margin-bottom: 16px; }
h3 { font-size: 16px; font-weight: 600; color: var(--blue-bright); margin-bottom: 10px; }
h4 { font-size: 13px; font-weight: 600; color: var(--muted); text-transform: uppercase; letter-spacing: 0.8px; margin-bottom: 8px; }
p { color: var(--muted); margin-bottom: 12px; }
p strong { color: var(--text); }
section { margin-bottom: 64px; scroll-margin-top: 64px; }
.badge {
display: inline-block;
padding: 2px 8px;
border-radius: 12px;
font-size: 11px;
font-weight: 600;
letter-spacing: 0.3px;
}
.badge-green { background: var(--green-dim); color: var(--green-bright); border: 1px solid var(--green); }
.badge-blue { background: var(--blue-dim); color: var(--blue-bright); border: 1px solid var(--blue); }
.badge-orange { background: var(--orange-dim); color: var(--orange-bright);border: 1px solid var(--orange); }
.badge-red { background: var(--red-dim); color: var(--red-bright); border: 1px solid var(--red); }
.badge-purple { background: var(--purple-dim); color: #b388ff; border: 1px solid var(--purple); }
/* ---- CARDS ---- */
.card {
background: var(--surface);
border: 1px solid var(--border);
border-radius: 10px;
padding: 20px 24px;
margin-bottom: 16px;
}
.card.green { border-left: 3px solid var(--green); }
.card.blue { border-left: 3px solid var(--blue); }
.card.orange { border-left: 3px solid var(--orange); }
.card.red { border-left: 3px solid var(--red); }
.card.purple { border-left: 3px solid var(--purple); }
.grid2 { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; }
.grid3 { display: grid; grid-template-columns: 1fr 1fr 1fr; gap: 16px; }
@media (max-width: 800px) { .grid2, .grid3 { grid-template-columns: 1fr; } }
/* ---- CODE ---- */
pre, code {
font-family: 'JetBrains Mono', 'Fira Code', 'Cascadia Code', monospace;
font-size: 12px;
}
pre {
background: #010409;
border: 1px solid var(--border);
border-radius: 8px;
padding: 16px;
overflow-x: auto;
margin-bottom: 16px;
line-height: 1.5;
}
code { background: var(--surface2); padding: 1px 5px; border-radius: 4px; color: var(--orange-bright); }
.kw { color: #ff7b72; }
.fn { color: var(--blue-bright); }
.st { color: var(--green-bright); }
.cm { color: var(--muted); font-style: italic; }
.nm { color: #ffa657; }
.py { color: #d2a8ff; }
/* ---- PIPELINE FLOW ---- */
.pipeline {
display: flex;
align-items: stretch;
gap: 0;
overflow-x: auto;
padding-bottom: 8px;
margin-bottom: 24px;
}
.pipe-step {
display: flex;
flex-direction: column;
align-items: center;
min-width: 110px;
flex: 1;
}
.pipe-box {
background: var(--surface);
border: 1px solid var(--border);
border-radius: 8px;
padding: 12px 10px;
text-align: center;
font-size: 12px;
font-weight: 600;
width: 100%;
position: relative;
transition: transform 0.15s;
cursor: default;
}
.pipe-box:hover { transform: translateY(-2px); }
.pipe-box .icon { font-size: 22px; margin-bottom: 4px; display: block; }
.pipe-box .label { color: var(--text); }
.pipe-box .sublabel { color: var(--muted); font-size: 10px; font-weight: 400; margin-top: 2px; }
.pipe-box.green { border-color: var(--green); background: var(--green-dim); }
.pipe-box.blue { border-color: var(--blue); background: var(--blue-dim); }
.pipe-box.orange { border-color: var(--orange); background: var(--orange-dim); }
.pipe-box.red { border-color: var(--red); background: var(--red-dim); }
.pipe-box.purple { border-color: var(--purple); background: var(--purple-dim); }
.pipe-box.teal { border-color: #1b6e6e; background: #0a2020; }
.pipe-arrow {
display: flex;
align-items: center;
justify-content: center;
padding: 0 4px;
color: var(--muted);
font-size: 18px;
flex-shrink: 0;
}
/* ---- TWO-CONTAINER DIAGRAM ---- */
.isolation-diagram {
display: grid;
grid-template-columns: 1fr auto 1fr;
gap: 0;
align-items: start;
margin-bottom: 24px;
}
.container-box {
border: 2px solid;
border-radius: 10px;
padding: 16px;
}
.container-box.agent { border-color: var(--green); background: var(--green-dim); }
.container-box.judge { border-color: var(--blue); background: var(--blue-dim); }
.container-header {
font-weight: 700;
font-size: 13px;
margin-bottom: 12px;
display: flex;
align-items: center;
gap: 8px;
}
.step-list { list-style: none; }
.step-list li {
display: flex;
align-items: flex-start;
gap: 8px;
padding: 6px 0;
border-bottom: 1px solid rgba(255,255,255,0.05);
font-size: 12px;
color: var(--text);
}
.step-list li:last-child { border-bottom: none; }
.step-num {
background: var(--surface2);
border-radius: 50%;
width: 20px;
height: 20px;
display: flex;
align-items: center;
justify-content: center;
font-size: 10px;
font-weight: 700;
flex-shrink: 0;
color: var(--muted);
}
.wall {
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
padding: 0 16px;
gap: 4px;
}
.wall-line {
width: 2px;
flex: 1;
background: repeating-linear-gradient(
to bottom,
var(--red) 0px, var(--red) 8px,
transparent 8px, transparent 16px
);
min-height: 80px;
}
.wall-label {
background: var(--red-dim);
border: 1px solid var(--red);
color: var(--red-bright);
font-size: 10px;
font-weight: 700;
padding: 4px 8px;
border-radius: 4px;
text-align: center;
white-space: nowrap;
}
/* ---- SCORE DIAGRAM ---- */
.score-bar-wrap { margin-bottom: 12px; }
.score-bar-label {
display: flex;
justify-content: space-between;
margin-bottom: 4px;
font-size: 12px;
}
.score-bar-track {
height: 24px;
background: var(--surface2);
border-radius: 4px;
overflow: hidden;
position: relative;
}
.score-bar-fill {
height: 100%;
border-radius: 4px;
display: flex;
align-items: center;
justify-content: flex-end;
padding-right: 8px;
font-size: 11px;
font-weight: 700;
color: #000;
transition: width 0.4s ease;
}
/* ---- ADAPTER CARDS ---- */
.adapter-grid {
display: grid;
grid-template-columns: repeat(3, 1fr);
gap: 16px;
margin-bottom: 24px;
}
@media (max-width: 900px) { .adapter-grid { grid-template-columns: 1fr; } }
.adapter-card {
background: var(--surface);
border: 1px solid var(--border);
border-radius: 10px;
overflow: hidden;
}
.adapter-header {
padding: 14px 16px;
font-weight: 700;
font-size: 14px;
display: flex;
align-items: center;
gap: 8px;
}
.adapter-header.cc { background: linear-gradient(135deg, #1a3a20 0%, #0f2010 100%); border-bottom: 1px solid var(--green); }
.adapter-header.ai { background: linear-gradient(135deg, #3a2700 0%, #1a1000 100%); border-bottom: 1px solid var(--orange); }
.adapter-header.cx { background: linear-gradient(135deg, #2a1a5e 0%, #15102d 100%); border-bottom: 1px solid var(--purple); }
.adapter-body { padding: 16px; }
.adapter-prop { margin-bottom: 8px; font-size: 12px; }
.adapter-prop .key { color: var(--muted); margin-bottom: 2px; }
.adapter-prop .val { color: var(--text); font-family: monospace; }
/* ---- DATA FLOW TABLE ---- */
table {
width: 100%;
border-collapse: collapse;
margin-bottom: 16px;
font-size: 12px;
}
th {
background: var(--surface2);
color: var(--muted);
text-align: left;
padding: 8px 12px;
font-size: 11px;
text-transform: uppercase;
letter-spacing: 0.5px;
border-bottom: 1px solid var(--border);
}
td {
padding: 8px 12px;
border-bottom: 1px solid var(--border);
vertical-align: top;
}
tr:last-child td { border-bottom: none; }
tr:hover td { background: var(--surface2); }
td code { font-size: 11px; }
/* ---- METRICS CARDS ---- */
.metric-card {
background: var(--surface);
border: 1px solid var(--border);
border-radius: 8px;
padding: 14px 16px;
text-align: center;
}
.metric-val { font-size: 28px; font-weight: 700; color: #fff; }
.metric-label { font-size: 11px; color: var(--muted); margin-top: 4px; }
/* ---- FILE TREE ---- */
.filetree {
background: #010409;
border: 1px solid var(--border);
border-radius: 8px;
padding: 16px;
font-family: monospace;
font-size: 12px;
line-height: 1.8;
}
.ft-dir { color: var(--blue-bright); font-weight: 600; }
.ft-file { color: var(--text); }
.ft-desc { color: var(--muted); font-style: italic; }
/* ---- LEADERBOARD ---- */
.leaderboard {
background: var(--surface);
border: 1px solid var(--border);
border-radius: 10px;
overflow: hidden;
}
.lb-row {
display: grid;
grid-template-columns: 40px 160px 80px 100px 100px 100px 120px;
align-items: center;
padding: 10px 16px;
border-bottom: 1px solid var(--border);
font-size: 13px;
transition: background 0.1s;
}
.lb-row:hover { background: var(--surface2); }
.lb-row.header {
background: var(--surface2);
color: var(--muted);
font-size: 11px;
text-transform: uppercase;
letter-spacing: 0.5px;
font-weight: 600;
cursor: default;
}
.lb-row:last-child { border-bottom: none; }
.lb-rank { color: var(--muted); font-weight: 700; }
.lb-rank.gold { color: #f1c40f; }
.lb-rank.silver { color: #95a5a6; }
.lb-score { font-weight: 700; }
.lb-score.perfect { color: var(--green-bright); }
.lb-score.good { color: var(--blue-bright); }
/* ---- FORMULA BOX ---- */
.formula {
background: #010409;
border: 1px solid var(--border);
border-left: 3px solid var(--purple);
border-radius: 0 8px 8px 0;
padding: 16px 20px;
font-family: monospace;
font-size: 13px;
margin-bottom: 16px;
color: var(--text);
}
.formula .fname { color: #d2a8ff; font-weight: 700; }
.formula .fnum { color: var(--orange-bright); }
/* ---- DIVIDER ---- */
.divider { border: none; border-top: 1px solid var(--border); margin: 40px 0; }
/* ---- TOOLTIP TABS ---- */
.tabs { display: flex; gap: 4px; margin-bottom: -1px; }
.tab {
padding: 6px 14px;
border-radius: 6px 6px 0 0;
border: 1px solid var(--border);
border-bottom: none;
background: var(--surface2);
color: var(--muted);
font-size: 12px;
cursor: pointer;
transition: all 0.15s;
}
.tab.active { background: var(--surface); color: var(--text); }
.tab-content { display: none; }
.tab-content.active { display: block; }
.tabpanel {
background: var(--surface);
border: 1px solid var(--border);
border-radius: 0 8px 8px 8px;
padding: 20px;
margin-bottom: 16px;
}
/* ---- SSE ANIMATION ---- */
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.3; }
}
.live-dot {
display: inline-block;
width: 8px; height: 8px;
background: var(--green-bright);
border-radius: 50%;
animation: pulse 1.5s ease-in-out infinite;
margin-right: 6px;
}
/* ---- SCROLLBAR ---- */
::-webkit-scrollbar { width: 6px; height: 6px; }
::-webkit-scrollbar-track { background: var(--bg); }
::-webkit-scrollbar-thumb { background: var(--border); border-radius: 3px; }
</style>
</head>
<body>
<!-- NAV -->
<nav>
<span class="brand">AgentArena</span>
<a href="#pipeline">Pipeline</a>
<a href="#isolation">Sicherheit</a>
<a href="#adapters">Adapter</a>
<a href="#judge">Judge</a>
<a href="#metrics">Metriken</a>
<a href="#sse">Live-Updates</a>
<a href="#filestructure">Dateien</a>
<a href="#results">Ergebnisse</a>
</nav>
<div class="page">
<!-- HEADER -->
<h1>AgentArena</h1>
<p style="font-size:16px; color:var(--text); margin-bottom:8px;">
<strong>Technische Architektur-Dokumentation</strong> &mdash; Stand Maerz 2026
</p>
<p>
AgentArena ist eine <strong>FastAPI + Docker + SQLite</strong> Benchmarking-Plattform fuer AI Coding Agents.
Das Alleinstellungsmerkmal: explizites <strong>Kosten-Tracking ($/Task)</strong> und ein
<strong>zweicontainer-isoliertes Bewertungssystem</strong>, das Manipulation durch den Agenten verhindert.
</p>
<div style="display:flex; gap:8px; flex-wrap:wrap; margin-bottom:40px;">
<span class="badge badge-green">FastAPI</span>
<span class="badge badge-blue">Docker</span>
<span class="badge badge-orange">SQLite</span>
<span class="badge badge-purple">htmx + SSE</span>
<span class="badge badge-red">2-Container Isolation</span>
</div>
<hr class="divider">
<!-- ==================== PIPELINE ==================== -->
<section id="pipeline">
<h2>1. Pipeline-Flow</h2>
<p>Jeder Benchmark-Run durchlaeuft diese 8 Phasen sequenziell. Der Agent-Container und der Judge-Container
existieren <strong>nie gleichzeitig</strong> &mdash; der Agent-Container wird zerstoert,
bevor der Judge-Container erzeugt wird.</p>
<div class="pipeline">
<div class="pipe-step">
<div class="pipe-box teal">
<span class="icon">&#128196;</span>
<span class="label">API Request</span>
<span class="sublabel">POST /runs</span>
</div>
</div>
<div class="pipe-arrow">&#8594;</div>
<div class="pipe-step">
<div class="pipe-box blue">
<span class="icon">&#127939;</span>
<span class="label">Runner</span>
<span class="sublabel">runner.py</span>
</div>
</div>
<div class="pipe-arrow">&#8594;</div>
<div class="pipe-step">
<div class="pipe-box green">
<span class="icon">&#128274;</span>
<span class="label">Agent Sandbox</span>
<span class="sublabel">Docker container</span>
</div>
</div>
<div class="pipe-arrow">&#8594;</div>
<div class="pipe-step">
<div class="pipe-box orange">
<span class="icon">&#128279;</span>
<span class="label">Adapter</span>
<span class="sublabel">Host subprocess</span>
</div>
</div>
<div class="pipe-arrow">&#8594;</div>
<div class="pipe-step">
<div class="pipe-box orange">
<span class="icon">&#128196;</span>
<span class="label">git diff</span>
<span class="sublabel">arena-checkpoint</span>
</div>
</div>
<div class="pipe-arrow">&#8594;</div>
<div class="pipe-step">
<div class="pipe-box red">
<span class="icon">&#9888;</span>
<span class="label">Destroy Agent</span>
<span class="sublabel">container rm -f</span>
</div>
</div>
<div class="pipe-arrow">&#8594;</div>
<div class="pipe-step">
<div class="pipe-box blue">
<span class="icon">&#128270;</span>
<span class="label">Judge Sandbox</span>
<span class="sublabel">frischer Clone</span>
</div>
</div>
<div class="pipe-arrow">&#8594;</div>
<div class="pipe-step">
<div class="pipe-box purple">
<span class="icon">&#127942;</span>
<span class="label">Score</span>
<span class="sublabel">4 Dimensionen</span>
</div>
</div>
<div class="pipe-arrow">&#8594;</div>
<div class="pipe-step">
<div class="pipe-box teal">
<span class="icon">&#127941;</span>
<span class="label">Leaderboard</span>
<span class="sublabel">SQLite + htmx</span>
</div>
</div>
</div>
<h3>Was passiert in jeder Phase?</h3>
<table>
<thead>
<tr>
<th>Phase</th>
<th>Komponente</th>
<th>Input</th>
<th>Output</th>
<th>Fehlerbehandlung</th>
</tr>
</thead>
<tbody>
<tr>
<td><strong>1. API Request</strong></td>
<td><code>main.py</code></td>
<td><code>task_id, agent_id, k</code></td>
<td>Neuer <code>Run</code> in DB mit Status <code>running</code></td>
<td>HTTP 422 bei fehlenden Parametern</td>
</tr>
<tr>
<td><strong>2. Sandbox erstellen</strong></td>
<td><code>sandbox.py</code></td>
<td><code>Task</code> (repo_url, repo_ref)</td>
<td>Laufender Docker-Container mit geklontem Repo + <code>arena-checkpoint</code> Git-Tag</td>
<td><code>RuntimeError</code> wenn Docker oder git clone fehlschlaegt</td>
</tr>
<tr>
<td><strong>3. Adapter ausfuehren</strong></td>
<td><code>adapters/</code></td>
<td><code>RunContext</code> (task_prompt, success_cmd, time_limit_s)</td>
<td><code>RunResult</code> (diff_text, tokens, cost_usd, trajectory)</td>
<td><code>subprocess.TimeoutExpired</code> &rarr; status=timeout</td>
</tr>
<tr>
<td><strong>4. Diff sammeln</strong></td>
<td><code>runner.py</code></td>
<td>Host-Tempdir oder Sandbox</td>
<td>Unified diff (arena-checkpoint bis HEAD)</td>
<td>Leerer Diff = kein Code generiert</td>
</tr>
<tr>
<td><strong>5. Agent zerstoeren</strong></td>
<td><code>sandbox.destroy()</code></td>
<td>Container-ID</td>
<td>&mdash; (Container weg)</td>
<td>Best-effort, ignoriert Fehler</td>
</tr>
<tr>
<td><strong>6. Judge Sandbox</strong></td>
<td><code>sandbox.create_judge_sandbox()</code></td>
<td>Task + diff_text</td>
<td>Frischer Container mit gepatchtem Code; <code>sandbox.patch_applied</code></td>
<td>Warning-Flag wenn Patch fehlschlaegt</td>
</tr>
<tr>
<td><strong>7. Judge</strong></td>
<td><code>judge.py</code></td>
<td>Judge-Container + success_cmd</td>
<td><code>JudgeResult</code> (4 Scores + Notes)</td>
<td>Neutral-Score 0.5 wenn kein Test-Runner gefunden</td>
</tr>
<tr>
<td><strong>8. Metriken + DB</strong></td>
<td><code>metrics.py</code>, <code>runner.py</code></td>
<td>JudgeResult + adapter_result + trajectory</td>
<td><code>Run</code> mit judge_score, metrics_json, cost_usd</td>
<td>Composite score = 0 wenn Funktionalitaet = 0</td>
</tr>
</tbody>
</table>
<h3>Aus dem Code: Runner-Kernlogik</h3>
<pre><span class="cm"># core/runner.py (vereinfacht)</span>
<span class="kw">async def</span> <span class="fn">run_single</span>(self, db, task, agent) -> Run:
<span class="cm"># 1. Sandbox erstellen (git clone + checkpoint)</span>
sandbox = <span class="kw">await</span> asyncio.to_thread(Sandbox.create, task)
<span class="cm"># 2. Adapter holen und ausfuehren (HOST subprocess, nicht im Container)</span>
adapter = get_adapter(agent.adapter_type, agent_config)
adapter_result = <span class="kw">await</span> adapter.run(ctx, sandbox)
<span class="cm"># 3. Diff aus Host-Tempdir oder Container holen</span>
diff_text = adapter_result.diff_text <span class="kw">or</span> <span class="kw">await</span> asyncio.to_thread(sandbox.get_diff)
<span class="cm"># 4. AGENT CONTAINER ZERSTOEREN (Security-Grenze!)</span>
<span class="kw">await</span> asyncio.to_thread(sandbox.destroy)
sandbox = <span class="nm">None</span>
<span class="cm"># 5. FRISCHEN Judge-Container erstellen + Diff einspielen</span>
judge_sandbox = <span class="kw">await</span> asyncio.to_thread(
Sandbox.create_judge_sandbox, task, diff_text
)
<span class="cm"># 6. Judge laeuft NUR im frischen Container</span>
judge_result = <span class="kw">await</span> asyncio.to_thread(
self.judge.judge_run, judge_sandbox, task.success_cmd
)</pre>
</section>
<!-- ==================== ISOLATION ==================== -->
<section id="isolation">
<h2>2. Sicherheitsmodell: 2-Container-Isolation</h2>
<p>Das kritischste Design-Entscheidung: Agent und Judge laufen in <strong>separaten, kurzlebigen Docker-Containern</strong>.
Der Agent hat <em>keine Moeglichkeit</em>, die Testumgebung zu manipulieren, weil der Judge-Container
erst nach Zerstoerung des Agent-Containers erstellt wird.</p>
<div class="isolation-diagram">
<!-- Container A -->
<div class="container-box agent">
<div class="container-header">
<span class="badge badge-green">Container A</span>
<span style="color: var(--green-bright);">Agent Sandbox</span>
</div>
<ul class="step-list">
<li><span class="step-num">1</span> <div><strong>docker run</strong> python:3.12-slim<br><span style="color:var(--muted)">--network=arena-net --memory=2g</span></div></li>
<li><span class="step-num">2</span> <div><strong>git clone</strong> --depth=1 repo_url /workspace<br><span style="color:var(--muted)">+ apt install git, pip install pytest</span></div></li>
<li><span class="step-num">3</span> <div><strong>git tag</strong> arena-checkpoint<br><span style="color:var(--muted)">Snapshot vor Agent-Arbeit</span></div></li>
<li><span class="step-num">4</span> <div><strong>Adapter laeuft auf HOST</strong><br><span style="color:var(--muted)">claude/aider als subprocess</span></div></li>
<li><span class="step-num">5</span> <div><strong>git diff</strong> arena-checkpoint<br><span style="color:var(--muted)">Unified diff gespeichert</span></div></li>
<li><span class="step-num" style="background:var(--red-dim);color:var(--red-bright);">X</span> <div><strong>docker rm -f</strong> container_id<br><span style="color:var(--red-bright)">Zerstoert. Kein shared state.</span></div></li>
</ul>
</div>
<!-- Wall -->
<div class="wall">
<div class="wall-line"></div>
<div class="wall-label">Security<br>Boundary<br>&#128274;</div>
<div class="wall-label" style="margin-top:8px; background:var(--orange-dim); border-color:var(--orange); color:var(--orange-bright);">nur<br>diff_text<br>&#8594;</div>
<div class="wall-line"></div>
</div>
<!-- Container B -->
<div class="container-box judge">
<div class="container-header">
<span class="badge badge-blue">Container B</span>
<span style="color: var(--blue-bright);">Judge Sandbox</span>
</div>
<ul class="step-list">
<li><span class="step-num">1</span> <div><strong>docker run</strong> python:3.12-slim<br><span style="color:var(--muted)">frischer Container, nichts vom Agenten</span></div></li>
<li><span class="step-num">2</span> <div><strong>git clone</strong> --depth=1 repo_url /workspace<br><span style="color:var(--muted)">+ apt install git patch</span></div></li>
<li><span class="step-num">3</span> <div><strong>docker cp</strong> agent.patch Container<br><span style="color:var(--muted)">Diff als temp file eingeschleust</span></div></li>
<li><span class="step-num">4</span> <div><strong>git apply</strong> agent.patch<br><span style="color:var(--muted)">Fallback: patch -p1</span></div></li>
<li><span class="step-num">5</span> <div><strong>pytest, ruff, radon</strong> ausfuehren<br><span style="color:var(--muted)">in sauberem Environment</span></div></li>
<li><span class="step-num" style="background:var(--blue-dim);color:var(--blue-bright);">&#10003;</span> <div><strong>Score berechnen</strong> + docker rm<br><span style="color:var(--blue-bright)">JudgeResult zurueck an Runner</span></div></li>
</ul>
</div>
</div>
<div class="grid2">
<div class="card red">
<h3 style="color:var(--red-bright)">Warum ist das wichtig?</h3>
<p>Ohne Isolation koennte ein Agent:</p>
<ul style="color:var(--muted); padding-left:20px; font-size:13px; margin-top:8px;">
<li>Test-Dateien so modifizieren, dass sie immer bestehen</li>
<li>pytest durch ein eigenes Script ersetzen</li>
<li>Git-Historie manipulieren um alte Fehler zu verstecken</li>
<li>Environment-Variablen setzen die den Judge beeinflussen</li>
</ul>
</div>
<div class="card green">
<h3 style="color:var(--green-bright)">Garantien des Modells</h3>
<ul style="color:var(--muted); padding-left:20px; font-size:13px; margin-top:8px;">
<li>Judge sieht <strong>nur</strong> den Diff (Text), nie den Agent-Container</li>
<li>Jeder Run startet mit einem <strong>frischen Repo-Clone</strong></li>
<li>Kein shared filesystem, kein shared network state zwischen Containern</li>
<li><code>patch_applied</code> Flag warnt wenn Diff nicht anwendbar</li>
</ul>
</div>
</div>
<h3>Sandbox.create_judge_sandbox() im Detail</h3>
<pre><span class="cm"># core/sandbox.py</span>
<span class="kw">@classmethod</span>
<span class="kw">def</span> <span class="fn">create_judge_sandbox</span>(cls, task, diff_text: <span class="py">str</span>) -> <span class="st">"Sandbox"</span>:
<span class="cm"># Frischer Container — kein Bezug zum Agent-Container</span>
container_id = docker_run(image=<span class="st">"python:3.12-slim"</span>)
sandbox = cls(container_id)
<span class="cm"># Frischer Clone direkt aus Remote</span>
sandbox.exec(<span class="st">f"git clone --depth=1 -b {repo_ref} {repo_url} ."</span>)
<span class="cm"># Diff als Datei in Container kopieren</span>
subprocess.run([<span class="st">"docker"</span>, <span class="st">"cp"</span>, tmp_path, <span class="st">f"{container_id}:/tmp/agent.patch"</span>])
<span class="cm"># Patch anwenden (git apply, Fallback: patch -p1)</span>
rc, out, err = sandbox.exec(
<span class="st">"git apply --whitespace=fix /tmp/agent.patch || patch -p1 &lt; /tmp/agent.patch"</span>
)
sandbox.patch_applied = rc == <span class="nm">0</span>
<span class="kw">return</span> sandbox</pre>
</section>
<!-- ==================== ADAPTERS ==================== -->
<section id="adapters">
<h2>3. Adapter-Architektur</h2>
<p>Alle Adapter implementieren das <code>AgentAdapter</code> ABC und laufen auf dem <strong>HOST-System</strong>,
nicht im Container. Der Grund: Claude Code und Aider koennen nicht sinnvoll innerhalb von Docker-Containern
ausgefuehrt werden (OAuth-Zustand, API-Keys, Dateisystem-Zugriff).</p>
<div class="adapter-grid">
<!-- Claude Code -->
<div class="adapter-card">
<div class="adapter-header cc">
<span>&#128640;</span>
<span style="color:var(--green-bright)">ClaudeCodeAdapter</span>
</div>
<div class="adapter-body">
<div class="adapter-prop">
<div class="key">CLI-Befehl</div>
<div class="val">claude --print --dangerously-skip-permissions --output-format json</div>
</div>
<div class="adapter-prop">
<div class="key">Auth</div>
<div class="val">OAuth (kein API-Key noetig)</div>
</div>
<div class="adapter-prop">
<div class="key">Repo-Strategie</div>
<div class="val">Klont in Host-Tempdir (arena-claude-*)</div>
</div>
<div class="adapter-prop">
<div class="key">Token-Tracking</div>
<div class="val">entry.usage.input_tokens, total_cost_usd direkt aus JSON</div>
</div>
<div class="adapter-prop">
<div class="key">Modell</div>
<div class="val">claude-sonnet-4-6 (aus modelUsage)</div>
</div>
<div class="adapter-prop">
<div class="key">Kosten (Run #5)</div>
<div class="val" style="color:var(--orange-bright)">~$0.15</div>
</div>
</div>
</div>
<!-- Aider -->
<div class="adapter-card">
<div class="adapter-header ai">
<span>&#9874;</span>
<span style="color:var(--orange-bright)">AiderAdapter</span>
</div>
<div class="adapter-body">
<div class="adapter-prop">
<div class="key">CLI-Befehl</div>
<div class="val">aider --yes-always --model openai/MiniMax-M2.5 --message ...</div>
</div>
<div class="adapter-prop">
<div class="key">Auth</div>
<div class="val">MINIMAX_API_KEY aus ~/.claude/.env</div>
</div>
<div class="adapter-prop">
<div class="key">API-Routing</div>
<div class="val">litellm &rarr; OPENAI_API_BASE=https://api.minimax.io/v1</div>
</div>
<div class="adapter-prop">
<div class="key">Token-Tracking</div>
<div class="val">Regex aus stdout: "Tokens: X sent, Y received, Cost: $Z"</div>
</div>
<div class="adapter-prop">
<div class="key">Modell</div>
<div class="val">MiniMax-M2.5 (via OpenAI-kompatibler Endpoint)</div>
</div>
<div class="adapter-prop">
<div class="key">Kosten (Run #10)</div>
<div class="val" style="color:var(--green-bright)">~$0.00 (MiniMax)</div>
</div>
</div>
</div>
<!-- Codex -->
<div class="adapter-card">
<div class="adapter-header cx">
<span>&#128187;</span>
<span style="color:#b388ff">CodexAdapter</span>
</div>
<div class="adapter-body">
<div class="adapter-prop">
<div class="key">CLI-Befehl</div>
<div class="val">codex --quiet --full-auto</div>
</div>
<div class="adapter-prop">
<div class="key">Status</div>
<div class="val" style="color:var(--orange-bright)">Noch nicht getestet</div>
</div>
<div class="adapter-prop">
<div class="key">Auth</div>
<div class="val">OPENAI_API_KEY</div>
</div>
<div class="adapter-prop">
<div class="key">Pricing</div>
<div class="val">gpt-4o: $2.50/$10.0 per 1M</div>
</div>
</div>
</div>
</div>
<h3>Gemeinsames Pattern: Host-Clone-Strategy</h3>
<p>Alle Adapter klonen das Repo in ein Host-Tempverzeichnis, setzen einen Git-Checkpoint-Tag,
fuehren dann den CLI-Prozess aus, und sammeln den Diff vor dem Cleanup.</p>
<pre><span class="cm"># Muster fuer alle Adapter (adapters/claude_code.py, adapters/aider.py)</span>
tmpdir = tempfile.mkdtemp(prefix=<span class="st">"arena-claude-"</span>) <span class="cm"># oder arena-aider-</span>
<span class="cm"># 1. Repo klonen</span>
subprocess.run([<span class="st">"git"</span>, <span class="st">"clone"</span>, <span class="st">"--depth=1"</span>, <span class="st">"-b"</span>, repo_ref, repo_url, tmpdir])
<span class="cm"># 2. Checkpoint vor Agent-Arbeit</span>
subprocess.run([<span class="st">"git"</span>, <span class="st">"tag"</span>, <span class="st">"arena-checkpoint"</span>], cwd=tmpdir)
<span class="cm"># 3. CLI ausfuehren (timeout = task.time_limit_s)</span>
proc = subprocess.run(cmd, cwd=tmpdir, capture_output=<span class="nm">True</span>, timeout=ctx.time_limit_s)
<span class="cm"># 4. Diff sammeln (excludiert __pycache__ und .pyc)</span>
diff = subprocess.run(
[<span class="st">"git"</span>, <span class="st">"diff"</span>, <span class="st">"arena-checkpoint"</span>, <span class="st">"--"</span>, <span class="st">"."</span>,
<span class="st">":!**/__pycache__/"</span>, <span class="st">":!**/*.pyc"</span>],
cwd=tmpdir
)
<span class="cm"># 5. Cleanup (always, via try/finally)</span>
shutil.rmtree(tmpdir, ignore_errors=<span class="nm">True</span>)</pre>
<h3>Preistabellen</h3>
<table>
<thead>
<tr><th>Modell</th><th>Input ($/1M)</th><th>Cache Write</th><th>Cache Read</th><th>Output ($/1M)</th></tr>
</thead>
<tbody>
<tr><td>claude-opus-4-6</td><td>$5.00</td><td>$6.25</td><td>$0.50</td><td>$25.00</td></tr>
<tr><td>claude-sonnet-4-6</td><td>$3.00</td><td>$3.75</td><td>$0.30</td><td>$15.00</td></tr>
<tr><td>claude-haiku-4-5</td><td>$1.00</td><td>$1.25</td><td>$0.10</td><td>$5.00</td></tr>
<tr><td>gpt-4o</td><td>$2.50</td><td>&mdash;</td><td>&mdash;</td><td>$10.00</td></tr>
<tr><td><strong>MiniMax-M2.5</strong></td><td><strong>$0.05</strong></td><td>&mdash;</td><td>&mdash;</td><td><strong>$0.05</strong></td></tr>
</tbody>
</table>
</section>
<!-- ==================== JUDGE ==================== -->
<section id="judge">
<h2>4. Judge-System: 4 Dimensionen</h2>
<p>Der Judge bewertet den Code in vier Dimensionen. <strong>Funktionalitaet wirkt als Gatekeeper</strong>:
besteht die Aufgabe nicht, ist der Overall-Score 0 unabhaengig von allen anderen Dimensionen.</p>
<div class="formula">
<span class="fname">overall_score</span> =
<span class="fname">functional_score</span> &times; <span class="fnum">0.50</span> +
<span class="fname">regression_score</span> &times; <span class="fnum">0.20</span> +
<span class="fname">lint_score</span> &times; <span class="fnum">0.15</span> +
<span class="fname">complexity_score</span> &times; <span class="fnum">0.15</span>
</div>
<div class="grid2">
<div class="card green">
<h3 style="color:var(--green-bright)">Functional Score (50%)</h3>
<p><strong>Messmethode:</strong> Fuehrt <code>success_cmd</code> (pytest) im Judge-Container aus.
Prueft Exit-Code und parst Testanzahl aus stdout.</p>
<p><strong>Scoring:</strong> Exit-Code 0 = 1.0, Exit-Code != 0 = 0.0.
Test-Count wird separat getrackt (tests_total, tests_passed).</p>
<p><strong>Beispiel:</strong> <code>5/5 tests passed. Exit 0 &rarr; score=1.0</code></p>
<pre style="font-size:11px; margin-bottom:0;"><span class="kw">if</span> rc == <span class="nm">0</span>:
result.functional_score = <span class="nm">1.0</span>
<span class="kw">else</span>:
result.functional_score = <span class="nm">0.0</span></pre>
</div>
<div class="card blue">
<h3 style="color:var(--blue-bright)">Regression Score (20%)</h3>
<p><strong>Messmethode:</strong> Fuehrt breiteren Test-Suite aus.
Kandidaten in Reihenfolge: <code>pytest --tb=no -q</code>, <code>python -m pytest</code>, <code>make test</code>.</p>
<p><strong>Scoring:</strong> <code>passed / total</code>. Neutralscore 0.5 wenn kein Test-Runner gefunden.</p>
<p><strong>Crash-Detection:</strong> <code>"X errors in Y.Zs"</code> (ohne "passed"/"failed") wird als
Sammelfehler erkannt und uebersprungen, nicht als echter Test-Ausfall gewertet.</p>
<pre style="font-size:11px; margin-bottom:0;"><span class="cm"># Crash detection</span>
is_crash = rc != <span class="nm">0</span> <span class="kw">and</span> passed == <span class="nm">0</span> \
<span class="kw">and</span> crash_match <span class="kw">and not</span> has_real_tests
<span class="kw">if</span> total > <span class="nm">0</span> <span class="kw">and not</span> is_crash:
result.regression_score = passed / total</pre>
</div>
<div class="card orange">
<h3 style="color:var(--orange-bright)">Lint Score (15%)</h3>
<p><strong>Messmethode:</strong> <code>ruff check --output-format=json</code> vor und nach Aenderungen.
Der <strong>git stash Trick</strong>: Stash die Agent-Aenderungen, zaehle Warnings im Original,
pop den Stash, zaehle erneut.</p>
<p><strong>Scoring:</strong> Delta-basiert. <code>delta &lt;= 0 = 1.0</code>.
Jede neue Warning kostet 0.1 Punkte. Fallback: <code>py_compile</code> bei fehlendem ruff.</p>
<pre style="font-size:11px; margin-bottom:0;"><span class="cm"># git stash trick fuer Before-State</span>
sandbox.exec(<span class="st">"git stash"</span>) <span class="cm"># temporaer zuruecksetzen</span>
before = count_ruff_warnings(sandbox, files)
sandbox.exec(<span class="st">"git stash pop"</span>) <span class="cm"># Agent-Aenderungen wiederherstellen</span>
after = count_ruff_warnings(sandbox, files)
<span class="cm"># score: max(0, 1.0 - delta * 0.1)</span></pre>
</div>
<div class="card purple">
<h3 style="color:#b388ff">Complexity Score (15%)</h3>
<p><strong>Messmethode:</strong> <code>radon cc --json</code> berechnet zyklomatische Komplexitaet
pro Funktion, gemittelt ueber alle geaenderten Dateien. Selber git-stash-Trick wie beim Lint.</p>
<p><strong>Fallback:</strong> Einfacher Proxy aus max. Einrueckungstiefe + Anzahl Funktionen/Klassen
pro Datei wenn radon nicht verfuegbar.</p>
<pre style="font-size:11px; margin-bottom:0;"><span class="cm"># Fallback-Proxy (simple_complexity)</span>
max_indent = <span class="fn">max</span>(len(l) - len(l.lstrip()) <span class="kw">for</span> l <span class="kw">in</span> lines)
func_count = <span class="fn">sum</span>(<span class="nm">1</span> <span class="kw">for</span> l <span class="kw">in</span> lines
<span class="kw">if</span> l.strip().startswith(<span class="st">'def '</span>)
<span class="kw">or</span> l.strip().startswith(<span class="st">'class '</span>))</pre>
</div>
</div>
<h3>pytest Output Parser</h3>
<p>Der Parser erkennt verschiedene pytest-Ausgabeformate robuster als ein einfaches Regex-Match:</p>
<pre><span class="cm"># core/metrics.py — parse_pytest_output()</span>
<span class="cm"># Erkennt: "5 passed", "3 failed, 2 passed", "1 error, 2 failed, 3 passed"</span>
passed_match = re.search(<span class="st">r"(\d+)\s+passed"</span>, output)
failed_match = re.search(<span class="st">r"(\d+)\s+failed"</span>, output)
error_match = re.search(<span class="st">r"(\d+)\s+error"</span>, output)
<span class="cm"># Fallback: "X/Y tests passed" (unittest-Stil)</span>
fraction = re.search(<span class="st">r"(\d+)/(\d+)\s+(?:tests?\s+)?passed"</span>, output)</pre>
</section>
<!-- ==================== METRICS ==================== -->
<section id="metrics">
<h2>5. Metriken-System</h2>
<div class="tabs">
<div class="tab active" onclick="showTab('traj')">Trajectory Analysis</div>
<div class="tab" onclick="showTab('diff')">Diff-Minimality</div>
<div class="tab" onclick="showTab('cost')">Cost Tracking</div>
<div class="tab" onclick="showTab('passk')">Pass@k</div>
<div class="tab" onclick="showTab('composite')">Composite Score</div>
</div>
<div class="tabpanel">
<div id="tab-traj" class="tab-content active">
<h3>Trajectory Analysis</h3>
<p>Eine Trajectory ist eine Liste von Tool-Use-Events mit <code>tool_name</code>, <code>input_preview</code>, <code>timestamp</code>.
Die Analyse erkennt ineffiziente Muster in der Problemloesungsstrategie des Agenten.</p>
<div class="grid2" style="margin-bottom:16px;">
<div>
<h4>Erkannte Metriken</h4>
<table>
<thead><tr><th>Metrik</th><th>Was wird gemessen</th></tr></thead>
<tbody>
<tr><td><code>directness_score</code></td><td>Wie direkt der Agent zur Loesung gelangt (0-1)</td></tr>
<tr><td><code>loop_count</code></td><td>Repetitive Tool-Call-Sequenzen (1er, 2er, 3er-Zyklen)</td></tr>
<tr><td><code>stall_rate</code></td><td>Anteil von Steps ohne Fortschritt (&ge;5 Read-Steps in Folge)</td></tr>
<tr><td><code>backtrack_count</code></td><td>Ueberschriebene Dateien (innerhalb 10 Steps)</td></tr>
<tr><td><code>self_correction_rate</code></td><td>Edit &rarr; Test &rarr; Edit-gleiche-Datei Pattern</td></tr>
<tr><td><code>exploration_ratio</code></td><td>Read/Grep/Glob / gesamt</td></tr>
<tr><td><code>turns_to_first_edit</code></td><td>Wieviele Steps bis zum ersten Edit/Write</td></tr>
</tbody>
</table>
</div>
<div>
<h4>Phasen-Klassifikation</h4>
<table>
<thead><tr><th>Phase</th><th>Tools</th></tr></thead>
<tbody>
<tr><td>exploration</td><td>Read, Grep, Glob, Bash(ls/find/cat)</td></tr>
<tr><td>implementation</td><td>Edit, Write, NotebookEdit</td></tr>
<tr><td>testing</td><td>Bash mit pytest/unittest/make test</td></tr>
<tr><td>debugging</td><td>Read/Edit <em>nach</em> fehlgeschlagenem Test</td></tr>
</tbody>
</table>
</div>
</div>
<pre><span class="cm"># Directness Score — kombiniert 5 Sub-Metriken</span>
score = (
<span class="nm">0.35</span> * trajectory_efficiency <span class="cm"># Edit/Write-Ratio</span>
+ <span class="nm">0.25</span> * early_edit_score <span class="cm"># wie frueh erster Edit</span>
- <span class="nm">0.20</span> * repeat_penalty <span class="cm"># gleiche Datei wiederholt gelesen</span>
- <span class="nm">0.10</span> * loop_penalty <span class="cm"># zyklische Tool-Patterns</span>
- <span class="nm">0.10</span> * stall_rate <span class="cm"># lange Read-Phasen ohne Edit</span>
)</pre>
</div>
<div id="tab-diff" class="tab-content">
<h3>Diff-Minimality Analyse</h3>
<p>Ein guter Agent aendert nur was noetig ist. Hoher Churn (viele Zeilen geaendert) deutet auf
Overengineering oder unnoetige Refactorings hin.</p>
<table>
<thead><tr><th>Metrik</th><th>Beschreibung</th><th>Berechnung</th></tr></thead>
<tbody>
<tr><td><code>lines_added</code></td><td>Zeilen mit +</td><td>Alle <code>+</code>-Zeilen ausser <code>+++</code>-Header</td></tr>
<tr><td><code>lines_removed</code></td><td>Zeilen mit -</td><td>Alle <code>-</code>-Zeilen ausser <code>---</code>-Header</td></tr>
<tr><td><code>files_changed</code></td><td>Distinct Files</td><td>aus <code>+++ b/</code> Headern</td></tr>
<tr><td><code>hunks</code></td><td>Diff-Bloecke</td><td>Anzahl <code>@@</code>-Marker</td></tr>
<tr><td><code>churn</code></td><td>Gesamtveraenderung</td><td>lines_added + lines_removed</td></tr>
</tbody>
</table>
<p>Im Composite Score: <code>diff_minimality = 1.0 / (1.0 + churn / 50.0)</code>
&mdash; Sigmoid-artige Normalisierung, churn=0 &rarr; 1.0, churn=50 &rarr; 0.5.</p>
</div>
<div id="tab-cost" class="tab-content">
<h3>Cost Tracking</h3>
<p>Jeder Run trackt Tokens und rechnet Kosten in USD um. Das ist das Alleinstellungsmerkmal von AgentArena
gegenueber anderen Benchmarks.</p>
<div class="grid2">
<div>
<h4>Claude Code (direkt aus JSON)</h4>
<pre style="font-size:11px;"><span class="cm"># --output-format json liefert:</span>
entry[<span class="st">"usage"</span>][<span class="st">"input_tokens"</span>]
entry[<span class="st">"usage"</span>][<span class="st">"output_tokens"</span>]
entry[<span class="st">"usage"</span>][<span class="st">"cache_creation_input_tokens"</span>]
entry[<span class="st">"usage"</span>][<span class="st">"cache_read_input_tokens"</span>]
entry[<span class="st">"total_cost_usd"</span>] <span class="cm"># direkt verfuegbar!</span></pre>
</div>
<div>
<h4>Aider (aus stdout geparst)</h4>
<pre style="font-size:11px;"><span class="cm"># Regex aus Aider-Output:</span>
_TOKEN_RE = re.compile(
<span class="st">r"Tokens:\s*(?P&lt;sent&gt;\d+)\s+sent,"</span>
<span class="st">r"\s*(?P&lt;recv&gt;\d+)\s+received"</span>
)
_COST_RE = re.compile(
<span class="st">r"Cost:\s*\$(?P&lt;cost&gt;[0-9]+\.[0-9]+)"</span>
)</pre>
</div>
</div>
<pre><span class="cm"># Kosten-Kalkulation wenn kein direkter Wert (Fallback)</span>
cost = (tokens_input * rate_in + tokens_output * rate_out) / <span class="nm">1_000_000</span>
<span class="cm"># Effizienz-Score fuer Composite (normalisiert auf 0-1)</span>
efficiency = <span class="nm">1.0</span> / (<span class="nm">1.0</span> + cost_usd / <span class="nm">0.50</span>) <span class="cm"># $0 = 1.0, $0.50 = 0.5</span></pre>
</div>
<div id="tab-passk" class="tab-content">
<h3>Pass@k Unbiased Estimator</h3>
<p>Pass@k misst die Wahrscheinlichkeit, dass mindestens einer von k unabhaengigen Runs erfolgreich ist.
Der unbiased Estimator vermeidet Overfitting gegenueber dem naiven Ansatz.</p>
<div class="formula">
<span class="fname">pass@k</span> = 1 &minus;
C(<span class="fname">n</span>&minus;<span class="fname">c</span>, <span class="fnum">k</span>) /
C(<span class="fname">n</span>, <span class="fnum">k</span>)
<br><br>
<span style="color:var(--muted)">n = gesamt Runs, c = korrekte Runs, k = Ziel-k</span>
</div>
<pre><span class="cm"># core/metrics.py</span>
<span class="kw">def</span> <span class="fn">compute_pass_at_k</span>(n: <span class="py">int</span>, c: <span class="py">int</span>, k: <span class="py">int</span>) -> <span class="py">float</span>:
<span class="kw">if</span> n < k: <span class="kw">return</span> <span class="nm">0.0</span>
<span class="kw">if</span> c == <span class="nm">0</span>: <span class="kw">return</span> <span class="nm">0.0</span>
<span class="kw">if</span> n - c < k: <span class="kw">return</span> <span class="nm">1.0</span>
<span class="kw">return</span> <span class="nm">1.0</span> - comb(n - c, k) / comb(n, k)</pre>
<p>Runner fuehrt k Runs <strong>parallel</strong> aus mittels <code>asyncio.gather()</code>,
jeder in eigenem DB-Session, getaggt mit gemeinsamer <code>pass_at_k_group</code> UUID.</p>
</div>
<div id="tab-composite" class="tab-content">
<h3>Composite Score</h3>
<p>Kombiniert Judge-Score, Trajectory-Metriken und Diff-Qualitaet zu einem Gesamtwert.
<strong>Funktionalitaet wirkt als Multiplikator</strong>: schlaegt der Test fehl, ist alles 0.</p>
<div class="formula">
<span class="fname">composite</span> = <span class="fname">functionality</span> &times; (<br>
&nbsp;&nbsp;<span class="fnum">0.25</span> &times; <span class="fname">directness</span> +<br>
&nbsp;&nbsp;<span class="fnum">0.25</span> &times; <span class="fname">efficiency</span> +<br>
&nbsp;&nbsp;<span class="fnum">0.20</span> &times; <span class="fname">diff_minimality</span> +<br>
&nbsp;&nbsp;<span class="fnum">0.15</span> &times; <span class="fname">no_loops</span> +<br>
&nbsp;&nbsp;<span class="fnum">0.15</span> &times; <span class="fname">low_correction</span><br>
)
</div>
<p><strong>Gatekeeper-Logik:</strong> Ist <code>functionality == 0.0</code>, wird sofort <code>0.0</code>
zurueckgegeben, ohne die anderen Dimensionen zu berechnen. Nur ein funktionierender Code kann
ein guter Code sein.</p>
</div>
</div>
<!-- Score Bars -->
<h3>Judge Score Visualisierung (Beispiel: Run #10, Aider + MiniMax)</h3>
<div style="background:var(--surface); border:1px solid var(--border); border-radius:10px; padding:20px; margin-bottom:16px;">
<div class="score-bar-wrap">
<div class="score-bar-label">
<span>Functional (50%)</span>
<span style="color:var(--green-bright)">1.0 &times; 0.50 = 0.50</span>
</div>
<div class="score-bar-track">
<div class="score-bar-fill" style="width:100%; background:var(--green)">1.00</div>
</div>
</div>
<div class="score-bar-wrap">
<div class="score-bar-label">
<span>Regression (20%)</span>
<span style="color:var(--green-bright)">1.0 &times; 0.20 = 0.20</span>
</div>
<div class="score-bar-track">
<div class="score-bar-fill" style="width:100%; background:var(--green)">1.00</div>
</div>
</div>
<div class="score-bar-wrap">
<div class="score-bar-label">
<span>Lint (15%)</span>
<span style="color:var(--green-bright)">1.0 &times; 0.15 = 0.15</span>
</div>
<div class="score-bar-track">
<div class="score-bar-fill" style="width:100%; background:var(--blue)">1.00</div>
</div>
</div>
<div class="score-bar-wrap">
<div class="score-bar-label">
<span>Complexity (15%)</span>
<span style="color:var(--green-bright)">1.0 &times; 0.15 = 0.15</span>
</div>
<div class="score-bar-track">
<div class="score-bar-fill" style="width:100%; background:var(--purple)">1.00</div>
</div>
</div>
<div style="border-top:2px solid var(--border); margin-top:12px; padding-top:12px; display:flex; justify-content:space-between; align-items:center;">
<span style="font-weight:700; color:#fff">Overall Score</span>
<span style="font-size:24px; font-weight:700; color:var(--green-bright)">1.00</span>
</div>
</div>
</section>
<!-- ==================== SSE ==================== -->
<section id="sse">
<h2>6. Live-Updates via Server-Sent Events</h2>
<p>Die Webapp streamt Fortschritts-Updates in Echtzeit ohne Polling. Kein WebSocket-Overhead,
kein periodisches Refetching &mdash; SSE ist ideal fuer unidirektionale Streams.</p>
<div class="card blue">
<div style="display:flex; align-items:center; gap:12px; margin-bottom:16px;">
<span class="live-dot"></span>
<h3 style="margin:0; color:var(--blue-bright)">SSE Endpoint: GET /runs/{id}/stream</h3>
</div>
<div class="grid2">
<div>
<h4>Event-Typen</h4>
<table>
<thead><tr><th>Phase</th><th>Status-Text</th></tr></thead>
<tbody>
<tr><td><code>starting</code></td><td>Run wurde erstellt</td></tr>
<tr><td><code>creating_sandbox</code></td><td>Docker-Container wird hochgefahren</td></tr>
<tr><td><code>running_agent</code></td><td>Adapter laeuft (kann Minuten dauern)</td></tr>
<tr><td><code>collecting_diff</code></td><td>git diff wird gesammelt</td></tr>
<tr><td><code>creating_judge_sandbox</code></td><td>Judge-Container wird erstellt</td></tr>
<tr><td><code>judging</code></td><td>4-Dimensionen-Bewertung laeuft</td></tr>
<tr><td><code>done</code></td><td>Score verfuegbar, Connection close</td></tr>
</tbody>
</table>
</div>
<div>
<h4>htmx-Integration</h4>
<pre style="font-size:11px; margin-bottom:0;">&lt;<span class="kw">div</span>
<span class="nm">hx-ext</span>=<span class="st">"sse"</span>
<span class="nm">sse-connect</span>=<span class="st">"/runs/{{id}}/stream"</span>
<span class="nm">sse-swap</span>=<span class="st">"message"</span>
<span class="nm">hx-target</span>=<span class="st">"#status-box"</span>&gt;
&lt;<span class="kw">div</span> <span class="nm">id</span>=<span class="st">"status-box"</span>&gt;Warte...&lt;/<span class="kw">div</span>&gt;
&lt;/<span class="kw">div</span>&gt;</pre>
</div>
</div>
</div>
<pre><span class="cm"># main.py — SSE Generator</span>
<span class="kw">async def</span> <span class="fn">run_status_stream</span>(run_id: <span class="py">int</span>):
<span class="kw">while</span> <span class="nm">True</span>:
progress = runner.get_progress(run_id)
<span class="kw">if</span> progress:
status = progress.get(<span class="st">"status"</span>, <span class="st">"unknown"</span>)
elapsed = time.time() - progress.get(<span class="st">"started_at"</span>, time.time())
<span class="kw">yield</span> <span class="st">f"data: {json.dumps({'status': status, 'elapsed': elapsed})}\n\n"</span>
<span class="kw">else</span>:
<span class="kw">yield</span> <span class="st">"data: {\"status\": \"done\"}\n\n"</span>
<span class="kw">break</span>
<span class="kw">await</span> asyncio.sleep(<span class="nm">0.5</span>)
<span class="kw">return</span> StreamingResponse(run_status_stream(run_id), media_type=<span class="st">"text/event-stream"</span>)</pre>
</section>
<!-- ==================== FILE STRUCTURE ==================== -->
<section id="filestructure">
<h2>7. Dateistruktur</h2>
<div class="filetree">
<span class="ft-dir">AgentArena/</span>
&#9500;&#9472;&#9472; <span class="ft-file">main.py</span> <span class="ft-desc"># FastAPI app, Routen, SSE-Endpoints, Jinja2-Templates</span>
&#9500;&#9472;&#9472; <span class="ft-file">models.py</span> <span class="ft-desc"># SQLAlchemy Models: Task, Agent, Run, RunStatus (Enum)</span>
&#9500;&#9472;&#9472; <span class="ft-file">database.py</span> <span class="ft-desc"># SessionLocal, engine, Base — SQLite via SQLAlchemy</span>
&#9500;&#9472;&#9472; <span class="ft-file">config.py</span> <span class="ft-desc"># Pricing-Tabellen, Docker-Config, CLI-Pfade, Pass@k Defaults</span>
&#9500;&#9472;&#9472; <span class="ft-dir">core/</span>
&#9474; &#9500;&#9472;&#9472; <span class="ft-file">runner.py</span> <span class="ft-desc"># Orchestrierung: run_single(), run_pass_at_k(), get_progress()</span>
&#9474; &#9500;&#9472;&#9472; <span class="ft-file">sandbox.py</span> <span class="ft-desc"># Docker-Lifecycle: create(), exec(), destroy(), create_judge_sandbox()</span>
&#9474; &#9500;&#9472;&#9472; <span class="ft-file">judge.py</span> <span class="ft-desc"># 4-Dimensionen Judge: functional, regression, lint, complexity</span>
&#9474; &#9492;&#9472;&#9472; <span class="ft-file">metrics.py</span> <span class="ft-desc"># Trajectory-Analyse, Diff-Analyse, Pass@k, Composite Score</span>
&#9500;&#9472;&#9472; <span class="ft-dir">adapters/</span>
&#9474; &#9500;&#9472;&#9472; <span class="ft-file">base.py</span> <span class="ft-desc"># AgentAdapter ABC, RunContext dataclass, RunResult dataclass</span>
&#9474; &#9500;&#9472;&#9472; <span class="ft-file">claude_code.py</span> <span class="ft-desc"># Claude Code CLI — OAuth, JSON output, Host-Tempdir</span>
&#9474; &#9500;&#9472;&#9472; <span class="ft-file">aider.py</span> <span class="ft-desc"># Aider CLI — MiniMax via litellm, Regex-Token-Parsing</span>
&#9474; &#9492;&#9472;&#9472; <span class="ft-file">codex.py</span> <span class="ft-desc"># Codex CLI — noch nicht getestet</span>
&#9500;&#9472;&#9472; <span class="ft-dir">templates/</span> <span class="ft-desc"># Jinja2 HTML-Templates</span>
&#9474; &#9500;&#9472;&#9472; <span class="ft-file">base.html</span> <span class="ft-desc"># Basis-Layout, Tailwind CDN, htmx, SSE-Extension</span>
&#9474; &#9500;&#9472;&#9472; <span class="ft-file">index.html</span> <span class="ft-desc"># Dashboard: Leaderboard, Run-Trigger</span>
&#9474; &#9492;&#9472;&#9472; <span class="ft-file">run_detail.html</span> <span class="ft-desc"># Run-Details: Diff, Judge-Breakdown, Trajectory, SSE</span>
&#9500;&#9472;&#9472; <span class="ft-dir">tasks/</span> <span class="ft-desc"># Task-Definitionen als YAML</span>
&#9474; &#9492;&#9472;&#9472; <span class="ft-file">*.yaml</span> <span class="ft-desc"># slug, title, repo_url, success_cmd, time_limit_s</span>
&#9500;&#9472;&#9472; <span class="ft-dir">docs/</span>
&#9474; &#9492;&#9472;&#9472; <span class="ft-file">architecture.html</span> <span class="ft-desc"># Diese Datei</span>
&#9492;&#9472;&#9472; <span class="ft-dir">static/</span> <span class="ft-desc"># CSS/JS Assets</span>
</div>
<h3 style="margin-top:20px;">Datenmodelle</h3>
<div class="grid3">
<div class="card blue">
<h4>Task</h4>
<table style="font-size:11px;">
<tbody>
<tr><td><code>slug</code></td><td>Unique Identifier</td></tr>
<tr><td><code>repo_url</code></td><td>Git-Repository URL</td></tr>
<tr><td><code>success_cmd</code></td><td>pytest Befehl</td></tr>
<tr><td><code>time_limit_s</code></td><td>Timeout (default 300s)</td></tr>
<tr><td><code>token_limit</code></td><td>Max Tokens (200k)</td></tr>
<tr><td><code>difficulty</code></td><td>easy/medium/hard</td></tr>
</tbody>
</table>
</div>
<div class="card green">
<h4>Agent</h4>
<table style="font-size:11px;">
<tbody>
<tr><td><code>name</code></td><td>Unique Name</td></tr>
<tr><td><code>adapter_type</code></td><td>claude_code / aider / codex</td></tr>
<tr><td><code>config_json</code></td><td>Adapter-Konfiguration</td></tr>
</tbody>
</table>
</div>
<div class="card orange">
<h4>Run</h4>
<table style="font-size:11px;">
<tbody>
<tr><td><code>status</code></td><td>pending/running/success/failure/error/timeout</td></tr>
<tr><td><code>judge_score</code></td><td>0.0 &ndash; 1.0</td></tr>
<tr><td><code>cost_usd</code></td><td>Kosten in Dollar</td></tr>
<tr><td><code>diff_text</code></td><td>Unified Diff</td></tr>
<tr><td><code>trajectory</code></td><td>JSON Array (Tool-Events)</td></tr>
<tr><td><code>metrics_json</code></td><td>Alle Metriken als JSON</td></tr>
<tr><td><code>pass_at_k_group</code></td><td>UUID fuer k-Gruppen</td></tr>
</tbody>
</table>
</div>
</div>
</section>
<!-- ==================== RESULTS ==================== -->
<section id="results">
<h2>8. Bisherige Ergebnisse</h2>
<div class="leaderboard" style="margin-bottom:24px; overflow-x:auto;">
<div class="lb-row header">
<div>Rank</div>
<div>Agent</div>
<div>Score</div>
<div>Tests</div>
<div>Kosten</div>
<div>Dauer</div>
<div>Modell</div>
</div>
<div class="lb-row">
<div class="lb-rank gold">1</div>
<div><strong>Aider</strong></div>
<div class="lb-score perfect">1.00</div>
<div>5/5</div>
<div style="color:var(--green-bright)"><strong>~$0.00</strong></div>
<div>~90s</div>
<div><code>MiniMax-M2.5</code></div>
</div>
<div class="lb-row">
<div class="lb-rank silver">2</div>
<div><strong>Claude Code</strong></div>
<div class="lb-score good">0.80</div>
<div>5/5</div>
<div style="color:var(--orange-bright)">~$0.15</div>
<div>~120s</div>
<div><code>claude-sonnet-4-6</code></div>
</div>
</div>
<div class="grid2">
<div class="card orange">
<h3 style="color:var(--orange-bright)">Wichtige Beobachtung</h3>
<p>Aider mit MiniMax-M2.5 erzielte einen <strong>perfekten Score bei nahezu keinen Kosten</strong>.
MiniMax-M2.5 kostet $0.05/1M Tokens &mdash; ca. 200x guenstiger als Claude Sonnet.</p>
<p>Dies unterstreicht die Notwendigkeit von <strong>explizitem Kosten-Tracking</strong>:
Reine Score-Vergleiche ohne Kostendimension sind irreführend.</p>
</div>
<div class="card blue">
<h3 style="color:var(--blue-bright)">Cost-Performance Ratio</h3>
<div style="display:grid; grid-template-columns:1fr 1fr; gap:12px; margin-top:8px;">
<div class="metric-card">
<div class="metric-val" style="color:var(--green-bright)">$0.00</div>
<div class="metric-label">Aider + MiniMax<br>Score 1.0</div>
</div>
<div class="metric-card">
<div class="metric-val" style="color:var(--orange-bright)">$0.15</div>
<div class="metric-label">Claude Code Sonnet<br>Score 0.80</div>
</div>
</div>
</div>
</div>
<h3>Was kommt als naechstes?</h3>
<div class="grid3">
<div class="card">
<h4>Mehr Agenten</h4>
<p>CodexAdapter testen. GPT-4o als Basis-Vergleich. Lokale Modelle via Ollama.</p>
</div>
<div class="card">
<h4>Mehr Tasks</h4>
<p>Schwerere Aufgaben (Refactoring, Multi-file, API Integration).
Task-YAML fuer einfaches Hinzufuegen.</p>
</div>
<div class="card">
<h4>Pass@k Analyse</h4>
<p>Parallele Runs fuer statistische Belastbarkeit.
Variance-Analyse: Welche Agenten sind konsistenter?</p>
</div>
</div>
</section>
<hr class="divider">
<div style="text-align:center; color:var(--muted); font-size:12px; padding:20px 0;">
AgentArena &mdash; Erstellt Maerz 2026 &mdash; FastAPI + Docker + SQLite
</div>
</div>
<script>
// Tab-Switcher
function showTab(id) {
document.querySelectorAll('.tab-content').forEach(el => el.classList.remove('active'));
document.querySelectorAll('.tab').forEach(el => el.classList.remove('active'));
document.getElementById('tab-' + id).classList.add('active');
event.target.classList.add('active');
}
// Smooth scroll for nav
document.querySelectorAll('nav a').forEach(a => {
a.addEventListener('click', e => {
e.preventDefault();
const target = document.querySelector(a.getAttribute('href'));
if (target) target.scrollIntoView({ behavior: 'smooth' });
});
});
// Active nav highlight on scroll
const sections = document.querySelectorAll('section[id]');
const navLinks = document.querySelectorAll('nav a');
const observer = new IntersectionObserver(entries => {
entries.forEach(entry => {
if (entry.isIntersecting) {
navLinks.forEach(a => {
a.style.background = a.getAttribute('href') === '#' + entry.target.id
? 'var(--surface2)' : '';
a.style.color = a.getAttribute('href') === '#' + entry.target.id
? 'var(--text)' : '';
});
}
});
}, { threshold: 0.3, rootMargin: '-60px 0px -60% 0px' });
sections.forEach(s => observer.observe(s));
// Animate score bars on load
window.addEventListener('load', () => {
document.querySelectorAll('.score-bar-fill').forEach(bar => {
const w = bar.style.width;
bar.style.width = '0%';
setTimeout(() => { bar.style.width = w; }, 200);
});
});
</script>
</body>
</html>