Download docs/architecture.html from smlflg/AgentArena: direct link, hf CLI and curl.
- Browser
- Download file 64.7 kB
-
https://huggingface.co/smlflg/AgentArena/resolve/main/docs/architecture.html
- Command line
-
hf download hf://smlflg/AgentArena/docs/architecture.html
-
curl -L -o architecture.html https://huggingface.co/smlflg/AgentArena/resolve/main/docs/architecture.html
64.7 kB
| <html lang="de"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <title>AgentArena — Technische Architektur</title> | |
| <style> | |
| :root { | |
| --bg: #0d1117; | |
| --surface: #161b22; | |
| --surface2: #21262d; | |
| --border: #30363d; | |
| --text: #c9d1d9; | |
| --muted: #8b949e; | |
| --green: #2ea043; | |
| --green-dim: #1a3a20; | |
| --green-bright: #56d364; | |
| --blue: #1f6feb; | |
| --blue-dim: #0d2a4a; | |
| --blue-bright: #58a6ff; | |
| --orange: #d29922; | |
| --orange-dim:#3a2700; | |
| --orange-bright: #e3b341; | |
| --red: #da3633; | |
| --red-dim: #3d0c09; | |
| --red-bright:#f85149; | |
| --purple: #8957e5; | |
| --purple-dim:#2a1a5e; | |
| --teal: #1b6e6e; | |
| --teal-bright:#4dd0d0; | |
| } | |
| * { box-sizing: border-box; margin: 0; padding: 0; } | |
| body { | |
| background: var(--bg); | |
| color: var(--text); | |
| font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, monospace; | |
| font-size: 14px; | |
| line-height: 1.6; | |
| } | |
| /* ---- NAV ---- */ | |
| nav { | |
| position: sticky; | |
| top: 0; | |
| z-index: 100; | |
| background: rgba(13,17,23,0.96); | |
| border-bottom: 1px solid var(--border); | |
| display: flex; | |
| align-items: center; | |
| gap: 2px; | |
| padding: 0 24px; | |
| height: 52px; | |
| backdrop-filter: blur(8px); | |
| } | |
| nav .brand { | |
| font-weight: 700; | |
| font-size: 15px; | |
| color: var(--green-bright); | |
| margin-right: 20px; | |
| white-space: nowrap; | |
| } | |
| nav a { | |
| color: var(--muted); | |
| text-decoration: none; | |
| padding: 6px 10px; | |
| border-radius: 6px; | |
| font-size: 12px; | |
| transition: all 0.15s; | |
| white-space: nowrap; | |
| } | |
| nav a:hover { background: var(--surface2); color: var(--text); } | |
| /* ---- LAYOUT ---- */ | |
| .page { max-width: 1200px; margin: 0 auto; padding: 40px 24px 80px; } | |
| h1 { font-size: 32px; font-weight: 700; color: #fff; margin-bottom: 8px; } | |
| h2 { font-size: 22px; font-weight: 600; color: #fff; margin-bottom: 16px; } | |
| h3 { font-size: 16px; font-weight: 600; color: var(--blue-bright); margin-bottom: 10px; } | |
| h4 { font-size: 13px; font-weight: 600; color: var(--muted); text-transform: uppercase; letter-spacing: 0.8px; margin-bottom: 8px; } | |
| p { color: var(--muted); margin-bottom: 12px; } | |
| p strong { color: var(--text); } | |
| section { margin-bottom: 64px; scroll-margin-top: 64px; } | |
| .badge { | |
| display: inline-block; | |
| padding: 2px 8px; | |
| border-radius: 12px; | |
| font-size: 11px; | |
| font-weight: 600; | |
| letter-spacing: 0.3px; | |
| } | |
| .badge-green { background: var(--green-dim); color: var(--green-bright); border: 1px solid var(--green); } | |
| .badge-blue { background: var(--blue-dim); color: var(--blue-bright); border: 1px solid var(--blue); } | |
| .badge-orange { background: var(--orange-dim); color: var(--orange-bright);border: 1px solid var(--orange); } | |
| .badge-red { background: var(--red-dim); color: var(--red-bright); border: 1px solid var(--red); } | |
| .badge-purple { background: var(--purple-dim); color: #b388ff; border: 1px solid var(--purple); } | |
| /* ---- CARDS ---- */ | |
| .card { | |
| background: var(--surface); | |
| border: 1px solid var(--border); | |
| border-radius: 10px; | |
| padding: 20px 24px; | |
| margin-bottom: 16px; | |
| } | |
| .card.green { border-left: 3px solid var(--green); } | |
| .card.blue { border-left: 3px solid var(--blue); } | |
| .card.orange { border-left: 3px solid var(--orange); } | |
| .card.red { border-left: 3px solid var(--red); } | |
| .card.purple { border-left: 3px solid var(--purple); } | |
| .grid2 { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; } | |
| .grid3 { display: grid; grid-template-columns: 1fr 1fr 1fr; gap: 16px; } | |
| @media (max-width: 800px) { .grid2, .grid3 { grid-template-columns: 1fr; } } | |
| /* ---- CODE ---- */ | |
| pre, code { | |
| font-family: 'JetBrains Mono', 'Fira Code', 'Cascadia Code', monospace; | |
| font-size: 12px; | |
| } | |
| pre { | |
| background: #010409; | |
| border: 1px solid var(--border); | |
| border-radius: 8px; | |
| padding: 16px; | |
| overflow-x: auto; | |
| margin-bottom: 16px; | |
| line-height: 1.5; | |
| } | |
| code { background: var(--surface2); padding: 1px 5px; border-radius: 4px; color: var(--orange-bright); } | |
| .kw { color: #ff7b72; } | |
| .fn { color: var(--blue-bright); } | |
| .st { color: var(--green-bright); } | |
| .cm { color: var(--muted); font-style: italic; } | |
| .nm { color: #ffa657; } | |
| .py { color: #d2a8ff; } | |
| /* ---- PIPELINE FLOW ---- */ | |
| .pipeline { | |
| display: flex; | |
| align-items: stretch; | |
| gap: 0; | |
| overflow-x: auto; | |
| padding-bottom: 8px; | |
| margin-bottom: 24px; | |
| } | |
| .pipe-step { | |
| display: flex; | |
| flex-direction: column; | |
| align-items: center; | |
| min-width: 110px; | |
| flex: 1; | |
| } | |
| .pipe-box { | |
| background: var(--surface); | |
| border: 1px solid var(--border); | |
| border-radius: 8px; | |
| padding: 12px 10px; | |
| text-align: center; | |
| font-size: 12px; | |
| font-weight: 600; | |
| width: 100%; | |
| position: relative; | |
| transition: transform 0.15s; | |
| cursor: default; | |
| } | |
| .pipe-box:hover { transform: translateY(-2px); } | |
| .pipe-box .icon { font-size: 22px; margin-bottom: 4px; display: block; } | |
| .pipe-box .label { color: var(--text); } | |
| .pipe-box .sublabel { color: var(--muted); font-size: 10px; font-weight: 400; margin-top: 2px; } | |
| .pipe-box.green { border-color: var(--green); background: var(--green-dim); } | |
| .pipe-box.blue { border-color: var(--blue); background: var(--blue-dim); } | |
| .pipe-box.orange { border-color: var(--orange); background: var(--orange-dim); } | |
| .pipe-box.red { border-color: var(--red); background: var(--red-dim); } | |
| .pipe-box.purple { border-color: var(--purple); background: var(--purple-dim); } | |
| .pipe-box.teal { border-color: #1b6e6e; background: #0a2020; } | |
| .pipe-arrow { | |
| display: flex; | |
| align-items: center; | |
| justify-content: center; | |
| padding: 0 4px; | |
| color: var(--muted); | |
| font-size: 18px; | |
| flex-shrink: 0; | |
| } | |
| /* ---- TWO-CONTAINER DIAGRAM ---- */ | |
| .isolation-diagram { | |
| display: grid; | |
| grid-template-columns: 1fr auto 1fr; | |
| gap: 0; | |
| align-items: start; | |
| margin-bottom: 24px; | |
| } | |
| .container-box { | |
| border: 2px solid; | |
| border-radius: 10px; | |
| padding: 16px; | |
| } | |
| .container-box.agent { border-color: var(--green); background: var(--green-dim); } | |
| .container-box.judge { border-color: var(--blue); background: var(--blue-dim); } | |
| .container-header { | |
| font-weight: 700; | |
| font-size: 13px; | |
| margin-bottom: 12px; | |
| display: flex; | |
| align-items: center; | |
| gap: 8px; | |
| } | |
| .step-list { list-style: none; } | |
| .step-list li { | |
| display: flex; | |
| align-items: flex-start; | |
| gap: 8px; | |
| padding: 6px 0; | |
| border-bottom: 1px solid rgba(255,255,255,0.05); | |
| font-size: 12px; | |
| color: var(--text); | |
| } | |
| .step-list li:last-child { border-bottom: none; } | |
| .step-num { | |
| background: var(--surface2); | |
| border-radius: 50%; | |
| width: 20px; | |
| height: 20px; | |
| display: flex; | |
| align-items: center; | |
| justify-content: center; | |
| font-size: 10px; | |
| font-weight: 700; | |
| flex-shrink: 0; | |
| color: var(--muted); | |
| } | |
| .wall { | |
| display: flex; | |
| flex-direction: column; | |
| align-items: center; | |
| justify-content: center; | |
| padding: 0 16px; | |
| gap: 4px; | |
| } | |
| .wall-line { | |
| width: 2px; | |
| flex: 1; | |
| background: repeating-linear-gradient( | |
| to bottom, | |
| var(--red) 0px, var(--red) 8px, | |
| transparent 8px, transparent 16px | |
| ); | |
| min-height: 80px; | |
| } | |
| .wall-label { | |
| background: var(--red-dim); | |
| border: 1px solid var(--red); | |
| color: var(--red-bright); | |
| font-size: 10px; | |
| font-weight: 700; | |
| padding: 4px 8px; | |
| border-radius: 4px; | |
| text-align: center; | |
| white-space: nowrap; | |
| } | |
| /* ---- SCORE DIAGRAM ---- */ | |
| .score-bar-wrap { margin-bottom: 12px; } | |
| .score-bar-label { | |
| display: flex; | |
| justify-content: space-between; | |
| margin-bottom: 4px; | |
| font-size: 12px; | |
| } | |
| .score-bar-track { | |
| height: 24px; | |
| background: var(--surface2); | |
| border-radius: 4px; | |
| overflow: hidden; | |
| position: relative; | |
| } | |
| .score-bar-fill { | |
| height: 100%; | |
| border-radius: 4px; | |
| display: flex; | |
| align-items: center; | |
| justify-content: flex-end; | |
| padding-right: 8px; | |
| font-size: 11px; | |
| font-weight: 700; | |
| color: #000; | |
| transition: width 0.4s ease; | |
| } | |
| /* ---- ADAPTER CARDS ---- */ | |
| .adapter-grid { | |
| display: grid; | |
| grid-template-columns: repeat(3, 1fr); | |
| gap: 16px; | |
| margin-bottom: 24px; | |
| } | |
| @media (max-width: 900px) { .adapter-grid { grid-template-columns: 1fr; } } | |
| .adapter-card { | |
| background: var(--surface); | |
| border: 1px solid var(--border); | |
| border-radius: 10px; | |
| overflow: hidden; | |
| } | |
| .adapter-header { | |
| padding: 14px 16px; | |
| font-weight: 700; | |
| font-size: 14px; | |
| display: flex; | |
| align-items: center; | |
| gap: 8px; | |
| } | |
| .adapter-header.cc { background: linear-gradient(135deg, #1a3a20 0%, #0f2010 100%); border-bottom: 1px solid var(--green); } | |
| .adapter-header.ai { background: linear-gradient(135deg, #3a2700 0%, #1a1000 100%); border-bottom: 1px solid var(--orange); } | |
| .adapter-header.cx { background: linear-gradient(135deg, #2a1a5e 0%, #15102d 100%); border-bottom: 1px solid var(--purple); } | |
| .adapter-body { padding: 16px; } | |
| .adapter-prop { margin-bottom: 8px; font-size: 12px; } | |
| .adapter-prop .key { color: var(--muted); margin-bottom: 2px; } | |
| .adapter-prop .val { color: var(--text); font-family: monospace; } | |
| /* ---- DATA FLOW TABLE ---- */ | |
| table { | |
| width: 100%; | |
| border-collapse: collapse; | |
| margin-bottom: 16px; | |
| font-size: 12px; | |
| } | |
| th { | |
| background: var(--surface2); | |
| color: var(--muted); | |
| text-align: left; | |
| padding: 8px 12px; | |
| font-size: 11px; | |
| text-transform: uppercase; | |
| letter-spacing: 0.5px; | |
| border-bottom: 1px solid var(--border); | |
| } | |
| td { | |
| padding: 8px 12px; | |
| border-bottom: 1px solid var(--border); | |
| vertical-align: top; | |
| } | |
| tr:last-child td { border-bottom: none; } | |
| tr:hover td { background: var(--surface2); } | |
| td code { font-size: 11px; } | |
| /* ---- METRICS CARDS ---- */ | |
| .metric-card { | |
| background: var(--surface); | |
| border: 1px solid var(--border); | |
| border-radius: 8px; | |
| padding: 14px 16px; | |
| text-align: center; | |
| } | |
| .metric-val { font-size: 28px; font-weight: 700; color: #fff; } | |
| .metric-label { font-size: 11px; color: var(--muted); margin-top: 4px; } | |
| /* ---- FILE TREE ---- */ | |
| .filetree { | |
| background: #010409; | |
| border: 1px solid var(--border); | |
| border-radius: 8px; | |
| padding: 16px; | |
| font-family: monospace; | |
| font-size: 12px; | |
| line-height: 1.8; | |
| } | |
| .ft-dir { color: var(--blue-bright); font-weight: 600; } | |
| .ft-file { color: var(--text); } | |
| .ft-desc { color: var(--muted); font-style: italic; } | |
| /* ---- LEADERBOARD ---- */ | |
| .leaderboard { | |
| background: var(--surface); | |
| border: 1px solid var(--border); | |
| border-radius: 10px; | |
| overflow: hidden; | |
| } | |
| .lb-row { | |
| display: grid; | |
| grid-template-columns: 40px 160px 80px 100px 100px 100px 120px; | |
| align-items: center; | |
| padding: 10px 16px; | |
| border-bottom: 1px solid var(--border); | |
| font-size: 13px; | |
| transition: background 0.1s; | |
| } | |
| .lb-row:hover { background: var(--surface2); } | |
| .lb-row.header { | |
| background: var(--surface2); | |
| color: var(--muted); | |
| font-size: 11px; | |
| text-transform: uppercase; | |
| letter-spacing: 0.5px; | |
| font-weight: 600; | |
| cursor: default; | |
| } | |
| .lb-row:last-child { border-bottom: none; } | |
| .lb-rank { color: var(--muted); font-weight: 700; } | |
| .lb-rank.gold { color: #f1c40f; } | |
| .lb-rank.silver { color: #95a5a6; } | |
| .lb-score { font-weight: 700; } | |
| .lb-score.perfect { color: var(--green-bright); } | |
| .lb-score.good { color: var(--blue-bright); } | |
| /* ---- FORMULA BOX ---- */ | |
| .formula { | |
| background: #010409; | |
| border: 1px solid var(--border); | |
| border-left: 3px solid var(--purple); | |
| border-radius: 0 8px 8px 0; | |
| padding: 16px 20px; | |
| font-family: monospace; | |
| font-size: 13px; | |
| margin-bottom: 16px; | |
| color: var(--text); | |
| } | |
| .formula .fname { color: #d2a8ff; font-weight: 700; } | |
| .formula .fnum { color: var(--orange-bright); } | |
| /* ---- DIVIDER ---- */ | |
| .divider { border: none; border-top: 1px solid var(--border); margin: 40px 0; } | |
| /* ---- TOOLTIP TABS ---- */ | |
| .tabs { display: flex; gap: 4px; margin-bottom: -1px; } | |
| .tab { | |
| padding: 6px 14px; | |
| border-radius: 6px 6px 0 0; | |
| border: 1px solid var(--border); | |
| border-bottom: none; | |
| background: var(--surface2); | |
| color: var(--muted); | |
| font-size: 12px; | |
| cursor: pointer; | |
| transition: all 0.15s; | |
| } | |
| .tab.active { background: var(--surface); color: var(--text); } | |
| .tab-content { display: none; } | |
| .tab-content.active { display: block; } | |
| .tabpanel { | |
| background: var(--surface); | |
| border: 1px solid var(--border); | |
| border-radius: 0 8px 8px 8px; | |
| padding: 20px; | |
| margin-bottom: 16px; | |
| } | |
| /* ---- SSE ANIMATION ---- */ | |
| @keyframes pulse { | |
| 0%, 100% { opacity: 1; } | |
| 50% { opacity: 0.3; } | |
| } | |
| .live-dot { | |
| display: inline-block; | |
| width: 8px; height: 8px; | |
| background: var(--green-bright); | |
| border-radius: 50%; | |
| animation: pulse 1.5s ease-in-out infinite; | |
| margin-right: 6px; | |
| } | |
| /* ---- SCROLLBAR ---- */ | |
| ::-webkit-scrollbar { width: 6px; height: 6px; } | |
| ::-webkit-scrollbar-track { background: var(--bg); } | |
| ::-webkit-scrollbar-thumb { background: var(--border); border-radius: 3px; } | |
| </style> | |
| </head> | |
| <body> | |
| <!-- NAV --> | |
| <nav> | |
| <span class="brand">AgentArena</span> | |
| <a href="#pipeline">Pipeline</a> | |
| <a href="#isolation">Sicherheit</a> | |
| <a href="#adapters">Adapter</a> | |
| <a href="#judge">Judge</a> | |
| <a href="#metrics">Metriken</a> | |
| <a href="#sse">Live-Updates</a> | |
| <a href="#filestructure">Dateien</a> | |
| <a href="#results">Ergebnisse</a> | |
| </nav> | |
| <div class="page"> | |
| <!-- HEADER --> | |
| <h1>AgentArena</h1> | |
| <p style="font-size:16px; color:var(--text); margin-bottom:8px;"> | |
| <strong>Technische Architektur-Dokumentation</strong> — Stand Maerz 2026 | |
| </p> | |
| <p> | |
| AgentArena ist eine <strong>FastAPI + Docker + SQLite</strong> Benchmarking-Plattform fuer AI Coding Agents. | |
| Das Alleinstellungsmerkmal: explizites <strong>Kosten-Tracking ($/Task)</strong> und ein | |
| <strong>zweicontainer-isoliertes Bewertungssystem</strong>, das Manipulation durch den Agenten verhindert. | |
| </p> | |
| <div style="display:flex; gap:8px; flex-wrap:wrap; margin-bottom:40px;"> | |
| <span class="badge badge-green">FastAPI</span> | |
| <span class="badge badge-blue">Docker</span> | |
| <span class="badge badge-orange">SQLite</span> | |
| <span class="badge badge-purple">htmx + SSE</span> | |
| <span class="badge badge-red">2-Container Isolation</span> | |
| </div> | |
| <hr class="divider"> | |
| <!-- ==================== PIPELINE ==================== --> | |
| <section id="pipeline"> | |
| <h2>1. Pipeline-Flow</h2> | |
| <p>Jeder Benchmark-Run durchlaeuft diese 8 Phasen sequenziell. Der Agent-Container und der Judge-Container | |
| existieren <strong>nie gleichzeitig</strong> — der Agent-Container wird zerstoert, | |
| bevor der Judge-Container erzeugt wird.</p> | |
| <div class="pipeline"> | |
| <div class="pipe-step"> | |
| <div class="pipe-box teal"> | |
| <span class="icon">📄</span> | |
| <span class="label">API Request</span> | |
| <span class="sublabel">POST /runs</span> | |
| </div> | |
| </div> | |
| <div class="pipe-arrow">→</div> | |
| <div class="pipe-step"> | |
| <div class="pipe-box blue"> | |
| <span class="icon">🏃</span> | |
| <span class="label">Runner</span> | |
| <span class="sublabel">runner.py</span> | |
| </div> | |
| </div> | |
| <div class="pipe-arrow">→</div> | |
| <div class="pipe-step"> | |
| <div class="pipe-box green"> | |
| <span class="icon">🔒</span> | |
| <span class="label">Agent Sandbox</span> | |
| <span class="sublabel">Docker container</span> | |
| </div> | |
| </div> | |
| <div class="pipe-arrow">→</div> | |
| <div class="pipe-step"> | |
| <div class="pipe-box orange"> | |
| <span class="icon">🔗</span> | |
| <span class="label">Adapter</span> | |
| <span class="sublabel">Host subprocess</span> | |
| </div> | |
| </div> | |
| <div class="pipe-arrow">→</div> | |
| <div class="pipe-step"> | |
| <div class="pipe-box orange"> | |
| <span class="icon">📄</span> | |
| <span class="label">git diff</span> | |
| <span class="sublabel">arena-checkpoint</span> | |
| </div> | |
| </div> | |
| <div class="pipe-arrow">→</div> | |
| <div class="pipe-step"> | |
| <div class="pipe-box red"> | |
| <span class="icon">⚠</span> | |
| <span class="label">Destroy Agent</span> | |
| <span class="sublabel">container rm -f</span> | |
| </div> | |
| </div> | |
| <div class="pipe-arrow">→</div> | |
| <div class="pipe-step"> | |
| <div class="pipe-box blue"> | |
| <span class="icon">🔎</span> | |
| <span class="label">Judge Sandbox</span> | |
| <span class="sublabel">frischer Clone</span> | |
| </div> | |
| </div> | |
| <div class="pipe-arrow">→</div> | |
| <div class="pipe-step"> | |
| <div class="pipe-box purple"> | |
| <span class="icon">🏆</span> | |
| <span class="label">Score</span> | |
| <span class="sublabel">4 Dimensionen</span> | |
| </div> | |
| </div> | |
| <div class="pipe-arrow">→</div> | |
| <div class="pipe-step"> | |
| <div class="pipe-box teal"> | |
| <span class="icon">🏅</span> | |
| <span class="label">Leaderboard</span> | |
| <span class="sublabel">SQLite + htmx</span> | |
| </div> | |
| </div> | |
| </div> | |
| <h3>Was passiert in jeder Phase?</h3> | |
| <table> | |
| <thead> | |
| <tr> | |
| <th>Phase</th> | |
| <th>Komponente</th> | |
| <th>Input</th> | |
| <th>Output</th> | |
| <th>Fehlerbehandlung</th> | |
| </tr> | |
| </thead> | |
| <tbody> | |
| <tr> | |
| <td><strong>1. API Request</strong></td> | |
| <td><code>main.py</code></td> | |
| <td><code>task_id, agent_id, k</code></td> | |
| <td>Neuer <code>Run</code> in DB mit Status <code>running</code></td> | |
| <td>HTTP 422 bei fehlenden Parametern</td> | |
| </tr> | |
| <tr> | |
| <td><strong>2. Sandbox erstellen</strong></td> | |
| <td><code>sandbox.py</code></td> | |
| <td><code>Task</code> (repo_url, repo_ref)</td> | |
| <td>Laufender Docker-Container mit geklontem Repo + <code>arena-checkpoint</code> Git-Tag</td> | |
| <td><code>RuntimeError</code> wenn Docker oder git clone fehlschlaegt</td> | |
| </tr> | |
| <tr> | |
| <td><strong>3. Adapter ausfuehren</strong></td> | |
| <td><code>adapters/</code></td> | |
| <td><code>RunContext</code> (task_prompt, success_cmd, time_limit_s)</td> | |
| <td><code>RunResult</code> (diff_text, tokens, cost_usd, trajectory)</td> | |
| <td><code>subprocess.TimeoutExpired</code> → status=timeout</td> | |
| </tr> | |
| <tr> | |
| <td><strong>4. Diff sammeln</strong></td> | |
| <td><code>runner.py</code></td> | |
| <td>Host-Tempdir oder Sandbox</td> | |
| <td>Unified diff (arena-checkpoint bis HEAD)</td> | |
| <td>Leerer Diff = kein Code generiert</td> | |
| </tr> | |
| <tr> | |
| <td><strong>5. Agent zerstoeren</strong></td> | |
| <td><code>sandbox.destroy()</code></td> | |
| <td>Container-ID</td> | |
| <td>— (Container weg)</td> | |
| <td>Best-effort, ignoriert Fehler</td> | |
| </tr> | |
| <tr> | |
| <td><strong>6. Judge Sandbox</strong></td> | |
| <td><code>sandbox.create_judge_sandbox()</code></td> | |
| <td>Task + diff_text</td> | |
| <td>Frischer Container mit gepatchtem Code; <code>sandbox.patch_applied</code></td> | |
| <td>Warning-Flag wenn Patch fehlschlaegt</td> | |
| </tr> | |
| <tr> | |
| <td><strong>7. Judge</strong></td> | |
| <td><code>judge.py</code></td> | |
| <td>Judge-Container + success_cmd</td> | |
| <td><code>JudgeResult</code> (4 Scores + Notes)</td> | |
| <td>Neutral-Score 0.5 wenn kein Test-Runner gefunden</td> | |
| </tr> | |
| <tr> | |
| <td><strong>8. Metriken + DB</strong></td> | |
| <td><code>metrics.py</code>, <code>runner.py</code></td> | |
| <td>JudgeResult + adapter_result + trajectory</td> | |
| <td><code>Run</code> mit judge_score, metrics_json, cost_usd</td> | |
| <td>Composite score = 0 wenn Funktionalitaet = 0</td> | |
| </tr> | |
| </tbody> | |
| </table> | |
| <h3>Aus dem Code: Runner-Kernlogik</h3> | |
| <pre><span class="cm"># core/runner.py (vereinfacht)</span> | |
| <span class="kw">async def</span> <span class="fn">run_single</span>(self, db, task, agent) -> Run: | |
| <span class="cm"># 1. Sandbox erstellen (git clone + checkpoint)</span> | |
| sandbox = <span class="kw">await</span> asyncio.to_thread(Sandbox.create, task) | |
| <span class="cm"># 2. Adapter holen und ausfuehren (HOST subprocess, nicht im Container)</span> | |
| adapter = get_adapter(agent.adapter_type, agent_config) | |
| adapter_result = <span class="kw">await</span> adapter.run(ctx, sandbox) | |
| <span class="cm"># 3. Diff aus Host-Tempdir oder Container holen</span> | |
| diff_text = adapter_result.diff_text <span class="kw">or</span> <span class="kw">await</span> asyncio.to_thread(sandbox.get_diff) | |
| <span class="cm"># 4. AGENT CONTAINER ZERSTOEREN (Security-Grenze!)</span> | |
| <span class="kw">await</span> asyncio.to_thread(sandbox.destroy) | |
| sandbox = <span class="nm">None</span> | |
| <span class="cm"># 5. FRISCHEN Judge-Container erstellen + Diff einspielen</span> | |
| judge_sandbox = <span class="kw">await</span> asyncio.to_thread( | |
| Sandbox.create_judge_sandbox, task, diff_text | |
| ) | |
| <span class="cm"># 6. Judge laeuft NUR im frischen Container</span> | |
| judge_result = <span class="kw">await</span> asyncio.to_thread( | |
| self.judge.judge_run, judge_sandbox, task.success_cmd | |
| )</pre> | |
| </section> | |
| <!-- ==================== ISOLATION ==================== --> | |
| <section id="isolation"> | |
| <h2>2. Sicherheitsmodell: 2-Container-Isolation</h2> | |
| <p>Das kritischste Design-Entscheidung: Agent und Judge laufen in <strong>separaten, kurzlebigen Docker-Containern</strong>. | |
| Der Agent hat <em>keine Moeglichkeit</em>, die Testumgebung zu manipulieren, weil der Judge-Container | |
| erst nach Zerstoerung des Agent-Containers erstellt wird.</p> | |
| <div class="isolation-diagram"> | |
| <!-- Container A --> | |
| <div class="container-box agent"> | |
| <div class="container-header"> | |
| <span class="badge badge-green">Container A</span> | |
| <span style="color: var(--green-bright);">Agent Sandbox</span> | |
| </div> | |
| <ul class="step-list"> | |
| <li><span class="step-num">1</span> <div><strong>docker run</strong> python:3.12-slim<br><span style="color:var(--muted)">--network=arena-net --memory=2g</span></div></li> | |
| <li><span class="step-num">2</span> <div><strong>git clone</strong> --depth=1 repo_url /workspace<br><span style="color:var(--muted)">+ apt install git, pip install pytest</span></div></li> | |
| <li><span class="step-num">3</span> <div><strong>git tag</strong> arena-checkpoint<br><span style="color:var(--muted)">Snapshot vor Agent-Arbeit</span></div></li> | |
| <li><span class="step-num">4</span> <div><strong>Adapter laeuft auf HOST</strong><br><span style="color:var(--muted)">claude/aider als subprocess</span></div></li> | |
| <li><span class="step-num">5</span> <div><strong>git diff</strong> arena-checkpoint<br><span style="color:var(--muted)">Unified diff gespeichert</span></div></li> | |
| <li><span class="step-num" style="background:var(--red-dim);color:var(--red-bright);">X</span> <div><strong>docker rm -f</strong> container_id<br><span style="color:var(--red-bright)">Zerstoert. Kein shared state.</span></div></li> | |
| </ul> | |
| </div> | |
| <!-- Wall --> | |
| <div class="wall"> | |
| <div class="wall-line"></div> | |
| <div class="wall-label">Security<br>Boundary<br>🔒</div> | |
| <div class="wall-label" style="margin-top:8px; background:var(--orange-dim); border-color:var(--orange); color:var(--orange-bright);">nur<br>diff_text<br>→</div> | |
| <div class="wall-line"></div> | |
| </div> | |
| <!-- Container B --> | |
| <div class="container-box judge"> | |
| <div class="container-header"> | |
| <span class="badge badge-blue">Container B</span> | |
| <span style="color: var(--blue-bright);">Judge Sandbox</span> | |
| </div> | |
| <ul class="step-list"> | |
| <li><span class="step-num">1</span> <div><strong>docker run</strong> python:3.12-slim<br><span style="color:var(--muted)">frischer Container, nichts vom Agenten</span></div></li> | |
| <li><span class="step-num">2</span> <div><strong>git clone</strong> --depth=1 repo_url /workspace<br><span style="color:var(--muted)">+ apt install git patch</span></div></li> | |
| <li><span class="step-num">3</span> <div><strong>docker cp</strong> agent.patch Container<br><span style="color:var(--muted)">Diff als temp file eingeschleust</span></div></li> | |
| <li><span class="step-num">4</span> <div><strong>git apply</strong> agent.patch<br><span style="color:var(--muted)">Fallback: patch -p1</span></div></li> | |
| <li><span class="step-num">5</span> <div><strong>pytest, ruff, radon</strong> ausfuehren<br><span style="color:var(--muted)">in sauberem Environment</span></div></li> | |
| <li><span class="step-num" style="background:var(--blue-dim);color:var(--blue-bright);">✓</span> <div><strong>Score berechnen</strong> + docker rm<br><span style="color:var(--blue-bright)">JudgeResult zurueck an Runner</span></div></li> | |
| </ul> | |
| </div> | |
| </div> | |
| <div class="grid2"> | |
| <div class="card red"> | |
| <h3 style="color:var(--red-bright)">Warum ist das wichtig?</h3> | |
| <p>Ohne Isolation koennte ein Agent:</p> | |
| <ul style="color:var(--muted); padding-left:20px; font-size:13px; margin-top:8px;"> | |
| <li>Test-Dateien so modifizieren, dass sie immer bestehen</li> | |
| <li>pytest durch ein eigenes Script ersetzen</li> | |
| <li>Git-Historie manipulieren um alte Fehler zu verstecken</li> | |
| <li>Environment-Variablen setzen die den Judge beeinflussen</li> | |
| </ul> | |
| </div> | |
| <div class="card green"> | |
| <h3 style="color:var(--green-bright)">Garantien des Modells</h3> | |
| <ul style="color:var(--muted); padding-left:20px; font-size:13px; margin-top:8px;"> | |
| <li>Judge sieht <strong>nur</strong> den Diff (Text), nie den Agent-Container</li> | |
| <li>Jeder Run startet mit einem <strong>frischen Repo-Clone</strong></li> | |
| <li>Kein shared filesystem, kein shared network state zwischen Containern</li> | |
| <li><code>patch_applied</code> Flag warnt wenn Diff nicht anwendbar</li> | |
| </ul> | |
| </div> | |
| </div> | |
| <h3>Sandbox.create_judge_sandbox() im Detail</h3> | |
| <pre><span class="cm"># core/sandbox.py</span> | |
| <span class="kw">@classmethod</span> | |
| <span class="kw">def</span> <span class="fn">create_judge_sandbox</span>(cls, task, diff_text: <span class="py">str</span>) -> <span class="st">"Sandbox"</span>: | |
| <span class="cm"># Frischer Container — kein Bezug zum Agent-Container</span> | |
| container_id = docker_run(image=<span class="st">"python:3.12-slim"</span>) | |
| sandbox = cls(container_id) | |
| <span class="cm"># Frischer Clone direkt aus Remote</span> | |
| sandbox.exec(<span class="st">f"git clone --depth=1 -b {repo_ref} {repo_url} ."</span>) | |
| <span class="cm"># Diff als Datei in Container kopieren</span> | |
| subprocess.run([<span class="st">"docker"</span>, <span class="st">"cp"</span>, tmp_path, <span class="st">f"{container_id}:/tmp/agent.patch"</span>]) | |
| <span class="cm"># Patch anwenden (git apply, Fallback: patch -p1)</span> | |
| rc, out, err = sandbox.exec( | |
| <span class="st">"git apply --whitespace=fix /tmp/agent.patch || patch -p1 < /tmp/agent.patch"</span> | |
| ) | |
| sandbox.patch_applied = rc == <span class="nm">0</span> | |
| <span class="kw">return</span> sandbox</pre> | |
| </section> | |
| <!-- ==================== ADAPTERS ==================== --> | |
| <section id="adapters"> | |
| <h2>3. Adapter-Architektur</h2> | |
| <p>Alle Adapter implementieren das <code>AgentAdapter</code> ABC und laufen auf dem <strong>HOST-System</strong>, | |
| nicht im Container. Der Grund: Claude Code und Aider koennen nicht sinnvoll innerhalb von Docker-Containern | |
| ausgefuehrt werden (OAuth-Zustand, API-Keys, Dateisystem-Zugriff).</p> | |
| <div class="adapter-grid"> | |
| <!-- Claude Code --> | |
| <div class="adapter-card"> | |
| <div class="adapter-header cc"> | |
| <span>🚀</span> | |
| <span style="color:var(--green-bright)">ClaudeCodeAdapter</span> | |
| </div> | |
| <div class="adapter-body"> | |
| <div class="adapter-prop"> | |
| <div class="key">CLI-Befehl</div> | |
| <div class="val">claude --print --dangerously-skip-permissions --output-format json</div> | |
| </div> | |
| <div class="adapter-prop"> | |
| <div class="key">Auth</div> | |
| <div class="val">OAuth (kein API-Key noetig)</div> | |
| </div> | |
| <div class="adapter-prop"> | |
| <div class="key">Repo-Strategie</div> | |
| <div class="val">Klont in Host-Tempdir (arena-claude-*)</div> | |
| </div> | |
| <div class="adapter-prop"> | |
| <div class="key">Token-Tracking</div> | |
| <div class="val">entry.usage.input_tokens, total_cost_usd direkt aus JSON</div> | |
| </div> | |
| <div class="adapter-prop"> | |
| <div class="key">Modell</div> | |
| <div class="val">claude-sonnet-4-6 (aus modelUsage)</div> | |
| </div> | |
| <div class="adapter-prop"> | |
| <div class="key">Kosten (Run #5)</div> | |
| <div class="val" style="color:var(--orange-bright)">~$0.15</div> | |
| </div> | |
| </div> | |
| </div> | |
| <!-- Aider --> | |
| <div class="adapter-card"> | |
| <div class="adapter-header ai"> | |
| <span>⚒</span> | |
| <span style="color:var(--orange-bright)">AiderAdapter</span> | |
| </div> | |
| <div class="adapter-body"> | |
| <div class="adapter-prop"> | |
| <div class="key">CLI-Befehl</div> | |
| <div class="val">aider --yes-always --model openai/MiniMax-M2.5 --message ...</div> | |
| </div> | |
| <div class="adapter-prop"> | |
| <div class="key">Auth</div> | |
| <div class="val">MINIMAX_API_KEY aus ~/.claude/.env</div> | |
| </div> | |
| <div class="adapter-prop"> | |
| <div class="key">API-Routing</div> | |
| <div class="val">litellm → OPENAI_API_BASE=https://api.minimax.io/v1</div> | |
| </div> | |
| <div class="adapter-prop"> | |
| <div class="key">Token-Tracking</div> | |
| <div class="val">Regex aus stdout: "Tokens: X sent, Y received, Cost: $Z"</div> | |
| </div> | |
| <div class="adapter-prop"> | |
| <div class="key">Modell</div> | |
| <div class="val">MiniMax-M2.5 (via OpenAI-kompatibler Endpoint)</div> | |
| </div> | |
| <div class="adapter-prop"> | |
| <div class="key">Kosten (Run #10)</div> | |
| <div class="val" style="color:var(--green-bright)">~$0.00 (MiniMax)</div> | |
| </div> | |
| </div> | |
| </div> | |
| <!-- Codex --> | |
| <div class="adapter-card"> | |
| <div class="adapter-header cx"> | |
| <span>💻</span> | |
| <span style="color:#b388ff">CodexAdapter</span> | |
| </div> | |
| <div class="adapter-body"> | |
| <div class="adapter-prop"> | |
| <div class="key">CLI-Befehl</div> | |
| <div class="val">codex --quiet --full-auto</div> | |
| </div> | |
| <div class="adapter-prop"> | |
| <div class="key">Status</div> | |
| <div class="val" style="color:var(--orange-bright)">Noch nicht getestet</div> | |
| </div> | |
| <div class="adapter-prop"> | |
| <div class="key">Auth</div> | |
| <div class="val">OPENAI_API_KEY</div> | |
| </div> | |
| <div class="adapter-prop"> | |
| <div class="key">Pricing</div> | |
| <div class="val">gpt-4o: $2.50/$10.0 per 1M</div> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| <h3>Gemeinsames Pattern: Host-Clone-Strategy</h3> | |
| <p>Alle Adapter klonen das Repo in ein Host-Tempverzeichnis, setzen einen Git-Checkpoint-Tag, | |
| fuehren dann den CLI-Prozess aus, und sammeln den Diff vor dem Cleanup.</p> | |
| <pre><span class="cm"># Muster fuer alle Adapter (adapters/claude_code.py, adapters/aider.py)</span> | |
| tmpdir = tempfile.mkdtemp(prefix=<span class="st">"arena-claude-"</span>) <span class="cm"># oder arena-aider-</span> | |
| <span class="cm"># 1. Repo klonen</span> | |
| subprocess.run([<span class="st">"git"</span>, <span class="st">"clone"</span>, <span class="st">"--depth=1"</span>, <span class="st">"-b"</span>, repo_ref, repo_url, tmpdir]) | |
| <span class="cm"># 2. Checkpoint vor Agent-Arbeit</span> | |
| subprocess.run([<span class="st">"git"</span>, <span class="st">"tag"</span>, <span class="st">"arena-checkpoint"</span>], cwd=tmpdir) | |
| <span class="cm"># 3. CLI ausfuehren (timeout = task.time_limit_s)</span> | |
| proc = subprocess.run(cmd, cwd=tmpdir, capture_output=<span class="nm">True</span>, timeout=ctx.time_limit_s) | |
| <span class="cm"># 4. Diff sammeln (excludiert __pycache__ und .pyc)</span> | |
| diff = subprocess.run( | |
| [<span class="st">"git"</span>, <span class="st">"diff"</span>, <span class="st">"arena-checkpoint"</span>, <span class="st">"--"</span>, <span class="st">"."</span>, | |
| <span class="st">":!**/__pycache__/"</span>, <span class="st">":!**/*.pyc"</span>], | |
| cwd=tmpdir | |
| ) | |
| <span class="cm"># 5. Cleanup (always, via try/finally)</span> | |
| shutil.rmtree(tmpdir, ignore_errors=<span class="nm">True</span>)</pre> | |
| <h3>Preistabellen</h3> | |
| <table> | |
| <thead> | |
| <tr><th>Modell</th><th>Input ($/1M)</th><th>Cache Write</th><th>Cache Read</th><th>Output ($/1M)</th></tr> | |
| </thead> | |
| <tbody> | |
| <tr><td>claude-opus-4-6</td><td>$5.00</td><td>$6.25</td><td>$0.50</td><td>$25.00</td></tr> | |
| <tr><td>claude-sonnet-4-6</td><td>$3.00</td><td>$3.75</td><td>$0.30</td><td>$15.00</td></tr> | |
| <tr><td>claude-haiku-4-5</td><td>$1.00</td><td>$1.25</td><td>$0.10</td><td>$5.00</td></tr> | |
| <tr><td>gpt-4o</td><td>$2.50</td><td>—</td><td>—</td><td>$10.00</td></tr> | |
| <tr><td><strong>MiniMax-M2.5</strong></td><td><strong>$0.05</strong></td><td>—</td><td>—</td><td><strong>$0.05</strong></td></tr> | |
| </tbody> | |
| </table> | |
| </section> | |
| <!-- ==================== JUDGE ==================== --> | |
| <section id="judge"> | |
| <h2>4. Judge-System: 4 Dimensionen</h2> | |
| <p>Der Judge bewertet den Code in vier Dimensionen. <strong>Funktionalitaet wirkt als Gatekeeper</strong>: | |
| besteht die Aufgabe nicht, ist der Overall-Score 0 unabhaengig von allen anderen Dimensionen.</p> | |
| <div class="formula"> | |
| <span class="fname">overall_score</span> = | |
| <span class="fname">functional_score</span> × <span class="fnum">0.50</span> + | |
| <span class="fname">regression_score</span> × <span class="fnum">0.20</span> + | |
| <span class="fname">lint_score</span> × <span class="fnum">0.15</span> + | |
| <span class="fname">complexity_score</span> × <span class="fnum">0.15</span> | |
| </div> | |
| <div class="grid2"> | |
| <div class="card green"> | |
| <h3 style="color:var(--green-bright)">Functional Score (50%)</h3> | |
| <p><strong>Messmethode:</strong> Fuehrt <code>success_cmd</code> (pytest) im Judge-Container aus. | |
| Prueft Exit-Code und parst Testanzahl aus stdout.</p> | |
| <p><strong>Scoring:</strong> Exit-Code 0 = 1.0, Exit-Code != 0 = 0.0. | |
| Test-Count wird separat getrackt (tests_total, tests_passed).</p> | |
| <p><strong>Beispiel:</strong> <code>5/5 tests passed. Exit 0 → score=1.0</code></p> | |
| <pre style="font-size:11px; margin-bottom:0;"><span class="kw">if</span> rc == <span class="nm">0</span>: | |
| result.functional_score = <span class="nm">1.0</span> | |
| <span class="kw">else</span>: | |
| result.functional_score = <span class="nm">0.0</span></pre> | |
| </div> | |
| <div class="card blue"> | |
| <h3 style="color:var(--blue-bright)">Regression Score (20%)</h3> | |
| <p><strong>Messmethode:</strong> Fuehrt breiteren Test-Suite aus. | |
| Kandidaten in Reihenfolge: <code>pytest --tb=no -q</code>, <code>python -m pytest</code>, <code>make test</code>.</p> | |
| <p><strong>Scoring:</strong> <code>passed / total</code>. Neutralscore 0.5 wenn kein Test-Runner gefunden.</p> | |
| <p><strong>Crash-Detection:</strong> <code>"X errors in Y.Zs"</code> (ohne "passed"/"failed") wird als | |
| Sammelfehler erkannt und uebersprungen, nicht als echter Test-Ausfall gewertet.</p> | |
| <pre style="font-size:11px; margin-bottom:0;"><span class="cm"># Crash detection</span> | |
| is_crash = rc != <span class="nm">0</span> <span class="kw">and</span> passed == <span class="nm">0</span> \ | |
| <span class="kw">and</span> crash_match <span class="kw">and not</span> has_real_tests | |
| <span class="kw">if</span> total > <span class="nm">0</span> <span class="kw">and not</span> is_crash: | |
| result.regression_score = passed / total</pre> | |
| </div> | |
| <div class="card orange"> | |
| <h3 style="color:var(--orange-bright)">Lint Score (15%)</h3> | |
| <p><strong>Messmethode:</strong> <code>ruff check --output-format=json</code> vor und nach Aenderungen. | |
| Der <strong>git stash Trick</strong>: Stash die Agent-Aenderungen, zaehle Warnings im Original, | |
| pop den Stash, zaehle erneut.</p> | |
| <p><strong>Scoring:</strong> Delta-basiert. <code>delta <= 0 = 1.0</code>. | |
| Jede neue Warning kostet 0.1 Punkte. Fallback: <code>py_compile</code> bei fehlendem ruff.</p> | |
| <pre style="font-size:11px; margin-bottom:0;"><span class="cm"># git stash trick fuer Before-State</span> | |
| sandbox.exec(<span class="st">"git stash"</span>) <span class="cm"># temporaer zuruecksetzen</span> | |
| before = count_ruff_warnings(sandbox, files) | |
| sandbox.exec(<span class="st">"git stash pop"</span>) <span class="cm"># Agent-Aenderungen wiederherstellen</span> | |
| after = count_ruff_warnings(sandbox, files) | |
| <span class="cm"># score: max(0, 1.0 - delta * 0.1)</span></pre> | |
| </div> | |
| <div class="card purple"> | |
| <h3 style="color:#b388ff">Complexity Score (15%)</h3> | |
| <p><strong>Messmethode:</strong> <code>radon cc --json</code> berechnet zyklomatische Komplexitaet | |
| pro Funktion, gemittelt ueber alle geaenderten Dateien. Selber git-stash-Trick wie beim Lint.</p> | |
| <p><strong>Fallback:</strong> Einfacher Proxy aus max. Einrueckungstiefe + Anzahl Funktionen/Klassen | |
| pro Datei wenn radon nicht verfuegbar.</p> | |
| <pre style="font-size:11px; margin-bottom:0;"><span class="cm"># Fallback-Proxy (simple_complexity)</span> | |
| max_indent = <span class="fn">max</span>(len(l) - len(l.lstrip()) <span class="kw">for</span> l <span class="kw">in</span> lines) | |
| func_count = <span class="fn">sum</span>(<span class="nm">1</span> <span class="kw">for</span> l <span class="kw">in</span> lines | |
| <span class="kw">if</span> l.strip().startswith(<span class="st">'def '</span>) | |
| <span class="kw">or</span> l.strip().startswith(<span class="st">'class '</span>))</pre> | |
| </div> | |
| </div> | |
| <h3>pytest Output Parser</h3> | |
| <p>Der Parser erkennt verschiedene pytest-Ausgabeformate robuster als ein einfaches Regex-Match:</p> | |
| <pre><span class="cm"># core/metrics.py — parse_pytest_output()</span> | |
| <span class="cm"># Erkennt: "5 passed", "3 failed, 2 passed", "1 error, 2 failed, 3 passed"</span> | |
| passed_match = re.search(<span class="st">r"(\d+)\s+passed"</span>, output) | |
| failed_match = re.search(<span class="st">r"(\d+)\s+failed"</span>, output) | |
| error_match = re.search(<span class="st">r"(\d+)\s+error"</span>, output) | |
| <span class="cm"># Fallback: "X/Y tests passed" (unittest-Stil)</span> | |
| fraction = re.search(<span class="st">r"(\d+)/(\d+)\s+(?:tests?\s+)?passed"</span>, output)</pre> | |
| </section> | |
| <!-- ==================== METRICS ==================== --> | |
| <section id="metrics"> | |
| <h2>5. Metriken-System</h2> | |
| <div class="tabs"> | |
| <div class="tab active" onclick="showTab('traj')">Trajectory Analysis</div> | |
| <div class="tab" onclick="showTab('diff')">Diff-Minimality</div> | |
| <div class="tab" onclick="showTab('cost')">Cost Tracking</div> | |
| <div class="tab" onclick="showTab('passk')">Pass@k</div> | |
| <div class="tab" onclick="showTab('composite')">Composite Score</div> | |
| </div> | |
| <div class="tabpanel"> | |
| <div id="tab-traj" class="tab-content active"> | |
| <h3>Trajectory Analysis</h3> | |
| <p>Eine Trajectory ist eine Liste von Tool-Use-Events mit <code>tool_name</code>, <code>input_preview</code>, <code>timestamp</code>. | |
| Die Analyse erkennt ineffiziente Muster in der Problemloesungsstrategie des Agenten.</p> | |
| <div class="grid2" style="margin-bottom:16px;"> | |
| <div> | |
| <h4>Erkannte Metriken</h4> | |
| <table> | |
| <thead><tr><th>Metrik</th><th>Was wird gemessen</th></tr></thead> | |
| <tbody> | |
| <tr><td><code>directness_score</code></td><td>Wie direkt der Agent zur Loesung gelangt (0-1)</td></tr> | |
| <tr><td><code>loop_count</code></td><td>Repetitive Tool-Call-Sequenzen (1er, 2er, 3er-Zyklen)</td></tr> | |
| <tr><td><code>stall_rate</code></td><td>Anteil von Steps ohne Fortschritt (≥5 Read-Steps in Folge)</td></tr> | |
| <tr><td><code>backtrack_count</code></td><td>Ueberschriebene Dateien (innerhalb 10 Steps)</td></tr> | |
| <tr><td><code>self_correction_rate</code></td><td>Edit → Test → Edit-gleiche-Datei Pattern</td></tr> | |
| <tr><td><code>exploration_ratio</code></td><td>Read/Grep/Glob / gesamt</td></tr> | |
| <tr><td><code>turns_to_first_edit</code></td><td>Wieviele Steps bis zum ersten Edit/Write</td></tr> | |
| </tbody> | |
| </table> | |
| </div> | |
| <div> | |
| <h4>Phasen-Klassifikation</h4> | |
| <table> | |
| <thead><tr><th>Phase</th><th>Tools</th></tr></thead> | |
| <tbody> | |
| <tr><td>exploration</td><td>Read, Grep, Glob, Bash(ls/find/cat)</td></tr> | |
| <tr><td>implementation</td><td>Edit, Write, NotebookEdit</td></tr> | |
| <tr><td>testing</td><td>Bash mit pytest/unittest/make test</td></tr> | |
| <tr><td>debugging</td><td>Read/Edit <em>nach</em> fehlgeschlagenem Test</td></tr> | |
| </tbody> | |
| </table> | |
| </div> | |
| </div> | |
| <pre><span class="cm"># Directness Score — kombiniert 5 Sub-Metriken</span> | |
| score = ( | |
| <span class="nm">0.35</span> * trajectory_efficiency <span class="cm"># Edit/Write-Ratio</span> | |
| + <span class="nm">0.25</span> * early_edit_score <span class="cm"># wie frueh erster Edit</span> | |
| - <span class="nm">0.20</span> * repeat_penalty <span class="cm"># gleiche Datei wiederholt gelesen</span> | |
| - <span class="nm">0.10</span> * loop_penalty <span class="cm"># zyklische Tool-Patterns</span> | |
| - <span class="nm">0.10</span> * stall_rate <span class="cm"># lange Read-Phasen ohne Edit</span> | |
| )</pre> | |
| </div> | |
| <div id="tab-diff" class="tab-content"> | |
| <h3>Diff-Minimality Analyse</h3> | |
| <p>Ein guter Agent aendert nur was noetig ist. Hoher Churn (viele Zeilen geaendert) deutet auf | |
| Overengineering oder unnoetige Refactorings hin.</p> | |
| <table> | |
| <thead><tr><th>Metrik</th><th>Beschreibung</th><th>Berechnung</th></tr></thead> | |
| <tbody> | |
| <tr><td><code>lines_added</code></td><td>Zeilen mit +</td><td>Alle <code>+</code>-Zeilen ausser <code>+++</code>-Header</td></tr> | |
| <tr><td><code>lines_removed</code></td><td>Zeilen mit -</td><td>Alle <code>-</code>-Zeilen ausser <code>---</code>-Header</td></tr> | |
| <tr><td><code>files_changed</code></td><td>Distinct Files</td><td>aus <code>+++ b/</code> Headern</td></tr> | |
| <tr><td><code>hunks</code></td><td>Diff-Bloecke</td><td>Anzahl <code>@@</code>-Marker</td></tr> | |
| <tr><td><code>churn</code></td><td>Gesamtveraenderung</td><td>lines_added + lines_removed</td></tr> | |
| </tbody> | |
| </table> | |
| <p>Im Composite Score: <code>diff_minimality = 1.0 / (1.0 + churn / 50.0)</code> | |
| — Sigmoid-artige Normalisierung, churn=0 → 1.0, churn=50 → 0.5.</p> | |
| </div> | |
| <div id="tab-cost" class="tab-content"> | |
| <h3>Cost Tracking</h3> | |
| <p>Jeder Run trackt Tokens und rechnet Kosten in USD um. Das ist das Alleinstellungsmerkmal von AgentArena | |
| gegenueber anderen Benchmarks.</p> | |
| <div class="grid2"> | |
| <div> | |
| <h4>Claude Code (direkt aus JSON)</h4> | |
| <pre style="font-size:11px;"><span class="cm"># --output-format json liefert:</span> | |
| entry[<span class="st">"usage"</span>][<span class="st">"input_tokens"</span>] | |
| entry[<span class="st">"usage"</span>][<span class="st">"output_tokens"</span>] | |
| entry[<span class="st">"usage"</span>][<span class="st">"cache_creation_input_tokens"</span>] | |
| entry[<span class="st">"usage"</span>][<span class="st">"cache_read_input_tokens"</span>] | |
| entry[<span class="st">"total_cost_usd"</span>] <span class="cm"># direkt verfuegbar!</span></pre> | |
| </div> | |
| <div> | |
| <h4>Aider (aus stdout geparst)</h4> | |
| <pre style="font-size:11px;"><span class="cm"># Regex aus Aider-Output:</span> | |
| _TOKEN_RE = re.compile( | |
| <span class="st">r"Tokens:\s*(?P<sent>\d+)\s+sent,"</span> | |
| <span class="st">r"\s*(?P<recv>\d+)\s+received"</span> | |
| ) | |
| _COST_RE = re.compile( | |
| <span class="st">r"Cost:\s*\$(?P<cost>[0-9]+\.[0-9]+)"</span> | |
| )</pre> | |
| </div> | |
| </div> | |
| <pre><span class="cm"># Kosten-Kalkulation wenn kein direkter Wert (Fallback)</span> | |
| cost = (tokens_input * rate_in + tokens_output * rate_out) / <span class="nm">1_000_000</span> | |
| <span class="cm"># Effizienz-Score fuer Composite (normalisiert auf 0-1)</span> | |
| efficiency = <span class="nm">1.0</span> / (<span class="nm">1.0</span> + cost_usd / <span class="nm">0.50</span>) <span class="cm"># $0 = 1.0, $0.50 = 0.5</span></pre> | |
| </div> | |
| <div id="tab-passk" class="tab-content"> | |
| <h3>Pass@k Unbiased Estimator</h3> | |
| <p>Pass@k misst die Wahrscheinlichkeit, dass mindestens einer von k unabhaengigen Runs erfolgreich ist. | |
| Der unbiased Estimator vermeidet Overfitting gegenueber dem naiven Ansatz.</p> | |
| <div class="formula"> | |
| <span class="fname">pass@k</span> = 1 − | |
| C(<span class="fname">n</span>−<span class="fname">c</span>, <span class="fnum">k</span>) / | |
| C(<span class="fname">n</span>, <span class="fnum">k</span>) | |
| <br><br> | |
| <span style="color:var(--muted)">n = gesamt Runs, c = korrekte Runs, k = Ziel-k</span> | |
| </div> | |
| <pre><span class="cm"># core/metrics.py</span> | |
| <span class="kw">def</span> <span class="fn">compute_pass_at_k</span>(n: <span class="py">int</span>, c: <span class="py">int</span>, k: <span class="py">int</span>) -> <span class="py">float</span>: | |
| <span class="kw">if</span> n < k: <span class="kw">return</span> <span class="nm">0.0</span> | |
| <span class="kw">if</span> c == <span class="nm">0</span>: <span class="kw">return</span> <span class="nm">0.0</span> | |
| <span class="kw">if</span> n - c < k: <span class="kw">return</span> <span class="nm">1.0</span> | |
| <span class="kw">return</span> <span class="nm">1.0</span> - comb(n - c, k) / comb(n, k)</pre> | |
| <p>Runner fuehrt k Runs <strong>parallel</strong> aus mittels <code>asyncio.gather()</code>, | |
| jeder in eigenem DB-Session, getaggt mit gemeinsamer <code>pass_at_k_group</code> UUID.</p> | |
| </div> | |
| <div id="tab-composite" class="tab-content"> | |
| <h3>Composite Score</h3> | |
| <p>Kombiniert Judge-Score, Trajectory-Metriken und Diff-Qualitaet zu einem Gesamtwert. | |
| <strong>Funktionalitaet wirkt als Multiplikator</strong>: schlaegt der Test fehl, ist alles 0.</p> | |
| <div class="formula"> | |
| <span class="fname">composite</span> = <span class="fname">functionality</span> × (<br> | |
| <span class="fnum">0.25</span> × <span class="fname">directness</span> +<br> | |
| <span class="fnum">0.25</span> × <span class="fname">efficiency</span> +<br> | |
| <span class="fnum">0.20</span> × <span class="fname">diff_minimality</span> +<br> | |
| <span class="fnum">0.15</span> × <span class="fname">no_loops</span> +<br> | |
| <span class="fnum">0.15</span> × <span class="fname">low_correction</span><br> | |
| ) | |
| </div> | |
| <p><strong>Gatekeeper-Logik:</strong> Ist <code>functionality == 0.0</code>, wird sofort <code>0.0</code> | |
| zurueckgegeben, ohne die anderen Dimensionen zu berechnen. Nur ein funktionierender Code kann | |
| ein guter Code sein.</p> | |
| </div> | |
| </div> | |
| <!-- Score Bars --> | |
| <h3>Judge Score Visualisierung (Beispiel: Run #10, Aider + MiniMax)</h3> | |
| <div style="background:var(--surface); border:1px solid var(--border); border-radius:10px; padding:20px; margin-bottom:16px;"> | |
| <div class="score-bar-wrap"> | |
| <div class="score-bar-label"> | |
| <span>Functional (50%)</span> | |
| <span style="color:var(--green-bright)">1.0 × 0.50 = 0.50</span> | |
| </div> | |
| <div class="score-bar-track"> | |
| <div class="score-bar-fill" style="width:100%; background:var(--green)">1.00</div> | |
| </div> | |
| </div> | |
| <div class="score-bar-wrap"> | |
| <div class="score-bar-label"> | |
| <span>Regression (20%)</span> | |
| <span style="color:var(--green-bright)">1.0 × 0.20 = 0.20</span> | |
| </div> | |
| <div class="score-bar-track"> | |
| <div class="score-bar-fill" style="width:100%; background:var(--green)">1.00</div> | |
| </div> | |
| </div> | |
| <div class="score-bar-wrap"> | |
| <div class="score-bar-label"> | |
| <span>Lint (15%)</span> | |
| <span style="color:var(--green-bright)">1.0 × 0.15 = 0.15</span> | |
| </div> | |
| <div class="score-bar-track"> | |
| <div class="score-bar-fill" style="width:100%; background:var(--blue)">1.00</div> | |
| </div> | |
| </div> | |
| <div class="score-bar-wrap"> | |
| <div class="score-bar-label"> | |
| <span>Complexity (15%)</span> | |
| <span style="color:var(--green-bright)">1.0 × 0.15 = 0.15</span> | |
| </div> | |
| <div class="score-bar-track"> | |
| <div class="score-bar-fill" style="width:100%; background:var(--purple)">1.00</div> | |
| </div> | |
| </div> | |
| <div style="border-top:2px solid var(--border); margin-top:12px; padding-top:12px; display:flex; justify-content:space-between; align-items:center;"> | |
| <span style="font-weight:700; color:#fff">Overall Score</span> | |
| <span style="font-size:24px; font-weight:700; color:var(--green-bright)">1.00</span> | |
| </div> | |
| </div> | |
| </section> | |
| <!-- ==================== SSE ==================== --> | |
| <section id="sse"> | |
| <h2>6. Live-Updates via Server-Sent Events</h2> | |
| <p>Die Webapp streamt Fortschritts-Updates in Echtzeit ohne Polling. Kein WebSocket-Overhead, | |
| kein periodisches Refetching — SSE ist ideal fuer unidirektionale Streams.</p> | |
| <div class="card blue"> | |
| <div style="display:flex; align-items:center; gap:12px; margin-bottom:16px;"> | |
| <span class="live-dot"></span> | |
| <h3 style="margin:0; color:var(--blue-bright)">SSE Endpoint: GET /runs/{id}/stream</h3> | |
| </div> | |
| <div class="grid2"> | |
| <div> | |
| <h4>Event-Typen</h4> | |
| <table> | |
| <thead><tr><th>Phase</th><th>Status-Text</th></tr></thead> | |
| <tbody> | |
| <tr><td><code>starting</code></td><td>Run wurde erstellt</td></tr> | |
| <tr><td><code>creating_sandbox</code></td><td>Docker-Container wird hochgefahren</td></tr> | |
| <tr><td><code>running_agent</code></td><td>Adapter laeuft (kann Minuten dauern)</td></tr> | |
| <tr><td><code>collecting_diff</code></td><td>git diff wird gesammelt</td></tr> | |
| <tr><td><code>creating_judge_sandbox</code></td><td>Judge-Container wird erstellt</td></tr> | |
| <tr><td><code>judging</code></td><td>4-Dimensionen-Bewertung laeuft</td></tr> | |
| <tr><td><code>done</code></td><td>Score verfuegbar, Connection close</td></tr> | |
| </tbody> | |
| </table> | |
| </div> | |
| <div> | |
| <h4>htmx-Integration</h4> | |
| <pre style="font-size:11px; margin-bottom:0;"><<span class="kw">div</span> | |
| <span class="nm">hx-ext</span>=<span class="st">"sse"</span> | |
| <span class="nm">sse-connect</span>=<span class="st">"/runs/{{id}}/stream"</span> | |
| <span class="nm">sse-swap</span>=<span class="st">"message"</span> | |
| <span class="nm">hx-target</span>=<span class="st">"#status-box"</span>> | |
| <<span class="kw">div</span> <span class="nm">id</span>=<span class="st">"status-box"</span>>Warte...</<span class="kw">div</span>> | |
| </<span class="kw">div</span>></pre> | |
| </div> | |
| </div> | |
| </div> | |
| <pre><span class="cm"># main.py — SSE Generator</span> | |
| <span class="kw">async def</span> <span class="fn">run_status_stream</span>(run_id: <span class="py">int</span>): | |
| <span class="kw">while</span> <span class="nm">True</span>: | |
| progress = runner.get_progress(run_id) | |
| <span class="kw">if</span> progress: | |
| status = progress.get(<span class="st">"status"</span>, <span class="st">"unknown"</span>) | |
| elapsed = time.time() - progress.get(<span class="st">"started_at"</span>, time.time()) | |
| <span class="kw">yield</span> <span class="st">f"data: {json.dumps({'status': status, 'elapsed': elapsed})}\n\n"</span> | |
| <span class="kw">else</span>: | |
| <span class="kw">yield</span> <span class="st">"data: {\"status\": \"done\"}\n\n"</span> | |
| <span class="kw">break</span> | |
| <span class="kw">await</span> asyncio.sleep(<span class="nm">0.5</span>) | |
| <span class="kw">return</span> StreamingResponse(run_status_stream(run_id), media_type=<span class="st">"text/event-stream"</span>)</pre> | |
| </section> | |
| <!-- ==================== FILE STRUCTURE ==================== --> | |
| <section id="filestructure"> | |
| <h2>7. Dateistruktur</h2> | |
| <div class="filetree"> | |
| <span class="ft-dir">AgentArena/</span> | |
| ├── <span class="ft-file">main.py</span> <span class="ft-desc"># FastAPI app, Routen, SSE-Endpoints, Jinja2-Templates</span> | |
| ├── <span class="ft-file">models.py</span> <span class="ft-desc"># SQLAlchemy Models: Task, Agent, Run, RunStatus (Enum)</span> | |
| ├── <span class="ft-file">database.py</span> <span class="ft-desc"># SessionLocal, engine, Base — SQLite via SQLAlchemy</span> | |
| ├── <span class="ft-file">config.py</span> <span class="ft-desc"># Pricing-Tabellen, Docker-Config, CLI-Pfade, Pass@k Defaults</span> | |
| ├── <span class="ft-dir">core/</span> | |
| │ ├── <span class="ft-file">runner.py</span> <span class="ft-desc"># Orchestrierung: run_single(), run_pass_at_k(), get_progress()</span> | |
| │ ├── <span class="ft-file">sandbox.py</span> <span class="ft-desc"># Docker-Lifecycle: create(), exec(), destroy(), create_judge_sandbox()</span> | |
| │ ├── <span class="ft-file">judge.py</span> <span class="ft-desc"># 4-Dimensionen Judge: functional, regression, lint, complexity</span> | |
| │ └── <span class="ft-file">metrics.py</span> <span class="ft-desc"># Trajectory-Analyse, Diff-Analyse, Pass@k, Composite Score</span> | |
| ├── <span class="ft-dir">adapters/</span> | |
| │ ├── <span class="ft-file">base.py</span> <span class="ft-desc"># AgentAdapter ABC, RunContext dataclass, RunResult dataclass</span> | |
| │ ├── <span class="ft-file">claude_code.py</span> <span class="ft-desc"># Claude Code CLI — OAuth, JSON output, Host-Tempdir</span> | |
| │ ├── <span class="ft-file">aider.py</span> <span class="ft-desc"># Aider CLI — MiniMax via litellm, Regex-Token-Parsing</span> | |
| │ └── <span class="ft-file">codex.py</span> <span class="ft-desc"># Codex CLI — noch nicht getestet</span> | |
| ├── <span class="ft-dir">templates/</span> <span class="ft-desc"># Jinja2 HTML-Templates</span> | |
| │ ├── <span class="ft-file">base.html</span> <span class="ft-desc"># Basis-Layout, Tailwind CDN, htmx, SSE-Extension</span> | |
| │ ├── <span class="ft-file">index.html</span> <span class="ft-desc"># Dashboard: Leaderboard, Run-Trigger</span> | |
| │ └── <span class="ft-file">run_detail.html</span> <span class="ft-desc"># Run-Details: Diff, Judge-Breakdown, Trajectory, SSE</span> | |
| ├── <span class="ft-dir">tasks/</span> <span class="ft-desc"># Task-Definitionen als YAML</span> | |
| │ └── <span class="ft-file">*.yaml</span> <span class="ft-desc"># slug, title, repo_url, success_cmd, time_limit_s</span> | |
| ├── <span class="ft-dir">docs/</span> | |
| │ └── <span class="ft-file">architecture.html</span> <span class="ft-desc"># Diese Datei</span> | |
| └── <span class="ft-dir">static/</span> <span class="ft-desc"># CSS/JS Assets</span> | |
| </div> | |
| <h3 style="margin-top:20px;">Datenmodelle</h3> | |
| <div class="grid3"> | |
| <div class="card blue"> | |
| <h4>Task</h4> | |
| <table style="font-size:11px;"> | |
| <tbody> | |
| <tr><td><code>slug</code></td><td>Unique Identifier</td></tr> | |
| <tr><td><code>repo_url</code></td><td>Git-Repository URL</td></tr> | |
| <tr><td><code>success_cmd</code></td><td>pytest Befehl</td></tr> | |
| <tr><td><code>time_limit_s</code></td><td>Timeout (default 300s)</td></tr> | |
| <tr><td><code>token_limit</code></td><td>Max Tokens (200k)</td></tr> | |
| <tr><td><code>difficulty</code></td><td>easy/medium/hard</td></tr> | |
| </tbody> | |
| </table> | |
| </div> | |
| <div class="card green"> | |
| <h4>Agent</h4> | |
| <table style="font-size:11px;"> | |
| <tbody> | |
| <tr><td><code>name</code></td><td>Unique Name</td></tr> | |
| <tr><td><code>adapter_type</code></td><td>claude_code / aider / codex</td></tr> | |
| <tr><td><code>config_json</code></td><td>Adapter-Konfiguration</td></tr> | |
| </tbody> | |
| </table> | |
| </div> | |
| <div class="card orange"> | |
| <h4>Run</h4> | |
| <table style="font-size:11px;"> | |
| <tbody> | |
| <tr><td><code>status</code></td><td>pending/running/success/failure/error/timeout</td></tr> | |
| <tr><td><code>judge_score</code></td><td>0.0 – 1.0</td></tr> | |
| <tr><td><code>cost_usd</code></td><td>Kosten in Dollar</td></tr> | |
| <tr><td><code>diff_text</code></td><td>Unified Diff</td></tr> | |
| <tr><td><code>trajectory</code></td><td>JSON Array (Tool-Events)</td></tr> | |
| <tr><td><code>metrics_json</code></td><td>Alle Metriken als JSON</td></tr> | |
| <tr><td><code>pass_at_k_group</code></td><td>UUID fuer k-Gruppen</td></tr> | |
| </tbody> | |
| </table> | |
| </div> | |
| </div> | |
| </section> | |
| <!-- ==================== RESULTS ==================== --> | |
| <section id="results"> | |
| <h2>8. Bisherige Ergebnisse</h2> | |
| <div class="leaderboard" style="margin-bottom:24px; overflow-x:auto;"> | |
| <div class="lb-row header"> | |
| <div>Rank</div> | |
| <div>Agent</div> | |
| <div>Score</div> | |
| <div>Tests</div> | |
| <div>Kosten</div> | |
| <div>Dauer</div> | |
| <div>Modell</div> | |
| </div> | |
| <div class="lb-row"> | |
| <div class="lb-rank gold">1</div> | |
| <div><strong>Aider</strong></div> | |
| <div class="lb-score perfect">1.00</div> | |
| <div>5/5</div> | |
| <div style="color:var(--green-bright)"><strong>~$0.00</strong></div> | |
| <div>~90s</div> | |
| <div><code>MiniMax-M2.5</code></div> | |
| </div> | |
| <div class="lb-row"> | |
| <div class="lb-rank silver">2</div> | |
| <div><strong>Claude Code</strong></div> | |
| <div class="lb-score good">0.80</div> | |
| <div>5/5</div> | |
| <div style="color:var(--orange-bright)">~$0.15</div> | |
| <div>~120s</div> | |
| <div><code>claude-sonnet-4-6</code></div> | |
| </div> | |
| </div> | |
| <div class="grid2"> | |
| <div class="card orange"> | |
| <h3 style="color:var(--orange-bright)">Wichtige Beobachtung</h3> | |
| <p>Aider mit MiniMax-M2.5 erzielte einen <strong>perfekten Score bei nahezu keinen Kosten</strong>. | |
| MiniMax-M2.5 kostet $0.05/1M Tokens — ca. 200x guenstiger als Claude Sonnet.</p> | |
| <p>Dies unterstreicht die Notwendigkeit von <strong>explizitem Kosten-Tracking</strong>: | |
| Reine Score-Vergleiche ohne Kostendimension sind irreführend.</p> | |
| </div> | |
| <div class="card blue"> | |
| <h3 style="color:var(--blue-bright)">Cost-Performance Ratio</h3> | |
| <div style="display:grid; grid-template-columns:1fr 1fr; gap:12px; margin-top:8px;"> | |
| <div class="metric-card"> | |
| <div class="metric-val" style="color:var(--green-bright)">$0.00</div> | |
| <div class="metric-label">Aider + MiniMax<br>Score 1.0</div> | |
| </div> | |
| <div class="metric-card"> | |
| <div class="metric-val" style="color:var(--orange-bright)">$0.15</div> | |
| <div class="metric-label">Claude Code Sonnet<br>Score 0.80</div> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| <h3>Was kommt als naechstes?</h3> | |
| <div class="grid3"> | |
| <div class="card"> | |
| <h4>Mehr Agenten</h4> | |
| <p>CodexAdapter testen. GPT-4o als Basis-Vergleich. Lokale Modelle via Ollama.</p> | |
| </div> | |
| <div class="card"> | |
| <h4>Mehr Tasks</h4> | |
| <p>Schwerere Aufgaben (Refactoring, Multi-file, API Integration). | |
| Task-YAML fuer einfaches Hinzufuegen.</p> | |
| </div> | |
| <div class="card"> | |
| <h4>Pass@k Analyse</h4> | |
| <p>Parallele Runs fuer statistische Belastbarkeit. | |
| Variance-Analyse: Welche Agenten sind konsistenter?</p> | |
| </div> | |
| </div> | |
| </section> | |
| <hr class="divider"> | |
| <div style="text-align:center; color:var(--muted); font-size:12px; padding:20px 0;"> | |
| AgentArena — Erstellt Maerz 2026 — FastAPI + Docker + SQLite | |
| </div> | |
| </div> | |
| <script> | |
| // Tab-Switcher | |
| function showTab(id) { | |
| document.querySelectorAll('.tab-content').forEach(el => el.classList.remove('active')); | |
| document.querySelectorAll('.tab').forEach(el => el.classList.remove('active')); | |
| document.getElementById('tab-' + id).classList.add('active'); | |
| event.target.classList.add('active'); | |
| } | |
| // Smooth scroll for nav | |
| document.querySelectorAll('nav a').forEach(a => { | |
| a.addEventListener('click', e => { | |
| e.preventDefault(); | |
| const target = document.querySelector(a.getAttribute('href')); | |
| if (target) target.scrollIntoView({ behavior: 'smooth' }); | |
| }); | |
| }); | |
| // Active nav highlight on scroll | |
| const sections = document.querySelectorAll('section[id]'); | |
| const navLinks = document.querySelectorAll('nav a'); | |
| const observer = new IntersectionObserver(entries => { | |
| entries.forEach(entry => { | |
| if (entry.isIntersecting) { | |
| navLinks.forEach(a => { | |
| a.style.background = a.getAttribute('href') === '#' + entry.target.id | |
| ? 'var(--surface2)' : ''; | |
| a.style.color = a.getAttribute('href') === '#' + entry.target.id | |
| ? 'var(--text)' : ''; | |
| }); | |
| } | |
| }); | |
| }, { threshold: 0.3, rootMargin: '-60px 0px -60% 0px' }); | |
| sections.forEach(s => observer.observe(s)); | |
| // Animate score bars on load | |
| window.addEventListener('load', () => { | |
| document.querySelectorAll('.score-bar-fill').forEach(bar => { | |
| const w = bar.style.width; | |
| bar.style.width = '0%'; | |
| setTimeout(() => { bar.style.width = w; }, 200); | |
| }); | |
| }); | |
| </script> | |
| </body> | |
| </html> | |