Publish verified EvalArc 483f84999ee1

coding/index.html

@@ -5,12 +5,13 @@
  content="default-src 'none'; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'">
 <title>EvalArc &#183; Grader audit</title>
 <style>
+
 :root{color-scheme:dark;font:16px/1.6 system-ui,sans-serif;background:#101618;color:#e0ece7}
 body{max-width:1080px;margin:60px auto;padding:0 24px}
 .eyebrow{color:#86dbaf;letter-spacing:.18em;font-size:12px;font-weight:700}
 h1{font-size:clamp(32px,6vw,64px);letter-spacing:-.045em;line-height:1.1;margin:16px 0}
 p{color:#acbeb5;max-width:760px}.cards{display:flex;gap:20px;flex-wrap:wrap;margin:36px 0}
-.card{border:1px solid #33463d;border-radius:12px;padding:18px 26px;flex:1;min-width:180px}
+.card{border:1px solid #33463d;border-radius:12px;padding:18px 26px;flex:1;min-width:150px}
 .number{font-size:38px;color:#96e8b9;font-weight:650}.label{color:#b7c6bf}
 .scroll{overflow:auto}table{border-collapse:collapse;width:100%;font-size:14px}
 td,th{text-align:left;padding:14px;border-bottom:1px solid #33463d}th{color:#86dbaf}
@@ -21,9 +22,231 @@ footer{margin-top:48px;border-top:1px solid #33463d;padding-top:20px;font-size:1
 .metadata{overflow-wrap:anywhere;font-size:13px}
 pre{overflow:auto;padding:18px;background:#17221c;font-size:12px;max-height:480px}
 summary{cursor:pointer;color:#96e8b9}details{margin:16px 0}
+summary{min-height:44px;padding:10px 0}
+a:focus-visible,summary:focus-visible{outline:3px solid #96e8b9;outline-offset:4px}
+.audit-control{border:1px solid #33463d;border-radius:12px;padding:12px 18px}
+.audit-control summary{overflow-wrap:anywhere}.audit-control p{overflow-wrap:anywhere}
+.audit-coverage{border-left:3px solid #edb68d;padding:12px 20px;background:#17221c}
+@media(max-width:640px){body{margin:28px auto;padding:0 16px}.cards{gap:12px}
+.card{min-width:0;flex-basis:100%}.dimension{flex-wrap:wrap}.audit-control{padding:10px 14px}}
+a{color:#96e8b9}.failed,.agent_error,.environment_error{color:#edb68d}
+.passed{color:#96e8b9}td{overflow-wrap:anywhere}
+.scroll table{min-width:680px}td:first-child{min-width:170px;overflow-wrap:normal}
+.change-cards{display:none}.change-card{border:1px solid #33463d;border-radius:12px;padding:16px}
+.change-card h3{font-size:16px;margin:0;overflow-wrap:anywhere}.change-card p{margin:10px 0 0}
+@media(max-width:640px){.change-table{display:none}.change-cards{display:grid;gap:16px}}
+
 </style>
 <div class="eyebrow">EVALARC / DEVELOPMENT AUDIT</div>
 <h1>Test the grader.<br>Then trust the signal.</h1>
 <p>Behavioral controls for task-specific agent evaluations. Correct and faulty
 submissions are evaluated against the same externally enforced contract.</p>
-<div class="cards"><div class="card"><div class="number">8/8</div><div class="label">declared faults detected</div></div><div class="card"><div class="number">1.000</div><div class="label">reference correctness</div></div><div class="card"><div class="number">docker</div><div class="label">execution backend</div></div></div><h2>Does the grader detect plausible defects?</h2><div class="scroll"><table><thead><tr><th>Negative control</th><th>Target</th><th>Candidate score</th><th>Result</th><th>Evidence</th></tr></thead><tbody><tr><td><code>ack-without-work</code></td><td>basic</td><td>0.000</td><td>Detected</td><td>json-types-and-overwrite, read-write-delete, seeded-state-machine</td></tr><tr><td><code>memory-only</code></td><td>persistence</td><td>0.783</td><td>Detected</td><td>restart-preserves-acknowledged-write, restart-preserves-cas, restart-preserves-transaction</td></tr><tr><td><code>partial-batch</code></td><td>transactions</td><td>0.900</td><td>Detected</td><td>rollback-batch</td></tr><tr><td><code>cas-always-wins</code></td><td>compare_swap</td><td>0.850</td><td>Detected</td><td>cas-match-and-mismatch, cas-type-sensitivity</td></tr><tr><td><code>boolean-equals-one</code></td><td>compare_swap</td><td>0.925</td><td>Detected</td><td>cas-type-sensitivity</td></tr><tr><td><code>delete-noop</code></td><td>basic</td><td>0.650</td><td>Detected</td><td>read-write-delete, seeded-state-machine</td></tr><tr><td><code>accept-nonstring-keys</code></td><td>validation</td><td>0.925</td><td>Detected</td><td>reject-and-continue</td></tr><tr><td><code>commit-on-exit</code></td><td>crash_recovery</td><td>0.900</td><td>Detected</td><td>sigkill-after-acknowledgement, sigkill-after-atomic-batch, sigkill-after-delete</td></tr></tbody></table></div><h2>Positive control</h2><div class="dimension"><span>basic</span><meter min="0" max="1" value="1.0">1.00</meter><span>3/3</span></div><div class="dimension"><span>validation</span><meter min="0" max="1" value="1.0">1.00</meter><span>2/2</span></div><div class="dimension"><span>transactions</span><meter min="0" max="1" value="1.0">1.00</meter><span>2/2</span></div><div class="dimension"><span>compare_swap</span><meter min="0" max="1" value="1.0">1.00</meter><span>2/2</span></div><div class="dimension"><span>persistence</span><meter min="0" max="1" value="1.0">1.00</meter><span>3/3</span></div><div class="dimension"><span>crash_recovery</span><meter min="0" max="1" value="1.0">1.00</meter><span>3/3</span></div><h2>Reproduction record</h2><p class="metadata">Task: durable-kv v0.1.0<br>Seeds: [17]<br>Grader SHA-256: 52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d<br>Cases SHA-256: 842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a<br>Image ID: sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea<br>Created: 2026-09-14T01:29:51.125110+00:00</p><p><a href="audit.json">Complete JSON evidence for all controls</a></p><footer><strong>Scope:</strong> This is a scripted grader audit, not an AI leaderboard. Public seeds are not held-out evaluation data. Detection rates apply only to the listed fault models. No human time horizon, model quality, or RL improvement is inferred.</footer></html>
+<div class="cards"><div class="card"><div class="number">8/8</div><div class="label">declared faults detected</div></div><div class="card"><div class="number">1.000</div><div class="label">reference correctness</div></div><div class="card"><div class="number">docker</div><div class="label">execution backend</div></div></div><section class="audit-coverage" aria-labelledby="coverage-title"><h2 id="coverage-title">Which faults depend on one case?</h2><p><strong>3 of 8</strong> declared faults have exactly one distinct detecting case in the assessed records. Repeating a case under more seeds does not increase this margin. A surviving fault has margin zero; an environment failure remains unassessed.</p><p>Margins below are derived from the saved failing case IDs. Rendering an older audit does not rerun its controls.</p></section><h2>Does the grader detect plausible defects?</h2><div class="scroll" tabindex="0" role="region" aria-label="Declared fault coverage"><table><thead><tr><th>Negative control</th><th>Target</th><th>Candidate score</th><th>Result</th><th title="Cases that caught this fault independently. One means the suite loses this fault if that case changes.">Margin</th><th>Evidence</th></tr></thead><tbody><tr><td><code>ack-without-work</code></td><td>basic</td><td>0.000</td><td>Detected</td><td>3</td><td><a href="#fault-0">Inspect recorded cases</a></td></tr><tr><td><code>memory-only</code></td><td>persistence</td><td>0.783</td><td>Detected</td><td>3</td><td><a href="#fault-1">Inspect recorded cases</a></td></tr><tr><td><code>partial-batch</code></td><td>transactions</td><td>0.900</td><td>Detected</td><td>1</td><td><a href="#fault-2">Inspect recorded cases</a></td></tr><tr><td><code>cas-always-wins</code></td><td>compare_swap</td><td>0.850</td><td>Detected</td><td>2</td><td><a href="#fault-3">Inspect recorded cases</a></td></tr><tr><td><code>boolean-equals-one</code></td><td>compare_swap</td><td>0.925</td><td>Detected</td><td>1</td><td><a href="#fault-4">Inspect recorded cases</a></td></tr><tr><td><code>delete-noop</code></td><td>basic</td><td>0.650</td><td>Detected</td><td>2</td><td><a href="#fault-5">Inspect recorded cases</a></td></tr><tr><td><code>accept-nonstring-keys</code></td><td>validation</td><td>0.925</td><td>Detected</td><td>1</td><td><a href="#fault-6">Inspect recorded cases</a></td></tr><tr><td><code>commit-on-exit</code></td><td>crash_recovery</td><td>0.900</td><td>Detected</td><td>3</td><td><a href="#fault-7">Inspect recorded cases</a></td></tr></tbody></table></div><h2>Inspect the detecting cases</h2><details class="audit-control"><summary>ack-without-work &#183; 3 detecting cases</summary><div id="fault-0"><p>Target: basic. Detecting case IDs: json-types-and-overwrite, read-write-delete, seeded-state-machine.</p><details><summary>read-write-delete &#183; seed 17 &#183; failed</summary><pre>{
+  &quot;seed&quot;: 17,
+  &quot;case_id&quot;: &quot;read-write-delete&quot;,
+  &quot;dimension&quot;: &quot;basic&quot;,
+  &quot;checks&quot;: {
+    &quot;basic&quot;: false
+  },
+  &quot;passed&quot;: false,
+  &quot;status&quot;: &quot;failed&quot;,
+  &quot;checks_completed&quot;: 0,
+  &quot;duration_seconds&quot;: 0.608378,
+  &quot;transcript_sha256&quot;: &quot;e9d1c1a4ce952f925f5a3d00a84348bf482007670ba13602f73d0129cc422a60&quot;,
+  &quot;error&quot;: &quot;response mismatch at request 1; expected {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}, got {\&quot;ok\&quot;: true}&quot;
+}</pre></details><details><summary>json-types-and-overwrite &#183; seed 17 &#183; failed</summary><pre>{
+  &quot;seed&quot;: 17,
+  &quot;case_id&quot;: &quot;json-types-and-overwrite&quot;,
+  &quot;dimension&quot;: &quot;basic&quot;,
+  &quot;checks&quot;: {
+    &quot;basic&quot;: false
+  },
+  &quot;passed&quot;: false,
+  &quot;status&quot;: &quot;failed&quot;,
+  &quot;checks_completed&quot;: 1,
+  &quot;duration_seconds&quot;: 0.563549,
+  &quot;transcript_sha256&quot;: &quot;ad684a2e4e62254b35236e279cda7ec03bfe6bf6a01e3c0b24a1810e6c8088d2&quot;,
+  &quot;error&quot;: &quot;response mismatch at request 2; expected {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: null}, got {\&quot;ok\&quot;: true}&quot;
+}</pre></details><details><summary>seeded-state-machine &#183; seed 17 &#183; failed</summary><pre>{
+  &quot;seed&quot;: 17,
+  &quot;case_id&quot;: &quot;seeded-state-machine&quot;,
+  &quot;dimension&quot;: &quot;basic&quot;,
+  &quot;checks&quot;: {
+    &quot;basic&quot;: false
+  },
+  &quot;passed&quot;: false,
+  &quot;status&quot;: &quot;failed&quot;,
+  &quot;checks_completed&quot;: 2,
+  &quot;duration_seconds&quot;: 0.497166,
+  &quot;transcript_sha256&quot;: &quot;ce6e264c70b36102aabe44145d2a7558766913a2725261b8f120efea98fa8659&quot;,
+  &quot;error&quot;: &quot;response mismatch at request 3; expected {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}, got {\&quot;ok\&quot;: true}&quot;
+}</pre></details></div></details><details class="audit-control"><summary>memory-only &#183; 3 detecting cases</summary><div id="fault-1"><p>Target: persistence. Detecting case IDs: restart-preserves-acknowledged-write, restart-preserves-cas, restart-preserves-transaction.</p><details><summary>restart-preserves-acknowledged-write &#183; seed 17 &#183; failed</summary><pre>{
+  &quot;seed&quot;: 17,
+  &quot;case_id&quot;: &quot;restart-preserves-acknowledged-write&quot;,
+  &quot;dimension&quot;: &quot;persistence&quot;,
+  &quot;checks&quot;: {
+    &quot;persistence&quot;: false
+  },
+  &quot;passed&quot;: false,
+  &quot;status&quot;: &quot;failed&quot;,
+  &quot;checks_completed&quot;: 1,
+  &quot;duration_seconds&quot;: 0.976095,
+  &quot;transcript_sha256&quot;: &quot;f62cc3fb4db094e24ad23141f7cc50908d24647156c4d2ba2386a2487c078e80&quot;,
+  &quot;error&quot;: &quot;response mismatch at request 2; expected {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: {\&quot;items\&quot;: [true, null, \&quot;\\u03bb\\n\\u4e16\\u754c\&quot;], \&quot;version\&quot;: 5991}}, got {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}&quot;
+}</pre></details><details><summary>restart-preserves-transaction &#183; seed 17 &#183; failed</summary><pre>{
+  &quot;seed&quot;: 17,
+  &quot;case_id&quot;: &quot;restart-preserves-transaction&quot;,
+  &quot;dimension&quot;: &quot;persistence&quot;,
+  &quot;checks&quot;: {
+    &quot;persistence&quot;: false
+  },
+  &quot;passed&quot;: false,
+  &quot;status&quot;: &quot;failed&quot;,
+  &quot;checks_completed&quot;: 1,
+  &quot;duration_seconds&quot;: 0.996261,
+  &quot;transcript_sha256&quot;: &quot;43cf3772fd4bd95486b1824b1d73c4e99859fb85088a7f62192d8325b4581b29&quot;,
+  &quot;error&quot;: &quot;response mismatch at request 2; expected {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: {\&quot;items\&quot;: [true, null, \&quot;\\u03bb\\n\\u4e16\\u754c\&quot;], \&quot;version\&quot;: 5991}}, got {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}&quot;
+}</pre></details><details><summary>restart-preserves-cas &#183; seed 17 &#183; failed</summary><pre>{
+  &quot;seed&quot;: 17,
+  &quot;case_id&quot;: &quot;restart-preserves-cas&quot;,
+  &quot;dimension&quot;: &quot;persistence&quot;,
+  &quot;checks&quot;: {
+    &quot;persistence&quot;: false
+  },
+  &quot;passed&quot;: false,
+  &quot;status&quot;: &quot;failed&quot;,
+  &quot;checks_completed&quot;: 2,
+  &quot;duration_seconds&quot;: 0.997537,
+  &quot;transcript_sha256&quot;: &quot;497082839111501c6aa4912312fe30686befcab05beb84ebbcb99be0d1e8aef1&quot;,
+  &quot;error&quot;: &quot;response mismatch at request 3; expected {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: 321}, got {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}&quot;
+}</pre></details></div></details><details class="audit-control"><summary>partial-batch &#183; single-case dependency</summary><div id="fault-2"><p>Target: transactions. Detecting case IDs: rollback-batch.</p><details><summary>rollback-batch &#183; seed 17 &#183; failed</summary><pre>{
+  &quot;seed&quot;: 17,
+  &quot;case_id&quot;: &quot;rollback-batch&quot;,
+  &quot;dimension&quot;: &quot;transactions&quot;,
+  &quot;checks&quot;: {
+    &quot;transactions&quot;: false
+  },
+  &quot;passed&quot;: false,
+  &quot;status&quot;: &quot;failed&quot;,
+  &quot;checks_completed&quot;: 2,
+  &quot;duration_seconds&quot;: 0.497347,
+  &quot;transcript_sha256&quot;: &quot;f6015865dce884224cf6ccccc9fbdd367baad17842dde00fbee476f06b1aab0d&quot;,
+  &quot;error&quot;: &quot;response mismatch at request 3; expected {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: {\&quot;items\&quot;: [true, null, \&quot;\\u03bb\\n\\u4e16\\u754c\&quot;], \&quot;version\&quot;: 5991}}, got {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: \&quot;must roll back\&quot;}&quot;
+}</pre></details></div></details><details class="audit-control"><summary>cas-always-wins &#183; 2 detecting cases</summary><div id="fault-3"><p>Target: compare_swap. Detecting case IDs: cas-match-and-mismatch, cas-type-sensitivity.</p><details><summary>cas-match-and-mismatch &#183; seed 17 &#183; failed</summary><pre>{
+  &quot;seed&quot;: 17,
+  &quot;case_id&quot;: &quot;cas-match-and-mismatch&quot;,
+  &quot;dimension&quot;: &quot;compare_swap&quot;,
+  &quot;checks&quot;: {
+    &quot;compare_swap&quot;: false
+  },
+  &quot;passed&quot;: false,
+  &quot;status&quot;: &quot;failed&quot;,
+  &quot;checks_completed&quot;: 1,
+  &quot;duration_seconds&quot;: 0.557099,
+  &quot;transcript_sha256&quot;: &quot;71199e89272547e9c1a1961a85a650b93c30e2f848912b6be28ea262907eda56&quot;,
+  &quot;error&quot;: &quot;response mismatch at request 2; expected {\&quot;ok\&quot;: true, \&quot;swapped\&quot;: false}, got {\&quot;ok\&quot;: true, \&quot;swapped\&quot;: true}&quot;
+}</pre></details><details><summary>cas-type-sensitivity &#183; seed 17 &#183; failed</summary><pre>{
+  &quot;seed&quot;: 17,
+  &quot;case_id&quot;: &quot;cas-type-sensitivity&quot;,
+  &quot;dimension&quot;: &quot;compare_swap&quot;,
+  &quot;checks&quot;: {
+    &quot;compare_swap&quot;: false
+  },
+  &quot;passed&quot;: false,
+  &quot;status&quot;: &quot;failed&quot;,
+  &quot;checks_completed&quot;: 1,
+  &quot;duration_seconds&quot;: 0.565552,
+  &quot;transcript_sha256&quot;: &quot;70231a539612a6973a06489411646982ea651ca401223316f7f670e6560a18bf&quot;,
+  &quot;error&quot;: &quot;response mismatch at request 2; expected {\&quot;ok\&quot;: true, \&quot;swapped\&quot;: false}, got {\&quot;ok\&quot;: true, \&quot;swapped\&quot;: true}&quot;
+}</pre></details></div></details><details class="audit-control"><summary>boolean-equals-one &#183; single-case dependency</summary><div id="fault-4"><p>Target: compare_swap. Detecting case IDs: cas-type-sensitivity.</p><details><summary>cas-type-sensitivity &#183; seed 17 &#183; failed</summary><pre>{
+  &quot;seed&quot;: 17,
+  &quot;case_id&quot;: &quot;cas-type-sensitivity&quot;,
+  &quot;dimension&quot;: &quot;compare_swap&quot;,
+  &quot;checks&quot;: {
+    &quot;compare_swap&quot;: false
+  },
+  &quot;passed&quot;: false,
+  &quot;status&quot;: &quot;failed&quot;,
+  &quot;checks_completed&quot;: 1,
+  &quot;duration_seconds&quot;: 0.57604,
+  &quot;transcript_sha256&quot;: &quot;70231a539612a6973a06489411646982ea651ca401223316f7f670e6560a18bf&quot;,
+  &quot;error&quot;: &quot;response mismatch at request 2; expected {\&quot;ok\&quot;: true, \&quot;swapped\&quot;: false}, got {\&quot;ok\&quot;: true, \&quot;swapped\&quot;: true}&quot;
+}</pre></details></div></details><details class="audit-control"><summary>delete-noop &#183; 2 detecting cases</summary><div id="fault-5"><p>Target: basic. Detecting case IDs: read-write-delete, seeded-state-machine.</p><details><summary>read-write-delete &#183; seed 17 &#183; failed</summary><pre>{
+  &quot;seed&quot;: 17,
+  &quot;case_id&quot;: &quot;read-write-delete&quot;,
+  &quot;dimension&quot;: &quot;basic&quot;,
+  &quot;checks&quot;: {
+    &quot;basic&quot;: false
+  },
+  &quot;passed&quot;: false,
+  &quot;status&quot;: &quot;failed&quot;,
+  &quot;checks_completed&quot;: 4,
+  &quot;duration_seconds&quot;: 0.557506,
+  &quot;transcript_sha256&quot;: &quot;4122b55bc205e4703a3a9f62a5397de448a833f424e54edc914bbbf5465c70d1&quot;,
+  &quot;error&quot;: &quot;response mismatch at request 5; expected {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}, got {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: {\&quot;items\&quot;: [true, null, \&quot;\\u03bb\\n\\u4e16\\u754c\&quot;], \&quot;version\&quot;: 5991}}&quot;
+}</pre></details><details><summary>seeded-state-machine &#183; seed 17 &#183; failed</summary><pre>{
+  &quot;seed&quot;: 17,
+  &quot;case_id&quot;: &quot;seeded-state-machine&quot;,
+  &quot;dimension&quot;: &quot;basic&quot;,
+  &quot;checks&quot;: {
+    &quot;basic&quot;: false
+  },
+  &quot;passed&quot;: false,
+  &quot;status&quot;: &quot;failed&quot;,
+  &quot;checks_completed&quot;: 8,
+  &quot;duration_seconds&quot;: 0.556806,
+  &quot;transcript_sha256&quot;: &quot;0132892f9b4bac6264eb6ef2dae9da8b7bac7a6a158b56357f1319c853615de7&quot;,
+  &quot;error&quot;: &quot;response mismatch at request 9; expected {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}, got {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: {\&quot;items\&quot;: [true, null, \&quot;\\u03bb\\n\\u4e16\\u754c\&quot;], \&quot;version\&quot;: 5991}}&quot;
+}</pre></details></div></details><details class="audit-control"><summary>accept-nonstring-keys &#183; single-case dependency</summary><div id="fault-6"><p>Target: validation. Detecting case IDs: reject-and-continue.</p><details><summary>reject-and-continue &#183; seed 17 &#183; failed</summary><pre>{
+  &quot;seed&quot;: 17,
+  &quot;case_id&quot;: &quot;reject-and-continue&quot;,
+  &quot;dimension&quot;: &quot;validation&quot;,
+  &quot;checks&quot;: {
+    &quot;validation&quot;: false
+  },
+  &quot;passed&quot;: false,
+  &quot;status&quot;: &quot;failed&quot;,
+  &quot;checks_completed&quot;: 4,
+  &quot;duration_seconds&quot;: 0.519073,
+  &quot;transcript_sha256&quot;: &quot;dc07c86f871afd6b2f208e9751bdd459e52368d54d008ca0622a8edd61a7008b&quot;,
+  &quot;error&quot;: &quot;response mismatch at request 5; expected {\&quot;error\&quot;: \&quot;invalid_request\&quot;, \&quot;ok\&quot;: false}, got {\&quot;ok\&quot;: true}&quot;
+}</pre></details></div></details><details class="audit-control"><summary>commit-on-exit &#183; 3 detecting cases</summary><div id="fault-7"><p>Target: crash_recovery. Detecting case IDs: sigkill-after-acknowledgement, sigkill-after-atomic-batch, sigkill-after-delete.</p><details><summary>sigkill-after-acknowledgement &#183; seed 17 &#183; failed</summary><pre>{
+  &quot;seed&quot;: 17,
+  &quot;case_id&quot;: &quot;sigkill-after-acknowledgement&quot;,
+  &quot;dimension&quot;: &quot;crash_recovery&quot;,
+  &quot;checks&quot;: {
+    &quot;crash_recovery&quot;: false
+  },
+  &quot;passed&quot;: false,
+  &quot;status&quot;: &quot;failed&quot;,
+  &quot;checks_completed&quot;: 1,
+  &quot;duration_seconds&quot;: 0.998141,
+  &quot;transcript_sha256&quot;: &quot;f62cc3fb4db094e24ad23141f7cc50908d24647156c4d2ba2386a2487c078e80&quot;,
+  &quot;error&quot;: &quot;response mismatch at request 2; expected {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: {\&quot;items\&quot;: [true, null, \&quot;\\u03bb\\n\\u4e16\\u754c\&quot;], \&quot;version\&quot;: 5991}}, got {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}&quot;
+}</pre></details><details><summary>sigkill-after-atomic-batch &#183; seed 17 &#183; failed</summary><pre>{
+  &quot;seed&quot;: 17,
+  &quot;case_id&quot;: &quot;sigkill-after-atomic-batch&quot;,
+  &quot;dimension&quot;: &quot;crash_recovery&quot;,
+  &quot;checks&quot;: {
+    &quot;crash_recovery&quot;: false
+  },
+  &quot;passed&quot;: false,
+  &quot;status&quot;: &quot;failed&quot;,
+  &quot;checks_completed&quot;: 1,
+  &quot;duration_seconds&quot;: 0.978949,
+  &quot;transcript_sha256&quot;: &quot;43cf3772fd4bd95486b1824b1d73c4e99859fb85088a7f62192d8325b4581b29&quot;,
+  &quot;error&quot;: &quot;response mismatch at request 2; expected {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: {\&quot;items\&quot;: [true, null, \&quot;\\u03bb\\n\\u4e16\\u754c\&quot;], \&quot;version\&quot;: 5991}}, got {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}&quot;
+}</pre></details><details><summary>sigkill-after-delete &#183; seed 17 &#183; failed</summary><pre>{
+  &quot;seed&quot;: 17,
+  &quot;case_id&quot;: &quot;sigkill-after-delete&quot;,
+  &quot;dimension&quot;: &quot;crash_recovery&quot;,
+  &quot;checks&quot;: {
+    &quot;crash_recovery&quot;: false
+  },
+  &quot;passed&quot;: false,
+  &quot;status&quot;: &quot;failed&quot;,
+  &quot;checks_completed&quot;: 2,
+  &quot;duration_seconds&quot;: 1.47289,
+  &quot;transcript_sha256&quot;: &quot;679e7ab1f7f833fb12551b29af4fdc1ad8e1051810e594a4da85fdc3420b8c42&quot;,
+  &quot;error&quot;: &quot;response mismatch at request 3; expected {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}, got {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: {\&quot;items\&quot;: [true, null, \&quot;\\u03bb\\n\\u4e16\\u754c\&quot;], \&quot;version\&quot;: 5991}}&quot;
+}</pre></details></div></details><h2>Positive control</h2><div class="dimension"><span>basic</span><meter min="0" max="1" value="1.0">1.00</meter><span>3/3</span></div><div class="dimension"><span>validation</span><meter min="0" max="1" value="1.0">1.00</meter><span>2/2</span></div><div class="dimension"><span>transactions</span><meter min="0" max="1" value="1.0">1.00</meter><span>2/2</span></div><div class="dimension"><span>compare_swap</span><meter min="0" max="1" value="1.0">1.00</meter><span>2/2</span></div><div class="dimension"><span>persistence</span><meter min="0" max="1" value="1.0">1.00</meter><span>3/3</span></div><div class="dimension"><span>crash_recovery</span><meter min="0" max="1" value="1.0">1.00</meter><span>3/3</span></div><h2>Reproduction record</h2><p class="metadata">Task: durable-kv v0.1.0<br>Seeds: [17]<br>Grader SHA-256: 52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d<br>Cases SHA-256: 842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a<br>Image ID: sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea<br>Created: 2026-09-14T01:29:51.125110+00:00</p><p><a href="audit.json">Complete JSON evidence for all controls</a></p><footer><strong>Scope:</strong> This is a scripted grader audit, not an AI leaderboard. Public seeds are not held-out evaluation data. Detection rates apply only to the listed fault models. No human time horizon, model quality, or RL improvement is inferred.</footer></html>

index.html

@@ -44,7 +44,14 @@
       <div><strong>JSON</strong><span>downloadable evidence</span></div>
       <div><strong>CPU</strong><span>reproduce without a model API</span></div>
     </div>
-    <nav class="section-nav" aria-label="Evidence sections"><a href="#suite">Acceptance gates</a><a href="#repeat">Repeated attempts</a><a href="#regression">Revision comparison</a><a href="#explorer">Case explorer</a></nav>
+    <nav class="section-nav" aria-label="Evidence sections"><a href="#coverage">Audit coverage</a><a href="#suite">Acceptance gates</a><a href="#repeat">Repeated attempts</a><a href="#regression">Revision comparison</a><a href="#explorer">Case explorer</a></nav>
+    <section id="coverage" aria-labelledby="coverage-title">
+      <p class="eyebrow">THREE TASK PACKS / DECLARED FAULTS</p>
+      <h2 id="coverage-title">A perfect score.<br>How much coverage remains?</h2>
+      <p class="muted">A fault detected by one case loses its coverage if that case is removed or weakened. Open each dependency to inspect the recorded checks, status and seed. Margins count distinct case IDs, not repeated runs.</p>
+      <div class="coverage-grid"><article class="coverage-card"><h3>durable-kv</h3><p><strong>8/8</strong> declared faults detected</p><p class="coverage-fragile">3 single-case dependencies</p><ul><li><a href="coding/index.html#fault-2">partial-batch</a><span> Sole case: rollback-batch</span></li><li><a href="coding/index.html#fault-4">boolean-equals-one</a><span> Sole case: cas-type-sensitivity</span></li><li><a href="coding/index.html#fault-6">accept-nonstring-keys</a><span> Sole case: reject-and-continue</span></li></ul><a href="coding/index.html">Inspect all controls &#8599;</a></article><article class="coverage-card"><h3>support-routing</h3><p><strong>7/7</strong> declared faults detected</p><p class="coverage-fragile">2 single-case dependencies</p><ul><li><a href="support/index.html#fault-5">skip-retry</a><span> Sole case: retry-before-commit</span></li><li><a href="support/index.html#fault-6">new-key-on-retry</a><span> Sole case: retry-after-commit</span></li></ul><a href="support/index.html">Inspect all controls &#8599;</a></article><article class="coverage-card"><h3>robot-evidence-review</h3><p><strong>6/6</strong> declared faults detected</p><p class="coverage-fragile">1 single-case dependencies</p><ul><li><a href="robot/index.html#fault-3">assume-complete</a><span> Sole case: incomplete-recording</span></li></ul><a href="robot/index.html">Inspect all controls &#8599;</a></article></div>
+      <p class="muted">These summaries derive from the original scripted audit records. They are not new agent runs or a guarantee against unseen faults.</p>
+    </section>
     <section id="suite" class="suite">
       <div class="section-top"><div><p class="eyebrow">NEW IN 0.5 / DECLARE YOUR ACCEPTANCE RULES</p><h2>Same score. Different gate.</h2></div><p>One frozen policy.<br>Two explicit acceptance rules.</p></div>
       <p class="muted">Both support jobs score 93.75% and resolve 0/2 attempts. A deliberately permissive gate accepts the result; requiring every notes check to pass rejects it. The task outcome stays the same.</p>

manifest.json

@@ -1,7 +1,7 @@
 {
   "schema": "evalarc.site.v1",
   "source_repository": "https://github.com/noteflowai/evalarc",
-  "source_commit": "754f2bd9bae7507239f8388551ccfe3e7b80ef6e",
+  "source_commit": "483f84999ee124c75f574e434b988b47f603cc3b",
   "source_dirty": false,
   "evidence": "Scripted grader controls and separately labeled recorded GPU model pilots; no live inference.",
   "files": {
@@ -10,7 +10,7 @@
     "README.md": "42469dc640f89338e95b9a185fd55624effaef8ad39ec2ade3951ca18f0c482f",
     "app.js": "efc8c50e0b449fde523780ad9f1c09b0651b70673d51419aa9b01b725657be4f",
     "coding/audit.json": "49aca7663ec3313eb6cacccfdea69a0380de62b7f9886bdd76add63e65b12c08",
-    "coding/index.html": "0e71ef2840443bc4e2a60e78802b1417eef68d6450af233a6964656c4e5def3d",
+    "coding/index.html": "044aa35be57c714cc4aeaaa6d61f0eba76c143a4a3b047d7313ced7fe11a2bc1",
     "comparison/baseline.json": "4ec043dd1de0b0d852737cf6e8f2ebe04d517d0c945335e686773e00a8f970b5",
     "comparison/comparison.json": "6a5478a96995a502aa78ff9a6c6a0cd32832ae5160f6ce271f19ff3fa5ea1e28",
     "comparison/current.json": "40d0726cd8378b237c514d69b4fda9e4f00550dc3012f4c276a360f4deed25ad",
@@ -18,7 +18,7 @@
     "evaluation/evaluation.json": "40d0726cd8378b237c514d69b4fda9e4f00550dc3012f4c276a360f4deed25ad",
     "evaluation/index.html": "d50a498960c71cf8b1a3a7074cb0de1bb73dc974a8cd57af5c4ea1d8c6cc75d6",
     "icon.svg": "722d0197e1953770ed2a2f470d066db9c79ac0549149c755fa60d7489ef582e7",
-    "index.html": "9e8ac3cdb9b655c59b6699c28c88b754a387a46277383fffd3ef03d85ba3d07f",
+    "index.html": "6e196a31e78d34567f30ee8aa0077c7d4bf57617ff82cdc211a95f35fea615b7",
     "repeat/faulty/attempts/0001/evaluation.json": "c9ed9542f1240a6706499f46ab80542e4963097e43aa7e21c5191730d32bbfe6",
     "repeat/faulty/attempts/0001/index.html": "cdf30b2f9f4bbbff8254b719f136752d43854c4a64e42f08b7cbd46d2224515c",
     "repeat/faulty/attempts/0002/evaluation.json": "bbb3a86918a6d88aca4d906dce2e0a45c6104861874f763a6897834658b6f2e2",
@@ -418,6 +418,8 @@
     "research/skill-impact-matched/experiment.json": "f2b8776c2533ccf4951a902516e06e3db3d521eb25990f7d44adb63696ee47c1",
     "research/skill-impact-matched/readiness-regrade.json": "6683a7128e5770859069d7779a89305fcdaba9e55738da4dc74458246467ba28",
     "research/skill-impact-matched/summary.json": "fc52cb8a729b162b34bb7827290a209c59f9ccd74679d182d88a2f83af9b09d6",
+    "robot/audit.json": "43f13a031ad5af63c632e61bcfb93f76dd9678aa29904904fa1ade3a6454d9ec",
+    "robot/index.html": "914197a55f5833e616b6878f5db2ed1f92ee53d807e7dc961147d2a2d6e0b918",
     "skill-impact/METHODS.md": "3ef572c799aa424407f09f47e8954c34cfa550fd0c03d7e547211be7fcf5e0cf",
     "skill-impact/ROBOT_DATA_LICENSE.txt": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4",
     "skill-impact/ROBOT_DATA_NOTICE.md": "cd9081380cd230008c7e96f4092272195055e3e4500817d3bfe105280359f0a7",
@@ -540,7 +542,7 @@
     "skill-impact/trials/skill-impact-matched/09-direct-97/main.py": "8d3e71c32ed880ea69b349d6f87007024fcb791747b49e655d75306f868af992",
     "skill-impact/trials/skill-impact-matched/09-direct-97/trajectory.atif.json": "24095e183f30163e09b8d0cdc417779c342e2a8c6eb11f35bad2b5d1527051e3",
     "skill-impact/trials/skill-impact-matched/09-direct-97/trial.json": "adcf116180b38925dbfe97497980cf867a90cc1ec32a392328bcaa84de2bb2f4",
-    "style.css": "85dfb0ddd0a08ff370244f4d921d5ea6e9921fb23e4163b5f7623726f925ca34",
+    "style.css": "eb92316a189a5f750eef969eaa277d7a8f418eb394683fcafab08143810fe73b",
     "suite/events.jsonl": "459d723abae60522199750dcdd88c076fda29d9661a1005cd3254b304a9d3995",
     "suite/index.html": "1bf1815ad7750dba9891fa775c8d5e2165cd3900b963ebc0e947277ab1560612",
     "suite/jobs/coding-reference/attempts/0001/evaluation.json": "83269e0338abca0d0eeda4f338a0843edcd90ea15c35620ef59a46582d45f842",
@@ -568,6 +570,6 @@
     "suite/suite.toml": "163e2103b9bd7b540219b140ab73d06d6870362cc4e83c04753eca232da02fa5",
     "suite-evidence.zip": "ebb68a7393e351a5c60b1bb03cc7cb193bf94c03fca7e5377537b10041997115",
     "support/audit.json": "596393f262a4b0d411ffd776bb83b80ef37b36663ee9bd29b447cf472ae20c95",
-    "support/index.html": "adf5cebedab31e336c88d984c99e3685fcebb22a9ff986767080777ef3ca7cd1"
+    "support/index.html": "7d86c92e35e9bb642b37863ce5415a14378e191171627cccc45c71a76c853e40"
   }
 }