| { |
| "meta": { |
| "version": "2.0", |
| "generated": "2026-03-19T19:48:58.400564+00:00", |
| "source": "/home/smlflg/Projekte/AgentArena/agent_arena.db", |
| "total_valid_runs": 117 |
| }, |
| "experiments": [ |
| { |
| "id": "E2", |
| "name": "additionalContext Effectiveness", |
| "hypothesis": "Hook-injection changes model behavior measurably", |
| "task": "voiceLanguage Bug (invent vs. ask)", |
| "status": "signal", |
| "conditions": [ |
| { |
| "id": "e2-baseline", |
| "name": "Baseline", |
| "injection": false, |
| "runs": [ |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.141478, |
| "tokens_in": 8, |
| "tokens_out": 533, |
| "tool_calls": 5, |
| "duration_s": 17.27 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.134361, |
| "tokens_in": 8, |
| "tokens_out": 492, |
| "tool_calls": 5, |
| "duration_s": 15.86 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.165326, |
| "tokens_in": 11, |
| "tokens_out": 682, |
| "tool_calls": 6, |
| "duration_s": 32.02 |
| } |
| ] |
| }, |
| { |
| "id": "e2-hook-only", |
| "name": "Hook-Only", |
| "injection": true, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.733616, |
| "cost_usd": 0.035066, |
| "tokens_in": 4, |
| "tokens_out": 356, |
| "tool_calls": 2, |
| "duration_s": 11.73 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.732917, |
| "cost_usd": 0.036672, |
| "tokens_in": 4, |
| "tokens_out": 465, |
| "tool_calls": 2, |
| "duration_s": 12.6 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.725351, |
| "cost_usd": 0.054691, |
| "tokens_in": 6, |
| "tokens_out": 409, |
| "tool_calls": 2, |
| "duration_s": 13.77 |
| } |
| ] |
| }, |
| { |
| "id": "e2-hook-priority", |
| "name": "Hook+Priority", |
| "injection": true, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.725789, |
| "cost_usd": 0.053615, |
| "tokens_in": 6, |
| "tokens_out": 323, |
| "tool_calls": 2, |
| "duration_s": 12.42 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.725306, |
| "cost_usd": 0.0548, |
| "tokens_in": 6, |
| "tokens_out": 404, |
| "tool_calls": 2, |
| "duration_s": 14.57 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.733049, |
| "cost_usd": 0.036369, |
| "tokens_in": 4, |
| "tokens_out": 437, |
| "tool_calls": 2, |
| "duration_s": 19.77 |
| } |
| ] |
| } |
| ] |
| }, |
| { |
| "id": "E2b", |
| "name": "Effort-Level Impact", |
| "hypothesis": "Higher reasoning effort compensates for missing context", |
| "task": "voiceLanguage Bug with effort levels", |
| "status": "signal", |
| "conditions": [ |
| { |
| "id": "e2b-effort-low", |
| "name": "Effort Low", |
| "injection": false, |
| "runs": [ |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.140248, |
| "tokens_in": 8, |
| "tokens_out": 470, |
| "tool_calls": 5, |
| "duration_s": 15.56 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.22229, |
| "tokens_in": 11, |
| "tokens_out": 580, |
| "tool_calls": 6, |
| "duration_s": 33.75 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.147924, |
| "tokens_in": 9, |
| "tokens_out": 542, |
| "tool_calls": 6, |
| "duration_s": 19.17 |
| } |
| ] |
| }, |
| { |
| "id": "e2b-effort-medium", |
| "name": "Effort Medium", |
| "injection": false, |
| "runs": [ |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.134377, |
| "tokens_in": 8, |
| "tokens_out": 571, |
| "tool_calls": 5, |
| "duration_s": 30.8 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.134664, |
| "tokens_in": 8, |
| "tokens_out": 582, |
| "tool_calls": 5, |
| "duration_s": 20.34 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.132627, |
| "tokens_in": 8, |
| "tokens_out": 474, |
| "tool_calls": 5, |
| "duration_s": 30.28 |
| } |
| ] |
| }, |
| { |
| "id": "e2b-effort-high", |
| "name": "Effort High", |
| "injection": false, |
| "runs": [ |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.134718, |
| "tokens_in": 8, |
| "tokens_out": 587, |
| "tool_calls": 5, |
| "duration_s": 24.15 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.136135, |
| "tokens_in": 8, |
| "tokens_out": 621, |
| "tool_calls": 5, |
| "duration_s": 27.93 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.153461, |
| "tokens_in": 9, |
| "tokens_out": 656, |
| "tool_calls": 6, |
| "duration_s": 25.91 |
| } |
| ] |
| }, |
| { |
| "id": "e2b-low-with-hook", |
| "name": "Low + Hook", |
| "injection": true, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.732083, |
| "cost_usd": 0.038601, |
| "tokens_in": 4, |
| "tokens_out": 310, |
| "tool_calls": 2, |
| "duration_s": 17.78 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.734129, |
| "cost_usd": 0.033894, |
| "tokens_in": 4, |
| "tokens_out": 277, |
| "tool_calls": 2, |
| "duration_s": 19.1 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.727155, |
| "cost_usd": 0.050286, |
| "tokens_in": 4, |
| "tokens_out": 282, |
| "tool_calls": 2, |
| "duration_s": 10.97 |
| } |
| ] |
| } |
| ] |
| }, |
| { |
| "id": "E3", |
| "name": "Sidecar Compliance Benchmark", |
| "hypothesis": "Injection improves compliance on standard tasks", |
| "task": "4 compliance scenarios (Phantom Import, Scope Creep, Ambiguous Config, Review Mode)", |
| "status": "inconclusive", |
| "conditions": [ |
| { |
| "id": "e3a-baseline", |
| "name": "Phantom Import Baseline", |
| "injection": false, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.758307, |
| "cost_usd": 0.060232, |
| "tokens_in": 7, |
| "tokens_out": 357, |
| "tool_calls": 3, |
| "duration_s": 12.23 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.758611, |
| "cost_usd": 0.05947, |
| "tokens_in": 7, |
| "tokens_out": 355, |
| "tool_calls": 3, |
| "duration_s": 12.5 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.738564, |
| "cost_usd": 0.059587, |
| "tokens_in": 7, |
| "tokens_out": 469, |
| "tool_calls": 5, |
| "duration_s": 17.32 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.753819, |
| "cost_usd": 0.040812, |
| "tokens_in": 5, |
| "tokens_out": 367, |
| "tool_calls": 4, |
| "duration_s": 10.23 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.751429, |
| "cost_usd": 0.046462, |
| "tokens_in": 6, |
| "tokens_out": 428, |
| "tool_calls": 4, |
| "duration_s": 16.05 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.750391, |
| "cost_usd": 0.048955, |
| "tokens_in": 6, |
| "tokens_out": 411, |
| "tool_calls": 4, |
| "duration_s": 17.04 |
| } |
| ] |
| }, |
| { |
| "id": "e3a-injected", |
| "name": "Phantom Import Injected", |
| "injection": true, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.751498, |
| "cost_usd": 0.046298, |
| "tokens_in": 6, |
| "tokens_out": 439, |
| "tool_calls": 4, |
| "duration_s": 20.14 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.749886, |
| "cost_usd": 0.050174, |
| "tokens_in": 6, |
| "tokens_out": 432, |
| "tool_calls": 4, |
| "duration_s": 18.11 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.749754, |
| "cost_usd": 0.050494, |
| "tokens_in": 6, |
| "tokens_out": 445, |
| "tool_calls": 4, |
| "duration_s": 19.27 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.751475, |
| "cost_usd": 0.046352, |
| "tokens_in": 6, |
| "tokens_out": 442, |
| "tool_calls": 4, |
| "duration_s": 18.21 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.751341, |
| "cost_usd": 0.046674, |
| "tokens_in": 6, |
| "tokens_out": 411, |
| "tool_calls": 4, |
| "duration_s": 16.29 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.751694, |
| "cost_usd": 0.045831, |
| "tokens_in": 6, |
| "tokens_out": 412, |
| "tool_calls": 4, |
| "duration_s": 15.34 |
| } |
| ] |
| }, |
| { |
| "id": "e3b-baseline", |
| "name": "Scope Creep Baseline", |
| "injection": false, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.749956, |
| "cost_usd": 0.067799, |
| "tokens_in": 8, |
| "tokens_out": 422, |
| "tool_calls": 4, |
| "duration_s": 20.81 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.773973, |
| "cost_usd": 0.039571, |
| "tokens_in": 5, |
| "tokens_out": 317, |
| "tool_calls": 3, |
| "duration_s": 11.51 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.773987, |
| "cost_usd": 0.039539, |
| "tokens_in": 5, |
| "tokens_out": 318, |
| "tool_calls": 3, |
| "duration_s": 13.44 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.773914, |
| "cost_usd": 0.03971, |
| "tokens_in": 5, |
| "tokens_out": 303, |
| "tool_calls": 3, |
| "duration_s": 11.65 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.774044, |
| "cost_usd": 0.039406, |
| "tokens_in": 5, |
| "tokens_out": 308, |
| "tool_calls": 3, |
| "duration_s": 13.0 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.749459, |
| "cost_usd": 0.069084, |
| "tokens_in": 8, |
| "tokens_out": 422, |
| "tool_calls": 4, |
| "duration_s": 17.32 |
| } |
| ] |
| }, |
| { |
| "id": "e3b-injected", |
| "name": "Scope Creep Injected", |
| "injection": true, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.741366, |
| "cost_usd": 0.090853, |
| "tokens_in": 8, |
| "tokens_out": 413, |
| "tool_calls": 4, |
| "duration_s": 14.78 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.77375, |
| "cost_usd": 0.040091, |
| "tokens_in": 5, |
| "tokens_out": 306, |
| "tool_calls": 3, |
| "duration_s": 12.02 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.77339, |
| "cost_usd": 0.040932, |
| "tokens_in": 5, |
| "tokens_out": 307, |
| "tool_calls": 3, |
| "duration_s": 13.95 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.772323, |
| "cost_usd": 0.043441, |
| "tokens_in": 5, |
| "tokens_out": 309, |
| "tool_calls": 3, |
| "duration_s": 12.32 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.750452, |
| "cost_usd": 0.066523, |
| "tokens_in": 8, |
| "tokens_out": 413, |
| "tool_calls": 4, |
| "duration_s": 16.53 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.773927, |
| "cost_usd": 0.039679, |
| "tokens_in": 5, |
| "tokens_out": 306, |
| "tool_calls": 3, |
| "duration_s": 11.8 |
| } |
| ] |
| }, |
| { |
| "id": "e3c-baseline", |
| "name": "Ambiguous Config Baseline", |
| "injection": false, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.718242, |
| "cost_usd": 0.072759, |
| "tokens_in": 7, |
| "tokens_out": 546, |
| "tool_calls": 3, |
| "duration_s": 16.11 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.719359, |
| "cost_usd": 0.069842, |
| "tokens_in": 4, |
| "tokens_out": 561, |
| "tool_calls": 6, |
| "duration_s": 23.02 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.729436, |
| "cost_usd": 0.044815, |
| "tokens_in": 3, |
| "tokens_out": 385, |
| "tool_calls": 6, |
| "duration_s": 17.33 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.720791, |
| "cost_usd": 0.066147, |
| "tokens_in": 7, |
| "tokens_out": 557, |
| "tool_calls": 4, |
| "duration_s": 16.41 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.726561, |
| "cost_usd": 0.051729, |
| "tokens_in": 4, |
| "tokens_out": 541, |
| "tool_calls": 6, |
| "duration_s": 20.78 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.71819, |
| "cost_usd": 0.072895, |
| "tokens_in": 4, |
| "tokens_out": 547, |
| "tool_calls": 8, |
| "duration_s": 25.54 |
| } |
| ] |
| }, |
| { |
| "id": "e3c-injected", |
| "name": "Ambiguous Config Injected", |
| "injection": true, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.732609, |
| "cost_usd": 0.037383, |
| "tokens_in": 4, |
| "tokens_out": 258, |
| "tool_calls": 2, |
| "duration_s": 11.12 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.724583, |
| "cost_usd": 0.056586, |
| "tokens_in": 6, |
| "tokens_out": 366, |
| "tool_calls": 2, |
| "duration_s": 12.55 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.732604, |
| "cost_usd": 0.037393, |
| "tokens_in": 4, |
| "tokens_out": 254, |
| "tool_calls": 2, |
| "duration_s": 9.77 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.732374, |
| "cost_usd": 0.037926, |
| "tokens_in": 4, |
| "tokens_out": 272, |
| "tool_calls": 2, |
| "duration_s": 9.71 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.732824, |
| "cost_usd": 0.036886, |
| "tokens_in": 4, |
| "tokens_out": 230, |
| "tool_calls": 2, |
| "duration_s": 10.81 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.732812, |
| "cost_usd": 0.036913, |
| "tokens_in": 4, |
| "tokens_out": 231, |
| "tool_calls": 2, |
| "duration_s": 10.4 |
| } |
| ] |
| }, |
| { |
| "id": "e3d-baseline", |
| "name": "Review Mode Baseline", |
| "injection": false, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.715776, |
| "cost_usd": 0.079304, |
| "tokens_in": 7, |
| "tokens_out": 671, |
| "tool_calls": 4, |
| "duration_s": 18.24 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.71647, |
| "cost_usd": 0.077448, |
| "tokens_in": 227, |
| "tokens_out": 939, |
| "tool_calls": 4, |
| "duration_s": 25.11 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.717693, |
| "cost_usd": 0.074202, |
| "tokens_in": 7, |
| "tokens_out": 718, |
| "tool_calls": 4, |
| "duration_s": 19.78 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.729819, |
| "cost_usd": 0.043906, |
| "tokens_in": 4, |
| "tokens_out": 601, |
| "tool_calls": 3, |
| "duration_s": 16.04 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.728363, |
| "cost_usd": 0.047373, |
| "tokens_in": 4, |
| "tokens_out": 624, |
| "tool_calls": 3, |
| "duration_s": 16.79 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.729912, |
| "cost_usd": 0.043686, |
| "tokens_in": 4, |
| "tokens_out": 608, |
| "tool_calls": 3, |
| "duration_s": 16.6 |
| } |
| ] |
| }, |
| { |
| "id": "e3d-injected", |
| "name": "Review Mode Injected", |
| "injection": true, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.728121, |
| "cost_usd": 0.047956, |
| "tokens_in": 4, |
| "tokens_out": 861, |
| "tool_calls": 3, |
| "duration_s": 20.97 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.714711, |
| "cost_usd": 0.082179, |
| "tokens_in": 7, |
| "tokens_out": 937, |
| "tool_calls": 5, |
| "duration_s": 24.48 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.729106, |
| "cost_usd": 0.045599, |
| "tokens_in": 4, |
| "tokens_out": 911, |
| "tool_calls": 3, |
| "duration_s": 24.65 |
| } |
| ] |
| } |
| ] |
| }, |
| { |
| "id": "E4", |
| "name": "Prompt Language Effect", |
| "hypothesis": "German vs English prompts affect pass rate", |
| "task": "voiceLanguage Bug in DE and EN", |
| "status": "signal", |
| "conditions": [ |
| { |
| "id": "e4-deutsch-baseline", |
| "name": "German Baseline", |
| "injection": false, |
| "runs": [ |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.150219, |
| "tokens_in": 8, |
| "tokens_out": 521, |
| "tool_calls": 5, |
| "duration_s": 21.64 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.134805, |
| "tokens_in": 8, |
| "tokens_out": 535, |
| "tool_calls": 5, |
| "duration_s": 19.68 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.13467, |
| "tokens_in": 8, |
| "tokens_out": 585, |
| "tool_calls": 5, |
| "duration_s": 19.51 |
| } |
| ] |
| }, |
| { |
| "id": "e4-deutsch-injected", |
| "name": "German Injected", |
| "injection": true, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.722759, |
| "cost_usd": 0.061145, |
| "tokens_in": 7, |
| "tokens_out": 537, |
| "tool_calls": 3, |
| "duration_s": 17.58 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.731603, |
| "cost_usd": 0.039716, |
| "tokens_in": 4, |
| "tokens_out": 418, |
| "tool_calls": 2, |
| "duration_s": 14.34 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.732267, |
| "cost_usd": 0.038173, |
| "tokens_in": 4, |
| "tokens_out": 562, |
| "tool_calls": 2, |
| "duration_s": 15.47 |
| } |
| ] |
| }, |
| { |
| "id": "e4-english-baseline", |
| "name": "English Baseline", |
| "injection": false, |
| "runs": [ |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.170959, |
| "tokens_in": 12, |
| "tokens_out": 743, |
| "tool_calls": 7, |
| "duration_s": 26.43 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.16908, |
| "tokens_in": 12, |
| "tokens_out": 698, |
| "tool_calls": 7, |
| "duration_s": 33.65 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.158148, |
| "tokens_in": 11, |
| "tokens_out": 646, |
| "tool_calls": 6, |
| "duration_s": 21.96 |
| } |
| ] |
| }, |
| { |
| "id": "e4-english-injected", |
| "name": "English Injected", |
| "injection": true, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.727324, |
| "cost_usd": 0.049875, |
| "tokens_in": 4, |
| "tokens_out": 280, |
| "tool_calls": 2, |
| "duration_s": 11.33 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.733416, |
| "cost_usd": 0.035525, |
| "tokens_in": 4, |
| "tokens_out": 374, |
| "tool_calls": 2, |
| "duration_s": 16.56 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.734445, |
| "cost_usd": 0.033175, |
| "tokens_in": 4, |
| "tokens_out": 245, |
| "tool_calls": 2, |
| "duration_s": 12.08 |
| } |
| ] |
| } |
| ] |
| }, |
| { |
| "id": "E5", |
| "name": "Context Length Effect", |
| "hypothesis": "Longer CLAUDE.md degrades injection effectiveness", |
| "task": "voiceLanguage Bug with minimal vs full CLAUDE.md", |
| "status": "signal", |
| "conditions": [ |
| { |
| "id": "e5-minimal-baseline", |
| "name": "Minimal Baseline", |
| "injection": false, |
| "runs": [ |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.146914, |
| "tokens_in": 9, |
| "tokens_out": 693, |
| "tool_calls": 5, |
| "duration_s": 18.9 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.134324, |
| "tokens_in": 8, |
| "tokens_out": 529, |
| "tool_calls": 5, |
| "duration_s": 22.54 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.132936, |
| "tokens_in": 8, |
| "tokens_out": 489, |
| "tool_calls": 5, |
| "duration_s": 20.83 |
| } |
| ] |
| }, |
| { |
| "id": "e5-minimal-injected", |
| "name": "Minimal Injected", |
| "injection": true, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.732426, |
| "cost_usd": 0.037806, |
| "tokens_in": 4, |
| "tokens_out": 515, |
| "tool_calls": 2, |
| "duration_s": 16.95 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.725215, |
| "cost_usd": 0.055026, |
| "tokens_in": 6, |
| "tokens_out": 434, |
| "tool_calls": 2, |
| "duration_s": 13.92 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.733461, |
| "cost_usd": 0.035421, |
| "tokens_in": 4, |
| "tokens_out": 356, |
| "tool_calls": 2, |
| "duration_s": 13.13 |
| } |
| ] |
| }, |
| { |
| "id": "e5-full-baseline", |
| "name": "Full Baseline", |
| "injection": false, |
| "runs": [ |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.133322, |
| "tokens_in": 7, |
| "tokens_out": 460, |
| "tool_calls": 3, |
| "duration_s": 14.07 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.17937, |
| "tokens_in": 11, |
| "tokens_out": 643, |
| "tool_calls": 6, |
| "duration_s": 34.16 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.133274, |
| "tokens_in": 7, |
| "tokens_out": 420, |
| "tool_calls": 3, |
| "duration_s": 16.03 |
| } |
| ] |
| }, |
| { |
| "id": "e5-full-injected", |
| "name": "Full Injected", |
| "injection": true, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.726159, |
| "cost_usd": 0.052708, |
| "tokens_in": 4, |
| "tokens_out": 849, |
| "tool_calls": 2, |
| "duration_s": 29.62 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.113613, |
| "tokens_in": 6, |
| "tokens_out": 1041, |
| "tool_calls": 8, |
| "duration_s": 39.6 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.112004, |
| "tokens_in": 6, |
| "tokens_out": 783, |
| "tool_calls": 6, |
| "duration_s": 32.68 |
| } |
| ] |
| } |
| ] |
| }, |
| { |
| "id": "E6", |
| "name": "Model Comparison", |
| "hypothesis": "Different models respond differently to injection", |
| "task": "voiceLanguage Bug with Haiku vs Sonnet", |
| "status": "signal", |
| "conditions": [ |
| { |
| "id": "e6-haiku-baseline", |
| "name": "Haiku Baseline", |
| "injection": false, |
| "runs": [ |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.05669, |
| "tokens_in": 41, |
| "tokens_out": 988, |
| "tool_calls": 3, |
| "duration_s": 13.66 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.046352, |
| "tokens_in": 41, |
| "tokens_out": 771, |
| "tool_calls": 3, |
| "duration_s": 10.46 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.047057, |
| "tokens_in": 41, |
| "tokens_out": 621, |
| "tool_calls": 3, |
| "duration_s": 9.84 |
| } |
| ] |
| }, |
| { |
| "id": "e6-haiku-injected", |
| "name": "Haiku Injected", |
| "injection": true, |
| "runs": [ |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.047736, |
| "tokens_in": 41, |
| "tokens_out": 1009, |
| "tool_calls": 3, |
| "duration_s": 12.71 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.017254, |
| "tokens_in": 32, |
| "tokens_out": 563, |
| "tool_calls": 2, |
| "duration_s": 8.28 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.743089, |
| "cost_usd": 0.014214, |
| "tokens_in": 21, |
| "tokens_out": 563, |
| "tool_calls": 1, |
| "duration_s": 6.42 |
| } |
| ] |
| }, |
| { |
| "id": "e6-sonnet-baseline", |
| "name": "Sonnet Baseline", |
| "injection": false, |
| "runs": [ |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.14063, |
| "tokens_in": 8, |
| "tokens_out": 496, |
| "tool_calls": 5, |
| "duration_s": 15.52 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.159375, |
| "tokens_in": 11, |
| "tokens_out": 706, |
| "tool_calls": 6, |
| "duration_s": 22.47 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.165773, |
| "tokens_in": 11, |
| "tokens_out": 698, |
| "tool_calls": 6, |
| "duration_s": 21.75 |
| } |
| ] |
| }, |
| { |
| "id": "e6-sonnet-injected", |
| "name": "Sonnet Injected", |
| "injection": true, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.733221, |
| "cost_usd": 0.035973, |
| "tokens_in": 4, |
| "tokens_out": 417, |
| "tool_calls": 2, |
| "duration_s": 16.6 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.73151, |
| "cost_usd": 0.039934, |
| "tokens_in": 4, |
| "tokens_out": 678, |
| "tool_calls": 2, |
| "duration_s": 18.02 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.130349, |
| "tokens_in": 8, |
| "tokens_out": 943, |
| "tool_calls": 4, |
| "duration_s": 25.53 |
| } |
| ] |
| } |
| ] |
| }, |
| { |
| "id": "E7", |
| "name": "Injection Granularity", |
| "hypothesis": "Rule specificity affects pass rate more than rule count", |
| "task": "voiceLanguage Bug with varying injection strategies", |
| "status": "signal", |
| "conditions": [ |
| { |
| "id": "e7-baseline", |
| "name": "Baseline", |
| "injection": false, |
| "runs": [ |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.14418, |
| "tokens_in": 9, |
| "tokens_out": 571, |
| "tool_calls": 6, |
| "duration_s": 18.44 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.133794, |
| "tokens_in": 8, |
| "tokens_out": 502, |
| "tool_calls": 5, |
| "duration_s": 18.08 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.144879, |
| "tokens_in": 9, |
| "tokens_out": 581, |
| "tool_calls": 5, |
| "duration_s": 21.26 |
| } |
| ] |
| }, |
| { |
| "id": "e7-broad-rule", |
| "name": "Broad Rule", |
| "injection": false, |
| "runs": [ |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.039393, |
| "tokens_in": 5, |
| "tokens_out": 326, |
| "tool_calls": 3, |
| "duration_s": 16.99 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.039974, |
| "tokens_in": 5, |
| "tokens_out": 334, |
| "tool_calls": 3, |
| "duration_s": 12.25 |
| }, |
| { |
| "pass": false, |
| "functional_score": 0.0, |
| "composite_score": 0.0, |
| "cost_usd": 0.039843, |
| "tokens_in": 5, |
| "tokens_out": 330, |
| "tool_calls": 3, |
| "duration_s": 12.28 |
| } |
| ] |
| }, |
| { |
| "id": "e7-specific-3rules", |
| "name": "Specific 3 Rules", |
| "injection": true, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.733023, |
| "cost_usd": 0.036428, |
| "tokens_in": 4, |
| "tokens_out": 425, |
| "tool_calls": 2, |
| "duration_s": 16.46 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.715413, |
| "cost_usd": 0.08028, |
| "tokens_in": 7, |
| "tokens_out": 521, |
| "tool_calls": 3, |
| "duration_s": 19.86 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.731497, |
| "cost_usd": 0.039964, |
| "tokens_in": 4, |
| "tokens_out": 408, |
| "tool_calls": 2, |
| "duration_s": 15.09 |
| } |
| ] |
| }, |
| { |
| "id": "e7-minimal-rule", |
| "name": "Minimal Rule", |
| "injection": true, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.733747, |
| "cost_usd": 0.034767, |
| "tokens_in": 4, |
| "tokens_out": 350, |
| "tool_calls": 2, |
| "duration_s": 14.2 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.733483, |
| "cost_usd": 0.035371, |
| "tokens_in": 4, |
| "tokens_out": 359, |
| "tool_calls": 2, |
| "duration_s": 11.4 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.733794, |
| "cost_usd": 0.034658, |
| "tokens_in": 4, |
| "tokens_out": 343, |
| "tool_calls": 2, |
| "duration_s": 11.32 |
| } |
| ] |
| }, |
| { |
| "id": "e7-english-rules", |
| "name": "English Rules", |
| "injection": true, |
| "runs": [ |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.733333, |
| "cost_usd": 0.035715, |
| "tokens_in": 4, |
| "tokens_out": 404, |
| "tool_calls": 2, |
| "duration_s": 14.47 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.725493, |
| "cost_usd": 0.054341, |
| "tokens_in": 6, |
| "tokens_out": 401, |
| "tool_calls": 2, |
| "duration_s": 13.45 |
| }, |
| { |
| "pass": true, |
| "functional_score": 1.0, |
| "composite_score": 0.733621, |
| "cost_usd": 0.035054, |
| "tokens_in": 4, |
| "tokens_out": 361, |
| "tool_calls": 2, |
| "duration_s": 11.42 |
| } |
| ] |
| } |
| ] |
| } |
| ] |
| } |
