File size: 1,910 Bytes
4ff5147 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 | {
"schema_version": 1,
"physical_cases_per_system": 32,
"independent_test_groups": 8,
"latency_comparable_across_systems": false,
"latency_note": "Observed wall latency is retained for execution audit only. ASR/OCR observations are memoized across systems in requested order, so later systems exclude analyzer work and their latency must not be compared with the first system.",
"systems": {
"always_vlm": {
"cases": 32,
"quality_pass_rate": 0.6875,
"state_accuracy": 0.71875,
"exact_state_accuracy": 0.6875,
"answer_fact_coverage": 0.96875,
"citation_validity": 0.875,
"mean_answer_char_f1": 0.6598538614163614,
"visual_backend_call_rate": 1.0,
"text_answer_backend_call_rate": 1.0,
"backend_calls_per_case": 2.0,
"mean_observed_wall_latency_ms": 2576.251625374425,
"backend_call_savings_vs_always": 0.0
},
"learned": {
"cases": 32,
"quality_pass_rate": 0.65625,
"state_accuracy": 0.6875,
"exact_state_accuracy": 0.6875,
"answer_fact_coverage": 0.96875,
"citation_validity": 0.84375,
"mean_answer_char_f1": 0.6769566939286767,
"visual_backend_call_rate": 0.90625,
"text_answer_backend_call_rate": 1.0,
"backend_calls_per_case": 1.90625,
"mean_observed_wall_latency_ms": 12376.805822888855,
"backend_call_savings_vs_always": 0.046875
},
"never_vlm": {
"cases": 32,
"quality_pass_rate": 0.78125,
"state_accuracy": 0.8125,
"exact_state_accuracy": 0.8125,
"answer_fact_coverage": 0.96875,
"citation_validity": 0.84375,
"mean_answer_char_f1": 0.6423656732759995,
"visual_backend_call_rate": 0.0,
"text_answer_backend_call_rate": 1.0,
"backend_calls_per_case": 1.0,
"mean_observed_wall_latency_ms": 1510.641771601513,
"backend_call_savings_vs_always": 0.5
}
}
} |