| { |
| "format": "tiny_receipt_vqa_split_kv_onnx_v2", |
| "canonical_heldout_report": "eval/fp32_int8_heldout_summary.json", |
| "heldout_summary": { |
| "fp32": { |
| "address": { |
| "n": 1032, |
| "answer_exact": 0.9534883720930233, |
| "recomputed_n": 1032, |
| "recomputed_answer_exact": 0.9525193798449613, |
| "target_exact": 0.015503875968992248 |
| }, |
| "overall": { |
| "n": 2000, |
| "answer_exact": 0.9725, |
| "recomputed_n": 2000, |
| "recomputed_answer_exact": 0.972, |
| "target_exact": 0.4735 |
| }, |
| "phone_number": { |
| "n": 968, |
| "answer_exact": 0.9927685950413223, |
| "recomputed_n": 968, |
| "recomputed_answer_exact": 0.9927685950413223, |
| "target_exact": 0.9617768595041323 |
| } |
| }, |
| "int8": { |
| "address": { |
| "n": 1032, |
| "answer_exact": 0.9505813953488372, |
| "recomputed_n": 1032, |
| "recomputed_answer_exact": 0.9505813953488372, |
| "target_exact": 0.014534883720930232 |
| }, |
| "overall": { |
| "n": 2000, |
| "answer_exact": 0.971, |
| "recomputed_n": 2000, |
| "recomputed_answer_exact": 0.971, |
| "target_exact": 0.472 |
| }, |
| "phone_number": { |
| "n": 968, |
| "answer_exact": 0.9927685950413223, |
| "recomputed_n": 968, |
| "recomputed_answer_exact": 0.9927685950413223, |
| "target_exact": 0.9597107438016529 |
| } |
| } |
| }, |
| "fp32_int8_comparison": { |
| "n": 2000, |
| "prediction_exact_agreement": 0.7445, |
| "answer_agreement": 0.9795, |
| "selected_family_agreement": 1.0, |
| "changed_predictions": 511 |
| }, |
| "heldout_timing": { |
| "fp32": { |
| "encoder_seconds": 243.63899111794308, |
| "decoder_seconds": 29.627758088114206, |
| "total_seconds": 273.2667492060573, |
| "decoder_calls": 2953, |
| "providers": [ |
| "CPUExecutionProvider" |
| ], |
| "encoder": "encoder_model.onnx", |
| "decoder": "decoder_model.onnx" |
| }, |
| "int8": { |
| "encoder_seconds": 195.71248534298502, |
| "decoder_seconds": 61.439259737788234, |
| "total_seconds": 257.15174508077325, |
| "decoder_calls": 2934, |
| "providers": [ |
| "CPUExecutionProvider" |
| ], |
| "encoder": "encoder_model_int8.onnx", |
| "decoder": "decoder_model_int8.onnx" |
| } |
| }, |
| "runtime_benchmark": { |
| "method": "20 heldout VQAs, CPUExecutionProvider, one thread, one warmup and three measured repetitions per VQA; session creation and input preprocessing excluded", |
| "fp32": { |
| "report": "eval/runtime_benchmark_fp32_cpu.json", |
| "report_sha256": "5aab96400c0ff6892dca8b8e7eebff71f45e419485c2e557b57aa9f4907b571f", |
| "e2e_median_of_vqa_medians_ms": 288.50183452595957 |
| }, |
| "int8_w8a8": { |
| "report": "eval/runtime_benchmark_int8_cpu.json", |
| "report_sha256": "0389b525014e8af0771050a1526815b8692889ca66b8810bdf43585434d96625", |
| "e2e_median_of_vqa_medians_ms": 173.67218298022635 |
| } |
| } |
| } |
