{
  "format": "tiny_receipt_vqa_split_kv_onnx_v2",
  "canonical_heldout_report": "eval/fp32_int8_heldout_summary.json",
  "heldout_summary": {
    "fp32": {
      "address": {
        "n": 1032,
        "answer_exact": 0.9534883720930233,
        "recomputed_n": 1032,
        "recomputed_answer_exact": 0.9525193798449613,
        "target_exact": 0.015503875968992248
      },
      "overall": {
        "n": 2000,
        "answer_exact": 0.9725,
        "recomputed_n": 2000,
        "recomputed_answer_exact": 0.972,
        "target_exact": 0.4735
      },
      "phone_number": {
        "n": 968,
        "answer_exact": 0.9927685950413223,
        "recomputed_n": 968,
        "recomputed_answer_exact": 0.9927685950413223,
        "target_exact": 0.9617768595041323
      }
    },
    "int8": {
      "address": {
        "n": 1032,
        "answer_exact": 0.9505813953488372,
        "recomputed_n": 1032,
        "recomputed_answer_exact": 0.9505813953488372,
        "target_exact": 0.014534883720930232
      },
      "overall": {
        "n": 2000,
        "answer_exact": 0.971,
        "recomputed_n": 2000,
        "recomputed_answer_exact": 0.971,
        "target_exact": 0.472
      },
      "phone_number": {
        "n": 968,
        "answer_exact": 0.9927685950413223,
        "recomputed_n": 968,
        "recomputed_answer_exact": 0.9927685950413223,
        "target_exact": 0.9597107438016529
      }
    }
  },
  "fp32_int8_comparison": {
    "n": 2000,
    "prediction_exact_agreement": 0.7445,
    "answer_agreement": 0.9795,
    "selected_family_agreement": 1.0,
    "changed_predictions": 511
  },
  "heldout_timing": {
    "fp32": {
      "encoder_seconds": 243.63899111794308,
      "decoder_seconds": 29.627758088114206,
      "total_seconds": 273.2667492060573,
      "decoder_calls": 2953,
      "providers": [
        "CPUExecutionProvider"
      ],
      "encoder": "encoder_model.onnx",
      "decoder": "decoder_model.onnx"
    },
    "int8": {
      "encoder_seconds": 195.71248534298502,
      "decoder_seconds": 61.439259737788234,
      "total_seconds": 257.15174508077325,
      "decoder_calls": 2934,
      "providers": [
        "CPUExecutionProvider"
      ],
      "encoder": "encoder_model_int8.onnx",
      "decoder": "decoder_model_int8.onnx"
    }
  },
  "runtime_benchmark": {
    "method": "20 heldout VQAs, CPUExecutionProvider, one thread, one warmup and three measured repetitions per VQA; session creation and input preprocessing excluded",
    "fp32": {
      "report": "eval/runtime_benchmark_fp32_cpu.json",
      "report_sha256": "5aab96400c0ff6892dca8b8e7eebff71f45e419485c2e557b57aa9f4907b571f",
      "e2e_median_of_vqa_medians_ms": 288.50183452595957
    },
    "int8_w8a8": {
      "report": "eval/runtime_benchmark_int8_cpu.json",
      "report_sha256": "0389b525014e8af0771050a1526815b8692889ca66b8810bdf43585434d96625",
      "e2e_median_of_vqa_medians_ms": 173.67218298022635
    }
  }
}