eval_summary.json

{
 "model": "decosa-cell-pointer-modernbert-base",
 "default": "v2-blind (root)",
 "twin": "v1-aware (aware/)",
 "date": "2026-09-27",
 "setup": "CSR number checker end to end, only the pointer changes; BM25 table retrieval on every row; held-out sets never trained on; thresholds and model choice on the model's own dev split",
 "baseline": "Qwen3.8-27B as the pointer, values shown",
 "sets": {
  "synthetic_test": {
   "desc": "12 fictional CSRs from the training generator (different seeds), 96 planted errors",
   "qwen3.8-27b": {
    "caught": 93,
    "planted": 96,
    "false_flags_per_100_numbers": 0.11,
    "right_cell": 0.9898,
    "right_cell_planted": 0.8542,
    "s_per_100_pages": 79.3
   },
   "v2-blind": {
    "caught": 96,
    "planted": 96,
    "false_flags_per_100_numbers": 0.0,
    "right_cell": 1.0,
    "right_cell_planted": 1.0,
    "s_per_100_pages": 69.1
   },
   "v1-aware": {
    "caught": 96,
    "planted": 96,
    "false_flags_per_100_numbers": 0.0,
    "right_cell": 1.0,
    "right_cell_planted": 1.0,
    "s_per_100_pages": 139
   },
   "cascade_v2_plus_qwen_below_0.6": {
    "caught": 95,
    "planted": 96,
    "false_flags_per_100_numbers": 0.0,
    "right_cell": 0.999,
    "right_cell_planted": 0.99
   }
  },
  "rewritten_test": {
   "desc": "the same reports with paragraphs rewritten by Qwen3.8-27B, 96 planted errors",
   "qwen3.8-27b": {
    "caught": 93,
    "planted": 96,
    "false_flags_per_100_numbers": 0.0,
    "right_cell": 0.9869,
    "right_cell_planted": 0.8211
   },
   "v2-blind": {
    "caught": 95,
    "planted": 96,
    "false_flags_per_100_numbers": 0.0,
    "right_cell": 1.0,
    "right_cell_planted": 1.0,
    "s_per_100_pages": 71.5
   },
   "v1-aware": {
    "caught": 95,
    "planted": 96,
    "false_flags_per_100_numbers": 0.0,
    "right_cell": 0.999,
    "right_cell_planted": 0.989,
    "s_per_100_pages": 156
   },
   "cascade_v2_plus_qwen_below_0.6": {
    "caught": 93,
    "planted": 96,
    "false_flags_per_100_numbers": 0.0,
    "right_cell": 0.996,
    "right_cell_planted": 0.958
   }
  },
  "clinicaltrials_gov": {
   "desc": "30 real phase 3 trials (conditions not in training), posted results as report tables with template narratives, 178 planted errors",
   "qwen3.8-27b": {
    "caught": 172,
    "planted": 178,
    "false_flags_per_100_numbers": 2.79,
    "right_cell": 0.9717,
    "right_cell_planted": 0.8757,
    "s_per_100_pages": 307.2
   },
   "v2-blind": {
    "caught": 176,
    "planted": 178,
    "false_flags_per_100_numbers": 4.08,
    "right_cell": 0.9778,
    "right_cell_planted": 0.9774,
    "s_per_100_pages": 31.3
   },
   "v1-aware": {
    "caught": 176,
    "planted": 178,
    "false_flags_per_100_numbers": 3.65,
    "right_cell": 0.9817,
    "right_cell_planted": 0.9774,
    "s_per_100_pages": 61.7
   },
   "cascade_v2_plus_qwen_below_0.6": {
    "caught": 176,
    "planted": 178,
    "false_flags_per_100_numbers": 3.76,
    "right_cell": 0.981,
    "right_cell_planted": 0.977
   }
  }
 },
 "transfer_10k_mdna": {
  "desc": "two FY2025 10-Ks not trained on plus a synthetic sample, 69 figures; gold from a value-aware code tie-out",
  "right_cell": {
   "qwen3.8-27b_values_shown": 0.957,
   "qwen3.8-27b_values_hidden": 0.71,
   "v1-blind": 0.391,
   "v1-aware": 0.435,
   "v2-blind": 0.565
  },
  "note": "v1-aware pointed at the cell holding the written value on a planted swapped-period figure; the blind models did not"
 },
 "cascade_verdict": "does not help; use the model alone",
 "latency": "0.2-0.45 s CPU per checked number, shared server, ONNX Runtime 4 threads",
 "source": "docs/evals/cell-pointer.md in the Decosa API repository"
}