#!/usr/bin/env python3 """Report every method at the SAME false-positive rate, not the same threshold. Why this exists. Our headline table sets each method's threshold on validation at the most permissive point with FPR <= 1%, then applies it unchanged to test. That is the right protocol for a deployable number -- nothing about the test set influences the operating point -- but it makes the precision and recall columns hard to read across methods, because each one lands somewhere different inside the 1% budget: Lookback Lens 0.658% FPR precision 0.6693 recall 0.1997 LatticeSpike 0.765% precision 0.6492 recall 0.2045 Linear probe 0.842% precision 0.6331 recall 0.2178 Ranked by FPR, the precision column is in exactly reverse order. That comparison is measuring where each method's validation calibration happened to land, not which method detects better. A single shared threshold VALUE cannot fix this: an RSNN anomaly score and a logistic probability are not the same units, so one number means different things per method. What is comparable is the operating point itself. This script sweeps each method's own scores to hit a fixed FPR exactly, and reports recall there. HONEST LIMITATION, and it must be stated wherever these numbers appear: the threshold here is chosen ON TEST, so recall@FPR is a ranking-quality metric in the family of AP and AUROC -- not an achievable deployed operating point. It answers "if every method were tuned to the same alarm budget, which finds the most hallucinations", which is a different and fairer question than the headline table's. Report both; neither replaces the other. Usage: python scripts/recall_at_fixed_fpr.py \ --predictions outputs/ragtruth_l2_7b_s42/predictions.jsonl \ --predictions outputs/rt_baselines/a_no_attention/predictions.jsonl \ --predictions outputs/rt_baselines/b_lookback/predictions.jsonl \ --fpr 0.01 0.005 0.001 """ from __future__ import annotations import argparse import collections import json from pathlib import Path import numpy as np def parse_args(argv=None): p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) p.add_argument("--predictions", type=Path, action="append", required=True) p.add_argument( "--fpr", type=float, nargs="+", default=[0.01, 0.005, 0.001], help="Target false-positive rates. 1%% matches the headline table's budget.", ) p.add_argument("--split", default="eval", help="Only score rows from this split.") return p.parse_args(argv) def load(paths, split): """method -> (scores, labels), using the runner's own token mask.""" buckets = collections.defaultdict(lambda: ([], [])) for path in paths: if not path.exists(): print(f"!! missing: {path}") continue with path.open(encoding="utf-8") as handle: for line in handle: line = line.strip() if not line: continue row = json.loads(line) meta = row.get("metadata") or {} # LatticeSpike prediction rows carry no split field; they are # test rows by construction. if meta.get("split", split) != split: continue method = row.get("method") or "lattice_spike_rsnn_anomaly" scores = row.get("scores") or [] labels = row.get("labels") or [] if not labels or len(labels) != len(scores): continue mask = row.get("comparison_mask") or [True] * len(scores) s, y = buckets[method] for score, label, keep in zip(scores, labels, mask): # label 2 is the unlabelled class and never enters a metric. if not keep or int(label) == 2: continue s.append(float(score)) y.append(int(label)) return buckets def recall_at_fpr(scores, labels, target): """Sweep to the largest FPR not exceeding target; report recall there. Ties matter: with a coarse score distribution the achievable FPR can sit well below the target, so the achieved value is returned alongside rather than assumed equal to it. """ s = np.asarray(scores, dtype=float) y = np.asarray(labels, dtype=int) n_pos, n_neg = int((y == 1).sum()), int((y == 0).sum()) if not n_pos or not n_neg: return None order = np.argsort(-s, kind="mergesort") y_sorted = y[order] tp = np.cumsum(y_sorted == 1) fp = np.cumsum(y_sorted == 0) fpr = fp / n_neg feasible = np.nonzero(fpr <= target)[0] if feasible.size == 0: return {"recall": 0.0, "achieved_fpr": 0.0, "precision": float("nan")} i = feasible[-1] denom = tp[i] + fp[i] return { "recall": float(tp[i] / n_pos), "achieved_fpr": float(fpr[i]), "precision": float(tp[i] / denom) if denom else float("nan"), "n_pos": n_pos, "n_neg": n_neg, } def main(argv=None): args = parse_args(argv) buckets = load(args.predictions, args.split) if not buckets: raise SystemExit("no prediction rows loaded") supports = {len(v[0]) for v in buckets.values()} print("# Recall at matched false-positive rate\n") print("Threshold set on TEST to hit each FPR exactly -- a ranking metric,") print("not a deployable operating point. See the headline table for that.\n") for target in args.fpr: print(f"## FPR <= {target:.3%}\n") rows = [] for method, (scores, labels) in buckets.items(): result = recall_at_fpr(scores, labels, target) if result: rows.append((result["recall"], method, result)) rows.sort(reverse=True) print(f"{'method':<32}{'recall':>10}{'precision':>12}{'achieved FPR':>14}") print("-" * 68) for recall, method, result in rows: print( f"{method:<32}{recall:>10.4f}{result['precision']:>12.4f}" f"{result['achieved_fpr']:>13.4%}" ) print() if len(supports) == 1: n = supports.pop() print(f"support: every method scored {n:,} tokens — comparable") else: print(f"!! SUPPORT MISMATCH: {sorted(supports)} — rows are NOT comparable") return 0 if __name__ == "__main__": raise SystemExit(main())