Download scripts/recall_at_fixed_fpr.py from Anonymous1611/Lattice_Spike: direct link, hf CLI and curl.
- Browser
- Download file 6.53 kB
-
https://huggingface.co/Anonymous1611/Lattice_Spike/resolve/main/scripts/recall_at_fixed_fpr.py
- Command line
-
hf download hf://Anonymous1611/Lattice_Spike/scripts/recall_at_fixed_fpr.py
-
curl -L -o recall_at_fixed_fpr.py https://huggingface.co/Anonymous1611/Lattice_Spike/resolve/main/scripts/recall_at_fixed_fpr.py
6.53 kB
| #!/usr/bin/env python3 | |
| """Report every method at the SAME false-positive rate, not the same threshold. | |
| Why this exists. Our headline table sets each method's threshold on validation | |
| at the most permissive point with FPR <= 1%, then applies it unchanged to test. | |
| That is the right protocol for a deployable number -- nothing about the test set | |
| influences the operating point -- but it makes the precision and recall columns | |
| hard to read across methods, because each one lands somewhere different inside | |
| the 1% budget: | |
| Lookback Lens 0.658% FPR precision 0.6693 recall 0.1997 | |
| LatticeSpike 0.765% precision 0.6492 recall 0.2045 | |
| Linear probe 0.842% precision 0.6331 recall 0.2178 | |
| Ranked by FPR, the precision column is in exactly reverse order. That comparison | |
| is measuring where each method's validation calibration happened to land, not | |
| which method detects better. | |
| A single shared threshold VALUE cannot fix this: an RSNN anomaly score and a | |
| logistic probability are not the same units, so one number means different | |
| things per method. What is comparable is the operating point itself. This script | |
| sweeps each method's own scores to hit a fixed FPR exactly, and reports recall | |
| there. | |
| HONEST LIMITATION, and it must be stated wherever these numbers appear: the | |
| threshold here is chosen ON TEST, so recall@FPR is a ranking-quality metric in | |
| the family of AP and AUROC -- not an achievable deployed operating point. It | |
| answers "if every method were tuned to the same alarm budget, which finds the | |
| most hallucinations", which is a different and fairer question than the headline | |
| table's. Report both; neither replaces the other. | |
| Usage: | |
| python scripts/recall_at_fixed_fpr.py \ | |
| --predictions outputs/ragtruth_l2_7b_s42/predictions.jsonl \ | |
| --predictions outputs/rt_baselines/a_no_attention/predictions.jsonl \ | |
| --predictions outputs/rt_baselines/b_lookback/predictions.jsonl \ | |
| --fpr 0.01 0.005 0.001 | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import collections | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| def parse_args(argv=None): | |
| p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) | |
| p.add_argument("--predictions", type=Path, action="append", required=True) | |
| p.add_argument( | |
| "--fpr", | |
| type=float, | |
| nargs="+", | |
| default=[0.01, 0.005, 0.001], | |
| help="Target false-positive rates. 1%% matches the headline table's budget.", | |
| ) | |
| p.add_argument("--split", default="eval", help="Only score rows from this split.") | |
| return p.parse_args(argv) | |
| def load(paths, split): | |
| """method -> (scores, labels), using the runner's own token mask.""" | |
| buckets = collections.defaultdict(lambda: ([], [])) | |
| for path in paths: | |
| if not path.exists(): | |
| print(f"!! missing: {path}") | |
| continue | |
| with path.open(encoding="utf-8") as handle: | |
| for line in handle: | |
| line = line.strip() | |
| if not line: | |
| continue | |
| row = json.loads(line) | |
| meta = row.get("metadata") or {} | |
| # LatticeSpike prediction rows carry no split field; they are | |
| # test rows by construction. | |
| if meta.get("split", split) != split: | |
| continue | |
| method = row.get("method") or "lattice_spike_rsnn_anomaly" | |
| scores = row.get("scores") or [] | |
| labels = row.get("labels") or [] | |
| if not labels or len(labels) != len(scores): | |
| continue | |
| mask = row.get("comparison_mask") or [True] * len(scores) | |
| s, y = buckets[method] | |
| for score, label, keep in zip(scores, labels, mask): | |
| # label 2 is the unlabelled class and never enters a metric. | |
| if not keep or int(label) == 2: | |
| continue | |
| s.append(float(score)) | |
| y.append(int(label)) | |
| return buckets | |
| def recall_at_fpr(scores, labels, target): | |
| """Sweep to the largest FPR not exceeding target; report recall there. | |
| Ties matter: with a coarse score distribution the achievable FPR can sit | |
| well below the target, so the achieved value is returned alongside rather | |
| than assumed equal to it. | |
| """ | |
| s = np.asarray(scores, dtype=float) | |
| y = np.asarray(labels, dtype=int) | |
| n_pos, n_neg = int((y == 1).sum()), int((y == 0).sum()) | |
| if not n_pos or not n_neg: | |
| return None | |
| order = np.argsort(-s, kind="mergesort") | |
| y_sorted = y[order] | |
| tp = np.cumsum(y_sorted == 1) | |
| fp = np.cumsum(y_sorted == 0) | |
| fpr = fp / n_neg | |
| feasible = np.nonzero(fpr <= target)[0] | |
| if feasible.size == 0: | |
| return {"recall": 0.0, "achieved_fpr": 0.0, "precision": float("nan")} | |
| i = feasible[-1] | |
| denom = tp[i] + fp[i] | |
| return { | |
| "recall": float(tp[i] / n_pos), | |
| "achieved_fpr": float(fpr[i]), | |
| "precision": float(tp[i] / denom) if denom else float("nan"), | |
| "n_pos": n_pos, | |
| "n_neg": n_neg, | |
| } | |
| def main(argv=None): | |
| args = parse_args(argv) | |
| buckets = load(args.predictions, args.split) | |
| if not buckets: | |
| raise SystemExit("no prediction rows loaded") | |
| supports = {len(v[0]) for v in buckets.values()} | |
| print("# Recall at matched false-positive rate\n") | |
| print("Threshold set on TEST to hit each FPR exactly -- a ranking metric,") | |
| print("not a deployable operating point. See the headline table for that.\n") | |
| for target in args.fpr: | |
| print(f"## FPR <= {target:.3%}\n") | |
| rows = [] | |
| for method, (scores, labels) in buckets.items(): | |
| result = recall_at_fpr(scores, labels, target) | |
| if result: | |
| rows.append((result["recall"], method, result)) | |
| rows.sort(reverse=True) | |
| print(f"{'method':<32}{'recall':>10}{'precision':>12}{'achieved FPR':>14}") | |
| print("-" * 68) | |
| for recall, method, result in rows: | |
| print( | |
| f"{method:<32}{recall:>10.4f}{result['precision']:>12.4f}" | |
| f"{result['achieved_fpr']:>13.4%}" | |
| ) | |
| print() | |
| if len(supports) == 1: | |
| n = supports.pop() | |
| print(f"support: every method scored {n:,} tokens — comparable") | |
| else: | |
| print(f"!! SUPPORT MISMATCH: {sorted(supports)} — rows are NOT comparable") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |