Lattice_Spike / scripts /recall_at_fixed_fpr.py
Md Saiful Islam Sajol
Upload Lattice_Spike project with full LFS weights
e2c9d42 verified
Raw History Blame Contribute Delete
6.53 kB
#!/usr/bin/env python3
"""Report every method at the SAME false-positive rate, not the same threshold.
Why this exists. Our headline table sets each method's threshold on validation
at the most permissive point with FPR <= 1%, then applies it unchanged to test.
That is the right protocol for a deployable number -- nothing about the test set
influences the operating point -- but it makes the precision and recall columns
hard to read across methods, because each one lands somewhere different inside
the 1% budget:
Lookback Lens 0.658% FPR precision 0.6693 recall 0.1997
LatticeSpike 0.765% precision 0.6492 recall 0.2045
Linear probe 0.842% precision 0.6331 recall 0.2178
Ranked by FPR, the precision column is in exactly reverse order. That comparison
is measuring where each method's validation calibration happened to land, not
which method detects better.
A single shared threshold VALUE cannot fix this: an RSNN anomaly score and a
logistic probability are not the same units, so one number means different
things per method. What is comparable is the operating point itself. This script
sweeps each method's own scores to hit a fixed FPR exactly, and reports recall
there.
HONEST LIMITATION, and it must be stated wherever these numbers appear: the
threshold here is chosen ON TEST, so recall@FPR is a ranking-quality metric in
the family of AP and AUROC -- not an achievable deployed operating point. It
answers "if every method were tuned to the same alarm budget, which finds the
most hallucinations", which is a different and fairer question than the headline
table's. Report both; neither replaces the other.
Usage:
python scripts/recall_at_fixed_fpr.py \
--predictions outputs/ragtruth_l2_7b_s42/predictions.jsonl \
--predictions outputs/rt_baselines/a_no_attention/predictions.jsonl \
--predictions outputs/rt_baselines/b_lookback/predictions.jsonl \
--fpr 0.01 0.005 0.001
"""
from __future__ import annotations
import argparse
import collections
import json
from pathlib import Path
import numpy as np
def parse_args(argv=None):
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
p.add_argument("--predictions", type=Path, action="append", required=True)
p.add_argument(
"--fpr",
type=float,
nargs="+",
default=[0.01, 0.005, 0.001],
help="Target false-positive rates. 1%% matches the headline table's budget.",
)
p.add_argument("--split", default="eval", help="Only score rows from this split.")
return p.parse_args(argv)
def load(paths, split):
"""method -> (scores, labels), using the runner's own token mask."""
buckets = collections.defaultdict(lambda: ([], []))
for path in paths:
if not path.exists():
print(f"!! missing: {path}")
continue
with path.open(encoding="utf-8") as handle:
for line in handle:
line = line.strip()
if not line:
continue
row = json.loads(line)
meta = row.get("metadata") or {}
# LatticeSpike prediction rows carry no split field; they are
# test rows by construction.
if meta.get("split", split) != split:
continue
method = row.get("method") or "lattice_spike_rsnn_anomaly"
scores = row.get("scores") or []
labels = row.get("labels") or []
if not labels or len(labels) != len(scores):
continue
mask = row.get("comparison_mask") or [True] * len(scores)
s, y = buckets[method]
for score, label, keep in zip(scores, labels, mask):
# label 2 is the unlabelled class and never enters a metric.
if not keep or int(label) == 2:
continue
s.append(float(score))
y.append(int(label))
return buckets
def recall_at_fpr(scores, labels, target):
"""Sweep to the largest FPR not exceeding target; report recall there.
Ties matter: with a coarse score distribution the achievable FPR can sit
well below the target, so the achieved value is returned alongside rather
than assumed equal to it.
"""
s = np.asarray(scores, dtype=float)
y = np.asarray(labels, dtype=int)
n_pos, n_neg = int((y == 1).sum()), int((y == 0).sum())
if not n_pos or not n_neg:
return None
order = np.argsort(-s, kind="mergesort")
y_sorted = y[order]
tp = np.cumsum(y_sorted == 1)
fp = np.cumsum(y_sorted == 0)
fpr = fp / n_neg
feasible = np.nonzero(fpr <= target)[0]
if feasible.size == 0:
return {"recall": 0.0, "achieved_fpr": 0.0, "precision": float("nan")}
i = feasible[-1]
denom = tp[i] + fp[i]
return {
"recall": float(tp[i] / n_pos),
"achieved_fpr": float(fpr[i]),
"precision": float(tp[i] / denom) if denom else float("nan"),
"n_pos": n_pos,
"n_neg": n_neg,
}
def main(argv=None):
args = parse_args(argv)
buckets = load(args.predictions, args.split)
if not buckets:
raise SystemExit("no prediction rows loaded")
supports = {len(v[0]) for v in buckets.values()}
print("# Recall at matched false-positive rate\n")
print("Threshold set on TEST to hit each FPR exactly -- a ranking metric,")
print("not a deployable operating point. See the headline table for that.\n")
for target in args.fpr:
print(f"## FPR <= {target:.3%}\n")
rows = []
for method, (scores, labels) in buckets.items():
result = recall_at_fpr(scores, labels, target)
if result:
rows.append((result["recall"], method, result))
rows.sort(reverse=True)
print(f"{'method':<32}{'recall':>10}{'precision':>12}{'achieved FPR':>14}")
print("-" * 68)
for recall, method, result in rows:
print(
f"{method:<32}{recall:>10.4f}{result['precision']:>12.4f}"
f"{result['achieved_fpr']:>13.4%}"
)
print()
if len(supports) == 1:
n = supports.pop()
print(f"support: every method scored {n:,} tokens — comparable")
else:
print(f"!! SUPPORT MISMATCH: {sorted(supports)} — rows are NOT comparable")
return 0
if __name__ == "__main__":
raise SystemExit(main())