Spaces:
Running on Zero
Running on Zero
File size: 4,837 Bytes
e8c5cc2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 | """Render results/metrics.json into the README results table.
results/metrics.json is the source of truth. This script rewrites the block
between the RESULTS markers in README.md so the two can never drift.
python scripts/render_readme.py
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.rerank.evaluate import ROW_LABELS, ROW_ORDER
START = "<!-- RESULTS:START -->"
END = "<!-- RESULTS:END -->"
NL = "\n"
def _fmt(value: float | None, places: int = 4) -> str:
return "β" if value is None else f"{value:.{places}f}"
def build_table(rows: dict) -> str:
"""Markdown table in the exact row order the spec defines."""
lines = [
"| Setup | nDCG@10 | Recall@50 | p95 latency | Params |",
"|---|---|---|---|---|",
]
for row_id in ROW_ORDER:
row = rows.get(row_id)
label = ROW_LABELS[row_id]
if not row:
lines.append(f"| {label} | β | β | β | β |")
continue
metrics = row.get("metrics", {})
latency = row.get("latency", {})
p95 = latency.get("p95_ms")
lines.append(
f"| {label} "
f"| {_fmt(metrics.get('ndcg@10'))} "
f"| {_fmt(metrics.get('recall@50'))} "
f"| {'β' if p95 is None else f'{p95:.0f} ms'} "
f"| {row.get('params') or 'β'} |"
)
return NL.join(lines)
def build_delta(rows: dict, sig_path: str = "results/significance.json") -> str:
"""The headline findings, with the significance test attached.
A delta reported without its confidence interval is a claim, not a result.
"""
hard = rows.get("finetuned_hard_negatives", {}).get("metrics", {})
rand = rows.get("finetuned_random_negatives", {}).get("metrics", {})
off = rows.get("offshelf_reranker", {}).get("metrics", {})
if not hard or not rand:
return ""
parts: list[str] = []
if off:
gain = hard.get("ndcg@10", 0.0) - off.get("ndcg@10", 0.0)
parts.append(
f"**Fine-tuning works.** The fine-tuned reranker scores nDCG@10 "
f"{hard.get('ndcg@10'):.4f} against {off.get('ndcg@10'):.4f} off the shelf "
f"({gain:+.4f}), and the off-the-shelf model is itself *below* the "
f"retrieval baseline it is meant to improve."
)
delta = hard.get("ndcg@10", 0.0) - rand.get("ndcg@10", 0.0)
sig_file = Path(sig_path)
if sig_file.exists():
sig = json.loads(sig_file.read_text(encoding="utf-8"))
low, high = sig["bootstrap_95_ci"]
verdict = (
"statistically significant"
if sig["significant_at_0.05"]
else "NOT statistically significant"
)
per_query = sig["per_query"]
parts.append(
f"**Hard vs. random negatives: {delta:+.4f} nDCG@10 β {verdict}.** "
f"Paired permutation p = {sig['permutation_p_value']:.3f}; 95% bootstrap CI "
f"[{low:+.4f}, {high:+.4f}] over {sig['n_queries']} queries. Per query: "
f"{per_query['treatment_better']} better, {per_query['control_better']} worse, "
f"{per_query['tied']} identical. Same seed, same hyperparameters, same epochs β "
f"only the negatives file differs. Reproduce with `scripts/significance.py`."
)
else:
parts.append(
f"**Hard vs. random negatives:** {delta:+.4f} nDCG@10 "
f"(no significance test has been run)."
)
return NL + (NL * 2).join(parts) + NL
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--metrics", default="results/metrics.json")
parser.add_argument("--significance", default="results/significance.json")
parser.add_argument("--readme", default="README.md")
args = parser.parse_args()
metrics_path = Path(args.metrics)
if not metrics_path.exists():
raise SystemExit(f"{metrics_path} not found - run the pipeline stages first.")
rows = json.loads(metrics_path.read_text(encoding="utf-8")).get("rows", {})
block = START + NL * 2 + build_table(rows) + NL + build_delta(rows, args.significance) + NL + END
readme_path = Path(args.readme)
if not readme_path.exists():
raise SystemExit(f"{readme_path} not found.")
text = readme_path.read_text(encoding="utf-8")
if START not in text or END not in text:
raise SystemExit(f"README.md is missing the {START} / {END} markers.")
before = text.split(START)[0]
after = text.split(END, 1)[1]
readme_path.write_text(before + block + after, encoding="utf-8")
print(f"[readme] rendered {len(rows)} row(s) into {readme_path}")
if __name__ == "__main__":
main()
|