ResearchOS / scripts /render_readme.py
Kagzy's picture
ResearchOS
e8c5cc2 verified
Raw History Blame Contribute Delete
4.84 kB
"""Render results/metrics.json into the README results table.
results/metrics.json is the source of truth. This script rewrites the block
between the RESULTS markers in README.md so the two can never drift.
python scripts/render_readme.py
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.rerank.evaluate import ROW_LABELS, ROW_ORDER
START = "<!-- RESULTS:START -->"
END = "<!-- RESULTS:END -->"
NL = "\n"
def _fmt(value: float | None, places: int = 4) -> str:
return "β€”" if value is None else f"{value:.{places}f}"
def build_table(rows: dict) -> str:
"""Markdown table in the exact row order the spec defines."""
lines = [
"| Setup | nDCG@10 | Recall@50 | p95 latency | Params |",
"|---|---|---|---|---|",
]
for row_id in ROW_ORDER:
row = rows.get(row_id)
label = ROW_LABELS[row_id]
if not row:
lines.append(f"| {label} | β€” | β€” | β€” | β€” |")
continue
metrics = row.get("metrics", {})
latency = row.get("latency", {})
p95 = latency.get("p95_ms")
lines.append(
f"| {label} "
f"| {_fmt(metrics.get('ndcg@10'))} "
f"| {_fmt(metrics.get('recall@50'))} "
f"| {'β€”' if p95 is None else f'{p95:.0f} ms'} "
f"| {row.get('params') or 'β€”'} |"
)
return NL.join(lines)
def build_delta(rows: dict, sig_path: str = "results/significance.json") -> str:
"""The headline findings, with the significance test attached.
A delta reported without its confidence interval is a claim, not a result.
"""
hard = rows.get("finetuned_hard_negatives", {}).get("metrics", {})
rand = rows.get("finetuned_random_negatives", {}).get("metrics", {})
off = rows.get("offshelf_reranker", {}).get("metrics", {})
if not hard or not rand:
return ""
parts: list[str] = []
if off:
gain = hard.get("ndcg@10", 0.0) - off.get("ndcg@10", 0.0)
parts.append(
f"**Fine-tuning works.** The fine-tuned reranker scores nDCG@10 "
f"{hard.get('ndcg@10'):.4f} against {off.get('ndcg@10'):.4f} off the shelf "
f"({gain:+.4f}), and the off-the-shelf model is itself *below* the "
f"retrieval baseline it is meant to improve."
)
delta = hard.get("ndcg@10", 0.0) - rand.get("ndcg@10", 0.0)
sig_file = Path(sig_path)
if sig_file.exists():
sig = json.loads(sig_file.read_text(encoding="utf-8"))
low, high = sig["bootstrap_95_ci"]
verdict = (
"statistically significant"
if sig["significant_at_0.05"]
else "NOT statistically significant"
)
per_query = sig["per_query"]
parts.append(
f"**Hard vs. random negatives: {delta:+.4f} nDCG@10 β€” {verdict}.** "
f"Paired permutation p = {sig['permutation_p_value']:.3f}; 95% bootstrap CI "
f"[{low:+.4f}, {high:+.4f}] over {sig['n_queries']} queries. Per query: "
f"{per_query['treatment_better']} better, {per_query['control_better']} worse, "
f"{per_query['tied']} identical. Same seed, same hyperparameters, same epochs β€” "
f"only the negatives file differs. Reproduce with `scripts/significance.py`."
)
else:
parts.append(
f"**Hard vs. random negatives:** {delta:+.4f} nDCG@10 "
f"(no significance test has been run)."
)
return NL + (NL * 2).join(parts) + NL
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--metrics", default="results/metrics.json")
parser.add_argument("--significance", default="results/significance.json")
parser.add_argument("--readme", default="README.md")
args = parser.parse_args()
metrics_path = Path(args.metrics)
if not metrics_path.exists():
raise SystemExit(f"{metrics_path} not found - run the pipeline stages first.")
rows = json.loads(metrics_path.read_text(encoding="utf-8")).get("rows", {})
block = START + NL * 2 + build_table(rows) + NL + build_delta(rows, args.significance) + NL + END
readme_path = Path(args.readme)
if not readme_path.exists():
raise SystemExit(f"{readme_path} not found.")
text = readme_path.read_text(encoding="utf-8")
if START not in text or END not in text:
raise SystemExit(f"README.md is missing the {START} / {END} markers.")
before = text.split(START)[0]
after = text.split(END, 1)[1]
readme_path.write_text(before + block + after, encoding="utf-8")
print(f"[readme] rendered {len(rows)} row(s) into {readme_path}")
if __name__ == "__main__":
main()