File size: 4,837 Bytes
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
"""Render results/metrics.json into the README results table.

results/metrics.json is the source of truth. This script rewrites the block
between the RESULTS markers in README.md so the two can never drift.

    python scripts/render_readme.py
"""

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from src.rerank.evaluate import ROW_LABELS, ROW_ORDER

START = "<!-- RESULTS:START -->"
END = "<!-- RESULTS:END -->"
NL = "\n"


def _fmt(value: float | None, places: int = 4) -> str:
    return "β€”" if value is None else f"{value:.{places}f}"


def build_table(rows: dict) -> str:
    """Markdown table in the exact row order the spec defines."""
    lines = [
        "| Setup | nDCG@10 | Recall@50 | p95 latency | Params |",
        "|---|---|---|---|---|",
    ]
    for row_id in ROW_ORDER:
        row = rows.get(row_id)
        label = ROW_LABELS[row_id]
        if not row:
            lines.append(f"| {label} | β€” | β€” | β€” | β€” |")
            continue
        metrics = row.get("metrics", {})
        latency = row.get("latency", {})
        p95 = latency.get("p95_ms")
        lines.append(
            f"| {label} "
            f"| {_fmt(metrics.get('ndcg@10'))} "
            f"| {_fmt(metrics.get('recall@50'))} "
            f"| {'β€”' if p95 is None else f'{p95:.0f} ms'} "
            f"| {row.get('params') or 'β€”'} |"
        )
    return NL.join(lines)


def build_delta(rows: dict, sig_path: str = "results/significance.json") -> str:
    """The headline findings, with the significance test attached.

    A delta reported without its confidence interval is a claim, not a result.
    """
    hard = rows.get("finetuned_hard_negatives", {}).get("metrics", {})
    rand = rows.get("finetuned_random_negatives", {}).get("metrics", {})
    off = rows.get("offshelf_reranker", {}).get("metrics", {})
    if not hard or not rand:
        return ""

    parts: list[str] = []

    if off:
        gain = hard.get("ndcg@10", 0.0) - off.get("ndcg@10", 0.0)
        parts.append(
            f"**Fine-tuning works.** The fine-tuned reranker scores nDCG@10 "
            f"{hard.get('ndcg@10'):.4f} against {off.get('ndcg@10'):.4f} off the shelf "
            f"({gain:+.4f}), and the off-the-shelf model is itself *below* the "
            f"retrieval baseline it is meant to improve."
        )

    delta = hard.get("ndcg@10", 0.0) - rand.get("ndcg@10", 0.0)
    sig_file = Path(sig_path)
    if sig_file.exists():
        sig = json.loads(sig_file.read_text(encoding="utf-8"))
        low, high = sig["bootstrap_95_ci"]
        verdict = (
            "statistically significant"
            if sig["significant_at_0.05"]
            else "NOT statistically significant"
        )
        per_query = sig["per_query"]
        parts.append(
            f"**Hard vs. random negatives: {delta:+.4f} nDCG@10 β€” {verdict}.** "
            f"Paired permutation p = {sig['permutation_p_value']:.3f}; 95% bootstrap CI "
            f"[{low:+.4f}, {high:+.4f}] over {sig['n_queries']} queries. Per query: "
            f"{per_query['treatment_better']} better, {per_query['control_better']} worse, "
            f"{per_query['tied']} identical. Same seed, same hyperparameters, same epochs β€” "
            f"only the negatives file differs. Reproduce with `scripts/significance.py`."
        )
    else:
        parts.append(
            f"**Hard vs. random negatives:** {delta:+.4f} nDCG@10 "
            f"(no significance test has been run)."
        )

    return NL + (NL * 2).join(parts) + NL


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--metrics", default="results/metrics.json")
    parser.add_argument("--significance", default="results/significance.json")
    parser.add_argument("--readme", default="README.md")
    args = parser.parse_args()

    metrics_path = Path(args.metrics)
    if not metrics_path.exists():
        raise SystemExit(f"{metrics_path} not found - run the pipeline stages first.")
    rows = json.loads(metrics_path.read_text(encoding="utf-8")).get("rows", {})

    block = START + NL * 2 + build_table(rows) + NL + build_delta(rows, args.significance) + NL + END

    readme_path = Path(args.readme)
    if not readme_path.exists():
        raise SystemExit(f"{readme_path} not found.")
    text = readme_path.read_text(encoding="utf-8")

    if START not in text or END not in text:
        raise SystemExit(f"README.md is missing the {START} / {END} markers.")

    before = text.split(START)[0]
    after = text.split(END, 1)[1]
    readme_path.write_text(before + block + after, encoding="utf-8")
    print(f"[readme] rendered {len(rows)} row(s) into {readme_path}")


if __name__ == "__main__":
    main()