scripts/figures.py

"""Article figures (PNG, 1400 px wide) from results/. Usage: python scripts/figures.py [results_dir]"""
import json
from decimal import ROUND_HALF_UP, Decimal
import sys
from pathlib import Path

import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import yaml
from matplotlib.ticker import FuncFormatter

RES = Path(sys.argv[1] if len(sys.argv) > 1 else "results")
OUT = Path("article/figures")
# dataviz reference palette, slots 1-3 (validated all-pairs on white; aqua < 3:1, so every series is direct-labeled)
DS = {"fiqa": ("FiQA", "#2a78d6"), "nfcorpus": ("NFCorpus", "#eb6834"), "scidocs": ("SCIDOCS", "#1baf7a")}
INK, INK2, MUTED, GRID, AXIS = "#0b0b0b", "#52514e", "#898781", "#e1e0d9", "#c3c2b7"
PCT = FuncFormatter(lambda v, _: f"%{v:.0f}")

plt.rcParams.update({
    "font.family": ["Segoe UI", "DejaVu Sans"], "font.size": 10, "text.color": INK, "axes.labelcolor": INK2,
    "xtick.color": MUTED, "ytick.color": MUTED, "axes.edgecolor": AXIS, "axes.grid": True, "grid.color": GRID,
    "grid.linewidth": 0.6, "axes.axisbelow": True, "axes.spines.top": False, "axes.spines.right": False,
    "figure.facecolor": "white", "axes.facecolor": "white", "lines.linewidth": 2, "lines.markersize": 6,
})


def tr(x, d=1):
    # half-up on the exact value, so figure labels match the article text (e.g. 49/2000 -> %2,5)
    return str(Decimal(repr(round(float(x), 10))).quantize(Decimal(1).scaleb(-d), ROUND_HALF_UP)).replace(".", ",")


def figure(title, subtitle):
    fig, ax = plt.subplots(figsize=(7, 4), dpi=200)
    fig.subplots_adjust(top=0.80, left=0.11, right=0.84, bottom=0.14)
    fig.text(0.02, 0.95, title, fontsize=12.5, fontweight="semibold", va="top")
    fig.text(0.02, 0.885, subtitle, fontsize=9, color=INK2, va="top")
    return fig, ax


def label_end(ax, x, y, text, color):
    ax.plot(x, y, "o", color=color, markeredgecolor="white", markeredgewidth=1.5)
    ax.annotate(text, (x, y), xytext=(8, 0), textcoords="offset points", va="center", color=INK, fontsize=9)


def sampling():
    """One system, 2000 golden sets of 100 questions: the same draws simulate.simulate uses for the population truth."""
    cfg = yaml.safe_load(Path("configs/default.yaml").read_text())
    pq = pd.read_parquet(RES / "per_query.parquet")
    v = pq[(pq.dataset == "fiqa") & (pq.system == "bge")]["recall@5"].to_numpy()
    n, R = 100, cfg["simulation"]["R"]
    rng = np.random.default_rng([cfg["seed"], cfg["datasets"].index("fiqa"), 0, n])
    est = v[rng.permuted(np.tile(np.arange(len(v)), (R, 1)), axis=1)[:, :n]].mean(1)
    true, tau = v.mean(), v.mean() - 0.02
    fail = (est < tau).mean() * 100
    fig, ax = figure("Aynı sistem, 2000 farklı 100 soruluk test seti",
                     "FiQA, bge-small · her çubuk: Recall@5'i o aralıkta ölçen test setlerinin sayısı")
    bins = tau + 0.01 * np.arange(-11, 16)  # edges aligned to the threshold, so no bin straddles it
    ax.hist(est[est < tau], bins=bins, color="#eb6834", edgecolor="white", linewidth=1)
    ax.hist(est[est >= tau], bins=bins, color="#2a78d6", edgecolor="white", linewidth=1)
    top = ax.get_ylim()[1] * 1.22
    ax.set_ylim(0, top)
    ax.axvline(true, color=INK, linewidth=1.2)
    ax.annotate(f"gerçek değer\n{tr(true, 3)}", (true, top * 0.97), xytext=(6, 0), textcoords="offset points",
                va="top", fontsize=8.5)
    ax.axvline(tau, color=INK2, linestyle="--", linewidth=1.2)
    ax.annotate(f"eşik\n{tr(tau, 3)}", (tau, top * 0.97), xytext=(-6, 0), textcoords="offset points", va="top",
                ha="right", fontsize=8.5, color=INK2)
    ax.annotate(f"FAIL: %{tr(fail, 0)}", (0.29, top * 0.55), fontsize=10, fontweight="semibold", ha="center")
    ax.annotate(f"PASS: %{tr(100 - fail, 0)}", (0.46, top * 0.55), fontsize=10, fontweight="semibold", ha="center")
    ax.set_xlabel("100 soruda ölçülen Recall@5")
    ax.xaxis.set_major_formatter(FuncFormatter(lambda x, _: tr(x, 2)))
    ax.grid(axis="x", visible=False)
    return fig


def flow():
    steps = [("Gerçek veri", "3 açık veri seti (BEIR) · 323–1.000 soru, her sorunun ilgili belgeleri belli"),
             ("Arama sistemleri ve değişiklikler", "BM25, 3 embedding modeli, HNSW · hiçbir şeyin değişmediği, "
                                                   "gerçekçi ve bilerek bozulmuş sürümler"),
             ("Soru başına skor", "Her sistem her soruyu yanıtlar · bütün soruların ortalaması = gerçek değer"),
             ("2000 sanal golden set", "Her biri 25–300 soru · her set bir CI koşusu yerine geçer"),
             ("Gate kararları", "Her sette 4 gate PASS/FAIL der · karar gerçekle uyuşuyor mu, sayılır")]
    fig = plt.figure(figsize=(7, 5.2), dpi=200)
    ax = fig.add_axes([0, 0, 1, 1])
    ax.set_axis_off()
    fig.text(0.04, 0.965, "Deney düzeneği: gate'leri test eden laboratuvar", fontsize=12.5, fontweight="semibold",
             va="top")
    h, gap, y = 0.13, 0.045, 0.86
    for i, (title, body) in enumerate(steps):
        top = y - i * (h + gap)
        ax.add_patch(plt.Rectangle((0.04, top - h), 0.92, h, facecolor="#f4f7fc", edgecolor="#cde2fb", linewidth=1.2))
        ax.add_patch(plt.Rectangle((0.04, top - h), 0.012, h, facecolor="#2a78d6", edgecolor="none"))
        ax.text(0.075, top - 0.035, f"{i + 1}. {title}", fontsize=10.5, fontweight="semibold", va="top")
        ax.text(0.075, top - 0.078, body, fontsize=8.6, color=INK2, va="top")
        if i < len(steps) - 1:
            ax.annotate("", (0.5, top - h - gap + 0.004), (0.5, top - h - 0.004),
                        arrowprops=dict(arrowstyle="-|>", color=MUTED, lw=1.2))
    return fig


def g1_error():
    t = pd.read_parquet(RES / "threshold.parquet")
    t = t[(t.truth == "population") & (t.metric == "recall@5") & np.isclose(t.d.abs(), 0.02)]
    t = t.assign(wrong=np.where(t.truth_pass, t.R - t.k_pass, t.k_pass))
    err = t.groupby(["dataset", "n"])[["wrong", "R"]].sum().pipe(lambda g: 100 * g.wrong / g.R)
    fig, ax = figure("Mutlak eşik gate'i (G1), eşiğe yakın sistemlerde sık yanılıyor",
                     "Gerçek Recall@5 eşikten 0,02 uzakken yanlış karar oranı · 2000 golden set · 4 sistem, eşiğin iki yönü")
    for ds, (name, c) in DS.items():
        s = err[ds]
        ax.plot(s.index, s.values, "-o", color=c, markeredgecolor="white", markeredgewidth=1.5)
        label_end(ax, s.index[-1], s.values[-1], f"{name} %{tr(s.values[-1])}", c)
    ax.set_xticks([25, 50, 100, 200, 300])
    ax.set_xlabel("Golden set boyutu (soru sayısı)")
    ax.yaxis.set_major_formatter(PCT)
    ax.set_ylim(0, 45)
    return fig


def headroom():
    p = pd.read_parquet(RES / "pairs.parquet")
    p = p[(p.truth == "population") & (p.metric == "recall@5") & (p.B == "ctl03") & (p.n == 100)]
    gates = [("G1", 0.05, "G1\nmutlak eşik"), ("G2", 0.02, "G2\nnaif delta"), ("G3", 0.02, "G3\neşleştirilmiş")]
    fig, ax = figure("Gerçek bir 0,03'lük düşüş: hangi gate geçiriyor?",
                     "PASS oranı (düşük olması iyi) · n = 100 · G1 eşiği baseline − 0,05 · G2/G3 marjı m = 0,02")
    w = 0.26
    for i, (ds, (name, c)) in enumerate(DS.items()):
        for j, (g, par, _) in enumerate(gates):
            r = p[(p.dataset == ds) & (p.gate == g) & np.isclose(p.param, par)].iloc[0]
            v = r.k_pass / r.R * 100
            ax.bar(j + (i - 1) * w, v, w, color=c, edgecolor="white", linewidth=1.5, label=name if j == 0 else None)
            ax.annotate(f"%{tr(v)}", (j + (i - 1) * w, v), xytext=(0, 3), textcoords="offset points",
                        ha="center", fontsize=8, color=INK)
    ax.axhline(5, color=MUTED, linestyle="--", linewidth=1)
    ax.annotate("%5", (2.5, 5), xytext=(4, 0), textcoords="offset points", va="center", color=INK2, fontsize=8.5,
                annotation_clip=False)
    ax.set_xticks(range(3), [g[2] for g in gates])
    ax.grid(axis="x", visible=False)
    ax.yaxis.set_major_formatter(PCT)
    ax.set_ylim(0, 100)
    ax.legend(frameon=False, loc="upper right", fontsize=9)
    return fig


def hnsw(hyp):
    efs = [16, 32, 64, 128]
    fig, ax = figure("Hiçbir şey değişmediğinde bile: HNSW ekleme sırası gürültüsü",
                     "Aynı vektörler ve ayarlar, 10 farklı ekleme sırası · Recall@10 farklarının 95. persentili")
    for u in hyp["H4"]["per_unit"]:
        name, c = DS[u["dataset"]]
        ax.plot(efs, u["p95_by_ef"], "-o", color=c, markeredgecolor="white", markeredgewidth=1.5)
        ax.annotate(name, (efs[0], u["p95_by_ef"][0]), xytext=(-10, 0), textcoords="offset points",
                    ha="right", va="center", fontsize=9, color=INK)
    ax.set_xscale("log", base=2)
    ax.set_xticks(efs, [str(e) for e in efs])
    ax.set_xlim(9, 150)
    ax.set_xlabel("ef_search (yüksek = daha geniş arama)")
    ax.set_yticks([0, 0.005, 0.01, 0.015, 0.02])
    ax.yaxis.set_major_formatter(FuncFormatter(lambda v, _: tr(v, 3)))
    ax.set_ylim(0, 0.02)
    return fig


def latency(hyp):
    names = {"bge": "bge-small, exact", "e5": "e5-small, exact", "minilm": "MiniLM, exact",
             "hnsw_ef16_s0": "HNSW ef = 16", "hnsw_ef128_s0": "HNSW ef = 128", "bm25": "BM25"}
    rows = sorted(hyp["H5"]["per_unit"], key=lambda u: u["width_ratio_median"])
    fig, ax = figure("P95, P50'den ne kadar daha belirsiz?",
                     "FiQA, 2 iş parçacığı · P95 güven aralığı genişliği / P50 güven aralığı genişliği (koşu içi medyan)")
    y = np.arange(len(rows))
    v = [u["width_ratio_median"] for u in rows]
    ax.barh(y, v, 0.6, color=DS["fiqa"][1])
    for yi, vi in zip(y, v):
        ax.annotate(tr(vi, 2), (vi, yi), xytext=(4, 0), textcoords="offset points", va="center", fontsize=8.5,
                    bbox=dict(boxstyle="square,pad=0.1", fc="white", ec="none"))
    ax.axvline(2, color=MUTED, linestyle="--", linewidth=1, zorder=0)
    ax.annotate("ön-kayıtlı eşik: 2×", (2, len(rows) - 0.5), xytext=(4, 0), textcoords="offset points",
                color=INK2, fontsize=8.5, va="center")
    ax.set_ylim(-0.6, len(rows))
    ax.set_xticks([0, 1, 2, 3, 4])
    ax.set_yticks(y, [names[u["system"]] for u in rows], color=INK2)
    ax.grid(axis="y", visible=False)
    ax.set_xlim(0, 4.3)
    ax.xaxis.set_major_formatter(FuncFormatter(lambda v, _: tr(v, 0) + "×"))
    fig.subplots_adjust(left=0.2)
    return fig


def main():
    OUT.mkdir(parents=True, exist_ok=True)
    hyp = json.loads((RES / "hypotheses.json").read_text(encoding="utf-8"))
    for name, fig in [("0a-orneklem", sampling()), ("0b-duzenek", flow()), ("1-g1-hata", g1_error()), ("2-headroom", headroom()), ("3-hnsw", hnsw(hyp)),
                      ("4-latency", latency(hyp))]:
        fig.savefig(OUT / f"fig{name}.png")
        plt.close(fig)
        print(OUT / f"fig{name}.png")


if __name__ == "__main__":
    main()