"""Article figures (PNG, 1400 px wide) from results/. Usage: python scripts/figures.py [results_dir]""" import json from decimal import ROUND_HALF_UP, Decimal import sys from pathlib import Path import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np import pandas as pd import yaml from matplotlib.ticker import FuncFormatter RES = Path(sys.argv[1] if len(sys.argv) > 1 else "results") OUT = Path("article/figures") # dataviz reference palette, slots 1-3 (validated all-pairs on white; aqua < 3:1, so every series is direct-labeled) DS = {"fiqa": ("FiQA", "#2a78d6"), "nfcorpus": ("NFCorpus", "#eb6834"), "scidocs": ("SCIDOCS", "#1baf7a")} INK, INK2, MUTED, GRID, AXIS = "#0b0b0b", "#52514e", "#898781", "#e1e0d9", "#c3c2b7" PCT = FuncFormatter(lambda v, _: f"%{v:.0f}") plt.rcParams.update({ "font.family": ["Segoe UI", "DejaVu Sans"], "font.size": 10, "text.color": INK, "axes.labelcolor": INK2, "xtick.color": MUTED, "ytick.color": MUTED, "axes.edgecolor": AXIS, "axes.grid": True, "grid.color": GRID, "grid.linewidth": 0.6, "axes.axisbelow": True, "axes.spines.top": False, "axes.spines.right": False, "figure.facecolor": "white", "axes.facecolor": "white", "lines.linewidth": 2, "lines.markersize": 6, }) def tr(x, d=1): # half-up on the exact value, so figure labels match the article text (e.g. 49/2000 -> %2,5) return str(Decimal(repr(round(float(x), 10))).quantize(Decimal(1).scaleb(-d), ROUND_HALF_UP)).replace(".", ",") def figure(title, subtitle): fig, ax = plt.subplots(figsize=(7, 4), dpi=200) fig.subplots_adjust(top=0.80, left=0.11, right=0.84, bottom=0.14) fig.text(0.02, 0.95, title, fontsize=12.5, fontweight="semibold", va="top") fig.text(0.02, 0.885, subtitle, fontsize=9, color=INK2, va="top") return fig, ax def label_end(ax, x, y, text, color): ax.plot(x, y, "o", color=color, markeredgecolor="white", markeredgewidth=1.5) ax.annotate(text, (x, y), xytext=(8, 0), textcoords="offset points", va="center", color=INK, fontsize=9) def sampling(): """One system, 2000 golden sets of 100 questions: the same draws simulate.simulate uses for the population truth.""" cfg = yaml.safe_load(Path("configs/default.yaml").read_text()) pq = pd.read_parquet(RES / "per_query.parquet") v = pq[(pq.dataset == "fiqa") & (pq.system == "bge")]["recall@5"].to_numpy() n, R = 100, cfg["simulation"]["R"] rng = np.random.default_rng([cfg["seed"], cfg["datasets"].index("fiqa"), 0, n]) est = v[rng.permuted(np.tile(np.arange(len(v)), (R, 1)), axis=1)[:, :n]].mean(1) true, tau = v.mean(), v.mean() - 0.02 fail = (est < tau).mean() * 100 fig, ax = figure("Aynı sistem, 2000 farklı 100 soruluk test seti", "FiQA, bge-small · her çubuk: Recall@5'i o aralıkta ölçen test setlerinin sayısı") bins = tau + 0.01 * np.arange(-11, 16) # edges aligned to the threshold, so no bin straddles it ax.hist(est[est < tau], bins=bins, color="#eb6834", edgecolor="white", linewidth=1) ax.hist(est[est >= tau], bins=bins, color="#2a78d6", edgecolor="white", linewidth=1) top = ax.get_ylim()[1] * 1.22 ax.set_ylim(0, top) ax.axvline(true, color=INK, linewidth=1.2) ax.annotate(f"gerçek değer\n{tr(true, 3)}", (true, top * 0.97), xytext=(6, 0), textcoords="offset points", va="top", fontsize=8.5) ax.axvline(tau, color=INK2, linestyle="--", linewidth=1.2) ax.annotate(f"eşik\n{tr(tau, 3)}", (tau, top * 0.97), xytext=(-6, 0), textcoords="offset points", va="top", ha="right", fontsize=8.5, color=INK2) ax.annotate(f"FAIL: %{tr(fail, 0)}", (0.29, top * 0.55), fontsize=10, fontweight="semibold", ha="center") ax.annotate(f"PASS: %{tr(100 - fail, 0)}", (0.46, top * 0.55), fontsize=10, fontweight="semibold", ha="center") ax.set_xlabel("100 soruda ölçülen Recall@5") ax.xaxis.set_major_formatter(FuncFormatter(lambda x, _: tr(x, 2))) ax.grid(axis="x", visible=False) return fig def flow(): steps = [("Gerçek veri", "3 açık veri seti (BEIR) · 323–1.000 soru, her sorunun ilgili belgeleri belli"), ("Arama sistemleri ve değişiklikler", "BM25, 3 embedding modeli, HNSW · hiçbir şeyin değişmediği, " "gerçekçi ve bilerek bozulmuş sürümler"), ("Soru başına skor", "Her sistem her soruyu yanıtlar · bütün soruların ortalaması = gerçek değer"), ("2000 sanal golden set", "Her biri 25–300 soru · her set bir CI koşusu yerine geçer"), ("Gate kararları", "Her sette 4 gate PASS/FAIL der · karar gerçekle uyuşuyor mu, sayılır")] fig = plt.figure(figsize=(7, 5.2), dpi=200) ax = fig.add_axes([0, 0, 1, 1]) ax.set_axis_off() fig.text(0.04, 0.965, "Deney düzeneği: gate'leri test eden laboratuvar", fontsize=12.5, fontweight="semibold", va="top") h, gap, y = 0.13, 0.045, 0.86 for i, (title, body) in enumerate(steps): top = y - i * (h + gap) ax.add_patch(plt.Rectangle((0.04, top - h), 0.92, h, facecolor="#f4f7fc", edgecolor="#cde2fb", linewidth=1.2)) ax.add_patch(plt.Rectangle((0.04, top - h), 0.012, h, facecolor="#2a78d6", edgecolor="none")) ax.text(0.075, top - 0.035, f"{i + 1}. {title}", fontsize=10.5, fontweight="semibold", va="top") ax.text(0.075, top - 0.078, body, fontsize=8.6, color=INK2, va="top") if i < len(steps) - 1: ax.annotate("", (0.5, top - h - gap + 0.004), (0.5, top - h - 0.004), arrowprops=dict(arrowstyle="-|>", color=MUTED, lw=1.2)) return fig def g1_error(): t = pd.read_parquet(RES / "threshold.parquet") t = t[(t.truth == "population") & (t.metric == "recall@5") & np.isclose(t.d.abs(), 0.02)] t = t.assign(wrong=np.where(t.truth_pass, t.R - t.k_pass, t.k_pass)) err = t.groupby(["dataset", "n"])[["wrong", "R"]].sum().pipe(lambda g: 100 * g.wrong / g.R) fig, ax = figure("Mutlak eşik gate'i (G1), eşiğe yakın sistemlerde sık yanılıyor", "Gerçek Recall@5 eşikten 0,02 uzakken yanlış karar oranı · 2000 golden set · 4 sistem, eşiğin iki yönü") for ds, (name, c) in DS.items(): s = err[ds] ax.plot(s.index, s.values, "-o", color=c, markeredgecolor="white", markeredgewidth=1.5) label_end(ax, s.index[-1], s.values[-1], f"{name} %{tr(s.values[-1])}", c) ax.set_xticks([25, 50, 100, 200, 300]) ax.set_xlabel("Golden set boyutu (soru sayısı)") ax.yaxis.set_major_formatter(PCT) ax.set_ylim(0, 45) return fig def headroom(): p = pd.read_parquet(RES / "pairs.parquet") p = p[(p.truth == "population") & (p.metric == "recall@5") & (p.B == "ctl03") & (p.n == 100)] gates = [("G1", 0.05, "G1\nmutlak eşik"), ("G2", 0.02, "G2\nnaif delta"), ("G3", 0.02, "G3\neşleştirilmiş")] fig, ax = figure("Gerçek bir 0,03'lük düşüş: hangi gate geçiriyor?", "PASS oranı (düşük olması iyi) · n = 100 · G1 eşiği baseline − 0,05 · G2/G3 marjı m = 0,02") w = 0.26 for i, (ds, (name, c)) in enumerate(DS.items()): for j, (g, par, _) in enumerate(gates): r = p[(p.dataset == ds) & (p.gate == g) & np.isclose(p.param, par)].iloc[0] v = r.k_pass / r.R * 100 ax.bar(j + (i - 1) * w, v, w, color=c, edgecolor="white", linewidth=1.5, label=name if j == 0 else None) ax.annotate(f"%{tr(v)}", (j + (i - 1) * w, v), xytext=(0, 3), textcoords="offset points", ha="center", fontsize=8, color=INK) ax.axhline(5, color=MUTED, linestyle="--", linewidth=1) ax.annotate("%5", (2.5, 5), xytext=(4, 0), textcoords="offset points", va="center", color=INK2, fontsize=8.5, annotation_clip=False) ax.set_xticks(range(3), [g[2] for g in gates]) ax.grid(axis="x", visible=False) ax.yaxis.set_major_formatter(PCT) ax.set_ylim(0, 100) ax.legend(frameon=False, loc="upper right", fontsize=9) return fig def hnsw(hyp): efs = [16, 32, 64, 128] fig, ax = figure("Hiçbir şey değişmediğinde bile: HNSW ekleme sırası gürültüsü", "Aynı vektörler ve ayarlar, 10 farklı ekleme sırası · Recall@10 farklarının 95. persentili") for u in hyp["H4"]["per_unit"]: name, c = DS[u["dataset"]] ax.plot(efs, u["p95_by_ef"], "-o", color=c, markeredgecolor="white", markeredgewidth=1.5) ax.annotate(name, (efs[0], u["p95_by_ef"][0]), xytext=(-10, 0), textcoords="offset points", ha="right", va="center", fontsize=9, color=INK) ax.set_xscale("log", base=2) ax.set_xticks(efs, [str(e) for e in efs]) ax.set_xlim(9, 150) ax.set_xlabel("ef_search (yüksek = daha geniş arama)") ax.set_yticks([0, 0.005, 0.01, 0.015, 0.02]) ax.yaxis.set_major_formatter(FuncFormatter(lambda v, _: tr(v, 3))) ax.set_ylim(0, 0.02) return fig def latency(hyp): names = {"bge": "bge-small, exact", "e5": "e5-small, exact", "minilm": "MiniLM, exact", "hnsw_ef16_s0": "HNSW ef = 16", "hnsw_ef128_s0": "HNSW ef = 128", "bm25": "BM25"} rows = sorted(hyp["H5"]["per_unit"], key=lambda u: u["width_ratio_median"]) fig, ax = figure("P95, P50'den ne kadar daha belirsiz?", "FiQA, 2 iş parçacığı · P95 güven aralığı genişliği / P50 güven aralığı genişliği (koşu içi medyan)") y = np.arange(len(rows)) v = [u["width_ratio_median"] for u in rows] ax.barh(y, v, 0.6, color=DS["fiqa"][1]) for yi, vi in zip(y, v): ax.annotate(tr(vi, 2), (vi, yi), xytext=(4, 0), textcoords="offset points", va="center", fontsize=8.5, bbox=dict(boxstyle="square,pad=0.1", fc="white", ec="none")) ax.axvline(2, color=MUTED, linestyle="--", linewidth=1, zorder=0) ax.annotate("ön-kayıtlı eşik: 2×", (2, len(rows) - 0.5), xytext=(4, 0), textcoords="offset points", color=INK2, fontsize=8.5, va="center") ax.set_ylim(-0.6, len(rows)) ax.set_xticks([0, 1, 2, 3, 4]) ax.set_yticks(y, [names[u["system"]] for u in rows], color=INK2) ax.grid(axis="y", visible=False) ax.set_xlim(0, 4.3) ax.xaxis.set_major_formatter(FuncFormatter(lambda v, _: tr(v, 0) + "×")) fig.subplots_adjust(left=0.2) return fig def main(): OUT.mkdir(parents=True, exist_ok=True) hyp = json.loads((RES / "hypotheses.json").read_text(encoding="utf-8")) for name, fig in [("0a-orneklem", sampling()), ("0b-duzenek", flow()), ("1-g1-hata", g1_error()), ("2-headroom", headroom()), ("3-hnsw", hnsw(hyp)), ("4-latency", latency(hyp))]: fig.savefig(OUT / f"fig{name}.png") plt.close(fig) print(OUT / f"fig{name}.png") if __name__ == "__main__": main()