| """Article figures (PNG, 1400 px wide) from results/. Usage: python scripts/figures.py [results_dir]"""
|
| import json
|
| from decimal import ROUND_HALF_UP, Decimal
|
| import sys
|
| from pathlib import Path
|
|
|
| import matplotlib
|
| matplotlib.use("Agg")
|
| import matplotlib.pyplot as plt
|
| import numpy as np
|
| import pandas as pd
|
| import yaml
|
| from matplotlib.ticker import FuncFormatter
|
|
|
| RES = Path(sys.argv[1] if len(sys.argv) > 1 else "results")
|
| OUT = Path("article/figures")
|
|
|
| DS = {"fiqa": ("FiQA", "#2a78d6"), "nfcorpus": ("NFCorpus", "#eb6834"), "scidocs": ("SCIDOCS", "#1baf7a")}
|
| INK, INK2, MUTED, GRID, AXIS = "#0b0b0b", "#52514e", "#898781", "#e1e0d9", "#c3c2b7"
|
| PCT = FuncFormatter(lambda v, _: f"%{v:.0f}")
|
|
|
| plt.rcParams.update({
|
| "font.family": ["Segoe UI", "DejaVu Sans"], "font.size": 10, "text.color": INK, "axes.labelcolor": INK2,
|
| "xtick.color": MUTED, "ytick.color": MUTED, "axes.edgecolor": AXIS, "axes.grid": True, "grid.color": GRID,
|
| "grid.linewidth": 0.6, "axes.axisbelow": True, "axes.spines.top": False, "axes.spines.right": False,
|
| "figure.facecolor": "white", "axes.facecolor": "white", "lines.linewidth": 2, "lines.markersize": 6,
|
| })
|
|
|
|
|
| def tr(x, d=1):
|
|
|
| return str(Decimal(repr(round(float(x), 10))).quantize(Decimal(1).scaleb(-d), ROUND_HALF_UP)).replace(".", ",")
|
|
|
|
|
| def figure(title, subtitle):
|
| fig, ax = plt.subplots(figsize=(7, 4), dpi=200)
|
| fig.subplots_adjust(top=0.80, left=0.11, right=0.84, bottom=0.14)
|
| fig.text(0.02, 0.95, title, fontsize=12.5, fontweight="semibold", va="top")
|
| fig.text(0.02, 0.885, subtitle, fontsize=9, color=INK2, va="top")
|
| return fig, ax
|
|
|
|
|
| def label_end(ax, x, y, text, color):
|
| ax.plot(x, y, "o", color=color, markeredgecolor="white", markeredgewidth=1.5)
|
| ax.annotate(text, (x, y), xytext=(8, 0), textcoords="offset points", va="center", color=INK, fontsize=9)
|
|
|
|
|
| def sampling():
|
| """One system, 2000 golden sets of 100 questions: the same draws simulate.simulate uses for the population truth."""
|
| cfg = yaml.safe_load(Path("configs/default.yaml").read_text())
|
| pq = pd.read_parquet(RES / "per_query.parquet")
|
| v = pq[(pq.dataset == "fiqa") & (pq.system == "bge")]["recall@5"].to_numpy()
|
| n, R = 100, cfg["simulation"]["R"]
|
| rng = np.random.default_rng([cfg["seed"], cfg["datasets"].index("fiqa"), 0, n])
|
| est = v[rng.permuted(np.tile(np.arange(len(v)), (R, 1)), axis=1)[:, :n]].mean(1)
|
| true, tau = v.mean(), v.mean() - 0.02
|
| fail = (est < tau).mean() * 100
|
| fig, ax = figure("Aynı sistem, 2000 farklı 100 soruluk test seti",
|
| "FiQA, bge-small · her çubuk: Recall@5'i o aralıkta ölçen test setlerinin sayısı")
|
| bins = tau + 0.01 * np.arange(-11, 16)
|
| ax.hist(est[est < tau], bins=bins, color="#eb6834", edgecolor="white", linewidth=1)
|
| ax.hist(est[est >= tau], bins=bins, color="#2a78d6", edgecolor="white", linewidth=1)
|
| top = ax.get_ylim()[1] * 1.22
|
| ax.set_ylim(0, top)
|
| ax.axvline(true, color=INK, linewidth=1.2)
|
| ax.annotate(f"gerçek değer\n{tr(true, 3)}", (true, top * 0.97), xytext=(6, 0), textcoords="offset points",
|
| va="top", fontsize=8.5)
|
| ax.axvline(tau, color=INK2, linestyle="--", linewidth=1.2)
|
| ax.annotate(f"eşik\n{tr(tau, 3)}", (tau, top * 0.97), xytext=(-6, 0), textcoords="offset points", va="top",
|
| ha="right", fontsize=8.5, color=INK2)
|
| ax.annotate(f"FAIL: %{tr(fail, 0)}", (0.29, top * 0.55), fontsize=10, fontweight="semibold", ha="center")
|
| ax.annotate(f"PASS: %{tr(100 - fail, 0)}", (0.46, top * 0.55), fontsize=10, fontweight="semibold", ha="center")
|
| ax.set_xlabel("100 soruda ölçülen Recall@5")
|
| ax.xaxis.set_major_formatter(FuncFormatter(lambda x, _: tr(x, 2)))
|
| ax.grid(axis="x", visible=False)
|
| return fig
|
|
|
|
|
| def flow():
|
| steps = [("Gerçek veri", "3 açık veri seti (BEIR) · 323–1.000 soru, her sorunun ilgili belgeleri belli"),
|
| ("Arama sistemleri ve değişiklikler", "BM25, 3 embedding modeli, HNSW · hiçbir şeyin değişmediği, "
|
| "gerçekçi ve bilerek bozulmuş sürümler"),
|
| ("Soru başına skor", "Her sistem her soruyu yanıtlar · bütün soruların ortalaması = gerçek değer"),
|
| ("2000 sanal golden set", "Her biri 25–300 soru · her set bir CI koşusu yerine geçer"),
|
| ("Gate kararları", "Her sette 4 gate PASS/FAIL der · karar gerçekle uyuşuyor mu, sayılır")]
|
| fig = plt.figure(figsize=(7, 5.2), dpi=200)
|
| ax = fig.add_axes([0, 0, 1, 1])
|
| ax.set_axis_off()
|
| fig.text(0.04, 0.965, "Deney düzeneği: gate'leri test eden laboratuvar", fontsize=12.5, fontweight="semibold",
|
| va="top")
|
| h, gap, y = 0.13, 0.045, 0.86
|
| for i, (title, body) in enumerate(steps):
|
| top = y - i * (h + gap)
|
| ax.add_patch(plt.Rectangle((0.04, top - h), 0.92, h, facecolor="#f4f7fc", edgecolor="#cde2fb", linewidth=1.2))
|
| ax.add_patch(plt.Rectangle((0.04, top - h), 0.012, h, facecolor="#2a78d6", edgecolor="none"))
|
| ax.text(0.075, top - 0.035, f"{i + 1}. {title}", fontsize=10.5, fontweight="semibold", va="top")
|
| ax.text(0.075, top - 0.078, body, fontsize=8.6, color=INK2, va="top")
|
| if i < len(steps) - 1:
|
| ax.annotate("", (0.5, top - h - gap + 0.004), (0.5, top - h - 0.004),
|
| arrowprops=dict(arrowstyle="-|>", color=MUTED, lw=1.2))
|
| return fig
|
|
|
|
|
| def g1_error():
|
| t = pd.read_parquet(RES / "threshold.parquet")
|
| t = t[(t.truth == "population") & (t.metric == "recall@5") & np.isclose(t.d.abs(), 0.02)]
|
| t = t.assign(wrong=np.where(t.truth_pass, t.R - t.k_pass, t.k_pass))
|
| err = t.groupby(["dataset", "n"])[["wrong", "R"]].sum().pipe(lambda g: 100 * g.wrong / g.R)
|
| fig, ax = figure("Mutlak eşik gate'i (G1), eşiğe yakın sistemlerde sık yanılıyor",
|
| "Gerçek Recall@5 eşikten 0,02 uzakken yanlış karar oranı · 2000 golden set · 4 sistem, eşiğin iki yönü")
|
| for ds, (name, c) in DS.items():
|
| s = err[ds]
|
| ax.plot(s.index, s.values, "-o", color=c, markeredgecolor="white", markeredgewidth=1.5)
|
| label_end(ax, s.index[-1], s.values[-1], f"{name} %{tr(s.values[-1])}", c)
|
| ax.set_xticks([25, 50, 100, 200, 300])
|
| ax.set_xlabel("Golden set boyutu (soru sayısı)")
|
| ax.yaxis.set_major_formatter(PCT)
|
| ax.set_ylim(0, 45)
|
| return fig
|
|
|
|
|
| def headroom():
|
| p = pd.read_parquet(RES / "pairs.parquet")
|
| p = p[(p.truth == "population") & (p.metric == "recall@5") & (p.B == "ctl03") & (p.n == 100)]
|
| gates = [("G1", 0.05, "G1\nmutlak eşik"), ("G2", 0.02, "G2\nnaif delta"), ("G3", 0.02, "G3\neşleştirilmiş")]
|
| fig, ax = figure("Gerçek bir 0,03'lük düşüş: hangi gate geçiriyor?",
|
| "PASS oranı (düşük olması iyi) · n = 100 · G1 eşiği baseline − 0,05 · G2/G3 marjı m = 0,02")
|
| w = 0.26
|
| for i, (ds, (name, c)) in enumerate(DS.items()):
|
| for j, (g, par, _) in enumerate(gates):
|
| r = p[(p.dataset == ds) & (p.gate == g) & np.isclose(p.param, par)].iloc[0]
|
| v = r.k_pass / r.R * 100
|
| ax.bar(j + (i - 1) * w, v, w, color=c, edgecolor="white", linewidth=1.5, label=name if j == 0 else None)
|
| ax.annotate(f"%{tr(v)}", (j + (i - 1) * w, v), xytext=(0, 3), textcoords="offset points",
|
| ha="center", fontsize=8, color=INK)
|
| ax.axhline(5, color=MUTED, linestyle="--", linewidth=1)
|
| ax.annotate("%5", (2.5, 5), xytext=(4, 0), textcoords="offset points", va="center", color=INK2, fontsize=8.5,
|
| annotation_clip=False)
|
| ax.set_xticks(range(3), [g[2] for g in gates])
|
| ax.grid(axis="x", visible=False)
|
| ax.yaxis.set_major_formatter(PCT)
|
| ax.set_ylim(0, 100)
|
| ax.legend(frameon=False, loc="upper right", fontsize=9)
|
| return fig
|
|
|
|
|
| def hnsw(hyp):
|
| efs = [16, 32, 64, 128]
|
| fig, ax = figure("Hiçbir şey değişmediğinde bile: HNSW ekleme sırası gürültüsü",
|
| "Aynı vektörler ve ayarlar, 10 farklı ekleme sırası · Recall@10 farklarının 95. persentili")
|
| for u in hyp["H4"]["per_unit"]:
|
| name, c = DS[u["dataset"]]
|
| ax.plot(efs, u["p95_by_ef"], "-o", color=c, markeredgecolor="white", markeredgewidth=1.5)
|
| ax.annotate(name, (efs[0], u["p95_by_ef"][0]), xytext=(-10, 0), textcoords="offset points",
|
| ha="right", va="center", fontsize=9, color=INK)
|
| ax.set_xscale("log", base=2)
|
| ax.set_xticks(efs, [str(e) for e in efs])
|
| ax.set_xlim(9, 150)
|
| ax.set_xlabel("ef_search (yüksek = daha geniş arama)")
|
| ax.set_yticks([0, 0.005, 0.01, 0.015, 0.02])
|
| ax.yaxis.set_major_formatter(FuncFormatter(lambda v, _: tr(v, 3)))
|
| ax.set_ylim(0, 0.02)
|
| return fig
|
|
|
|
|
| def latency(hyp):
|
| names = {"bge": "bge-small, exact", "e5": "e5-small, exact", "minilm": "MiniLM, exact",
|
| "hnsw_ef16_s0": "HNSW ef = 16", "hnsw_ef128_s0": "HNSW ef = 128", "bm25": "BM25"}
|
| rows = sorted(hyp["H5"]["per_unit"], key=lambda u: u["width_ratio_median"])
|
| fig, ax = figure("P95, P50'den ne kadar daha belirsiz?",
|
| "FiQA, 2 iş parçacığı · P95 güven aralığı genişliği / P50 güven aralığı genişliği (koşu içi medyan)")
|
| y = np.arange(len(rows))
|
| v = [u["width_ratio_median"] for u in rows]
|
| ax.barh(y, v, 0.6, color=DS["fiqa"][1])
|
| for yi, vi in zip(y, v):
|
| ax.annotate(tr(vi, 2), (vi, yi), xytext=(4, 0), textcoords="offset points", va="center", fontsize=8.5,
|
| bbox=dict(boxstyle="square,pad=0.1", fc="white", ec="none"))
|
| ax.axvline(2, color=MUTED, linestyle="--", linewidth=1, zorder=0)
|
| ax.annotate("ön-kayıtlı eşik: 2×", (2, len(rows) - 0.5), xytext=(4, 0), textcoords="offset points",
|
| color=INK2, fontsize=8.5, va="center")
|
| ax.set_ylim(-0.6, len(rows))
|
| ax.set_xticks([0, 1, 2, 3, 4])
|
| ax.set_yticks(y, [names[u["system"]] for u in rows], color=INK2)
|
| ax.grid(axis="y", visible=False)
|
| ax.set_xlim(0, 4.3)
|
| ax.xaxis.set_major_formatter(FuncFormatter(lambda v, _: tr(v, 0) + "×"))
|
| fig.subplots_adjust(left=0.2)
|
| return fig
|
|
|
|
|
| def main():
|
| OUT.mkdir(parents=True, exist_ok=True)
|
| hyp = json.loads((RES / "hypotheses.json").read_text(encoding="utf-8"))
|
| for name, fig in [("0a-orneklem", sampling()), ("0b-duzenek", flow()), ("1-g1-hata", g1_error()), ("2-headroom", headroom()), ("3-hnsw", hnsw(hyp)),
|
| ("4-latency", latency(hyp))]:
|
| fig.savefig(OUT / f"fig{name}.png")
|
| plt.close(fig)
|
| print(OUT / f"fig{name}.png")
|
|
|
|
|
| if __name__ == "__main__":
|
| main()
|
|
|
