scripts/figure_bootstrap.py

#!/usr/bin/env python
"""Figure: paired bootstrap CI distributions (Claim 3).

Two panels using the SAME paired-percentile-bootstrap machinery the paper
applies to Table 6:
  (left)  LIFO+LIT - SPT+SPT  : strong vs weak, CI far below 0 (clearly
          distinguishable) -- CPU-reproduced from the 70-instance sweep.
  (right) LIFO+LIT - 2nd-best rule : a small-Delta paired comparison whose CI
          spans 0 (indistinguishable) -- the CPU methodology analog of the
          paper's Qwen3-8B vs LIFO+LIT test.
The paper's REPORTED Qwen3-8B - LIFO+LIT result (Delta -0.099 pp,
95% CI [-0.531, 0.389]) is overlaid on the right panel as a labelled marker
(reported-not-run: no Qwen rollouts were produced here).
"""
from __future__ import annotations

import json
from pathlib import Path

import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt  # noqa: E402
import numpy as np  # noqa: E402

ROOT = Path("/home/ubuntu/samuel/dynasched-repro")
GAPS = ROOT / "outputs" / "pdr_gaps.json"
OUT = ROOT / "figs" / "bootstrap_ci.png"
N_BOOT = 100_000
SEED = 12345


def boot_dist(a, b, seed=SEED):
    a = np.asarray(a, float)
    b = np.asarray(b, float)
    d = a - b
    n = len(d)
    rng = np.random.default_rng(seed)
    idx = rng.integers(0, n, size=(N_BOOT, n))
    boot = d[idx].mean(axis=1)
    lo, hi = np.percentile(boot, [2.5, 97.5])
    return boot, float(d.mean()), float(lo), float(hi)


def panel(ax, boot, delta, lo, hi, title, color):
    ax.hist(boot, bins=80, color=color, alpha=0.75)
    ax.axvline(0, color="#111827", lw=1.2, ls="-", label="null (0)")
    ax.axvline(delta, color="#b91c1c", lw=1.5, ls="-", label=f"Δ={delta:.3f} pp")
    ax.axvline(lo, color="#334155", lw=1.0, ls="--")
    ax.axvline(
        hi, color="#334155", lw=1.0, ls="--", label=f"95% CI [{lo:.3f}, {hi:.3f}]"
    )
    ax.set_title(title, fontsize=10)
    ax.set_xlabel("bootstrap mean paired Δ (pp)")
    ax.legend(fontsize=8)


def main():
    g = json.loads(GAPS.read_text())
    lifo = g["lifo_lit_gap_vector"]
    spt = g["spt_spt_gap_vector"]
    ranked = g["ranked_rules"]
    gbi = g["gap_by_instance"]
    instances = g["instances"]
    second = ranked[1] if ranked[0] == "LIFO:LIT" else ranked[0]
    second_vec = [gbi[k][second] for k in instances]

    b1, d1, lo1, hi1 = boot_dist(lifo, spt)
    b2, d2, lo2, hi2 = boot_dist(lifo, second_vec)

    fig, (axl, axr) = plt.subplots(1, 2, figsize=(13, 5))
    panel(
        axl,
        b1,
        d1,
        lo1,
        hi1,
        "LIFO+LIT − SPT+SPT (strong vs weak)\nCI excludes 0 → distinguishable "
        "[CPU-reproduced, n=70]",
        "#93c5fd",
    )
    panel(
        axr,
        b2,
        d2,
        lo2,
        hi2,
        f"LIFO+LIT − {second.replace(':', '+')} (2nd-best)\nCI spans 0 → "
        "indistinguishable [CPU methodology demo]",
        "#5eead4",
    )

    # Overlay the reported Qwen3-8B - LIFO+LIT result on the right panel.
    q_delta, q_lo, q_hi = -0.099, -0.531, 0.389
    y = axr.get_ylim()[1] * 0.85
    axr.errorbar(
        [q_delta],
        [y],
        xerr=[[q_delta - q_lo], [q_hi - q_delta]],
        fmt="o",
        color="#7c3aed",
        capsize=4,
        lw=2,
        label=f"REPORTED Qwen3-8B−LIFO+LIT Δ={q_delta} [{q_lo}, {q_hi}]",
    )
    axr.legend(fontsize=8)
    axr.text(
        0.5,
        -0.22,
        "Purple = paper's REPORTED Qwen3-8B vs LIFO+LIT (no rollouts run here)",
        transform=axr.transAxes,
        ha="center",
        fontsize=8,
        color="#7c3aed",
        style="italic",
    )

    fig.suptitle(
        "Claim 3: paired percentile-bootstrap CIs (100k resamples, seed 12345)",
        fontsize=12,
    )
    plt.tight_layout(rect=(0, 0.03, 1, 0.97))
    OUT.parent.mkdir(parents=True, exist_ok=True)
    fig.savefig(OUT, dpi=130)
    print(f"Wrote {OUT}")


if __name__ == "__main__":
    main()