scripts/finalize_claim3.py

"""Claim 3 (heuristic dispatch-rule side) — CPU reproduction on the released DynaSched-Subset.
Reproduces the paper's central dispatch-rule finding: LIFO+LIT is the best of the 24 composite
priority dispatch rules by mean relative-to-best makespan gap.

HONEST SCOPE: 19 of 70 instances are 'hetero_dynamic_large' whose per-rollout time exceeded a
multi-minute budget on this box; they are excluded. The excluded instances are systematically the
largest/highest-gap ones, so the absolute gaps here are LOWER BOUNDS vs the paper's 70-instance
means (LIFO+LIT 1.11%, median 1.94%, worst 51.52%). The QUALITATIVE result (LIFO+LIT ranked #1)
reproduces on the completed subset.
"""

import json, statistics, os
import numpy as np
import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt

ROOT = "/home/ubuntu/samuel/dynasched-repro"
OP = ["SPT", "LPT", "MWKR", "LWKR", "MOPNR", "LOPNR", "FIFO", "LIFO"]
MAC = ["LIT", "LWL", "SPT"]
RULES = [f"{o}:{m}" for o in OP for m in MAC]

d = json.load(open(f"{ROOT}/outputs/pdr_makespans.json"))
inst = [k for k, v in d.items() if len(v) == 24]
N = len(inst)

best = {k: min(d[k][r] for r in RULES) for k in inst}
gap = {r: np.array([(d[k][r] / best[k] - 1.0) * 100.0 for k in inst]) for r in RULES}
mean_gap = {r: float(gap[r].mean()) for r in RULES}
ranked = sorted(mean_gap.items(), key=lambda x: x[1])

best_rule, best_val = ranked[0]
worst_rule, worst_val = ranked[-1]
median_val = float(np.median(list(mean_gap.values())))
lifolit = mean_gap["LIFO:LIT"]

# Bootstrap 95% CI of the paired difference between the best rule (LIFO:LIT) and the 2nd-best rule
# — a faithful reproduction of the paper's paired-bootstrap METHOD (Claim 3 applies it to
# Qwen3-8B vs LIFO+LIT; the Qwen rollouts are not run here, so we demonstrate the identical
# procedure on the heuristic pool and report the paper's Qwen-vs-LIFO+LIT CI as reported).
second_rule = ranked[1][0]
paired = gap["LIFO:LIT"] - gap[second_rule]  # per-instance paired differences
rng = np.random.default_rng(0)
boot = np.array([rng.choice(paired, size=N, replace=True).mean() for _ in range(20000)])
ci = (float(np.percentile(boot, 2.5)), float(np.percentile(boot, 97.5)))
paired_diff = float(paired.mean())

out = {
    "instances_completed": N,
    "instances_total": 70,
    "excluded": "19 hetero_dynamic_large instances (per-rollout time budget exceeded); excluded set is the largest/highest-gap -> gaps below are LOWER BOUNDS",
    "best_rule": best_rule,
    "best_rule_gap_pct": round(best_val, 3),
    "LIFO_LIT_gap_pct": round(lifolit, 3),
    "LIFO_LIT_is_best": best_rule == "LIFO:LIT",
    "median_rule_gap_pct": round(median_val, 3),
    "worst_rule": worst_rule,
    "worst_rule_gap_pct": round(worst_val, 3),
    "paper_targets": {"LIFO+LIT": 1.11, "median": 1.94, "worst": 51.52},
    "top5": [(r, round(g, 3)) for r, g in ranked[:5]],
    "paired_bootstrap_method_demo": {
        "comparison": f"LIFO:LIT vs 2nd-best ({second_rule})",
        "paired_diff_pp": round(paired_diff, 3),
        "ci95": [round(ci[0], 3), round(ci[1], 3)],
        "note": "reproduces the paper's paired-bootstrap CI PROCEDURE (Claim 3 applies it to Qwen3-8B vs LIFO+LIT: reported Delta=-0.099 pp, 95% CI [-0.531, 0.389]); Qwen rollouts not run here",
    },
}
json.dump(out, open(f"{ROOT}/outputs/claim3_gaps.json", "w"), indent=2)

# Figure 1: per-rule mean gap (sorted), LIFO+LIT highlighted
plt.figure(figsize=(9, 4.2))
rs = [r for r, _ in ranked]
gs = [g for _, g in ranked]
colors = ["#c0392b" if r == "LIFO:LIT" else "#5b8bd6" for r in rs]
plt.bar(range(len(rs)), gs, color=colors)
plt.axhline(
    1.11, color="green", ls="--", lw=1, label="paper LIFO+LIT (70 inst) = 1.11%"
)
plt.xticks(range(len(rs)), rs, rotation=90, fontsize=6)
plt.ylabel("mean relative makespan gap (%)")
plt.title(
    f"Claim 3: LIFO+LIT is best of 24 dispatch rules ({N}/70 instances; gap {lifolit:.2f}%)"
)
plt.legend(fontsize=8)
plt.tight_layout()
plt.savefig(f"{ROOT}/figs/claim3_pdr_gaps.png", dpi=130, bbox_inches="tight")
plt.close()

# Figure 2: bootstrap CI histogram
plt.figure(figsize=(6.5, 4))
plt.hist(boot, bins=60, color="#5b8bd6", alpha=0.85)
plt.axvline(paired_diff, color="k", lw=1.5, label=f"paired Δ = {paired_diff:.3f} pp")
plt.axvline(ci[0], color="r", ls="--", lw=1)
plt.axvline(ci[1], color="r", ls="--", lw=1, label=f"95% CI [{ci[0]:.3f}, {ci[1]:.3f}]")
plt.xlabel(f"bootstrap mean paired diff: LIFO:LIT − {second_rule} (pp)")
plt.ylabel("count")
plt.title(
    "Claim 3: paired-bootstrap CI method (paper applies it to Qwen3-8B vs LIFO+LIT)"
)
plt.legend(fontsize=8)
plt.tight_layout()
plt.savefig(f"{ROOT}/figs/claim3_bootstrap.png", dpi=130, bbox_inches="tight")
plt.close()

print(json.dumps(out, indent=2))