scripts/figure_observability.py

#!/usr/bin/env python
"""Figure: Observability Paradox + tail stability (Claims 1 & 2), REPORTED.

Bar chart of the paper's reported Table 4 mean and p95 relative-to-best gaps
across observability/reasoning configs. Clearly labelled REPORTED (the LLM
slice was not run here). Highlights L2 CoT (best mean + tightest tail) vs
L3 CoT (worse despite full structural priors) = the Observability Paradox.
"""
from __future__ import annotations

import json
from pathlib import Path

import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt  # noqa: E402
import numpy as np  # noqa: E402

ROOT = Path("/home/ubuntu/samuel/dynasched-repro")
SRC = ROOT / "outputs" / "observability_reported.json"
OUT = ROOT / "figs" / "observability_l2_vs_l3.png"


def main():
    d = json.loads(SRC.read_text())["table4_reported"]
    configs = ["L1 Direct", "L1+Tool", "L2 CoT", "L3 CoT"]
    means = [d[c]["mean"] for c in configs]
    p95 = [d[c]["p95"] for c in configs]

    x = np.arange(len(configs))
    w = 0.38
    colors_mean = ["#94a3b8", "#94a3b8", "#0f766e", "#b91c1c"]
    colors_p95 = ["#cbd5e1", "#cbd5e1", "#5eead4", "#fca5a5"]

    fig, ax = plt.subplots(figsize=(9, 5.5))
    b1 = ax.bar(x - w / 2, means, w, color=colors_mean, label="mean gap %")
    b2 = ax.bar(x + w / 2, p95, w, color=colors_p95, label="p95 gap %")
    ax.set_xticks(x)
    ax.set_xticklabels(configs)
    ax.set_ylabel("Relative-to-best makespan gap (%)")
    ax.set_title(
        "Observability Paradox (Table 4, REPORTED — LLM slice not run)\n"
        "L2 CoT beats L3 CoT on both mean (0.65<1.66) and p95 tail (2.01<2.92)"
    )
    for bars in (b1, b2):
        for bar in bars:
            ax.text(
                bar.get_x() + bar.get_width() / 2,
                bar.get_height() + 0.05,
                f"{bar.get_height():.2f}",
                ha="center",
                va="bottom",
                fontsize=8,
            )
    ax.legend()
    ax.text(
        0.99,
        0.97,
        "REPORTED — not reproduced (no LLM rollouts)",
        transform=ax.transAxes,
        ha="right",
        va="top",
        fontsize=9,
        color="#b91c1c",
        style="italic",
    )
    plt.tight_layout()
    OUT.parent.mkdir(parents=True, exist_ok=True)
    fig.savefig(OUT, dpi=130)
    print(f"Wrote {OUT}")


if __name__ == "__main__":
    main()