scripts/tail_stability_reported.py

#!/usr/bin/env python
"""Claim 2 - Tail stability p95 (Table 4), REPORTED-NOT-RUN.

p95 is the 95th percentile of the per-instance relative-to-best makespan gap
distribution. It requires the same L2/L3 CoT LLM rollouts as Claim 1, which
were NOT run here (vLLM/driver ABI mismatch). This script transcribes the
paper's reported Table 4 tail statistics and reproduces the ordering
(L2 CoT p95 < L3 CoT p95) so the claim is auditable, clearly labelled
REPORTED-NOT-RUN.

Paper: DynaSchedBench (arXiv 2605.27566), Table 4 / Section 7.3.
"""
from __future__ import annotations

import json
from pathlib import Path

OUT = Path("/home/ubuntu/samuel/dynasched-repro/outputs/tail_stability_reported.json")

# Table 4 tail percentiles (paper), per-instance relative-to-best gap (%).
TAIL = {
    "L1 Direct": {"p50": 0.63, "p90": 2.57, "p95": 3.27},
    "L1+Tool": {"p50": 0.52, "p90": 2.47, "p95": 4.27},
    "L2 CoT": {"p50": 0.25, "p90": 1.70, "p95": 2.01},
    "L3 CoT": {"p50": 0.35, "p90": 1.95, "p95": 2.92},
}


def main():
    l2 = TAIL["L2 CoT"]["p95"]
    l3 = TAIL["L3 CoT"]["p95"]
    tighter = l2 < l3

    print("=" * 66)
    print("CLAIM 2 - TAIL STABILITY p95 (Table 4)   [REPORTED-NOT-RUN]")
    print("=" * 66)
    print("LLM slice not run (vLLM/driver ABI mismatch); paper values below.\n")
    print(f"{'Config':<12}{'p50%':>8}{'p90%':>8}{'p95%':>8}")
    for k, v in TAIL.items():
        print(f"{k:<12}{v['p50']:>8.2f}{v['p90']:>8.2f}{v['p95']:>8.2f}")
    print()
    print(f"Tail check: L2 CoT p95 ({l2}%) < L3 CoT p95 ({l3}%) ?  -> {tighter}")
    print(
        f"  L2 (concise stats) has a tighter tail than L3 (structural priors) "
        f"by {l3 - l2:.2f} pp at p95."
    )
    print("  => matches the paper's reported 2.01% vs 2.92% tail-stability result.")

    OUT.write_text(
        json.dumps(
            {
                "status": "REPORTED-NOT-RUN",
                "reason": "Qwen3-8B L2/L3 CoT rollouts not produced (vLLM/driver ABI mismatch).",
                "tail_reported": TAIL,
                "L2_p95": l2,
                "L3_p95": l3,
                "L2_tighter_tail": tighter,
            },
            indent=2,
        )
    )
    print(f"\nWrote {OUT}")


if __name__ == "__main__":
    main()