scripts/observability_reported.py

#!/usr/bin/env python
"""Claim 1 - Observability Paradox (Table 4), REPORTED-NOT-RUN.

The observability-paradox claim requires L2-CoT vs L3-CoT LLM rollouts on the
70 DynaSched-Subset instances. Serving Qwen3-8B via vLLM in this environment
failed at engine-core init (vLLM 0.25.1 vs NVIDIA driver 570.124.06 ABI
mismatch), so this slice was NOT run. This script transcribes the paper's
reported Table 4 numbers and reproduces the *arithmetic* of the paradox
(the ordering L2 mean gap < L3 mean gap) so the claim is auditable, and it is
clearly labelled REPORTED-NOT-RUN.

Paper: DynaSchedBench (arXiv 2605.27566), Table 4 / Section 7.2.
"""
from __future__ import annotations

import json
from pathlib import Path

OUT = Path("/home/ubuntu/samuel/dynasched-repro/outputs/observability_reported.json")

# Table 4 (paper), columns = [mean, p50, p90, p95] of the per-instance
# relative-to-best makespan gap (%). Rows are observability x reasoning configs.
TABLE4 = {
    "L1 Direct": {"mean": 1.95, "p50": 0.63, "p90": 2.57, "p95": 3.27},
    "L1+Tool": {"mean": 2.03, "p50": 0.52, "p90": 2.47, "p95": 4.27},
    "L2 CoT": {"mean": 0.65, "p50": 0.25, "p90": 1.70, "p95": 2.01},
    "L3 CoT": {"mean": 1.66, "p50": 0.35, "p90": 1.95, "p95": 2.92},
}


def main():
    l2 = TABLE4["L2 CoT"]["mean"]
    l3 = TABLE4["L3 CoT"]["mean"]
    paradox = l2 < l3
    delta = l3 - l2

    print("=" * 68)
    print("CLAIM 1 - OBSERVABILITY PARADOX (Table 4)   [REPORTED-NOT-RUN]")
    print("=" * 68)
    print("LLM slice not run in this environment (vLLM/driver ABI mismatch);")
    print("numbers below are the paper's reported Table 4 values.\n")
    print(f"{'Config':<12}{'mean%':>8}{'p50%':>8}{'p90%':>8}{'p95%':>8}")
    for k, v in TABLE4.items():
        print(
            f"{k:<12}{v['mean']:>8.2f}{v['p50']:>8.2f}{v['p90']:>8.2f}{v['p95']:>8.2f}"
        )
    print()
    print(f"Paradox check: L2 CoT mean ({l2}%) < L3 CoT mean ({l3}%) ?  -> {paradox}")
    print(
        f"  Giving the agent full structural priors (L3) is WORSE by "
        f"{delta:.2f} pp than concise summaries (L2)."
    )
    print("  => matches the paper's reported 1.66% vs 0.65% Observability Paradox.")

    OUT.write_text(
        json.dumps(
            {
                "status": "REPORTED-NOT-RUN",
                "reason": "vLLM 0.25.1 engine-core init failed on NVIDIA driver 570.124.06; "
                "Qwen3-8B L2/L3 CoT rollouts not produced.",
                "table4_reported": TABLE4,
                "paradox_L2_lt_L3": paradox,
                "L2_mean": l2,
                "L3_mean": l3,
                "L3_minus_L2_pp": delta,
            },
            indent=2,
        )
    )
    print(f"\nWrote {OUT}")


if __name__ == "__main__":
    main()