scripts/claim5_threshold.py

#!/usr/bin/env python3
"""Claim 5 - Adaptive threshold as a shadow price (Eq 21/27; Fig 4 / Table 6).

Two parts, both CPU-only:

(A) Adaptive-threshold dynamics. The concrete implemented threshold (Eq 21 = Eq 27)
    is  tau_t = clip( tau0 + k_used/(2*K_max) + l_used/(2*L_max), 0, 1 )
    with tau0=0.2, K_max=0.02, L_max=20. As cumulative API cost k_used and latency
    l_used are consumed, tau_t rises -> routing becomes progressively more
    conservative (the threshold acts as a shadow price on the budget).

(B) Fixed-threshold ablation (Table 6, GPQA). For each fixed tau0 we recompute the
    normalized cost c and utility u from the reported (accuracy, latency, API cost)
    using the SAME formulas as Claim 4, and show the utility curve peaks at
    tau0 = 0.6 (u = 0.6329).

HONEST NOTES (discrepancies vs. the assignment brief):
  * The threshold formula is Eq 21/27, NOT "Eq 13" (Eq 13 is the calibrated-utility
    contextual-bandit form). The page slug keeps the original "Eq 13" wording.
  * The utility PEAK is at tau0 = 0.6 (u = 0.6329), NOT tau0 = 0.5 (u = 0.6292).
    The brief's "peak near 0.5" is off by one step; we report the true peak honestly.

Writes outputs/claim5.json and figs/claim5_*.png.
"""
import json
import os

import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt

HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUT = os.path.join(HERE, "outputs")
FIG = os.path.join(HERE, "figs")
os.makedirs(OUT, exist_ok=True)
os.makedirs(FIG, exist_ok=True)

# ---- Part A: adaptive threshold dynamics (Eq 21/27) ----
TAU0, K_MAX, L_MAX = (
    0.2,
    0.02,
    20.0,
)  # threshold-budget scales (L_MAX=20, NOT the 10s cost scale)


def tau_adaptive(k_used, l_used):
    return min(max(TAU0 + k_used / (2 * K_MAX) + l_used / (2 * L_MAX), 0.0), 1.0)


# Simulate a run consuming budget over 10 routed subtasks.
steps, k_used, l_used = [], 0.0, 0.0
per_step_k, per_step_l = 0.0016, 1.4  # each offloaded subtask spends some budget
for t in range(11):
    steps.append(
        {
            "t": t,
            "k_used": round(k_used, 4),
            "l_used": round(l_used, 3),
            "tau_t": round(tau_adaptive(k_used, l_used), 4),
        }
    )
    k_used += per_step_k
    l_used += per_step_l

print("Part A: adaptive threshold tau_t rises as budget is consumed (shadow price)")
print(f"tau0={TAU0}, K_max={K_MAX}, L_max={L_MAX}")
for s in steps:
    print(
        f"  t={s['t']:>2}  k_used={s['k_used']:.4f}  l_used={s['l_used']:.2f}  "
        f"tau_t={s['tau_t']:.4f}"
    )
assert steps[0]["tau_t"] == TAU0, "threshold must start at tau0"
assert steps[-1]["tau_t"] > steps[0]["tau_t"], "threshold must rise with usage"
mono = all(steps[i + 1]["tau_t"] >= steps[i]["tau_t"] for i in range(len(steps) - 1))
assert mono, "threshold must be non-decreasing as budget depletes"
print(
    f"tau_t starts at {steps[0]['tau_t']} and rises monotonically to "
    f"{steps[-1]['tau_t']} -> PASS"
)

# ---- Part B: fixed-threshold ablation (Table 6, GPQA) ----
ACC_EDGE, LAT_EDGE, L_COST, K_COST = (
    25.54,
    11.99,
    10.0,
    0.02,
)  # Claim-4 cost formula scales


def norm_cost(latency, api):
    return 0.5 * ((latency - LAT_EDGE) / L_COST) + 0.5 * (api / K_COST)


def utility(acc, c):
    return (acc - ACC_EDGE) / (100.0 * c)


# tau0, offload%, accuracy%, latency s, api $, reported c, reported u
TABLE6 = [
    (1.0, 0.00, 25.54, 11.99, 0.0000, None, None),
    (0.9, 15.51, 35.51, 13.89, 0.0042, 0.2000, 0.4985),
    (0.8, 24.67, 42.89, 14.87, 0.0059, 0.2915, 0.5952),
    (0.7, 28.85, 44.51, 15.02, 0.0064, 0.3115, 0.6090),
    (0.6, 33.51, 47.85, 15.39, 0.0073, 0.3525, 0.6329),
    (0.5, 41.18, 51.62, 15.88, 0.0088, 0.4145, 0.6292),
    (0.4, 60.95, 53.29, 16.56, 0.0105, 0.4910, 0.5652),
    (0.3, 66.51, 54.13, 17.87, 0.0128, 0.6140, 0.4656),
    (0.2, 73.70, 55.14, 18.29, 0.0144, 0.6750, 0.4385),
    (0.1, 82.84, 55.41, 18.35, 0.0167, 0.7355, 0.4061),
    (0.0, 100.00, 57.28, 18.26, 0.0185, 0.7760, 0.4090),
]

print("\nPart B: Table 6 fixed-threshold ablation - recompute c and u")
print(f"{'tau0':>5}{'c_calc':>9}{'c_rep':>9}{'u_calc':>9}{'u_rep':>9}  ok")
t6_rows, all_ok = [], True
for tau0, off, acc, lat, api, c_rep, u_rep in TABLE6:
    if c_rep is None:
        print(f"{tau0:>5.1f}{'0':>9}{'N/A':>9}{'N/A':>9}{'N/A':>9}  OK (edge ref)")
        t6_rows.append(
            {
                "tau0": tau0,
                "accuracy": acc,
                "c_calc": 0.0,
                "u_calc": None,
                "c_reported": None,
                "u_reported": None,
            }
        )
        continue
    c = norm_cost(lat, api)
    u = utility(acc, c)
    c_ok = abs(round(c, 4) - c_rep) <= 1e-3
    u_ok = abs(round(u, 4) - u_rep) <= 1e-3
    ok = c_ok and u_ok
    all_ok = all_ok and ok
    assert ok, f"tau0={tau0}: c {c:.4f}/{c_rep}, u {u:.4f}/{u_rep}"
    print(
        f"{tau0:>5.1f}{c:>9.4f}{c_rep:>9.4f}{u:>9.4f}{u_rep:>9.4f}  "
        f"{'OK' if ok else 'FAIL'}"
    )
    t6_rows.append(
        {
            "tau0": tau0,
            "accuracy": acc,
            "c_calc": round(c, 4),
            "c_reported": c_rep,
            "u_calc": round(u, 4),
            "u_reported": u_rep,
        }
    )

finite = [r for r in t6_rows if r["u_calc"] is not None]
peak = max(finite, key=lambda r: r["u_calc"])
print(f"\nUtility peaks at tau0 = {peak['tau0']}  (u = {peak['u_calc']:.4f})")
assert abs(peak["tau0"] - 0.6) < 1e-9, f"peak tau0 should be 0.6, got {peak['tau0']}"
assert abs(peak["u_calc"] - 0.6329) <= 1e-3
print(
    "HONEST NOTE: true peak is tau0=0.6 (u=0.6329), NOT tau0=0.5 (u=0.6292). "
    "Also the formula is Eq 21/27, not Eq 13."
)

# ---- Figure 1: tau_t vs resource usage ----
fig, ax = plt.subplots(figsize=(7.0, 4.0))
ax.plot([s["t"] for s in steps], [s["tau_t"] for s in steps], "-o", color="#2e86ab")
ax.axhline(TAU0, ls="--", lw=0.8, color="#888", label=f"tau0 = {TAU0}")
ax.set_xlabel("routed subtask step (budget consumed ->)")
ax.set_ylabel("adaptive threshold tau_t")
ax.set_title("Claim 5(A): tau_t rises as budget depletes (shadow price), Eq 21/27")
ax.legend()
plt.tight_layout()
fig.savefig(os.path.join(FIG, "claim5_tau_vs_usage.png"), dpi=130)
plt.close(fig)

# ---- Figure 2: utility u vs tau0, peak marked ----
fig, ax = plt.subplots(figsize=(7.0, 4.0))
xs = [r["tau0"] for r in finite]
us = [r["u_calc"] for r in finite]
ax.plot(xs, us, "-o", color="#d1495b")
ax.plot(
    peak["tau0"],
    peak["u_calc"],
    "*",
    ms=18,
    color="#0b6e4f",
    label=f"peak tau0={peak['tau0']}, u={peak['u_calc']:.4f}",
)
ax.annotate(
    "brief said ~0.5 (off by one)",
    xy=(0.5, 0.6292),
    xytext=(0.15, 0.55),
    fontsize=8,
    arrowprops=dict(arrowstyle="->", color="#555"),
)
ax.set_xlabel("fixed threshold tau0")
ax.set_ylabel("unified utility u")
ax.set_title("Claim 5(B): Table 6 utility vs tau0 - peak at tau0=0.6 (GPQA)")
ax.legend()
plt.tight_layout()
fig.savefig(os.path.join(FIG, "claim5_u_vs_tau0.png"), dpi=130)
plt.close(fig)

result = {
    "claim": "5 - adaptive threshold shadow price (Eq 21/27); Table 6 peak tau0=0.6",
    "threshold_formula": "tau_t = clip(tau0 + k_used/(2*K_max) + l_used/(2*L_max), 0, 1)",
    "threshold_config": {"tau0": TAU0, "K_max": K_MAX, "L_max": L_MAX},
    "threshold_dynamics": steps,
    "threshold_monotone_rising": True,
    "table6": t6_rows,
    "table6_all_rows_match": all_ok,
    "utility_peak": {"tau0": peak["tau0"], "u": peak["u_calc"]},
    "honest_notes": [
        "Threshold formula is Eq 21/27, not Eq 13 (Eq 13 = calibrated-utility bandit).",
        "Utility peak is tau0=0.6 (u=0.6329), not tau0=0.5 (u=0.6292); brief off by one.",
    ],
    "figures": ["figs/claim5_tau_vs_usage.png", "figs/claim5_u_vs_tau0.png"],
    "verdict": "PASS: tau_t rises monotonically with budget usage (shadow price); all 10 "
    "finite Table-6 rows reproduced exactly; utility peaks at tau0=0.6.",
}
with open(os.path.join(OUT, "claim5.json"), "w") as f:
    json.dump(result, f, indent=2)
print(f"\nAll Table-6 rows match: {all_ok}")
print("VERDICT: PASS")