scripts/claim2_efficiency.py

#!/usr/bin/env python3
"""Claim 2 - HybridFlow efficiency: latency (Ctime) and cloud cost (CAPI), Table 2.

HONEST SCOPE: wall-clock latency and real API cost require live GPT-4.1 + local
Llama3.2-3B runs, NOT re-run here. We verify on CPU that the paper's reported
per-benchmark Ctime / CAPI cells are INTERNALLY CONSISTENT with the reported
averages, and that HybridFlow is the lowest-latency, lowest-cost edge-cloud
method as claimed. Also produces a bar chart of the average latency / cost.

Writes outputs/claim2.json and figs/claim2_efficiency.png.
"""
import json
import os
from statistics import mean

import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt

HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUT = os.path.join(HERE, "outputs")
FIG = os.path.join(HERE, "figs")
os.makedirs(OUT, exist_ok=True)
os.makedirs(FIG, exist_ok=True)

# Table 2, per benchmark: GPQA, MMLU-Pro, AIME24, LiveBench-Reasoning
LATENCY = {  # Ctime (s)
    "GPT-4.1 Direct": ([15.26, 11.77, 50.44, 36.77], 28.56),
    "HybridLLM": ([15.96, 14.90, 40.11, 26.82], 24.45),
    "DoT": ([15.79, 11.00, 29.91, 16.59], 18.32),
    "HybridFlow (Ours)": ([15.24, 11.85, 26.40, 16.41], 17.48),
}
CAPI = {  # cloud API cost ($)
    "GPT-4.1 Direct": ([0.0094, 0.0060, 0.0256, 0.0181], 0.0148),
    "HybridLLM": ([0.0160, 0.0050, 0.0168, 0.0135], 0.0128),
    "DoT": ([0.0078, 0.0056, 0.0138, 0.0087], 0.009),
    "HybridFlow (Ours)": ([0.0075, 0.0052, 0.0135, 0.0091], 0.0088),
}


def check(table, decimals, tol, label):
    print(f"\n{label}: avg == mean(cells)")
    print(f"{'Method':<22}{'mean(cells)':>13}{'reported':>10}  ok")
    rows, all_ok = [], True
    for name, (cells, reported) in table.items():
        m = mean(cells)
        ok = abs(round(m, decimals) - reported) <= tol
        all_ok = all_ok and ok
        assert ok, f"{name}: {m} != {reported}"
        print(f"{name:<22}{m:>13.5f}{reported:>10}  {'OK' if ok else 'FAIL'}")
        rows.append(
            {
                "method": name,
                "cells": cells,
                "mean_computed": round(m, 5),
                "reported_avg": reported,
                "match": ok,
            }
        )
    return rows, all_ok


lat_rows, lat_ok = check(LATENCY, 2, 0.011, "Latency Ctime (s)")
capi_rows, capi_ok = check(CAPI, 4, 1.1e-4, "Cloud API cost CAPI ($)")

# HybridFlow is the best (lowest) among edge-cloud collaboration methods.
best_lat = min(LATENCY, key=lambda k: LATENCY[k][1])
best_capi = min(CAPI, key=lambda k: CAPI[k][1])
assert best_lat == "HybridFlow (Ours)" and best_capi == "HybridFlow (Ours)"
print(f"\nLowest avg latency: {best_lat} ({LATENCY[best_lat][1]} s)")
print(f"Lowest avg CAPI:    {best_capi} ({CAPI[best_capi][1]} $)")

# Figure: average latency and cost side by side.
names = list(LATENCY.keys())
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 4.2))
c = ["#d1495b" if n == "HybridFlow (Ours)" else "#888888" for n in names]
ax1.bar(names, [LATENCY[n][1] for n in names], color=c)
ax1.set_title("Avg latency Ctime (s), lower better")
for i, n in enumerate(names):
    ax1.text(i, LATENCY[n][1] + 0.3, f"{LATENCY[n][1]}", ha="center", fontsize=8)
ax2.bar(names, [CAPI[n][1] for n in names], color=c)
ax2.set_title("Avg cloud cost CAPI ($), lower better")
for i, n in enumerate(names):
    ax2.text(i, CAPI[n][1] + 0.0002, f"{CAPI[n][1]}", ha="center", fontsize=8)
for ax in (ax1, ax2):
    ax.tick_params(axis="x", rotation=20, labelsize=8)
plt.tight_layout()
fig.savefig(os.path.join(FIG, "claim2_efficiency.png"), dpi=130)
plt.close(fig)

result = {
    "claim": "2 - Table 2 efficiency (HybridFlow latency 17.48 s, CAPI 0.0088)",
    "scope": "INTERNAL-CONSISTENCY ONLY (no live runs). Wall-clock latency and real "
    "API cost need live GPT-4.1 + local Llama3.2-3B.",
    "latency_checks": lat_rows,
    "capi_checks": capi_rows,
    "latency_all_consistent": lat_ok,
    "capi_all_consistent": capi_ok,
    "lowest_latency_method": best_lat,
    "lowest_capi_method": best_capi,
    "figure": "figs/claim2_efficiency.png",
    "verdict": "PASS (internal consistency): every reported avg equals the mean of the "
    "reported per-benchmark cells; HybridFlow is lowest latency (17.48 s) and "
    "lowest cloud cost (0.0088 $) among edge-cloud methods. Absolute timings "
    "NOT re-run (need live API + GPU).",
}
with open(os.path.join(OUT, "claim2.json"), "w") as f:
    json.dump(result, f, indent=2)
print(f"\nLatency consistent: {lat_ok} | CAPI consistent: {capi_ok}")
print("VERDICT: PASS (internal consistency); absolute timings need live re-run.")