File size: 7,441 Bytes
8bf7463 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 | """Publishable figures + metrics (aggregates only - no rows leave the box).
Reads runs/baselines.json, runs/laya-*/eval.json and the saved test predictions, writes
results/metrics.json and results/*.png for the model card.
"""
from __future__ import annotations
import json
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from common import split
OUT = Path("results"); OUT.mkdir(exist_ok=True)
# Reference palette (dataviz skill), light surface. Validated: see docs/PROCESS.md.
SURFACE, INK, INK2, MUTED, GRID = "#fcfcfb", "#0b0b0b", "#52514e", "#8a8984", "#e6e5e1"
BLUE, ORANGE = "#2a78d6", "#eb6834"
plt.rcParams.update({"font.family": "DejaVu Sans", "font.size": 11, "axes.edgecolor": GRID,
"axes.labelcolor": INK2, "xtick.color": INK2, "ytick.color": INK2,
"figure.facecolor": SURFACE, "axes.facecolor": SURFACE, "savefig.facecolor": SURFACE})
base = json.loads(Path("runs/baselines.json").read_text())["paid_7d"]
evals = {p.parent.name: json.loads(p.read_text()) for p in sorted(Path("runs").glob("laya-*/eval.json"))}
MODELS = [ # (label, source, is_ours)
("Recency rule", base["recency_rule"], False),
("Logistic regression", base["logreg"], False),
("LightGBM (telemetry)", base["lightgbm"], False),
]
if "laya-text" in evals:
MODELS.append(("x402-demand · listing only (open)", evals["laya-text"]["paid_7d"], True))
if "laya-full" in evals:
MODELS.append(("x402-demand · listing + telemetry (API)", evals["laya-full"]["paid_7d"], True))
# Stacking pair: shown once, for the open listing-only model (the question it answers:
# does the free text score add anything on top of telemetry?).
st = evals.get("laya-text", {}).get("stacking_paid_7d", {})
if st:
MODELS.append(("LightGBM, calibration hosts only*", st["gbm_calib_only"], False))
MODELS.append((" … + listing-only score*", st["gbm_calib+laya"], True))
SEGS = [("all", "All test hosts"), ("new_listings", "New listings (cold start)"),
("small_hosts(<=20)", "Hosts with ≤ 20 listings")]
# ---------- Figure 1: AUC by segment, dot plot small multiples ----------
fig, axes = plt.subplots(1, 3, figsize=(13, 0.52 * len(MODELS) + 1.9), sharey=True)
y = np.arange(len(MODELS))[::-1]
for ax, (seg, title) in zip(axes, SEGS):
ax.axvline(0.5, color=MUTED, lw=1, ls=(0, (3, 3)), zorder=1)
ax.text(0.505, y.min() - 0.55, "chance", color=MUTED, ha="left", fontsize=9)
for yi, (label, src, ours) in zip(y, MODELS):
m = src.get(seg)
if not m or m.get("auc") is None:
continue
c = BLUE if ours else MUTED
ax.hlines(yi, 0.5, m["auc"], color=c, lw=2, zorder=2)
ax.scatter([m["auc"]], [yi], s=70, color=c, edgecolor=SURFACE, linewidth=2, zorder=3)
ax.text(m["auc"] + 0.008, yi, f"{m['auc']:.3f}", va="center", fontsize=9.5, color=INK)
ax.set_title(title, loc="left", color=INK, fontsize=12)
ax.set_xlim(0.38, 0.85); ax.set_ylim(y.min() - 0.8, y.max() + 0.5)
ax.grid(axis="x", color=GRID, lw=0.8); ax.set_axisbelow(True)
ax.spines[["top", "right", "left"]].set_visible(False)
ax.tick_params(axis="y", length=0)
ax.set_xlabel("ROC AUC, paid within 7 days")
axes[0].set_yticks(y, [m[0] for m in MODELS])
for t, (_, _, ours) in zip(axes[0].get_yticklabels(), MODELS):
t.set_color(INK if ours else INK2)
fig.suptitle("Held-out test: future dates × hosts never seen in training", x=0.01, ha="left",
color=INK, fontsize=13, fontweight="bold")
note = "Blue = uses the x402-demand checkpoint. Grey = baseline."
if any("*" in m[0] for m in MODELS):
note += " * both trained only on calibration hosts; the gap isolates the listing-only score's contribution."
fig.text(0.01, 0.005, note, color=INK2, fontsize=9)
fig.tight_layout(rect=(0, 0.03, 1, 0.95))
fig.savefig(OUT / "auc_by_segment.png", dpi=160)
plt.close(fig)
# ---------- Figure 2: reliability diagram + Figure 3: lift by decile ----------
df = pd.read_parquet("data/rows.parquet")
_, _, te = split(df, 7)
yt = te.paid_7d.to_numpy()
curves = [("LightGBM (telemetry)", np.load("runs/lightgbm_paid7d_test_pred.npy"), ORANGE)]
AQUA = "#1baf7a" # palette slot 3; below 3:1 on light surface -> legend labels carry identity
for name, lab, c in (("laya-text", "x402-demand · listing only (open)", AQUA),
("laya-full", "x402-demand · listing + telemetry (API, recalibrated)", BLUE)):
p = Path(f"runs/{name}/test_pred_paid7d_isotonic.npy")
if not p.exists():
p = Path(f"runs/{name}/test_pred_paid7d.npy")
if p.exists():
curves.append((lab, np.load(p), c))
fig, ax = plt.subplots(figsize=(6.2, 5.6))
ax.plot([0, 1], [0, 1], color=MUTED, lw=1, ls=(0, (3, 3)))
ax.text(0.83, 0.77, "perfect", color=MUTED, rotation=45, fontsize=9)
for label, p, c in curves:
q = np.quantile(p, np.linspace(0, 1, 11)); q[-1] += 1e-9
b = np.clip(np.digitize(p, q) - 1, 0, 9)
xs = [p[b == i].mean() for i in range(10) if (b == i).any()]
ys = [yt[b == i].mean() for i in range(10) if (b == i).any()]
ax.plot(xs, ys, color=c, lw=2, marker="o", ms=8, mec=SURFACE, mew=2, label=label)
ax.set_xlim(0, 1); ax.set_ylim(0, 1)
ax.set_xlabel("Predicted probability (decile mean)"); ax.set_ylabel("Observed share paid within 7 days")
ax.set_title("Calibration on the held-out test set", loc="left", color=INK, fontsize=12)
ax.grid(color=GRID, lw=0.8); ax.set_axisbelow(True); ax.spines[["top", "right"]].set_visible(False)
ax.legend(frameon=False, loc="upper left", fontsize=9.5, labelcolor=INK)
fig.tight_layout(); fig.savefig(OUT / "calibration.png", dpi=160); plt.close(fig)
best_label = "x402-demand · listing + telemetry"
best_p = np.load("runs/laya-full/test_pred_paid7d.npy")
q = np.quantile(best_p, np.linspace(0, 1, 11)); q[-1] += 1e-9
b = np.clip(np.digitize(best_p, q) - 1, 0, 9)
rates = [yt[b == i].mean() for i in range(10)]
fig, ax = plt.subplots(figsize=(7.5, 4.4))
bars = ax.bar(np.arange(1, 11), rates, width=0.78, color=BLUE, edgecolor=SURFACE, linewidth=2)
ax.axhline(yt.mean(), color=MUTED, lw=1, ls=(0, (3, 3)))
ax.text(0.6, yt.mean() + 0.012, f"base rate {yt.mean():.0%}", color=INK2, fontsize=9)
for r, bar in zip(rates, bars):
if r in (rates[0], rates[-1]) or bar is bars[4]:
ax.text(bar.get_x() + bar.get_width() / 2, r + 0.01, f"{r:.0%}", ha="center", fontsize=9.5, color=INK)
ax.set_xticks(range(1, 11)); ax.set_xlabel(f"Decile of predicted probability ({best_label})")
ax.set_ylabel("Share paid within 7 days"); ax.set_ylim(0, max(rates) * 1.18)
ax.set_title("Who actually gets paid, by predicted decile (test set)", loc="left", color=INK, fontsize=12)
ax.grid(axis="y", color=GRID, lw=0.8); ax.set_axisbelow(True); ax.spines[["top", "right"]].set_visible(False)
fig.tight_layout(); fig.savefig(OUT / "lift_by_decile.png", dpi=160); plt.close(fig)
metrics = {"task": "paid_7d / paid_14d", "test": "anchors 17-21 (7d) / 15 (14d), hosts in buckets 80-99",
"baselines": json.loads(Path("runs/baselines.json").read_text()),
"checkpoints": evals,
"lift_by_decile_paid7d": {"model": best_label, "share_paid": [round(float(r), 4) for r in rates],
"base_rate": round(float(yt.mean()), 4)}}
(OUT / "metrics.json").write_text(json.dumps(metrics, indent=1))
print("wrote", sorted(p.name for p in OUT.iterdir()))
|