Spaces:
Sleeping
Sleeping
Upload folder using huggingface_hub
Browse files
app.py
CHANGED
|
@@ -47,20 +47,20 @@ st.title("π Credit Scoring β Monitoring")
|
|
| 47 |
st.caption("PrΓͺt Γ DΓ©penser Β· prod observability + data drift")
|
| 48 |
|
| 49 |
with st.sidebar:
|
| 50 |
-
st.header("
|
| 51 |
hours = st.slider(
|
| 52 |
-
"
|
| 53 |
min_value=1,
|
| 54 |
max_value=168,
|
| 55 |
-
value=
|
| 56 |
-
help="
|
| 57 |
)
|
| 58 |
st.markdown("---")
|
| 59 |
st.markdown(
|
| 60 |
"**Sources**\n\n"
|
| 61 |
-
"- Logs
|
| 62 |
-
"- Drift
|
| 63 |
-
"-
|
| 64 |
)
|
| 65 |
|
| 66 |
|
|
@@ -82,36 +82,36 @@ except Exception:
|
|
| 82 |
|
| 83 |
if not DB_AVAILABLE:
|
| 84 |
st.warning(
|
| 85 |
-
"**
|
| 86 |
-
"
|
| 87 |
-
"
|
| 88 |
-
"
|
| 89 |
)
|
| 90 |
|
| 91 |
tab_ops, tab_drift, tab_business, tab_advanced = st.tabs(
|
| 92 |
-
["βοΈ
|
| 93 |
)
|
| 94 |
|
| 95 |
|
| 96 |
# ------------------------------------------------------------------ Drift --
|
| 97 |
with tab_drift:
|
| 98 |
-
st.subheader("
|
| 99 |
if DRIFT_REPORT_PATH.exists():
|
| 100 |
-
st.caption(f"Source
|
| 101 |
html = DRIFT_REPORT_PATH.read_text(encoding="utf-8")
|
| 102 |
st.components.v1.html(html, height=900, scrolling=True)
|
| 103 |
else:
|
| 104 |
st.info(
|
| 105 |
-
"
|
| 106 |
"`uv run python scripts/generate_drift_report.py --days 30`\n\n"
|
| 107 |
-
"
|
| 108 |
)
|
| 109 |
|
| 110 |
|
| 111 |
# --------------------------------------------------------------- Business --
|
| 112 |
with tab_business:
|
| 113 |
if _recent_df.empty:
|
| 114 |
-
st.warning("
|
| 115 |
else:
|
| 116 |
ok = _ok_df
|
| 117 |
c1, c2 = st.columns(2)
|
|
@@ -120,7 +120,10 @@ with tab_business:
|
|
| 120 |
decision_counts.columns = ["decision", "count"]
|
| 121 |
st.plotly_chart(
|
| 122 |
px.pie(
|
| 123 |
-
decision_counts,
|
|
|
|
|
|
|
|
|
|
| 124 |
),
|
| 125 |
use_container_width=True,
|
| 126 |
)
|
|
@@ -128,19 +131,19 @@ with tab_business:
|
|
| 128 |
known = (
|
| 129 |
ok["client_known"]
|
| 130 |
.value_counts()
|
| 131 |
-
.rename({True: "
|
| 132 |
)
|
| 133 |
st.plotly_chart(
|
| 134 |
px.pie(
|
| 135 |
pd.DataFrame({"type": known.index, "count": known.values}),
|
| 136 |
names="type",
|
| 137 |
values="count",
|
| 138 |
-
title="
|
| 139 |
),
|
| 140 |
use_container_width=True,
|
| 141 |
)
|
| 142 |
|
| 143 |
-
st.subheader("
|
| 144 |
st.dataframe(
|
| 145 |
ok[
|
| 146 |
[
|
|
@@ -161,9 +164,9 @@ with tab_business:
|
|
| 161 |
# --------------------------------------------------------- Advanced KPIs --
|
| 162 |
with tab_advanced:
|
| 163 |
st.caption(
|
| 164 |
-
"
|
| 165 |
-
"
|
| 166 |
-
"importance
|
| 167 |
)
|
| 168 |
|
| 169 |
proba_ref = load_proba_reference()
|
|
@@ -172,11 +175,11 @@ with tab_advanced:
|
|
| 172 |
drift_results = parse_drift_results(drift_json)
|
| 173 |
|
| 174 |
# ---------------------------------------------------- Output drift --
|
| 175 |
-
st.subheader("1. Output drift β
|
| 176 |
if proba_ref is None:
|
| 177 |
st.info(
|
| 178 |
-
"`dashboard/static/proba_reference.json`
|
| 179 |
-
"
|
| 180 |
)
|
| 181 |
else:
|
| 182 |
try:
|
|
@@ -184,13 +187,13 @@ with tab_advanced:
|
|
| 184 |
except Exception:
|
| 185 |
logger.exception("fetch_proba_distribution failed")
|
| 186 |
st.info(
|
| 187 |
-
"
|
| 188 |
-
"
|
| 189 |
)
|
| 190 |
current_proba = []
|
| 191 |
|
| 192 |
if not current_proba:
|
| 193 |
-
st.warning("
|
| 194 |
else:
|
| 195 |
ref_values = np.array(proba_ref.get("values", []))
|
| 196 |
cur_values = np.array(current_proba)
|
|
@@ -212,7 +215,7 @@ with tab_advanced:
|
|
| 212 |
c3.metric("K-S p-value", f"{ks_p:.2e}")
|
| 213 |
c4.metric(
|
| 214 |
"Output drift",
|
| 215 |
-
"β
|
| 216 |
delta_color="inverse" if detected else "normal",
|
| 217 |
)
|
| 218 |
|
|
@@ -242,29 +245,29 @@ with tab_advanced:
|
|
| 242 |
barmode="overlay",
|
| 243 |
xaxis_title="probability_default",
|
| 244 |
yaxis_title="density",
|
| 245 |
-
title="
|
| 246 |
height=350,
|
| 247 |
)
|
| 248 |
st.plotly_chart(fig, use_container_width=True)
|
| 249 |
|
| 250 |
st.caption(
|
| 251 |
-
"
|
| 252 |
-
"
|
| 253 |
-
"
|
| 254 |
-
"
|
| 255 |
)
|
| 256 |
|
| 257 |
# ------------------------------------------------- Critical features --
|
| 258 |
-
st.subheader("2.
|
| 259 |
if importance is None:
|
| 260 |
st.info(
|
| 261 |
-
"`dashboard/static/feature_importance.json`
|
| 262 |
-
"
|
| 263 |
)
|
| 264 |
elif not drift_results:
|
| 265 |
st.info(
|
| 266 |
-
"`dashboard/static/drift_report.json`
|
| 267 |
-
"
|
| 268 |
)
|
| 269 |
else:
|
| 270 |
top_n = 10
|
|
@@ -281,7 +284,7 @@ with tab_advanced:
|
|
| 281 |
"Rank": entry["rank"],
|
| 282 |
"Feature": feat,
|
| 283 |
"SHAP importance": round(imp, 4),
|
| 284 |
-
"Drift": "π΄
|
| 285 |
if detected
|
| 286 |
else ("π’ Stable" if detected is False else "β"),
|
| 287 |
"Drift score": (f"{score:.4f}" if score is not None else "β"),
|
|
@@ -290,28 +293,28 @@ with tab_advanced:
|
|
| 290 |
)
|
| 291 |
df_critical = pd.DataFrame(rows)
|
| 292 |
|
| 293 |
-
n_drifted = sum(1 for r in rows if "
|
| 294 |
c1, c2 = st.columns([1, 3])
|
| 295 |
c1.metric(
|
| 296 |
-
f"Drifted
|
| 297 |
f"{n_drifted}/{top_n}",
|
| 298 |
delta_color="inverse",
|
| 299 |
)
|
| 300 |
c2.caption(
|
| 301 |
-
f"
|
| 302 |
-
"
|
| 303 |
-
"
|
| 304 |
-
"
|
| 305 |
)
|
| 306 |
|
| 307 |
st.dataframe(df_critical, use_container_width=True, hide_index=True)
|
| 308 |
|
| 309 |
# -------------------------------------------------- Weighted drift --
|
| 310 |
-
st.subheader("3.
|
| 311 |
if importance is None or not drift_results:
|
| 312 |
st.info(
|
| 313 |
-
"
|
| 314 |
-
"`drift_report.json`
|
| 315 |
)
|
| 316 |
else:
|
| 317 |
total_importance = 0.0
|
|
@@ -335,27 +338,26 @@ with tab_advanced:
|
|
| 335 |
|
| 336 |
c1, c2, c3 = st.columns(3)
|
| 337 |
c1.metric(
|
| 338 |
-
"
|
| 339 |
f"{weighted_ratio:.1%}",
|
| 340 |
-
delta=f"
|
| 341 |
delta_color="inverse" if weighted_ratio >= threshold else "normal",
|
| 342 |
)
|
| 343 |
c2.metric(
|
| 344 |
-
"Importance
|
| 345 |
f"{n_features_seen} / {len(importance['top'])} features",
|
| 346 |
)
|
| 347 |
c3.metric(
|
| 348 |
"Verdict",
|
| 349 |
-
"π΄
|
| 350 |
)
|
| 351 |
|
| 352 |
st.caption(
|
| 353 |
-
"**
|
| 354 |
-
f"top-{len(importance['top'])}
|
| 355 |
-
"
|
| 356 |
-
"
|
| 357 |
-
"
|
| 358 |
-
"l'onglet Data Drift."
|
| 359 |
)
|
| 360 |
|
| 361 |
|
|
@@ -369,13 +371,13 @@ with tab_ops:
|
|
| 369 |
except Exception:
|
| 370 |
logger.exception("fetch_summary failed")
|
| 371 |
st.info(
|
| 372 |
-
"
|
| 373 |
-
"
|
| 374 |
)
|
| 375 |
st.stop()
|
| 376 |
|
| 377 |
if not summary["total"]:
|
| 378 |
-
st.warning(f"
|
| 379 |
st.stop()
|
| 380 |
|
| 381 |
# Headline: total server-side wall-clock = handler + DB log. The detail
|
|
@@ -388,9 +390,9 @@ with tab_ops:
|
|
| 388 |
)
|
| 389 |
|
| 390 |
cols = st.columns(6)
|
| 391 |
-
cols[0].metric("Total
|
| 392 |
cols[1].metric(
|
| 393 |
-
"
|
| 394 |
f"{summary['errors']:,}",
|
| 395 |
delta=f"{(summary['errors'] / summary['total']) * 100:.1f} %",
|
| 396 |
delta_color="inverse",
|
|
@@ -398,30 +400,30 @@ with tab_ops:
|
|
| 398 |
cols[2].metric(
|
| 399 |
"Total p50",
|
| 400 |
f"{_total_p50_top} ms",
|
| 401 |
-
help="
|
| 402 |
)
|
| 403 |
cols[3].metric(
|
| 404 |
"Total p95",
|
| 405 |
f"{_total_p95_top} ms",
|
| 406 |
-
help="
|
| 407 |
)
|
| 408 |
cols[4].metric(
|
| 409 |
"% REFUSED",
|
| 410 |
f"{(summary['refused'] / max(summary['total'], 1)) * 100:.1f} %",
|
| 411 |
)
|
| 412 |
cols[5].metric(
|
| 413 |
-
"%
|
| 414 |
f"{(summary['unknowns'] / max(summary['total'], 1)) * 100:.1f} %",
|
| 415 |
-
help="
|
| 416 |
)
|
| 417 |
|
| 418 |
-
st.subheader("Volume &
|
| 419 |
hourly = fetch_volume_by_hour(hours)
|
| 420 |
if not hourly.empty:
|
| 421 |
c1, c2 = st.columns(2)
|
| 422 |
with c1:
|
| 423 |
st.plotly_chart(
|
| 424 |
-
px.bar(hourly, x="hour", y="total", title="
|
| 425 |
use_container_width=True,
|
| 426 |
)
|
| 427 |
with c2:
|
|
@@ -430,11 +432,11 @@ with tab_ops:
|
|
| 430 |
x="hour",
|
| 431 |
y="value",
|
| 432 |
color="variable",
|
| 433 |
-
title="
|
| 434 |
)
|
| 435 |
st.plotly_chart(fig, use_container_width=True)
|
| 436 |
|
| 437 |
-
st.subheader("
|
| 438 |
|
| 439 |
def _ms(v) -> int:
|
| 440 |
"""Format helper β round to int ms, default 0 when SQL returns NULL."""
|
|
@@ -456,32 +458,33 @@ with tab_ops:
|
|
| 456 |
total_p95 = handler_p95 + db_log_p95
|
| 457 |
|
| 458 |
st.caption(
|
| 459 |
-
f"**
|
| 460 |
-
f"
|
| 461 |
-
"
|
| 462 |
-
"
|
| 463 |
-
"
|
| 464 |
-
"
|
| 465 |
-
"
|
| 466 |
-
"
|
| 467 |
-
"`finally`) β
|
|
|
|
| 468 |
)
|
| 469 |
|
| 470 |
cols_perf = st.columns(7)
|
| 471 |
cols_perf[0].metric(
|
| 472 |
"Total p50 / p95",
|
| 473 |
f"{total_p50} / {total_p95} ms",
|
| 474 |
-
help="
|
| 475 |
)
|
| 476 |
cols_perf[1].metric(
|
| 477 |
"Handler p50 / p95",
|
| 478 |
f"{handler_p50} / {handler_p95} ms",
|
| 479 |
-
help="`latency_ms` = assembly + inference + plumbing. **
|
| 480 |
)
|
| 481 |
cols_perf[2].metric(
|
| 482 |
"Feature assembly p50 / p95",
|
| 483 |
f"{asm_p50} / {asm_p95} ms",
|
| 484 |
-
help="
|
| 485 |
)
|
| 486 |
cols_perf[3].metric(
|
| 487 |
"Inference wall p50 / p95",
|
|
@@ -491,17 +494,17 @@ with tab_ops:
|
|
| 491 |
cols_perf[4].metric(
|
| 492 |
"Inference CPU p50 / p95",
|
| 493 |
f"{inf_cpu_p50} / {inf_cpu_p95} ms",
|
| 494 |
-
help="CPU time
|
| 495 |
)
|
| 496 |
cols_perf[5].metric(
|
| 497 |
"DB log p50 / p95",
|
| 498 |
f"{db_log_p50} / {db_log_p95} ms",
|
| 499 |
-
help="
|
| 500 |
)
|
| 501 |
cols_perf[6].metric(
|
| 502 |
"Plumbing Ξ p50 / p95",
|
| 503 |
f"{plumb_p50} / {plumb_p95} ms",
|
| 504 |
-
help="`latency_ms - feature_assembly_ms - inference_ms`.
|
| 505 |
)
|
| 506 |
|
| 507 |
breakdown = fetch_latency_breakdown(hours)
|
|
@@ -523,33 +526,33 @@ with tab_ops:
|
|
| 523 |
"plumbing_p50",
|
| 524 |
"db_log_p50",
|
| 525 |
],
|
| 526 |
-
var_name="
|
| 527 |
value_name="ms",
|
| 528 |
)
|
| 529 |
-
long_df["
|
| 530 |
{
|
| 531 |
"feature_assembly_p50": "Feature assembly",
|
| 532 |
"inference_p50": "Model inference",
|
| 533 |
-
"plumbing_p50": "
|
| 534 |
-
"db_log_p50": "DB log (
|
| 535 |
}
|
| 536 |
)
|
| 537 |
fig_breakdown = px.area(
|
| 538 |
long_df,
|
| 539 |
x="hour",
|
| 540 |
y="ms",
|
| 541 |
-
color="
|
| 542 |
-
title="
|
| 543 |
)
|
| 544 |
-
fig_breakdown.update_layout(yaxis_title="
|
| 545 |
st.plotly_chart(fig_breakdown, use_container_width=True)
|
| 546 |
else:
|
| 547 |
st.info(
|
| 548 |
-
"
|
| 549 |
-
"
|
| 550 |
)
|
| 551 |
|
| 552 |
-
st.subheader("
|
| 553 |
if not _ok_df.empty:
|
| 554 |
st.plotly_chart(
|
| 555 |
px.histogram(
|
|
@@ -557,7 +560,7 @@ with tab_ops:
|
|
| 557 |
x="probability_default",
|
| 558 |
nbins=40,
|
| 559 |
color="decision",
|
| 560 |
-
title="probability_default β split
|
| 561 |
),
|
| 562 |
use_container_width=True,
|
| 563 |
)
|
|
|
|
| 47 |
st.caption("PrΓͺt Γ DΓ©penser Β· prod observability + data drift")
|
| 48 |
|
| 49 |
with st.sidebar:
|
| 50 |
+
st.header("Filters")
|
| 51 |
hours = st.slider(
|
| 52 |
+
"Window (hours)",
|
| 53 |
min_value=1,
|
| 54 |
max_value=168,
|
| 55 |
+
value=168,
|
| 56 |
+
help="Time range for every metric. 24h = 1 day, 168h = 7 days.",
|
| 57 |
)
|
| 58 |
st.markdown("---")
|
| 59 |
st.markdown(
|
| 60 |
"**Sources**\n\n"
|
| 61 |
+
"- Logs: Supabase `predictions_log`\n"
|
| 62 |
+
"- Drift: `static/drift_report.html`\n"
|
| 63 |
+
"- Regenerate the report: `uv run python scripts/generate_drift_report.py`"
|
| 64 |
)
|
| 65 |
|
| 66 |
|
|
|
|
| 82 |
|
| 83 |
if not DB_AVAILABLE:
|
| 84 |
st.warning(
|
| 85 |
+
"**Degraded mode β log database unreachable.** The demo instance is most "
|
| 86 |
+
"likely asleep. The *Data Drift Report* and *Advanced Data Drift* tabs "
|
| 87 |
+
"remain available: they read frozen artefacts versioned with the "
|
| 88 |
+
"application."
|
| 89 |
)
|
| 90 |
|
| 91 |
tab_ops, tab_drift, tab_business, tab_advanced = st.tabs(
|
| 92 |
+
["βοΈ Operational", "π Data Drift Report", "πΌ Business", "π§ Advanced Data Drift"]
|
| 93 |
)
|
| 94 |
|
| 95 |
|
| 96 |
# ------------------------------------------------------------------ Drift --
|
| 97 |
with tab_drift:
|
| 98 |
+
st.subheader("Data Drift Report (Evidently)")
|
| 99 |
if DRIFT_REPORT_PATH.exists():
|
| 100 |
+
st.caption(f"Source: {DRIFT_REPORT_PATH.name}")
|
| 101 |
html = DRIFT_REPORT_PATH.read_text(encoding="utf-8")
|
| 102 |
st.components.v1.html(html, height=900, scrolling=True)
|
| 103 |
else:
|
| 104 |
st.info(
|
| 105 |
+
"No Evidently report available. Generate it with:\n\n"
|
| 106 |
"`uv run python scripts/generate_drift_report.py --days 30`\n\n"
|
| 107 |
+
"Then redeploy the Space or copy the HTML into `dashboard/static/`."
|
| 108 |
)
|
| 109 |
|
| 110 |
|
| 111 |
# --------------------------------------------------------------- Business --
|
| 112 |
with tab_business:
|
| 113 |
if _recent_df.empty:
|
| 114 |
+
st.warning("No data for this window.")
|
| 115 |
else:
|
| 116 |
ok = _ok_df
|
| 117 |
c1, c2 = st.columns(2)
|
|
|
|
| 120 |
decision_counts.columns = ["decision", "count"]
|
| 121 |
st.plotly_chart(
|
| 122 |
px.pie(
|
| 123 |
+
decision_counts,
|
| 124 |
+
names="decision",
|
| 125 |
+
values="count",
|
| 126 |
+
title="Decisions",
|
| 127 |
),
|
| 128 |
use_container_width=True,
|
| 129 |
)
|
|
|
|
| 131 |
known = (
|
| 132 |
ok["client_known"]
|
| 133 |
.value_counts()
|
| 134 |
+
.rename({True: "Known", False: "Unknown"})
|
| 135 |
)
|
| 136 |
st.plotly_chart(
|
| 137 |
px.pie(
|
| 138 |
pd.DataFrame({"type": known.index, "count": known.values}),
|
| 139 |
names="type",
|
| 140 |
values="count",
|
| 141 |
+
title="Known vs unknown clients",
|
| 142 |
),
|
| 143 |
use_container_width=True,
|
| 144 |
)
|
| 145 |
|
| 146 |
+
st.subheader("Latest calls")
|
| 147 |
st.dataframe(
|
| 148 |
ok[
|
| 149 |
[
|
|
|
|
| 164 |
# --------------------------------------------------------- Advanced KPIs --
|
| 165 |
with tab_advanced:
|
| 166 |
st.caption(
|
| 167 |
+
"Advanced indicators beyond per-feature drift: model output drift, "
|
| 168 |
+
"critical-feature tracking, and a drift score weighted by SHAP "
|
| 169 |
+
"importance."
|
| 170 |
)
|
| 171 |
|
| 172 |
proba_ref = load_proba_reference()
|
|
|
|
| 175 |
drift_results = parse_drift_results(drift_json)
|
| 176 |
|
| 177 |
# ---------------------------------------------------- Output drift --
|
| 178 |
+
st.subheader("1. Output drift β probability_default distribution")
|
| 179 |
if proba_ref is None:
|
| 180 |
st.info(
|
| 181 |
+
"`dashboard/static/proba_reference.json` not found. "
|
| 182 |
+
"Generate it with `uv run python scripts/build_monitoring_artefacts.py`."
|
| 183 |
)
|
| 184 |
else:
|
| 185 |
try:
|
|
|
|
| 187 |
except Exception:
|
| 188 |
logger.exception("fetch_proba_distribution failed")
|
| 189 |
st.info(
|
| 190 |
+
"Production distribution unavailable (log database unreachable). "
|
| 191 |
+
"The frozen reference is still shown below."
|
| 192 |
)
|
| 193 |
current_proba = []
|
| 194 |
|
| 195 |
if not current_proba:
|
| 196 |
+
st.warning("No logged prediction to compute the production distribution.")
|
| 197 |
else:
|
| 198 |
ref_values = np.array(proba_ref.get("values", []))
|
| 199 |
cur_values = np.array(current_proba)
|
|
|
|
| 215 |
c3.metric("K-S p-value", f"{ks_p:.2e}")
|
| 216 |
c4.metric(
|
| 217 |
"Output drift",
|
| 218 |
+
"β detected" if detected else "β stable",
|
| 219 |
delta_color="inverse" if detected else "normal",
|
| 220 |
)
|
| 221 |
|
|
|
|
| 245 |
barmode="overlay",
|
| 246 |
xaxis_title="probability_default",
|
| 247 |
yaxis_title="density",
|
| 248 |
+
title="Default probability distribution β reference vs current",
|
| 249 |
height=350,
|
| 250 |
)
|
| 251 |
st.plotly_chart(fig, use_container_width=True)
|
| 252 |
|
| 253 |
st.caption(
|
| 254 |
+
"The K-S test compares the shape of the two sample "
|
| 255 |
+
"distributions. Model output drift is the most direct signal of "
|
| 256 |
+
"altered model behaviour in production β it aggregates the "
|
| 257 |
+
"effect of every input drift at once."
|
| 258 |
)
|
| 259 |
|
| 260 |
# ------------------------------------------------- Critical features --
|
| 261 |
+
st.subheader("2. Critical features (top 10 SHAP)")
|
| 262 |
if importance is None:
|
| 263 |
st.info(
|
| 264 |
+
"`dashboard/static/feature_importance.json` not found. "
|
| 265 |
+
"Generate it with `uv run python scripts/build_monitoring_artefacts.py`."
|
| 266 |
)
|
| 267 |
elif not drift_results:
|
| 268 |
st.info(
|
| 269 |
+
"`dashboard/static/drift_report.json` not found. "
|
| 270 |
+
"Regenerate the drift report with `uv run python scripts/generate_drift_report.py`."
|
| 271 |
)
|
| 272 |
else:
|
| 273 |
top_n = 10
|
|
|
|
| 284 |
"Rank": entry["rank"],
|
| 285 |
"Feature": feat,
|
| 286 |
"SHAP importance": round(imp, 4),
|
| 287 |
+
"Drift": "π΄ Detected"
|
| 288 |
if detected
|
| 289 |
else ("π’ Stable" if detected is False else "β"),
|
| 290 |
"Drift score": (f"{score:.4f}" if score is not None else "β"),
|
|
|
|
| 293 |
)
|
| 294 |
df_critical = pd.DataFrame(rows)
|
| 295 |
|
| 296 |
+
n_drifted = sum(1 for r in rows if "Detected" in r["Drift"])
|
| 297 |
c1, c2 = st.columns([1, 3])
|
| 298 |
c1.metric(
|
| 299 |
+
f"Drifted in top {top_n}",
|
| 300 |
f"{n_drifted}/{top_n}",
|
| 301 |
delta_color="inverse",
|
| 302 |
)
|
| 303 |
c2.caption(
|
| 304 |
+
f"Method: {importance['method']} over {importance['sample_size']} "
|
| 305 |
+
"reference rows. How many critical features have drifted is the most "
|
| 306 |
+
"actionable indicator β drift on a top feature calls for retraining "
|
| 307 |
+
"as a priority."
|
| 308 |
)
|
| 309 |
|
| 310 |
st.dataframe(df_critical, use_container_width=True, hide_index=True)
|
| 311 |
|
| 312 |
# -------------------------------------------------- Weighted drift --
|
| 313 |
+
st.subheader("3. Importance-weighted drift score")
|
| 314 |
if importance is None or not drift_results:
|
| 315 |
st.info(
|
| 316 |
+
"Indicator unavailable until both `feature_importance.json` and "
|
| 317 |
+
"`drift_report.json` are present."
|
| 318 |
)
|
| 319 |
else:
|
| 320 |
total_importance = 0.0
|
|
|
|
| 338 |
|
| 339 |
c1, c2, c3 = st.columns(3)
|
| 340 |
c1.metric(
|
| 341 |
+
"Weighted drift",
|
| 342 |
f"{weighted_ratio:.1%}",
|
| 343 |
+
delta=f"threshold {threshold:.0%}",
|
| 344 |
delta_color="inverse" if weighted_ratio >= threshold else "normal",
|
| 345 |
)
|
| 346 |
c2.metric(
|
| 347 |
+
"Importance covered",
|
| 348 |
f"{n_features_seen} / {len(importance['top'])} features",
|
| 349 |
)
|
| 350 |
c3.metric(
|
| 351 |
"Verdict",
|
| 352 |
+
"π΄ Alert" if weighted_ratio >= threshold else "π’ OK",
|
| 353 |
)
|
| 354 |
|
| 355 |
st.caption(
|
| 356 |
+
"**Formula**: Ξ£(importance Γ drift_detected) / Ξ£(importance) over the "
|
| 357 |
+
f"top-{len(importance['top'])} SHAP features. Weights Evidently's "
|
| 358 |
+
"binary verdict by how much each feature actually drives the model. "
|
| 359 |
+
"Threshold: 30% of total importance drifting β alert. A finer signal "
|
| 360 |
+
"than the raw ratio Evidently shows in the Data Drift tab."
|
|
|
|
| 361 |
)
|
| 362 |
|
| 363 |
|
|
|
|
| 371 |
except Exception:
|
| 372 |
logger.exception("fetch_summary failed")
|
| 373 |
st.info(
|
| 374 |
+
"Operational metrics unavailable β see the banner at the top of the "
|
| 375 |
+
"page. The drift tabs remain available."
|
| 376 |
)
|
| 377 |
st.stop()
|
| 378 |
|
| 379 |
if not summary["total"]:
|
| 380 |
+
st.warning(f"No prediction logged over the last {hours} hours.")
|
| 381 |
st.stop()
|
| 382 |
|
| 383 |
# Headline: total server-side wall-clock = handler + DB log. The detail
|
|
|
|
| 390 |
)
|
| 391 |
|
| 392 |
cols = st.columns(6)
|
| 393 |
+
cols[0].metric("Total requests", f"{summary['total']:,}")
|
| 394 |
cols[1].metric(
|
| 395 |
+
"Errors",
|
| 396 |
f"{summary['errors']:,}",
|
| 397 |
delta=f"{(summary['errors'] / summary['total']) * 100:.1f} %",
|
| 398 |
delta_color="inverse",
|
|
|
|
| 400 |
cols[2].metric(
|
| 401 |
"Total p50",
|
| 402 |
f"{_total_p50_top} ms",
|
| 403 |
+
help="Full server wall-clock = handler (`latency_ms`) + DB log (`db_log_ms`). Details in the *Latency breakdown* section below.",
|
| 404 |
)
|
| 405 |
cols[3].metric(
|
| 406 |
"Total p95",
|
| 407 |
f"{_total_p95_top} ms",
|
| 408 |
+
help="Server wall-clock p95 = handler p95 + DB log p95.",
|
| 409 |
)
|
| 410 |
cols[4].metric(
|
| 411 |
"% REFUSED",
|
| 412 |
f"{(summary['refused'] / max(summary['total'], 1)) * 100:.1f} %",
|
| 413 |
)
|
| 414 |
cols[5].metric(
|
| 415 |
+
"% New clients",
|
| 416 |
f"{(summary['unknowns'] / max(summary['total'], 1)) * 100:.1f} %",
|
| 417 |
+
help="Share of clients with no entry in the feature store (no_history_template).",
|
| 418 |
)
|
| 419 |
|
| 420 |
+
st.subheader("Volume & latency per hour")
|
| 421 |
hourly = fetch_volume_by_hour(hours)
|
| 422 |
if not hourly.empty:
|
| 423 |
c1, c2 = st.columns(2)
|
| 424 |
with c1:
|
| 425 |
st.plotly_chart(
|
| 426 |
+
px.bar(hourly, x="hour", y="total", title="Requests / hour"),
|
| 427 |
use_container_width=True,
|
| 428 |
)
|
| 429 |
with c2:
|
|
|
|
| 432 |
x="hour",
|
| 433 |
y="value",
|
| 434 |
color="variable",
|
| 435 |
+
title="Latency (ms)",
|
| 436 |
)
|
| 437 |
st.plotly_chart(fig, use_container_width=True)
|
| 438 |
|
| 439 |
+
st.subheader("Latency breakdown")
|
| 440 |
|
| 441 |
def _ms(v) -> int:
|
| 442 |
"""Format helper β round to int ms, default 0 when SQL returns NULL."""
|
|
|
|
| 458 |
total_p95 = handler_p95 + db_log_p95
|
| 459 |
|
| 460 |
st.caption(
|
| 461 |
+
f"**Client-perceived latency β handler ({handler_p50} ms p50).** "
|
| 462 |
+
f"The **DB log** ({db_log_p50} ms p50) runs in a `BackgroundTask` after "
|
| 463 |
+
"the response is sent β it no longer affects the client (step 4). \n"
|
| 464 |
+
"The **handler** (`latency_ms`) covers assembly + inference + response "
|
| 465 |
+
"building. The **DB log** (`db_log_ms`) is measured separately in "
|
| 466 |
+
"`api/logger.py` around the Supabase INSERT, and is still shown as a "
|
| 467 |
+
"server health metric. The **plumbing Ξ** = `latency_ms - assembly - "
|
| 468 |
+
"inference` isolates the Python residual between sub-measurements "
|
| 469 |
+
"(variable init, return statement, entering the `finally`) β typically "
|
| 470 |
+
"< 1 ms."
|
| 471 |
)
|
| 472 |
|
| 473 |
cols_perf = st.columns(7)
|
| 474 |
cols_perf[0].metric(
|
| 475 |
"Total p50 / p95",
|
| 476 |
f"{total_p50} / {total_p95} ms",
|
| 477 |
+
help="Full server wall-clock = `latency_ms` (handler) + `db_log_ms` (INSERT). The real time spent server-side on a request.",
|
| 478 |
)
|
| 479 |
cols_perf[1].metric(
|
| 480 |
"Handler p50 / p95",
|
| 481 |
f"{handler_p50} / {handler_p95} ms",
|
| 482 |
+
help="`latency_ms` = assembly + inference + plumbing. **Excludes** the DB log.",
|
| 483 |
)
|
| 484 |
cols_perf[2].metric(
|
| 485 |
"Feature assembly p50 / p95",
|
| 486 |
f"{asm_p50} / {asm_p95} ms",
|
| 487 |
+
help="Feature store lookup + transforms + ratios + reindex.",
|
| 488 |
)
|
| 489 |
cols_perf[3].metric(
|
| 490 |
"Inference wall p50 / p95",
|
|
|
|
| 494 |
cols_perf[4].metric(
|
| 495 |
"Inference CPU p50 / p95",
|
| 496 |
f"{inf_cpu_p50} / {inf_cpu_p95} ms",
|
| 497 |
+
help="CPU time consumed during inference (can read 0 on very fast paths β `time.process_time` resolution).",
|
| 498 |
)
|
| 499 |
cols_perf[5].metric(
|
| 500 |
"DB log p50 / p95",
|
| 501 |
f"{db_log_p50} / {db_log_p95} ms",
|
| 502 |
+
help="Supabase INSERT measured around `conn.execute(insert(...))` in `api/logger.py`. Usually dominates the total overhead.",
|
| 503 |
)
|
| 504 |
cols_perf[6].metric(
|
| 505 |
"Plumbing Ξ p50 / p95",
|
| 506 |
f"{plumb_p50} / {plumb_p95} ms",
|
| 507 |
+
help="`latency_ms - feature_assembly_ms - inference_ms`. Python residual between sub-measurements (typically < 1 ms).",
|
| 508 |
)
|
| 509 |
|
| 510 |
breakdown = fetch_latency_breakdown(hours)
|
|
|
|
| 526 |
"plumbing_p50",
|
| 527 |
"db_log_p50",
|
| 528 |
],
|
| 529 |
+
var_name="component",
|
| 530 |
value_name="ms",
|
| 531 |
)
|
| 532 |
+
long_df["component"] = long_df["component"].map(
|
| 533 |
{
|
| 534 |
"feature_assembly_p50": "Feature assembly",
|
| 535 |
"inference_p50": "Model inference",
|
| 536 |
+
"plumbing_p50": "Python plumbing (residual)",
|
| 537 |
+
"db_log_p50": "DB log (Supabase INSERT)",
|
| 538 |
}
|
| 539 |
)
|
| 540 |
fig_breakdown = px.area(
|
| 541 |
long_df,
|
| 542 |
x="hour",
|
| 543 |
y="ms",
|
| 544 |
+
color="component",
|
| 545 |
+
title="p50 breakdown per hour (stacked = server wall-clock)",
|
| 546 |
)
|
| 547 |
+
fig_breakdown.update_layout(yaxis_title="p50 latency (ms)")
|
| 548 |
st.plotly_chart(fig_breakdown, use_container_width=True)
|
| 549 |
else:
|
| 550 |
st.info(
|
| 551 |
+
"No instrumented data on this window yet. "
|
| 552 |
+
"Send traffic with `scripts/seed_traffic.py` after deploying the step-4 API."
|
| 553 |
)
|
| 554 |
|
| 555 |
+
st.subheader("Probability distribution")
|
| 556 |
if not _ok_df.empty:
|
| 557 |
st.plotly_chart(
|
| 558 |
px.histogram(
|
|
|
|
| 560 |
x="probability_default",
|
| 561 |
nbins=40,
|
| 562 |
color="decision",
|
| 563 |
+
title="probability_default β split by decision",
|
| 564 |
),
|
| 565 |
use_container_width=True,
|
| 566 |
)
|