"""Credit Scoring Monitoring Dashboard. Four tabs: - Operational: volume, latency p50/p95, error rate, score distribution - Drift: embedded Evidently HTML + summary - Business: GRANTED vs REFUSED, top-driver features - Advanced: output drift, critical features, weighted drift score Reads from Supabase (predictions_log) β€” never touches the test table. """ from __future__ import annotations import logging from pathlib import Path import numpy as np import pandas as pd import plotly.express as px import plotly.graph_objects as go import streamlit as st from scipy import stats as scipy_stats from queries import ( fetch_latency_breakdown, fetch_proba_distribution, fetch_recent, fetch_summary, fetch_volume_by_hour, load_drift_report_json, load_feature_importance, load_proba_reference, parse_drift_results, ) logger = logging.getLogger(__name__) DRIFT_REPORT_PATH = Path(__file__).parent / "static" / "drift_report.html" st.set_page_config( page_title="OC P8 Monitoring", page_icon="πŸ“Š", layout="wide", ) st.title("πŸ“Š Credit Scoring β€” Monitoring") st.caption("PrΓͺt Γ  DΓ©penser Β· prod observability + data drift") with st.sidebar: st.header("Filters") hours = st.slider( "Window (hours)", min_value=1, max_value=168, value=168, help="Time range for every metric. 24h = 1 day, 168h = 7 days.", ) st.markdown("---") st.markdown( "**Sources**\n\n" "- Logs: Supabase `predictions_log`\n" "- Drift: `static/drift_report.html`\n" "- Regenerate the report: `uv run python scripts/generate_drift_report.py`" ) # Fetched once and reused across the Operational and Business tabs. The # @st.cache_data decorator on fetch_recent already deduplicates the DB # round-trip, but computing the boolean mask twice would still cost two # DataFrame allocations. try: _recent_df = fetch_recent(hours) _ok_df = _recent_df[_recent_df["status_code"] == 200] DB_AVAILABLE = True except Exception: # The connection string, host and driver internals belong in the Space # logs, never on screen: this page is public. logger.exception("predictions_log unreachable β€” rendering in degraded mode") _recent_df = pd.DataFrame() _ok_df = pd.DataFrame() DB_AVAILABLE = False if not DB_AVAILABLE: st.warning( "**Degraded mode β€” log database unreachable.** The demo instance is most " "likely asleep. The *Data Drift Report* and *Advanced Data Drift* tabs " "remain available: they read frozen artefacts versioned with the " "application." ) tab_ops, tab_drift, tab_business, tab_advanced = st.tabs( ["βš™οΈ Operational", "🌊 Data Drift Report", "πŸ’Ό Business", "🧠 Advanced Data Drift"] ) # ------------------------------------------------------------------ Drift -- with tab_drift: st.subheader("Data Drift Report (Evidently)") if DRIFT_REPORT_PATH.exists(): st.caption(f"Source: {DRIFT_REPORT_PATH.name}") html = DRIFT_REPORT_PATH.read_text(encoding="utf-8") st.components.v1.html(html, height=900, scrolling=True) else: st.info( "No Evidently report available. Generate it with:\n\n" "`uv run python scripts/generate_drift_report.py --days 30`\n\n" "Then redeploy the Space or copy the HTML into `dashboard/static/`." ) # --------------------------------------------------------------- Business -- with tab_business: if _recent_df.empty: st.warning("No data for this window.") else: ok = _ok_df c1, c2 = st.columns(2) with c1: decision_counts = ok["decision"].value_counts().reset_index() decision_counts.columns = ["decision", "count"] st.plotly_chart( px.pie( decision_counts, names="decision", values="count", title="Decisions", ), use_container_width=True, ) with c2: known = ( ok["client_known"] .value_counts() .rename({True: "Known", False: "Unknown"}) ) st.plotly_chart( px.pie( pd.DataFrame({"type": known.index, "count": known.values}), names="type", values="count", title="Known vs unknown clients", ), use_container_width=True, ) st.subheader("Latest calls") st.dataframe( ok[ [ "timestamp", "sk_id_curr", "client_known", "probability_default", "decision", "latency_ms", "model_version", ] ].head(50), use_container_width=True, hide_index=True, ) # --------------------------------------------------------- Advanced KPIs -- with tab_advanced: st.caption( "Advanced indicators beyond per-feature drift: model output drift, " "critical-feature tracking, and a drift score weighted by SHAP " "importance." ) proba_ref = load_proba_reference() importance = load_feature_importance() drift_json = load_drift_report_json() drift_results = parse_drift_results(drift_json) # ---------------------------------------------------- Output drift -- st.subheader("1. Output drift β€” probability_default distribution") if proba_ref is None: st.info( "`dashboard/static/proba_reference.json` not found. " "Generate it with `uv run python scripts/build_monitoring_artefacts.py`." ) else: try: current_proba = fetch_proba_distribution(limit=500) except Exception: logger.exception("fetch_proba_distribution failed") st.info( "Production distribution unavailable (log database unreachable). " "The frozen reference is still shown below." ) current_proba = [] if not current_proba: st.warning("No logged prediction to compute the production distribution.") else: ref_values = np.array(proba_ref.get("values", [])) cur_values = np.array(current_proba) # K-S test on raw samples β€” robust comparison of distributions. # scipy returns a KstestResult NamedTuple (statistic, pvalue); the # type stubs are weak, hence the ignore comment. ks_result = scipy_stats.ks_2samp(ref_values, cur_values) ks_p = float(ks_result.pvalue) # type: ignore[attr-defined] detected = ks_p < 0.05 c1, c2, c3, c4 = st.columns(4) c1.metric("Reference mean", f"{ref_values.mean():.3f}") c2.metric( "Current mean", f"{cur_values.mean():.3f}", delta=f"{(cur_values.mean() - ref_values.mean()):+.3f}", ) c3.metric("K-S p-value", f"{ks_p:.2e}") c4.metric( "Output drift", "βœ“ detected" if detected else "βœ— stable", delta_color="inverse" if detected else "normal", ) # Overlay histogram. fig = go.Figure() fig.add_trace( go.Histogram( x=ref_values, name="Reference (training)", opacity=0.55, nbinsx=40, histnorm="probability", marker_color="#888", ) ) fig.add_trace( go.Histogram( x=cur_values, name=f"Current (last {len(cur_values)})", opacity=0.7, nbinsx=40, histnorm="probability", marker_color="#e74c3c", ) ) fig.update_layout( barmode="overlay", xaxis_title="probability_default", yaxis_title="density", title="Default probability distribution β€” reference vs current", height=350, ) st.plotly_chart(fig, use_container_width=True) st.caption( "The K-S test compares the shape of the two sample " "distributions. Model output drift is the most direct signal of " "altered model behaviour in production β€” it aggregates the " "effect of every input drift at once." ) # ------------------------------------------------- Critical features -- st.subheader("2. Critical features (top 10 SHAP)") if importance is None: st.info( "`dashboard/static/feature_importance.json` not found. " "Generate it with `uv run python scripts/build_monitoring_artefacts.py`." ) elif not drift_results: st.info( "`dashboard/static/drift_report.json` not found. " "Regenerate the drift report with `uv run python scripts/generate_drift_report.py`." ) else: top_n = 10 rows = [] for entry in importance["top"][:top_n]: feat = entry["feature"] imp = entry["importance"] result = drift_results.get(feat, {}) detected = result.get("detected") score = result.get("score") stattest = result.get("stattest") or "β€”" rows.append( { "Rank": entry["rank"], "Feature": feat, "SHAP importance": round(imp, 4), "Drift": "πŸ”΄ Detected" if detected else ("🟒 Stable" if detected is False else "β€”"), "Drift score": (f"{score:.4f}" if score is not None else "β€”"), "Stat test": stattest, } ) df_critical = pd.DataFrame(rows) n_drifted = sum(1 for r in rows if "Detected" in r["Drift"]) c1, c2 = st.columns([1, 3]) c1.metric( f"Drifted in top {top_n}", f"{n_drifted}/{top_n}", delta_color="inverse", ) c2.caption( f"Method: {importance['method']} over {importance['sample_size']} " "reference rows. How many critical features have drifted is the most " "actionable indicator β€” drift on a top feature calls for retraining " "as a priority." ) st.dataframe(df_critical, use_container_width=True, hide_index=True) # -------------------------------------------------- Weighted drift -- st.subheader("3. Importance-weighted drift score") if importance is None or not drift_results: st.info( "Indicator unavailable until both `feature_importance.json` and " "`drift_report.json` are present." ) else: total_importance = 0.0 drifted_importance = 0.0 n_features_seen = 0 for entry in importance["top"]: feat = entry["feature"] imp = float(entry["importance"]) total_importance += imp result = drift_results.get(feat) if result is None: continue n_features_seen += 1 if result.get("detected"): drifted_importance += imp weighted_ratio = ( (drifted_importance / total_importance) if total_importance > 0 else 0.0 ) threshold = 0.30 c1, c2, c3 = st.columns(3) c1.metric( "Weighted drift", f"{weighted_ratio:.1%}", delta=f"threshold {threshold:.0%}", delta_color="inverse" if weighted_ratio >= threshold else "normal", ) c2.metric( "Importance covered", f"{n_features_seen} / {len(importance['top'])} features", ) c3.metric( "Verdict", "πŸ”΄ Alert" if weighted_ratio >= threshold else "🟒 OK", ) st.caption( "**Formula**: Ξ£(importance Γ— drift_detected) / Ξ£(importance) over the " f"top-{len(importance['top'])} SHAP features. Weights Evidently's " "binary verdict by how much each feature actually drives the model. " "Threshold: 30% of total importance drifting β†’ alert. A finer signal " "than the raw ratio Evidently shows in the Data Drift tab." ) # -------------------------------------------------------------------- Ops -- # Rendered last on purpose: it is the only tab that cannot show anything # without the database, and st.stop() halts the whole script run. Filling the # file-backed tabs first keeps a database outage local to this one. with tab_ops: try: summary = fetch_summary(hours) except Exception: logger.exception("fetch_summary failed") st.info( "Operational metrics unavailable β€” see the banner at the top of the " "page. The drift tabs remain available." ) st.stop() if not summary["total"]: st.warning(f"No prediction logged over the last {hours} hours.") st.stop() # Headline: total server-side wall-clock = handler + DB log. The detail # decomposition lives in the dedicated section below. _total_p50_top = int(round(float(summary["p50"] or 0))) + int( round(float(summary["db_log_p50"] or 0)) ) _total_p95_top = int(round(float(summary["p95"] or 0))) + int( round(float(summary["db_log_p95"] or 0)) ) cols = st.columns(6) cols[0].metric("Total requests", f"{summary['total']:,}") cols[1].metric( "Errors", f"{summary['errors']:,}", delta=f"{(summary['errors'] / summary['total']) * 100:.1f} %", delta_color="inverse", ) cols[2].metric( "Total p50", f"{_total_p50_top} ms", help="Full server wall-clock = handler (`latency_ms`) + DB log (`db_log_ms`). Details in the *Latency breakdown* section below.", ) cols[3].metric( "Total p95", f"{_total_p95_top} ms", help="Server wall-clock p95 = handler p95 + DB log p95.", ) cols[4].metric( "% REFUSED", f"{(summary['refused'] / max(summary['total'], 1)) * 100:.1f} %", ) cols[5].metric( "% New clients", f"{(summary['unknowns'] / max(summary['total'], 1)) * 100:.1f} %", help="Share of clients with no entry in the feature store (no_history_template).", ) st.subheader("Volume & latency per hour") hourly = fetch_volume_by_hour(hours) if not hourly.empty: c1, c2 = st.columns(2) with c1: st.plotly_chart( px.bar(hourly, x="hour", y="total", title="Requests / hour"), use_container_width=True, ) with c2: fig = px.line( hourly.melt(id_vars="hour", value_vars=["p50", "p95"]), x="hour", y="value", color="variable", title="Latency (ms)", ) st.plotly_chart(fig, use_container_width=True) st.subheader("Latency breakdown") def _ms(v) -> int: """Format helper β€” round to int ms, default 0 when SQL returns NULL.""" return 0 if v is None else int(round(float(v))) handler_p50 = _ms(summary["p50"]) handler_p95 = _ms(summary["p95"]) asm_p50 = _ms(summary["asm_p50"]) asm_p95 = _ms(summary["asm_p95"]) inf_p50 = _ms(summary["inf_p50"]) inf_p95 = _ms(summary["inf_p95"]) inf_cpu_p50 = _ms(summary["inf_cpu_p50"]) inf_cpu_p95 = _ms(summary["inf_cpu_p95"]) db_log_p50 = _ms(summary["db_log_p50"]) db_log_p95 = _ms(summary["db_log_p95"]) plumb_p50 = _ms(summary["plumbing_p50"]) plumb_p95 = _ms(summary["plumbing_p95"]) total_p50 = handler_p50 + db_log_p50 total_p95 = handler_p95 + db_log_p95 st.caption( f"**Client-perceived latency β‰ˆ handler ({handler_p50} ms p50).** " f"The **DB log** ({db_log_p50} ms p50) runs in a `BackgroundTask` after " "the response is sent β€” it no longer affects the client (step 4). \n" "The **handler** (`latency_ms`) covers assembly + inference + response " "building. The **DB log** (`db_log_ms`) is measured separately in " "`api/logger.py` around the Supabase INSERT, and is still shown as a " "server health metric. The **plumbing Ξ”** = `latency_ms - assembly - " "inference` isolates the Python residual between sub-measurements " "(variable init, return statement, entering the `finally`) β€” typically " "< 1 ms." ) cols_perf = st.columns(7) cols_perf[0].metric( "Total p50 / p95", f"{total_p50} / {total_p95} ms", help="Full server wall-clock = `latency_ms` (handler) + `db_log_ms` (INSERT). The real time spent server-side on a request.", ) cols_perf[1].metric( "Handler p50 / p95", f"{handler_p50} / {handler_p95} ms", help="`latency_ms` = assembly + inference + plumbing. **Excludes** the DB log.", ) cols_perf[2].metric( "Feature assembly p50 / p95", f"{asm_p50} / {asm_p95} ms", help="Feature store lookup + transforms + ratios + reindex.", ) cols_perf[3].metric( "Inference wall p50 / p95", f"{inf_p50} / {inf_p95} ms", help="`model.predict_proba` (wall-clock).", ) cols_perf[4].metric( "Inference CPU p50 / p95", f"{inf_cpu_p50} / {inf_cpu_p95} ms", help="CPU time consumed during inference (can read 0 on very fast paths β€” `time.process_time` resolution).", ) cols_perf[5].metric( "DB log p50 / p95", f"{db_log_p50} / {db_log_p95} ms", help="Supabase INSERT measured around `conn.execute(insert(...))` in `api/logger.py`. Usually dominates the total overhead.", ) cols_perf[6].metric( "Plumbing Ξ” p50 / p95", f"{plumb_p50} / {plumb_p95} ms", help="`latency_ms - feature_assembly_ms - inference_ms`. Python residual between sub-measurements (typically < 1 ms).", ) breakdown = fetch_latency_breakdown(hours) if not breakdown.empty: breakdown = breakdown.copy() # Plumbing per hour = handler - assembly - inference, clamped at 0 to # absorb sub-ms rounding artefacts. We then stack 4 components whose # total equals handler + db_log = full server wall-clock. breakdown["plumbing_p50"] = ( breakdown["total_p50"].fillna(0) - breakdown["feature_assembly_p50"].fillna(0) - breakdown["inference_p50"].fillna(0) ).clip(lower=0) long_df = breakdown.melt( id_vars="hour", value_vars=[ "feature_assembly_p50", "inference_p50", "plumbing_p50", "db_log_p50", ], var_name="component", value_name="ms", ) long_df["component"] = long_df["component"].map( { "feature_assembly_p50": "Feature assembly", "inference_p50": "Model inference", "plumbing_p50": "Python plumbing (residual)", "db_log_p50": "DB log (Supabase INSERT)", } ) fig_breakdown = px.area( long_df, x="hour", y="ms", color="component", title="p50 breakdown per hour (stacked = server wall-clock)", ) fig_breakdown.update_layout(yaxis_title="p50 latency (ms)") st.plotly_chart(fig_breakdown, use_container_width=True) else: st.info( "No instrumented data on this window yet. " "Send traffic with `scripts/seed_traffic.py` after deploying the step-4 API." ) st.subheader("Probability distribution") if not _ok_df.empty: st.plotly_chart( px.histogram( _ok_df, x="probability_default", nbins=40, color="decision", title="probability_default β€” split by decision", ), use_container_width=True, )