File size: 7,768 Bytes
ab6d9f6
 
 
 
f1ef656
 
 
 
 
 
ab6d9f6
 
 
 
f1ef656
 
ab6d9f6
 
 
 
 
 
 
 
 
f1ef656
ab6d9f6
 
 
 
 
 
 
 
 
 
 
 
2e0ef6a
 
 
 
 
 
ab6d9f6
 
 
 
 
 
 
 
 
 
 
 
 
2e0ef6a
 
 
f1ef656
 
 
 
 
 
ab6d9f6
 
 
 
 
f1ef656
2e0ef6a
f1ef656
 
ab6d9f6
2e0ef6a
ab6d9f6
 
 
f1ef656
 
ab6d9f6
 
 
f1ef656
 
2e0ef6a
 
 
f1ef656
ab6d9f6
 
 
 
f1ef656
 
 
 
 
 
ab6d9f6
f1ef656
 
ab6d9f6
 
 
f1ef656
ab6d9f6
 
f1ef656
 
 
 
 
 
 
 
 
ab6d9f6
 
 
 
 
 
f1ef656
 
 
 
 
 
ab6d9f6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f1ef656
ab6d9f6
 
 
 
 
f1ef656
 
 
 
 
 
 
 
 
 
 
2e0ef6a
 
 
 
f1ef656
 
 
 
 
ab6d9f6
f1ef656
ab6d9f6
f1ef656
ab6d9f6
 
f1ef656
 
 
 
 
 
 
2e0ef6a
f1ef656
 
 
 
 
 
ab6d9f6
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
"""Generate the Evidently data-drift HTML report.

Reads:
- Reference: ``data/reference_dataset.parquet`` (training sample, frozen)
- Current: last N production predictions from Supabase, ordered by most
  recent (ignores history beyond that window)

The "last N" approach makes the pipeline replayable: re-run seed_traffic
and re-run this script — the report reflects the most recent batch
without needing to truncate the DB.

Writes an HTML report consumable by the Streamlit dashboard.

Usage:
    uv run python scripts/generate_drift_report.py                  # last 100
    uv run python scripts/generate_drift_report.py --limit 500
    uv run python scripts/generate_drift_report.py --output dashboard/static/drift_report.html
"""

from __future__ import annotations

import argparse
import json
import logging
import os
import warnings
from pathlib import Path

import pandas as pd
from sqlalchemy import create_engine, text

logger = logging.getLogger("scripts.generate_drift_report")
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")

DEFAULT_REFERENCE = Path("data/reference_dataset.parquet")
DEFAULT_OUTPUT = Path("dashboard/static/drift_report.html")
DEFAULT_FEATURE_NAMES = Path("models/feature_names.json")

# A scheduled job replays examples/low_risk.json every two days so the free-tier
# database is never paused for inactivity. Those rows are all the same applicant:
# counted as production traffic they would manufacture drift that says nothing
# about real usage, so they are excluded from the current window.
KEEP_ALIVE_SK_ID_CURR = 282751


def _load_reference(path: Path, feature_names: list[str]) -> pd.DataFrame:
    df = pd.read_parquet(path)
    keep = [c for c in feature_names if c in df.columns]
    missing = set(feature_names) - set(df.columns)
    if missing:
        logger.warning(
            "%d feature(s) absent from reference dataset (drift skipped on those)",
            len(missing),
        )
    return df[keep].copy()


def _load_current(
    database_url: str, limit: int, feature_names: list[str]
) -> pd.DataFrame:
    """Pull the last ``limit`` successful predictions, ordered by most recent.

    Returns the JSONB feature payloads expanded into a wide DataFrame for
    Evidently. Newer rows come first in SQL, but order does not matter for
    drift (the test is distribution-based, not temporal).
    """
    engine = create_engine(database_url, future=True)
    with engine.connect() as conn:
        rows = conn.execute(
            text(
                "SELECT features FROM predictions_log "
                "WHERE status_code = 200 "
                "AND sk_id_curr <> :keep_alive "
                "ORDER BY timestamp DESC "
                "LIMIT :lim"
            ),
            {"lim": limit, "keep_alive": KEEP_ALIVE_SK_ID_CURR},
        ).all()
    if not rows:
        raise SystemExit(
            "No successful prediction rows in predictions_log — generate some "
            "traffic first (scripts/seed_traffic.py)."
        )
    flat = pd.json_normalize([r.features for r in rows])
    keep = [c for c in feature_names if c in flat.columns]
    logger.info(
        "Loaded %d production rows (last %d requested), %d features available",
        len(flat),
        limit,
        len(keep),
    )
    return flat[keep].copy()


def _build_report(reference: pd.DataFrame, current: pd.DataFrame):
    """Import Evidently lazily so the script still works when only inspecting.

    Returns the evaluation object produced by ``Report.run``. The 0.7+ API
    no longer mutates ``report`` in place — ``.run()`` returns a separate
    snapshot that carries ``save_html``.
    """
    try:
        from evidently import Report
        from evidently.presets import DataDriftPreset
    except ImportError as exc:
        raise SystemExit(
            "evidently not installed. Add it to dashboard/requirements.txt "
            "or `uv add --group dev 'evidently>=0.7'`."
        ) from exc

    report = Report([DataDriftPreset()])
    # Evidently emits dozens of "divide by zero" RuntimeWarnings from scipy
    # when statistical tests run on near-constant columns. These are benign
    # (the resulting NaN/inf is interpreted as "skip this feature" downstream)
    # and they drown out real log output. Silence them only inside this call.
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore", category=RuntimeWarning)
        warnings.filterwarnings("ignore", category=FutureWarning)
        return report.run(reference, current)


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--reference", type=Path, default=DEFAULT_REFERENCE)
    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
    parser.add_argument(
        "--limit",
        type=int,
        default=100,
        help="Compare against the last N successful predictions (default %(default)s).",
    )
    parser.add_argument("--feature-names", type=Path, default=DEFAULT_FEATURE_NAMES)
    parser.add_argument("--database-url", default=os.getenv("DATABASE_URL"))
    args = parser.parse_args()

    if not args.database_url:
        # Allow loading from database/.env for local runs.
        try:
            from dotenv import load_dotenv

            load_dotenv(Path("database/.env"))
            args.database_url = os.getenv("DATABASE_URL")
        except ImportError:
            pass
    if not args.database_url:
        raise SystemExit("DATABASE_URL is required (env var or database/.env).")

    feature_names = json.loads(args.feature_names.read_text())
    reference = _load_reference(args.reference, feature_names)
    current = _load_current(args.database_url, args.limit, feature_names)

    # Align columns: drop any column not in BOTH frames (Evidently expects parity)
    common = [c for c in reference.columns if c in current.columns]
    reference = reference[common]
    current = current[common]
    logger.info("Aligned on %d common features", len(common))

    # Evidently rejects all-NaN columns ("empty column ... was provided for drift
    # calculation"). Drop any column that is fully null on either side.
    empty_ref = {c for c in reference.columns if reference[c].isna().all()}
    empty_cur = {c for c in current.columns if current[c].isna().all()}
    empty = empty_ref | empty_cur
    if empty:
        sample = ", ".join(sorted(empty)[:5])
        logger.warning(
            "Dropping %d all-NaN column(s) (empty in ref=%d, in current=%d). Sample: %s%s",
            len(empty),
            len(empty_ref),
            len(empty_cur),
            sample,
            " ..." if len(empty) > 5 else "",
        )
        reference = reference.drop(columns=list(empty))
        current = current.drop(columns=list(empty))
    logger.info("Comparing on %d features after NaN filter", len(reference.columns))

    evaluation = _build_report(reference, current)
    args.output.parent.mkdir(parents=True, exist_ok=True)
    evaluation.save_html(str(args.output))
    logger.info("Saved drift report to %s", args.output)

    # Also dump a JSON snapshot of the run so the dashboard can parse
    # per-feature drift scores without scraping the HTML.
    json_path = args.output.with_suffix(".json")
    try:
        json_payload = evaluation.json()
        if not isinstance(json_payload, str):
            import json as _json

            json_payload = _json.dumps(json_payload)
        json_path.write_text(json_payload, encoding="utf-8")
        logger.info("Saved drift report JSON to %s", json_path)
    except Exception as exc:  # noqa: BLE001
        logger.warning("Could not save JSON snapshot of the drift report: %s", exc)


if __name__ == "__main__":
    main()