| """Generate docs/MODEL_CARD.md and the auto-generated metrics block in README.md from metrics.json.""" |
|
|
| import re |
| from pathlib import Path |
|
|
| START, END = "<!-- METRICS:START -->", "<!-- METRICS:END -->" |
|
|
|
|
| def _fmt(m, key, pct=False, money=False): |
| d = m["test"][key] |
| v, ci = d["value"], d["ci95"] |
| f = (lambda x: f"{x:,.0f}") if money else ((lambda x: f"{x:.1%}") if pct else (lambda x: f"{x:.3f}")) |
| return f"{f(v)} [{f(ci[0])}, {f(ci[1])}]" if ci else f(v) |
|
|
|
|
| def _banner(metrics) -> str: |
| if metrics.get("synthetic_data"): |
| return ("> **WARNING: these results come from SYNTHETIC data** (generated offline by `src/synthetic.py`). " |
| "They demonstrate that the pipeline works; they say nothing about real-world fraud performance. " |
| "Re-train on the real dataset (`python scripts/download_data.py && python -m src.train`) " |
| "and these numbers will be replaced.\n") |
| return "" |
|
|
|
|
| def metrics_table(metrics, metadata) -> str: |
| cm = metrics["test_confusion"] |
| d = metrics["data"] |
| b = metrics["test_cost_baselines"] |
| cv_rows = "\n".join( |
| f"| {s}{' (selected)' if s == metrics['selected_strategy'] else ''} | " |
| f"{v['pr_auc_mean']:.3f} +- {v['pr_auc_std']:.3f} | {v['recall_at_precision_mean']:.3f} |" |
| for s, v in metrics["cv"].items()) |
| return f"""{_banner(metrics)} |
| Model `{metrics['model_version']}` - strategy **{metrics['selected_strategy']}**, calibration **{metrics['calibration']['method']}**, |
| decision threshold **{metrics['threshold']:.4f}** (cost-optimal: FN cost {metrics['costs']['false_negative']:.2f}, FP cost {metrics['costs']['false_positive']:.2f}). |
| |
| Data: {d['rows']:,} rows / {d['frauds']:,} frauds - time-ordered split train/val/test = {d['split_rows']['train']:,} / {d['split_rows']['val']:,} / {d['split_rows']['test']:,} rows |
| ({d['split_frauds']['train']} / {d['split_frauds']['val']} / {d['split_frauds']['test']} frauds). Source: {d['source']}. |
| |
| **Held-out test split, mean [95% bootstrap CI]** |
| |
| | Metric | Value | |
| |---|---| |
| | PR-AUC (primary) | {_fmt(metrics, 'pr_auc')} | |
| | ROC-AUC | {_fmt(metrics, 'roc_auc')} | |
| | Recall at precision >= {metadata['config']['cv']['min_precision']:.0%} | {_fmt(metrics, 'recall_at_precision')} | |
| | Precision @ threshold | {_fmt(metrics, 'precision')} | |
| | Recall @ threshold | {_fmt(metrics, 'recall')} | |
| | F1 @ threshold | {_fmt(metrics, 'f1')} | |
| | Expected cost on test | {_fmt(metrics, 'expected_cost', money=True)} | |
| |
| Confusion at threshold: TN {cm['tn']:,}, FP {cm['fp']:,}, FN {cm['fn']:,}, TP {cm['tp']:,}. |
| Cost baselines on test: flag nothing = {b['cost_flag_nothing']:,.0f}; fixed 0.5 threshold = {b['cost_at_threshold_0.5']:,.0f}; |
| chosen threshold = {b['cost_at_chosen_threshold']:,.0f}. |
| |
| **5-fold CV on the training split (model selection)** |
| |
| | Strategy | PR-AUC (mean +- sd) | Recall @ precision | |
| |---|---|---| |
| {cv_rows} |
| """ |
|
|
|
|
| def write_model_card(path: Path, metrics, metadata, importance): |
| top = sorted(importance.items(), key=lambda kv: kv[1], reverse=True)[:10] |
| top_txt = "\n".join(f"- `{k}`: {v:.3f}" for k, v in top) |
| cal = metrics["calibration"] |
| text = f"""# Model card - SENTINEL fraud classifier |
| |
| _Generated by `python -m src.train` from `model/metrics.json`. Do not edit by hand._ |
| |
| {metrics_table(metrics, metadata)} |
| |
| ## Intended use |
| Ranking and flagging card transactions for **human review** in a demo / research setting. |
| It is not validated for automated blocking, credit decisions, or any regulated production use. |
| |
| ## Model |
| XGBoost gradient-boosted trees on V1-V28 (anonymised PCA components), log-amount (standardised, fitted on train only) |
| and hour-of-day (sin/cos). Imbalance handled by **{metrics['selected_strategy']}**, chosen by CV PR-AUC on the training split only. |
| |
| ## Calibration and decision rule |
| Method: {cal['method']}, fitted on the validation split. Brier on test: raw {cal['brier_raw_test']:.5f} -> calibrated {cal['brier_calibrated_test']:.5f}. |
| Mean predicted fraud rate on test: raw {cal['mean_pred_raw_test']:.4f}, calibrated {cal['mean_pred_calibrated_test']:.4f}, |
| observed {cal['observed_fraud_rate_test']:.4f}. Reliability plot: `docs/figures/reliability.png`. |
| A transaction is flagged when calibrated probability >= {metrics['threshold']:.4f}, the threshold minimising |
| `FN * {metrics['costs']['false_negative']:.2f} + FP * {metrics['costs']['false_positive']:.2f}` on the validation split. |
| Risk bands: LOW < t/2 <= MEDIUM < t <= HIGH. |
| |
| ## Top features (mean |TreeSHAP|, log-odds) |
| {top_txt} |
| |
| ## Limitations and known failure modes |
| - **Test frauds are few** ({metrics['data']['split_frauds']['test']}); see the confidence intervals above - they are wide. |
| - The dashboard/Kafka replay sample holds every test fraud, the legitimate rows the model scores as suspicious (weight 1) and a weighted random sample of the rest. The threshold tuner reconstructs the full test split from it (exact for thresholds above the lowest suspicious score, estimated below). Its raw fraud rate is inflated, so do not compute precision from raw replay rows. |
| - Time-ordered split: performance on later data reflects drift, but only over the span of this dataset (about 2 days for the Kaggle set). |
| - Fraud patterns change; the model will degrade without retraining. `/monitoring/drift` reports PSI on live inputs but cannot detect label drift. |
| - Features are anonymised PCA components, so explanations (reason codes) describe the model, not a real-world cause. |
| - Threshold depends on the configured costs; change `config/train.yaml` costs and re-train if your costs differ. |
| - Single-process in-memory monitoring and rate limiting; not suitable for multi-replica deployment as-is. |
| - No fairness / bias assessment is possible: the dataset has no demographic attributes. |
| |
| ## Regulatory note |
| This project provides per-transaction reason codes and a documented evaluation. It does **not** by itself satisfy |
| GDPR Art. 22 or EU AI Act requirements (which also need governance, human oversight, logging/retention, DPIA, |
| conformity assessment, etc.). Treat it as a technical building block, not a compliance claim. |
| |
| ## Provenance |
| Data sha256 `{metadata['data']['sha256'][:16]}...`, trained {metadata['trained_at']}, libraries {metadata['library_versions']}. |
| """ |
| path.parent.mkdir(parents=True, exist_ok=True) |
| path.write_text(text, encoding="utf-8") |
|
|
|
|
| def update_readme_block(readme: Path, metrics, metadata): |
| text = readme.read_text(encoding="utf-8") |
| block = f"{START}\n{metrics_table(metrics, metadata)}\n{END}" |
| if START in text and END in text: |
| text = re.sub(re.escape(START) + r".*?" + re.escape(END), lambda _: block, text, flags=re.S) |
| readme.write_text(text, encoding="utf-8") |
|
|
