File size: 3,396 Bytes
dddb6a8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
"""
Track 2.2 — serving-gate drift monitoring.

Compares the live feature distribution (logged per-request by
app/logging_pipeline/feature_log.py) against the training distribution using
scipy.stats.ks_2samp. Intended to run periodically — a scheduled GitHub
Action (see .github/workflows/drift-check.yml) or a manual notebook run.

Expect this to fire early and often at first: chat audio characteristics
diverging from RAVDESS/IEMOCAP/CREMA-D is not a bug, it's the exact gap the
cross-corpus numbers already predict.

Usage:
    python scripts/drift_check.py
"""
import json
import sys

from scipy.stats import ks_2samp

from app.alerts.webhook import send_alert
from app.config import get_settings
from app.logging_pipeline.feature_log import build_feature_log
from app.logging_pipeline.review_queue import queue_stats


def load_reference_distribution(path: str) -> dict:
    """
    Reference distribution built once (e.g. from your RAVDESS/IEMOCAP/CREMA-D
    training set) via scripts/build_drift_reference.py — see that script's
    docstring for the expected format. Kept separate from this script since
    it only needs to run once per training set, not on every drift check.
    """
    with open(path) as f:
        return json.load(f)


def main():
    settings = get_settings()
    feature_log = build_feature_log(settings)
    if feature_log is None:
        print("Feature logging disabled (FEATURE_LOG_BACKEND=none); nothing to check.")
        return

    live = feature_log.feature_distribution()
    try:
        reference = load_reference_distribution(settings.drift_reference_path)
    except FileNotFoundError:
        print(f"No reference distribution at {settings.drift_reference_path}. "
              "Run scripts/build_drift_reference.py against your training set first.")
        sys.exit(1)

    results = {}
    fired = False
    for feature_name, live_values in live.items():
        ref_values = reference.get(feature_name, [])
        if len(live_values) < 20 or len(ref_values) < 20:
            results[feature_name] = {"skipped": "insufficient samples"}
            continue
        stat, p_value = ks_2samp(live_values, ref_values)
        results[feature_name] = {"ks_statistic": float(stat), "p_value": float(p_value)}
        if p_value < settings.drift_p_value_threshold:
            fired = True

    stats = queue_stats(feature_log)
    low_conf_rate = None
    total_logged = sum(len(v) for v in live.values()) or 1
    if stats.unreviewed_count:
        low_conf_rate = stats.unreviewed_count / max(total_logged, 1)

    print(json.dumps({"drift": results, "review_queue_unreviewed": stats.unreviewed_count,
                       "low_confidence_rate": low_conf_rate}, indent=2))

    if fired:
        send_alert(
            settings.alert_webhook_url,
            f":rotating_light: SER drift check: KS-test fired (p < {settings.drift_p_value_threshold}) "
            f"on one or more features. Details: {json.dumps(results)}",
        )
    if low_conf_rate is not None and low_conf_rate > settings.low_confidence_rate_threshold:
        send_alert(
            settings.alert_webhook_url,
            f":warning: SER low-confidence rate {low_conf_rate:.1%} exceeds threshold "
            f"{settings.low_confidence_rate_threshold:.0%}. Review queue depth: {stats.unreviewed_count}.",
        )


if __name__ == "__main__":
    main()