""" utils/classification.py Failure-horizon classification for h in {10, 20, 30} cycles (brief, Task 4). Logistic Regression is the required transparent baseline; Random Forest is the nonlinear comparison. Every model is then probability- calibrated on the VALIDATION split (never on test) with isotonic regression, because raw tree/logistic scores are not automatically well-calibrated probabilities. """ from __future__ import annotations import numpy as np import pandas as pd from sklearn.linear_model import LogisticRegression from sklearn.ensemble import RandomForestClassifier from sklearn.calibration import CalibratedClassifierCV from sklearn.frozen import FrozenEstimator from sklearn.metrics import ( precision_score, recall_score, f1_score, average_precision_score, confusion_matrix, brier_score_loss, precision_recall_curve, ) HORIZONS = (10, 20, 30) def train_classifiers( train_feat: pd.DataFrame, feature_cols: list[str], horizons=HORIZONS, seed: int = 42, max_tree_rows: int | None = None, ) -> dict: """Returns {horizon: {'logistic': model, 'random_forest': model}} (uncalibrated).""" X_train = train_feat[feature_cols].to_numpy() # Same runtime trade-off as the regression random forest: cap the rows # the (slower) random forest trains on for the larger FD002 subset. if max_tree_rows is not None and len(X_train) > max_tree_rows: rng = np.random.default_rng(seed) tree_idx = rng.choice(len(X_train), size=max_tree_rows, replace=False) else: tree_idx = np.arange(len(X_train)) out = {} for h in horizons: y_train = train_feat[f"failure_within_{h}"].to_numpy() logit = LogisticRegression( max_iter=2000, class_weight="balanced", random_state=seed ) logit.fit(X_train, y_train) rf = RandomForestClassifier( n_estimators=80, max_depth=8, min_samples_leaf=15, max_samples=0.5, class_weight="balanced", random_state=seed, n_jobs=-1, ) rf.fit(X_train[tree_idx], y_train[tree_idx]) out[h] = {"logistic": logit, "random_forest": rf} return out def calibrate_classifiers( models: dict, val_feat: pd.DataFrame, feature_cols: list[str], horizons=HORIZONS ) -> dict: """Wraps every fitted model with isotonic calibration fit on VALIDATION rows.""" X_val = val_feat[feature_cols].to_numpy() calibrated = {} for h in horizons: calibrated[h] = {} y_val = val_feat[f"failure_within_{h}"].to_numpy() for name, model in models[h].items(): cal = CalibratedClassifierCV(FrozenEstimator(model), method="isotonic") cal.fit(X_val, y_val) calibrated[h][name] = cal return calibrated def predict_proba_all( models: dict, feat: pd.DataFrame, feature_cols: list[str], horizons=HORIZONS ) -> dict: X = feat[feature_cols].to_numpy() out = {} for h in horizons: out[h] = {} for name, model in models[h].items(): out[h][name] = model.predict_proba(X)[:, 1] return out def classification_report_table( models: dict, feat: pd.DataFrame, feature_cols: list[str], horizons=HORIZONS, threshold: float = 0.5, ) -> pd.DataFrame: """One row per (horizon, model) with the full required metric set.""" proba = predict_proba_all(models, feat, feature_cols, horizons) rows = [] for h in horizons: y_true = feat[f"failure_within_{h}"].to_numpy() for name, p in proba[h].items(): y_pred = (p >= threshold).astype(int) rows.append( { "horizon": h, "model": name, "precision": precision_score(y_true, y_pred, zero_division=0), "recall": recall_score(y_true, y_pred, zero_division=0), "f1": f1_score(y_true, y_pred, zero_division=0), "pr_auc": average_precision_score(y_true, p), "brier": brier_score_loss(y_true, p), "positive_rate": float(y_true.mean()), } ) return pd.DataFrame(rows) def confusion_matrices( models: dict, feat: pd.DataFrame, feature_cols: list[str], horizons=HORIZONS, threshold: float = 0.5, ) -> dict: proba = predict_proba_all(models, feat, feature_cols, horizons) out = {} for h in horizons: y_true = feat[f"failure_within_{h}"].to_numpy() out[h] = {} for name, p in proba[h].items(): y_pred = (p >= threshold).astype(int) out[h][name] = confusion_matrix(y_true, y_pred) return out def pr_curves( models: dict, feat: pd.DataFrame, feature_cols: list[str], horizons=HORIZONS ) -> dict: proba = predict_proba_all(models, feat, feature_cols, horizons) out = {} for h in horizons: y_true = feat[f"failure_within_{h}"].to_numpy() out[h] = {} for name, p in proba[h].items(): precision, recall, _ = precision_recall_curve(y_true, p) out[h][name] = (precision, recall) return out def choose_cost_optimal_threshold( y_true: np.ndarray, proba: np.ndarray, cost_fp: float = 1.0, cost_fn: float = 5.0 ) -> float: """ Picks the probability threshold (on VALIDATION data) minimizing cost_fp * false_positives + cost_fn * false_negatives. Missing a real near-failure warning (false negative) is set to cost more than an unnecessary inspection (false positive) by default, matching the project's overall "0.50 is not automatically optimal" guidance. """ candidate_thresholds = np.linspace(0.01, 0.99, 99) best_t, best_cost = 0.5, np.inf for t in candidate_thresholds: y_pred = (proba >= t).astype(int) fp = int(((y_pred == 1) & (y_true == 0)).sum()) fn = int(((y_pred == 0) & (y_true == 1)).sum()) cost = cost_fp * fp + cost_fn * fn if cost < best_cost: best_cost = cost best_t = t return float(best_t)