feat: implement automated training diagnostics, dataset bias analysis, and revision figure generation pipelines

app/training/ablation_flatness.py

@@ -0,0 +1,133 @@
+"""
+Spectral flatness ablation study (reviewer priority #3).
+
+The paper reports spectral_flatness_std as the single most important
+LightGBM feature and attributes AI/human separability to it. The reviewer
+asks for a direct ablation: does performance meaningfully depend on this one
+feature, or would the model do just as well without it (in which case the
+"AI music has flatter spectra" narrative should be stated more cautiously)?
+
+Three configurations, each evaluated with 5-fold stratified CV + LightGBM:
+  (a) all 47 features
+  (b) 46 features, spectral_flatness_std removed
+  (c) spectral_flatness_std alone (1 feature)
+
+Usage:
+    python -m app.training.ablation_flatness
+"""
+
+from __future__ import annotations
+
+import csv
+import sys
+import warnings
+from pathlib import Path
+
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+import lightgbm as lgb
+from sklearn.exceptions import ConvergenceWarning
+from sklearn.metrics import balanced_accuracy_score, f1_score, roc_auc_score, roc_curve
+from sklearn.model_selection import StratifiedKFold, cross_val_predict
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler
+
+from app.training.evaluate import load_features_csv
+
+FEATURES_CSV = Path("D:/CrownCode/DataSet/features.csv")
+OUTPUT_CSV = Path(__file__).resolve().parents[3] / "docs/academic/paper/real_tables/flatness_ablation.csv"
+
+_EXCLUDED_COLUMNS = {"file_path", "label_int", "duration_sec", "sample_rate"}
+TARGET_FEATURE = "spectral_flatness_std"
+
+_LGBM_PARAMS = dict(
+    n_estimators=300, max_depth=-1, learning_rate=0.05, num_leaves=31,
+    subsample=0.8, colsample_bytree=0.8, min_child_samples=20,
+    reg_alpha=0.1, reg_lambda=1.0, class_weight="balanced",
+    random_state=42, verbose=-1,
+)
+
+
+def _feature_columns() -> list[str]:
+    with open(FEATURES_CSV, "r", encoding="utf-8") as f:
+        reader = csv.DictReader(f)
+        return [c for c in (reader.fieldnames or []) if c not in _EXCLUDED_COLUMNS]
+
+
+def _optimal_threshold(y_true: np.ndarray, y_prob: np.ndarray) -> float:
+    fpr, tpr, thresholds = roc_curve(y_true, y_prob)
+    return float(thresholds[np.argmax(tpr - fpr)])
+
+
+def _evaluate_feature_set(X: np.ndarray, y: np.ndarray, n_folds: int = 5) -> dict:
+    cv = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=42)
+    pipeline = Pipeline([
+        ("scaler", StandardScaler()),
+        ("model", lgb.LGBMClassifier(**_LGBM_PARAMS)),
+    ])
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", category=ConvergenceWarning)
+        y_prob = cross_val_predict(pipeline, X, y, cv=cv, method="predict_proba")[:, 1]
+
+    threshold = _optimal_threshold(y, y_prob)
+    y_pred = (y_prob >= threshold).astype(int)
+
+    return {
+        "roc_auc": round(float(roc_auc_score(y, y_prob)), 4),
+        "f1": round(float(f1_score(y, y_pred, zero_division=0)), 4),
+        "balanced_accuracy": round(float(balanced_accuracy_score(y, y_pred)), 4),
+        "threshold": round(threshold, 4),
+    }
+
+
+def run() -> None:
+    X_full, y = load_features_csv(FEATURES_CSV)
+    X_full = np.nan_to_num(X_full, nan=0.0, posinf=1.0, neginf=-1.0)
+    feature_cols = _feature_columns()
+
+    if TARGET_FEATURE not in feature_cols:
+        raise RuntimeError(f"{TARGET_FEATURE} not found in {FEATURES_CSV} columns: {feature_cols}")
+
+    target_idx = feature_cols.index(TARGET_FEATURE)
+    keep_idx = [i for i in range(len(feature_cols)) if i != target_idx]
+
+    configs = {
+        "47_features_all": X_full,
+        "46_features_without_flatness_std": X_full[:, keep_idx],
+        "1_feature_flatness_std_only": X_full[:, [target_idx]],
+    }
+
+    results = []
+    for name, X in configs.items():
+        print(f"\nEvaluating: {name} ({X.shape[1]} features)")
+        metrics = _evaluate_feature_set(X, y)
+        metrics["config"] = name
+        metrics["n_features"] = X.shape[1]
+        results.append(metrics)
+        print(f"  AUC={metrics['roc_auc']:.4f} F1={metrics['f1']:.4f} "
+              f"BalAcc={metrics['balanced_accuracy']:.4f}")
+
+    OUTPUT_CSV.parent.mkdir(parents=True, exist_ok=True)
+    fieldnames = ["config", "n_features", "roc_auc", "f1", "balanced_accuracy", "threshold"]
+    with open(OUTPUT_CSV, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        for r in results:
+            writer.writerow({k: r[k] for k in fieldnames})
+
+    print(f"\nOutput: {OUTPUT_CSV}")
+
+    full_auc = next(r["roc_auc"] for r in results if r["config"] == "47_features_all")
+    no_flatness_auc = next(r["roc_auc"] for r in results if r["config"] == "46_features_without_flatness_std")
+    flatness_only_auc = next(r["roc_auc"] for r in results if r["config"] == "1_feature_flatness_std_only")
+    print("\n" + "=" * 60)
+    print(f"  47 features:              AUC={full_auc:.4f}")
+    print(f"  46 features (no flatness): AUC={no_flatness_auc:.4f} (diff={full_auc - no_flatness_auc:+.4f})")
+    print(f"  flatness_std alone:       AUC={flatness_only_auc:.4f}")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    run()

app/training/bias_analysis.py

@@ -0,0 +1,306 @@
+"""
+Dataset and genre bias analysis for AURIS (reviewer priority #2), plus a
+segment/track-level leakage check that feeds into the nested-CV / independent
+test split design (reviewer priority #4).
+
+Three outputs:
+  1. dataset_bias_table.csv   — per-source composition (n, class, genre mix,
+                                 duration stats, sample rate) so a reader can
+                                 see whether sources differ in ways unrelated
+                                 to AI-vs-human content.
+  2. genre_bias_test.csv      — LightGBM performance on the full (genre-
+                                 imbalanced) test set vs. a genre-balanced
+                                 subsample, to check for a genre-as-shortcut
+                                 effect.
+  3. segment_leakage_report.txt — whether any track appears to be split
+                                 across multiple audio files (same base name
+                                 with numbered suffixes), which would leak
+                                 information across CV folds.
+
+Usage:
+    python -m app.training.bias_analysis
+"""
+
+from __future__ import annotations
+
+import csv
+import re
+import sys
+import warnings
+from collections import Counter, defaultdict
+from pathlib import Path
+
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+import lightgbm as lgb
+from sklearn.exceptions import ConvergenceWarning
+from sklearn.metrics import balanced_accuracy_score, f1_score, roc_auc_score, roc_curve
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import StandardScaler
+
+DATASET_DIR = Path("D:/CrownCode/DataSet")
+FEATURES_WITH_META = DATASET_DIR / "features_with_meta.csv"
+TABLES_DIR = Path(__file__).resolve().parents[3] / "docs/academic/paper/real_tables"
+
+_EXCLUDED_COLUMNS = {
+    "file_path", "label_int", "duration_sec", "sample_rate",
+    "genre", "generator", "ai_model", "meta_sample_rate", "meta_duration_sec",
+    "match_source",
+}
+
+_LGBM_PARAMS = dict(
+    n_estimators=300, max_depth=-1, learning_rate=0.05, num_leaves=31,
+    subsample=0.8, colsample_bytree=0.8, min_child_samples=20,
+    reg_alpha=0.1, reg_lambda=1.0, class_weight="balanced",
+    random_state=42, verbose=-1,
+)
+
+
+def _load_rows() -> list[dict]:
+    with open(FEATURES_WITH_META, "r", encoding="utf-8") as f:
+        return list(csv.DictReader(f))
+
+
+def dataset_bias_table(rows: list[dict]) -> None:
+    """Per-source composition: n, class, genre mix, duration, sample rate."""
+    by_source: dict[str, list[dict]] = defaultdict(list)
+    for r in rows:
+        by_source[r["generator"]].append(r)
+
+    out_rows = []
+    for source, source_rows in sorted(by_source.items()):
+        durations = [float(r["duration_sec"]) for r in source_rows]
+        sample_rates = Counter(r["sample_rate"] for r in source_rows)
+        genres = Counter(r["genre"] for r in source_rows if r["genre"])
+        labels = Counter(r["label_int"] for r in source_rows)
+        top_genre = genres.most_common(1)[0] if genres else ("n/a", 0)
+
+        out_rows.append({
+            "source": source,
+            "n": len(source_rows),
+            "label": "AI" if labels.get("1", 0) > labels.get("0", 0) else "Human",
+            "n_ai": labels.get("1", 0),
+            "n_human": labels.get("0", 0),
+            "n_distinct_genres": len(genres),
+            "top_genre": top_genre[0],
+            "top_genre_frac": round(top_genre[1] / len(source_rows), 3) if source_rows else 0,
+            "duration_mean_sec": round(float(np.mean(durations)), 2),
+            "duration_std_sec": round(float(np.std(durations)), 2),
+            "duration_min_sec": round(float(np.min(durations)), 2),
+            "duration_max_sec": round(float(np.max(durations)), 2),
+            "sample_rate_modes": ";".join(f"{sr}={n}" for sr, n in sample_rates.most_common(3)),
+        })
+
+    out_path = TABLES_DIR / "dataset_bias_table.csv"
+    with open(out_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=list(out_rows[0].keys()))
+        writer.writeheader()
+        writer.writerows(out_rows)
+
+    print(f"Dataset bias table written: {out_path}")
+    for r in out_rows:
+        print(
+            f"  {r['source']:25s} n={r['n']:5d} label={r['label']:5s} "
+            f"dur={r['duration_mean_sec']:.1f}±{r['duration_std_sec']:.1f}s "
+            f"genres={r['n_distinct_genres']:2d} top={r['top_genre']}({r['top_genre_frac']})"
+        )
+
+
+def genre_bias_test(rows: list[dict]) -> None:
+    """
+    Compare LightGBM performance on the natural (genre-imbalanced) test split
+    vs. a genre-balanced test split, to check whether the model is exploiting
+    genre as a shortcut for the AI/human label rather than learning
+    AI-vs-human acoustic differences.
+    """
+    genre_rows = [r for r in rows if r["genre"]]
+    print(f"\nRows with genre metadata: {len(genre_rows)} / {len(rows)}")
+
+    feature_cols = [c for c in rows[0].keys() if c not in _EXCLUDED_COLUMNS]
+
+    def to_xy(row_subset: list[dict]) -> tuple[np.ndarray, np.ndarray]:
+        X = np.array([[float(r[c]) for c in feature_cols] for r in row_subset], dtype=np.float32)
+        X = np.nan_to_num(X, nan=0.0, posinf=1.0, neginf=-1.0)
+        y = np.array([int(r["label_int"]) for r in row_subset], dtype=np.int32)
+        return X, y
+
+    # Train on everything EXCEPT genre_rows' test portion — use a standard
+    # stratified split of genre_rows only, so genre is controlled for on
+    # both sides.
+    train_rows, test_rows = train_test_split(
+        genre_rows, test_size=0.3, stratify=[r["label_int"] for r in genre_rows], random_state=42,
+    )
+
+    X_train, y_train = to_xy(train_rows)
+    scaler = StandardScaler()
+    X_train_scaled = scaler.fit_transform(X_train)
+
+    model = lgb.LGBMClassifier(**_LGBM_PARAMS)
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", category=ConvergenceWarning)
+        model.fit(X_train_scaled, y_train)
+
+    def eval_subset(subset_rows: list[dict], label: str) -> dict:
+        if len(subset_rows) < 10 or len(set(r["label_int"] for r in subset_rows)) < 2:
+            return {"subset": label, "n": len(subset_rows), "note": "too small / single class"}
+        X, y = to_xy(subset_rows)
+        X_scaled = scaler.transform(X)
+        y_prob = model.predict_proba(X_scaled)[:, 1]
+        fpr, tpr, thr = roc_curve(y, y_prob)
+        threshold = float(thr[np.argmax(tpr - fpr)])
+        y_pred = (y_prob >= threshold).astype(int)
+        return {
+            "subset": label,
+            "n": len(subset_rows),
+            "roc_auc": round(float(roc_auc_score(y, y_prob)), 4),
+            "f1": round(float(f1_score(y, y_pred, zero_division=0)), 4),
+            "balanced_accuracy": round(float(balanced_accuracy_score(y, y_pred)), 4),
+        }
+
+    results = [eval_subset(test_rows, "natural_test_split")]
+
+    # Genre-balanced subsample: for each genre, cap AI and human counts to
+    # min(n_ai, n_human) in that genre so the AI/human ratio is ~1:1 within
+    # every genre — removes genre as a usable shortcut.
+    by_genre_label: dict[tuple[str, str], list[dict]] = defaultdict(list)
+    for r in test_rows:
+        by_genre_label[(r["genre"], r["label_int"])].append(r)
+
+    balanced_rows: list[dict] = []
+    rng = np.random.RandomState(42)
+    genres_present = set(g for g, _ in by_genre_label.keys())
+    for genre in genres_present:
+        ai_rows = by_genre_label.get((genre, "1"), [])
+        human_rows = by_genre_label.get((genre, "0"), [])
+        cap = min(len(ai_rows), len(human_rows))
+        if cap == 0:
+            continue
+        ai_sample = list(rng.choice(len(ai_rows), size=cap, replace=False))
+        human_sample = list(rng.choice(len(human_rows), size=cap, replace=False))
+        balanced_rows.extend(ai_rows[i] for i in ai_sample)
+        balanced_rows.extend(human_rows[i] for i in human_sample)
+
+    results.append(eval_subset(balanced_rows, "genre_balanced_test_split"))
+
+    out_path = TABLES_DIR / "genre_bias_test.csv"
+    fieldnames = ["subset", "n", "roc_auc", "f1", "balanced_accuracy", "note"]
+    with open(out_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        for r in results:
+            writer.writerow({k: r.get(k) for k in fieldnames})
+
+    print(f"\nGenre bias test written: {out_path}")
+    for r in results:
+        print(f"  {r}")
+
+    # Per-genre breakdown for the paper's genre table
+    per_genre_path = TABLES_DIR / "genre_distribution.csv"
+    genre_label_counts = Counter((r["genre"], r["label_int"]) for r in genre_rows)
+    genres_sorted = sorted(set(g for g, _ in genre_label_counts.keys()))
+    with open(per_genre_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=["genre", "n_human", "n_ai", "ai_fraction"])
+        writer.writeheader()
+        for genre in genres_sorted:
+            n_human = genre_label_counts.get((genre, "0"), 0)
+            n_ai = genre_label_counts.get((genre, "1"), 0)
+            total = n_human + n_ai
+            writer.writerow({
+                "genre": genre, "n_human": n_human, "n_ai": n_ai,
+                "ai_fraction": round(n_ai / total, 3) if total else 0,
+            })
+    print(f"Genre distribution table written: {per_genre_path}")
+
+
+_SEGMENT_PATTERN = re.compile(
+    r"^(?P<base>.+?)[_\-](?:part|seg|segment|chunk|p)[_\-]?0*(?P<idx>\d+)$",
+    re.IGNORECASE,
+)
+
+
+def segment_leakage_report(rows: list[dict]) -> None:
+    """
+    Detect filenames that look like numbered segments of the same source
+    track (e.g. track_part1.wav, track_part2.wav) — if such groups exist and
+    the current pipeline does group-unaware stratified CV, segments of the
+    same track could land in both train and validation folds.
+    """
+    stems = []
+    for r in rows:
+        fname = Path(r["file_path"]).stem
+        stems.append(fname)
+
+    groups: dict[str, list[str]] = defaultdict(list)
+    ungrouped = 0
+    for stem in stems:
+        m = _SEGMENT_PATTERN.match(stem)
+        if m:
+            groups[m.group("base")].append(stem)
+        else:
+            ungrouped += 1
+
+    multi_segment_groups = {k: v for k, v in groups.items() if len(v) > 1}
+
+    lines = []
+    lines.append("Segment/track-level leakage check")
+    lines.append("=" * 50)
+    lines.append(f"Total files: {len(stems)}")
+    lines.append(f"Files matching a segment-numbering pattern (base_part1, base_seg2, ...): "
+                 f"{sum(len(v) for v in groups.values())}")
+    lines.append(f"Distinct base tracks with >1 segment file: {len(multi_segment_groups)}")
+    lines.append(f"Files with no recognizable segment suffix: {ungrouped}")
+    lines.append("")
+    if multi_segment_groups:
+        lines.append("Sample multi-segment groups (up to 10):")
+        for base, members in list(multi_segment_groups.items())[:10]:
+            lines.append(f"  {base}: {members}")
+        lines.append("")
+        lines.append(
+            "FINDING: multi-segment groups exist. The current pipeline uses "
+            "StandardScaler + StratifiedKFold with no GroupKFold — segments "
+            "of the same base track can be split across train/validation, "
+            "which would leak track-level information (recording conditions, "
+            "mastering, etc.) across the CV boundary. This affects the "
+            "independent-test-set design in nested_cv.py, which groups by "
+            "base track before splitting."
+        )
+    else:
+        lines.append(
+            "FINDING: no filename-based segment groups detected. Filenames "
+            "appear to be one-file-per-track (e.g. gtzan_blues_0000.wav), "
+            "so standard stratified splitting does not leak track identity "
+            "via segment numbering. This does not rule out leakage from "
+            "duplicate or near-duplicate source audio uploaded under "
+            "different filenames, which is not filename-detectable."
+        )
+
+    out_path = TABLES_DIR / "segment_leakage_report.txt"
+    out_path.write_text("\n".join(lines), encoding="utf-8")
+    print(f"\nSegment leakage report written: {out_path}")
+    print("\n".join(lines))
+
+
+def run() -> None:
+    rows = _load_rows()
+    TABLES_DIR.mkdir(parents=True, exist_ok=True)
+
+    print("=" * 70)
+    print("STEP 1/3 — Dataset bias table (per-source composition)")
+    print("=" * 70)
+    dataset_bias_table(rows)
+
+    print("\n" + "=" * 70)
+    print("STEP 2/3 — Genre bias test (natural vs. genre-balanced)")
+    print("=" * 70)
+    genre_bias_test(rows)
+
+    print("\n" + "=" * 70)
+    print("STEP 3/3 — Segment/track leakage check")
+    print("=" * 70)
+    segment_leakage_report(rows)
+
+
+if __name__ == "__main__":
+    run()

app/training/build_master_dataset.py

@@ -0,0 +1,158 @@
+"""
+Joins the raw feature matrix with source/genre/generator metadata.
+
+The training pipeline (train_classifier.py) reads DataSet/features.csv, which
+contains only the 47 acoustic features plus file_path/label_int. It has no
+notion of which AI generator or genre a sample came from, so none of the
+reviewer-requested analyses (Leave-One-Generator-Out, dataset/genre bias,
+segment-leakage checks) can be run against it directly.
+
+DataSet/metadata.csv carries that provenance (source=generator, genre,
+ai_model) but was produced by a separate pass and only covers a subset of
+files. This script joins the two by filename and writes
+DataSet/features_with_meta.csv, reporting anything that fails to join so the
+gap is visible rather than silently dropped.
+
+Usage:
+    python -m app.training.build_master_dataset
+"""
+
+from __future__ import annotations
+
+import csv
+import re
+import sys
+from pathlib import Path
+
+DATASET_DIR = Path("D:/CrownCode/DataSet")
+FEATURES_CSV = DATASET_DIR / "features.csv"
+METADATA_CSV = DATASET_DIR / "metadata.csv"
+OUTPUT_CSV = DATASET_DIR / "features_with_meta.csv"
+UNMATCHED_REPORT_CSV = DATASET_DIR / "features_with_meta_unmatched_report.csv"
+LABEL_MISMATCH_CSV = DATASET_DIR / "features_with_meta_label_mismatches.csv"
+
+# Files under DataSet/*/ai/echoes_<generator>_*.{wav,mp3} have no row in
+# metadata.csv, but the generator is recoverable from the filename prefix.
+_ECHOES_PREFIX = re.compile(r"^echoes_([a-zA-Z0-9]+)_", re.IGNORECASE)
+
+
+def _filename(path: str) -> str:
+    return path.replace("\\", "/").rsplit("/", 1)[-1]
+
+
+def _infer_generator_from_filename(filename: str, label_int: int) -> tuple[str, str] | None:
+    """Best-effort (generator, ai_model) recovery for files absent from metadata.csv."""
+    m = _ECHOES_PREFIX.match(filename)
+    if m:
+        return "echoes", f"echoes_{m.group(1).lower()}"
+    if label_int == 0:
+        return "archive_org", ""
+    return None
+
+
+def build() -> None:
+    with open(FEATURES_CSV, "r", encoding="utf-8") as f:
+        reader = csv.DictReader(f)
+        feature_fieldnames = reader.fieldnames or []
+        feature_rows = list(reader)
+
+    with open(METADATA_CSV, "r", encoding="utf-8") as f:
+        meta_rows = list(csv.DictReader(f))
+
+    meta_by_filename: dict[str, dict] = {}
+    for row in meta_rows:
+        meta_by_filename[_filename(row["path"])] = row
+
+    matched_direct = 0
+    matched_inferred = 0
+    truly_unmatched: list[dict] = []
+    label_mismatches: list[dict] = []
+    out_fieldnames = feature_fieldnames + [
+        "genre", "generator", "ai_model", "meta_sample_rate", "meta_duration_sec",
+        "match_source",
+    ]
+
+    with open(OUTPUT_CSV, "w", newline="", encoding="utf-8") as f_out:
+        writer = csv.DictWriter(f_out, fieldnames=out_fieldnames)
+        writer.writeheader()
+
+        for row in feature_rows:
+            fname = _filename(row["file_path"])
+            meta = meta_by_filename.get(fname)
+
+            if meta is not None:
+                meta_label_int = 1 if meta["label"] == "ai" else 0
+                if int(row["label_int"]) != meta_label_int:
+                    mismatch_row = dict(row)
+                    mismatch_row["features_label_int"] = row["label_int"]
+                    mismatch_row["metadata_label"] = meta["label"]
+                    mismatch_row["generator"] = meta["source"]
+                    mismatch_row["genre"] = meta["genre"]
+                    label_mismatches.append(mismatch_row)
+                    # features.csv (the file the training pipeline actually
+                    # reads) is treated as the ground-truth label; metadata's
+                    # label disagreement is recorded but does not override it.
+
+                out_row = dict(row)
+                out_row["genre"] = meta["genre"]
+                out_row["generator"] = meta["source"]
+                out_row["ai_model"] = meta["ai_model"]
+                out_row["meta_sample_rate"] = meta["sample_rate"]
+                out_row["meta_duration_sec"] = meta["duration_sec"]
+                out_row["match_source"] = "metadata_csv"
+                writer.writerow(out_row)
+                matched_direct += 1
+                continue
+
+            inferred = _infer_generator_from_filename(fname, int(row["label_int"]))
+            if inferred is not None:
+                generator, ai_model = inferred
+                out_row = dict(row)
+                out_row["genre"] = ""
+                out_row["generator"] = generator
+                out_row["ai_model"] = ai_model
+                out_row["meta_sample_rate"] = ""
+                out_row["meta_duration_sec"] = ""
+                out_row["match_source"] = "filename_inferred"
+                writer.writerow(out_row)
+                matched_inferred += 1
+                continue
+
+            truly_unmatched.append(row)
+
+    if truly_unmatched:
+        with open(UNMATCHED_REPORT_CSV, "w", newline="", encoding="utf-8") as f_out:
+            writer = csv.DictWriter(f_out, fieldnames=feature_fieldnames)
+            writer.writeheader()
+            writer.writerows(truly_unmatched)
+
+    if label_mismatches:
+        mismatch_fieldnames = feature_fieldnames + [
+            "features_label_int", "metadata_label", "generator", "genre",
+        ]
+        with open(LABEL_MISMATCH_CSV, "w", newline="", encoding="utf-8") as f_out:
+            writer = csv.DictWriter(f_out, fieldnames=mismatch_fieldnames)
+            writer.writeheader()
+            writer.writerows(label_mismatches)
+
+    total_matched = matched_direct + matched_inferred
+    print(f"features.csv rows:            {len(feature_rows)}")
+    print(f"metadata.csv rows:            {len(meta_rows)}")
+    print(f"matched via metadata.csv:     {matched_direct}")
+    print(f"matched via filename inference: {matched_inferred}")
+    print(f"total matched:                {total_matched} / {len(feature_rows)}")
+    print(f"truly unmatched:              {len(truly_unmatched)}")
+    print(f"label mismatches (features.csv vs metadata.csv label): {len(label_mismatches)}")
+    print(f"\nOutput:            {OUTPUT_CSV}")
+    if truly_unmatched:
+        print(f"Unmatched report:  {UNMATCHED_REPORT_CSV}")
+    if label_mismatches:
+        print(f"Label mismatches:  {LABEL_MISMATCH_CSV}")
+        gens = {}
+        for m in label_mismatches:
+            gens[m["generator"]] = gens.get(m["generator"], 0) + 1
+        print(f"  by generator: {gens}")
+
+
+if __name__ == "__main__":
+    build()

app/training/calibration_analysis.py

@@ -0,0 +1,202 @@
+"""
+Calibration analysis for LightGBM (reviewer priority — Brier alone is not
+enough evidence of good calibration).
+
+Produces:
+  1. calibration_diagnostics.csv — ECE, calibration slope, calibration
+     intercept, Brier score, alongside the reliability-diagram bins.
+  2. extended_metrics_table.csv  — Table 3 (all 11 models) extended with
+     PR-AUC, Balanced Accuracy, and MCC, computed from the same OOF
+     predictions stored in models/training_results.json (no retraining).
+
+Usage:
+    python -m app.training.calibration_analysis
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+import sys
+from pathlib import Path
+
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sklearn.calibration import calibration_curve
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import (
+    average_precision_score,
+    balanced_accuracy_score,
+    matthews_corrcoef,
+)
+
+MODELS_DIR = Path(__file__).resolve().parents[2] / "models"
+TABLES_DIR = Path(__file__).resolve().parents[3] / "docs/academic/paper/real_tables"
+
+
+def _expected_calibration_error(y_true: np.ndarray, y_prob: np.ndarray, n_bins: int = 10) -> float:
+    bin_edges = np.linspace(0.0, 1.0, n_bins + 1)
+    bin_indices = np.digitize(y_prob, bin_edges[1:-1])
+    ece = 0.0
+    n = len(y_true)
+    for b in range(n_bins):
+        mask = bin_indices == b
+        if not np.any(mask):
+            continue
+        bin_conf = float(np.mean(y_prob[mask]))
+        bin_acc = float(np.mean(y_true[mask]))
+        ece += (np.sum(mask) / n) * abs(bin_conf - bin_acc)
+    return ece
+
+
+def _calibration_slope_intercept(y_true: np.ndarray, y_prob: np.ndarray) -> tuple[float, float]:
+    """Logistic recalibration: y ~ sigmoid(slope * logit(p) + intercept).
+    slope=1, intercept=0 is perfect calibration."""
+    eps = 1e-6
+    p_clipped = np.clip(y_prob, eps, 1 - eps)
+    logit_p = np.log(p_clipped / (1 - p_clipped)).reshape(-1, 1)
+    lr = LogisticRegression()
+    lr.fit(logit_p, y_true)
+    slope = float(lr.coef_[0][0])
+    intercept = float(lr.intercept_[0])
+    return slope, intercept
+
+
+def run() -> None:
+    results_path = MODELS_DIR / "training_results.json"
+    with open(results_path, "r", encoding="utf-8") as f:
+        training_results = json.load(f)
+
+    # ── Extended metrics table (Table 3 + PR-AUC/BalAcc/MCC) ──
+    # training_results.json's per-model entries were saved without the
+    # y_true/y_pred/y_prob arrays (train_classifier.py strips those before
+    # writing JSON) — so PR-AUC/BalAcc/MCC must come from re-running
+    # cross_val_predict, matching evaluate_predictions()'s new metrics.
+    # Simpler and fully consistent: reuse train_classifier's train() output
+    # in-process is out of scope here; instead recompute from the model
+    # pickles + features.csv using the same OOF fold logic as ensemble_model.py.
+    print("Computing extended metrics (PR-AUC, Balanced Accuracy, MCC) for all 11 models...")
+    from sklearn.model_selection import StratifiedKFold
+    from sklearn.preprocessing import StandardScaler
+    from sklearn.base import clone
+    from sklearn.metrics import roc_auc_score, f1_score, accuracy_score, roc_curve
+    import pickle
+    import warnings
+    from sklearn.exceptions import ConvergenceWarning
+
+    from app.training.evaluate import load_features_csv
+
+    FEATURES_CSV = Path("D:/CrownCode/DataSet/features.csv")
+    DL_OOF_NPZ = MODELS_DIR / "dl_oof_probs.npz"
+    X, y = load_features_csv(FEATURES_CSV)
+    X = np.nan_to_num(X, nan=0.0, posinf=1.0, neginf=-1.0)
+
+    ml_files = {
+        "Logistic Regression": "model_logistic_regression.pkl",
+        "Random Forest": "model_random_forest.pkl",
+        "Gradient Boosting": "model_gradient_boosting.pkl",
+        "SVM (RBF)": "model_svm_rbf.pkl",
+        "MLP Neural Network": "model_mlp_neural_network.pkl",
+        "XGBoost": "model_xgboost.pkl",
+        "LightGBM": "model_lightgbm.pkl",
+    }
+    dl_npz_keys = {
+        "Deep MLP (512-256-128-64)": "Deep_MLP_512_256_128_64",
+        "1D-CNN": "1D_CNN",
+        "Residual MLP (3 blocks)": "Residual_MLP_3_blocks",
+        "Attention MLP": "Attention_MLP",
+    }
+
+    if not DL_OOF_NPZ.exists():
+        raise RuntimeError(f"{DL_OOF_NPZ} not found — run dump_dl_oof.py first.")
+    dl_npz = np.load(DL_OOF_NPZ)
+    if not np.array_equal(dl_npz["y"], y):
+        raise RuntimeError("dl_oof_probs.npz label order does not match features.csv load order.")
+
+    ml_models_raw = {}
+    for name, fname in ml_files.items():
+        with open(MODELS_DIR / fname, "rb") as f:
+            ml_models_raw[name] = pickle.load(f)
+
+    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
+    fold_assignments = list(cv.split(X, y))
+    n = len(y)
+    oof_probs: dict[str, np.ndarray] = {name: np.zeros(n) for name in list(ml_files) + list(dl_npz_keys)}
+    for name, npz_key in dl_npz_keys.items():
+        oof_probs[name] = dl_npz[npz_key]
+
+    for fold_idx, (train_idx, test_idx) in enumerate(fold_assignments, start=1):
+        print(f"  Fold {fold_idx}/5 (ML models) ...")
+        X_train, y_train = X[train_idx], y[train_idx]
+        X_test, y_test = X[test_idx], y[test_idx]
+
+        scaler = StandardScaler()
+        X_train_scaled = scaler.fit_transform(X_train)
+        X_test_scaled = scaler.transform(X_test)
+
+        for name in ml_files:
+            model = clone(ml_models_raw[name])
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", category=ConvergenceWarning)
+                model.fit(X_train_scaled, y_train)
+            oof_probs[name][test_idx] = model.predict_proba(X_test_scaled)[:, 1]
+
+    extended_rows = []
+    for name, probs in oof_probs.items():
+        fpr, tpr, thr = roc_curve(y, probs)
+        threshold = float(thr[np.argmax(tpr - fpr)])
+        y_pred = (probs >= threshold).astype(int)
+        extended_rows.append({
+            "model": name,
+            "roc_auc": round(float(roc_auc_score(y, probs)), 4),
+            "pr_auc": round(float(average_precision_score(y, probs)), 4),
+            "accuracy": round(float(accuracy_score(y, y_pred)), 4),
+            "f1": round(float(f1_score(y, y_pred, zero_division=0)), 4),
+            "balanced_accuracy": round(float(balanced_accuracy_score(y, y_pred)), 4),
+            "mcc": round(float(matthews_corrcoef(y, y_pred)), 4),
+            "threshold": round(threshold, 4),
+        })
+
+    extended_rows.sort(key=lambda r: -r["roc_auc"])
+    TABLES_DIR.mkdir(parents=True, exist_ok=True)
+    ext_path = TABLES_DIR / "extended_metrics_table.csv"
+    with open(ext_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=list(extended_rows[0].keys()))
+        writer.writeheader()
+        writer.writerows(extended_rows)
+    print(f"\nExtended metrics table written: {ext_path}")
+    for r in extended_rows:
+        print(f"  {r['model']:28s} AUC={r['roc_auc']:.4f} PR-AUC={r['pr_auc']:.4f} "
+              f"BalAcc={r['balanced_accuracy']:.4f} MCC={r['mcc']:.4f}")
+
+    # ── Calibration diagnostics for LightGBM specifically ──
+    lgbm_probs = oof_probs["LightGBM"]
+    prob_true, prob_pred = calibration_curve(y, lgbm_probs, n_bins=10, strategy="uniform")
+    ece = _expected_calibration_error(y, lgbm_probs, n_bins=10)
+    slope, intercept = _calibration_slope_intercept(y, lgbm_probs)
+    brier = float(np.mean((lgbm_probs - y) ** 2))
+
+    calib_path = TABLES_DIR / "calibration_diagnostics.csv"
+    with open(calib_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.writer(f)
+        writer.writerow(["metric", "value"])
+        writer.writerow(["brier_score", round(brier, 4)])
+        writer.writerow(["ece_10bin", round(ece, 4)])
+        writer.writerow(["calibration_slope", round(slope, 4)])
+        writer.writerow(["calibration_intercept", round(intercept, 4)])
+        writer.writerow([])
+        writer.writerow(["bin_mean_predicted", "bin_fraction_positive"])
+        for pp, pt in zip(prob_pred, prob_true):
+            writer.writerow([round(float(pp), 4), round(float(pt), 4)])
+
+    print(f"\nCalibration diagnostics written: {calib_path}")
+    print(f"  Brier score:            {brier:.4f}")
+    print(f"  ECE (10-bin):           {ece:.4f}")
+    print(f"  Calibration slope:      {slope:.4f} (1.0 = perfect)")
+    print(f"  Calibration intercept:  {intercept:.4f} (0.0 = perfect)")
+
+
+if __name__ == "__main__":
+    run()

app/training/dump_dl_oof.py

@@ -0,0 +1,72 @@
+"""
+Minimal script: run the DL 5-fold CV exactly as train_deep_classifiers.py
+does, but also save the raw out-of-fold probability array per model to
+models/dl_oof_probs.npz, so ensemble_model.py can build a real ensemble
+without retraining DL models a second time.
+
+Usage:
+    python -m app.training.dump_dl_oof
+"""
+
+from __future__ import annotations
+
+import sys
+import time
+from pathlib import Path
+
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sklearn.metrics import roc_auc_score
+from sklearn.model_selection import StratifiedKFold
+
+from app.training.train_deep_classifiers import (
+    DeepMLP, Conv1DClassifier, ResidualMLP, AttentionMLP,
+    load_data, train_one_fold, set_seed, SEED, N_FOLDS, DEVICE,
+)
+
+MODELS_DIR = Path(__file__).resolve().parents[2] / "models"
+FEATURES_CSV = Path("D:/CrownCode/DataSet/features.csv")
+
+MODEL_CLASSES = {
+    "Deep MLP (512-256-128-64)": DeepMLP,
+    "1D-CNN": Conv1DClassifier,
+    "Residual MLP (3 blocks)": ResidualMLP,
+    "Attention MLP": AttentionMLP,
+}
+
+
+def main() -> None:
+    print(f"Device: {DEVICE}", flush=True)
+    X, y, feature_cols = load_data(FEATURES_CSV)
+    print(f"Samples: {len(y)}, Features: {X.shape[1]}", flush=True)
+
+    cv = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)
+    fold_assignments = list(cv.split(X, y))
+
+    oof: dict[str, np.ndarray] = {}
+    for name, cls in MODEL_CLASSES.items():
+        print(f"\n{'='*60}\n  {name}\n{'='*60}", flush=True)
+        t0 = time.time()
+        all_probs = np.zeros(len(y))
+        for fold, (train_idx, val_idx) in enumerate(fold_assignments):
+            set_seed(SEED + fold)
+            model = cls(X.shape[1])
+            fold_t0 = time.time()
+            auc, probs = train_one_fold(model, X[train_idx], y[train_idx], X[val_idx], y[val_idx])
+            all_probs[val_idx] = probs
+            print(f"    Fold {fold+1}/{N_FOLDS}: AUC={auc:.4f} ({time.time()-fold_t0:.1f}s)", flush=True)
+        oof[name] = all_probs
+        total_auc = roc_auc_score(y, all_probs)
+        print(f"  => Overall OOF AUC={total_auc:.4f}  ({time.time()-t0:.1f}s total)", flush=True)
+
+    out_path = MODELS_DIR / "dl_oof_probs.npz"
+    np.savez(out_path, y=y, **{name.replace(" ", "_").replace("(", "").replace(")", "").replace("-", "_"): probs
+                                for name, probs in oof.items()})
+    print(f"\nSaved: {out_path}", flush=True)
+    print("Keys:", list(np.load(out_path).keys()))
+
+
+if __name__ == "__main__":
+    main()

app/training/ensemble_model.py

@@ -0,0 +1,208 @@
+"""
+Real ensemble construction (reviewer priority #5).
+
+The paper's title claims "Ensemble Learning" but the reported result is a
+single best model (LightGBM) chosen from eleven independently-trained
+candidates — no prediction combination happens. The reviewer explicitly asks
+that this either be fixed (build an actual ensemble and report its score
+against LightGBM) or the terminology be revised.
+
+This script loads the saved ML models (models/model_*.pkl, scaled input) and
+DL models (models/model_dl_*.pkl, TorchSklearnWrapper — raw input, wrapper
+scales internally), gets out-of-fold-style predictions via a fresh 5-fold CV
+using the SAME fold assignment as train_classifier.py (StratifiedKFold,
+random_state=42), and combines them two ways:
+  - soft voting (unweighted mean of probabilities)
+  - stacking (logistic regression meta-learner on the base probabilities)
+
+Both are compared against the best single model (LightGBM) on identical
+folds, so the comparison is apples-to-apples.
+
+Usage:
+    python -m app.training.ensemble_model
+"""
+
+from __future__ import annotations
+
+import csv
+import sys
+import warnings
+from pathlib import Path
+
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+import pickle
+from sklearn.exceptions import ConvergenceWarning
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import (
+    accuracy_score,
+    balanced_accuracy_score,
+    f1_score,
+    matthews_corrcoef,
+    roc_auc_score,
+    roc_curve,
+)
+from sklearn.model_selection import StratifiedKFold
+from sklearn.preprocessing import StandardScaler
+
+from app.training.evaluate import load_features_csv
+
+FEATURES_CSV = Path("D:/CrownCode/DataSet/features.csv")
+MODELS_DIR = Path(__file__).resolve().parents[2] / "models"
+TABLES_DIR = Path(__file__).resolve().parents[3] / "docs/academic/paper/real_tables"
+DL_OOF_NPZ = MODELS_DIR / "dl_oof_probs.npz"
+
+# DL out-of-fold probabilities come from dump_dl_oof.py, which retrains each
+# of the 4 architectures per fold with the SAME StratifiedKFold(random_state
+# =42) split used below — so they are genuinely held-out and directly
+# comparable to the ML models' per-fold-refit predictions.
+_DL_NPZ_KEYS = {
+    "Deep MLP": "Deep_MLP_512_256_128_64",
+    "1D-CNN": "1D_CNN",
+    "Residual MLP": "Residual_MLP_3_blocks",
+    "Attention MLP": "Attention_MLP",
+}
+_ML_MODEL_FILES = {
+    "Logistic Regression": "model_logistic_regression.pkl",
+    "Random Forest": "model_random_forest.pkl",
+    "Gradient Boosting": "model_gradient_boosting.pkl",
+    "SVM (RBF)": "model_svm_rbf.pkl",
+    "MLP Neural Network": "model_mlp_neural_network.pkl",
+    "XGBoost": "model_xgboost.pkl",
+    "LightGBM": "model_lightgbm.pkl",
+}
+
+
+def _load_model(filename: str):
+    with open(MODELS_DIR / filename, "rb") as f:
+        return pickle.load(f)
+
+
+def _optimal_threshold(y_true: np.ndarray, y_prob: np.ndarray) -> float:
+    fpr, tpr, thresholds = roc_curve(y_true, y_prob)
+    return float(thresholds[np.argmax(tpr - fpr)])
+
+
+def _metrics(y_true: np.ndarray, y_prob: np.ndarray) -> dict:
+    threshold = _optimal_threshold(y_true, y_prob)
+    y_pred = (y_prob >= threshold).astype(int)
+    return {
+        "roc_auc": round(float(roc_auc_score(y_true, y_prob)), 4),
+        "accuracy": round(float(accuracy_score(y_true, y_pred)), 4),
+        "f1": round(float(f1_score(y_true, y_pred, zero_division=0)), 4),
+        "balanced_accuracy": round(float(balanced_accuracy_score(y_true, y_pred)), 4),
+        "mcc": round(float(matthews_corrcoef(y_true, y_pred)), 4),
+        "threshold": round(threshold, 4),
+    }
+
+
+def run() -> None:
+    X, y = load_features_csv(FEATURES_CSV)
+    X = np.nan_to_num(X, nan=0.0, posinf=1.0, neginf=-1.0)
+
+    if not DL_OOF_NPZ.exists():
+        raise RuntimeError(
+            f"{DL_OOF_NPZ} not found — run `python -m app.training.dump_dl_oof` first "
+            "to generate genuinely held-out DL predictions."
+        )
+
+    print("ML models are refit per fold below (fast). DL out-of-fold predictions")
+    print(f"are loaded from {DL_OOF_NPZ.name} (produced by dump_dl_oof.py, which")
+    print("retrains each DL architecture per fold on the identical fold split).\n")
+
+    ml_models_raw = {name: _load_model(fname) for name, fname in _ML_MODEL_FILES.items()}
+    all_names = list(_ML_MODEL_FILES.keys()) + list(_DL_NPZ_KEYS.keys())
+
+    dl_npz = np.load(DL_OOF_NPZ)
+    y_npz = dl_npz["y"]
+
+    # Uses the SAME StratifiedKFold(random_state=42) as train_classifier.py's
+    # 5-fold CV and the SAME SEED=42 as train_deep_classifiers.py / dump_dl_oof.py,
+    # so this fold assignment matches the folds the DL OOF array was computed on.
+    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
+    fold_assignments = list(cv.split(X, y))
+
+    n = len(y)
+    if not np.array_equal(y_npz, y):
+        raise RuntimeError(
+            "Label array in dl_oof_probs.npz does not match the current "
+            "features.csv load order — DL OOF predictions would be "
+            "misaligned with ML predictions. Re-run dump_dl_oof.py."
+        )
+
+    oof_probs = {name: np.zeros(n) for name in all_names}
+    for name, npz_key in _DL_NPZ_KEYS.items():
+        oof_probs[name] = dl_npz[npz_key]
+
+    for fold_idx, (train_idx, test_idx) in enumerate(fold_assignments, start=1):
+        print(f"Fold {fold_idx}/5 (ML models) ...")
+        X_train, y_train = X[train_idx], y[train_idx]
+        X_test, y_test = X[test_idx], y[test_idx]
+
+        scaler = StandardScaler()
+        X_train_scaled = scaler.fit_transform(X_train)
+        X_test_scaled = scaler.transform(X_test)
+
+        from sklearn.base import clone
+        for name, fname in _ML_MODEL_FILES.items():
+            base_model = ml_models_raw[name]
+            model = clone(base_model)
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", category=ConvergenceWarning)
+                model.fit(X_train_scaled, y_train)
+            oof_probs[name][test_idx] = model.predict_proba(X_test_scaled)[:, 1]
+
+    print("\nPer-model OOF metrics (all models refit per fold, held-out predictions):")
+    per_model_results = []
+    for name in all_names:
+        m = _metrics(y, oof_probs[name])
+        m["model"] = name
+        per_model_results.append(m)
+        print(f"  {name:25s} AUC={m['roc_auc']:.4f} F1={m['f1']:.4f}")
+
+    # ── Soft voting: unweighted mean of all 11 base-model probabilities ──
+    prob_matrix = np.column_stack([oof_probs[name] for name in all_names])
+    soft_vote_prob = prob_matrix.mean(axis=1)
+    soft_vote_metrics = _metrics(y, soft_vote_prob)
+    soft_vote_metrics["model"] = "Ensemble (soft voting, 11 models)"
+    print(f"\n  {'Ensemble (soft voting)':25s} AUC={soft_vote_metrics['roc_auc']:.4f} "
+          f"F1={soft_vote_metrics['f1']:.4f}")
+
+    # ── Stacking: logistic regression meta-learner, 5-fold on base OOF probs ──
+    meta_cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)
+    stack_prob = np.zeros(n)
+    for train_idx, test_idx in meta_cv.split(prob_matrix, y):
+        meta = LogisticRegression(max_iter=1000)
+        meta.fit(prob_matrix[train_idx], y[train_idx])
+        stack_prob[test_idx] = meta.predict_proba(prob_matrix[test_idx])[:, 1]
+    stack_metrics = _metrics(y, stack_prob)
+    stack_metrics["model"] = "Ensemble (stacking, LR meta-learner)"
+    print(f"  {'Ensemble (stacking)':25s} AUC={stack_metrics['roc_auc']:.4f} "
+          f"F1={stack_metrics['f1']:.4f}")
+
+    best_single = max(per_model_results, key=lambda r: r["roc_auc"])
+    print("\n" + "=" * 70)
+    print(f"  Best single model:      {best_single['model']} AUC={best_single['roc_auc']:.4f}")
+    print(f"  Ensemble (soft voting): AUC={soft_vote_metrics['roc_auc']:.4f} "
+          f"(diff={soft_vote_metrics['roc_auc'] - best_single['roc_auc']:+.4f})")
+    print(f"  Ensemble (stacking):    AUC={stack_metrics['roc_auc']:.4f} "
+          f"(diff={stack_metrics['roc_auc'] - best_single['roc_auc']:+.4f})")
+    print("=" * 70)
+
+    TABLES_DIR.mkdir(parents=True, exist_ok=True)
+    out_path = TABLES_DIR / "ensemble_comparison.csv"
+    fieldnames = ["model", "roc_auc", "accuracy", "f1", "balanced_accuracy", "mcc", "threshold"]
+    all_results = per_model_results + [soft_vote_metrics, stack_metrics]
+    with open(out_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        for r in sorted(all_results, key=lambda r: -r["roc_auc"]):
+            writer.writerow({k: r[k] for k in fieldnames})
+
+    print(f"\nOutput: {out_path}")
+
+
+if __name__ == "__main__":
+    run()

app/training/evaluate.py

@@ -24,6 +24,9 @@ try:
         recall_score,
         f1_score,
         roc_auc_score,
+        average_precision_score,
+        balanced_accuracy_score,
+        matthews_corrcoef,
         confusion_matrix,
         classification_report,
     )
@@ -92,12 +95,16 @@ def evaluate_predictions(
     prec = precision_score(y_true, y_pred, zero_division=0)
     rec = recall_score(y_true, y_pred, zero_division=0)
     f1 = f1_score(y_true, y_pred, zero_division=0)
+    bal_acc = balanced_accuracy_score(y_true, y_pred)
+    mcc = matthews_corrcoef(y_true, y_pred) if len(set(y_pred.tolist())) > 1 else 0.0
 
     metrics = {
         "accuracy": round(acc, 4),
         "precision": round(prec, 4),
         "recall": round(rec, 4),
         "f1_score": round(f1, 4),
+        "balanced_accuracy": round(bal_acc, 4),
+        "mcc": round(mcc, 4),
     }
 
     if y_prob is not None:
@@ -106,6 +113,11 @@ def evaluate_predictions(
             metrics["roc_auc"] = round(auc, 4)
         except ValueError:
             metrics["roc_auc"] = None
+        try:
+            pr_auc = average_precision_score(y_true, y_prob)
+            metrics["pr_auc"] = round(pr_auc, 4)
+        except ValueError:
+            metrics["pr_auc"] = None
 
     cm = confusion_matrix(y_true, y_pred)
 
@@ -117,8 +129,12 @@ def evaluate_predictions(
     print(f"  Precision: {prec:.4f}")
     print(f"  Recall:    {rec:.4f}")
     print(f"  F1 Score:  {f1:.4f}")
+    print(f"  Balanced Accuracy: {bal_acc:.4f}")
+    print(f"  MCC:       {mcc:.4f}")
     if "roc_auc" in metrics and metrics["roc_auc"] is not None:
         print(f"  ROC-AUC:   {metrics['roc_auc']:.4f}")
+    if "pr_auc" in metrics and metrics["pr_auc"] is not None:
+        print(f"  PR-AUC:    {metrics['pr_auc']:.4f}")
 
     print(f"\n  Confusion Matrix:")
     print(f"                 Predicted")

app/training/generate_revision_figures.py

@@ -0,0 +1,274 @@
+"""
+Reviewer-revision experiments için TÜRKÇE ve makalenin mevcut görsel stiliyle
+BİREBİR uyumlu figürler üretir.
+
+Stil kaynak: docs/academic/paper/regenerate_tr_figures.py — aynı renk
+paleti (GOLD/HUMAN/AIRED), aynı font (Times New Roman), aynı saydam arka
+plan (BG="none", savefig.transparent=True), aynı grid.alpha=0.15.
+
+Kullanım:
+    python -m app.training.generate_revision_figures
+"""
+
+from __future__ import annotations
+
+import csv
+import sys
+from pathlib import Path
+
+import numpy as np
+
+try:
+    import matplotlib
+    matplotlib.use("Agg")
+    import matplotlib.pyplot as plt
+except ImportError:
+    print("matplotlib gerekli")
+    sys.exit(1)
+
+TABLES = Path(__file__).resolve().parents[3] / "docs/academic/paper/real_tables"
+OUT = Path(__file__).resolve().parents[3] / "docs/academic/figures"
+OUT.mkdir(parents=True, exist_ok=True)
+
+# ── Stil — regenerate_tr_figures.py ile birebir aynı ──
+GOLD = "#C99347"
+BG = "none"
+HUMAN = "#3cb44b"
+AIRED = "#e6194b"
+BLUE = "#4363d8"
+ORANGE = "#f58231"
+PURPLE = "#911eb4"
+CYAN = "#42d4f4"
+BROWN = "#a64b3c"
+DARKGOLD = "#6b4a1e"
+
+plt.rcParams.update({
+    "font.family": "Times New Roman",
+    "font.size": 10,
+    "font.weight": "normal",
+    "axes.labelweight": "normal",
+    "axes.titleweight": "normal",
+    "figure.dpi": 150,
+    "savefig.dpi": 300,
+    "savefig.bbox": "tight",
+    "savefig.transparent": True,
+    "axes.grid": True,
+    "grid.alpha": 0.15,
+    "grid.linewidth": 0.5,
+    "axes.spines.top": False,
+    "axes.spines.right": False,
+    "axes.edgecolor": "black",
+    "text.color": "black",
+    "axes.labelcolor": "black",
+    "xtick.color": "black",
+    "ytick.color": "black",
+})
+
+# Model adları — makalenin Tablo 2/3'ündeki TAM Türkçe karşılıklar
+MODEL_TR = {
+    "Logistic Regression": "Lojistik Regresyon",
+    "Random Forest": "Rastgele Orman",
+    "Gradient Boosting": "Gradyan Artırma",
+    "SVM (RBF)": "SVM-RBF",
+    "MLP Neural Network": "ÇKA Sinir Ağı",
+    "XGBoost": "XGBoost",
+    "LightGBM": "LightGBM",
+    "Deep MLP": "Derin ÇKA",
+    "Deep MLP (512-256-128-64)": "Derin ÇKA",
+    "1D-CNN": "1B-ESA",
+    "Residual MLP": "Artık ÇKA",
+    "Residual MLP (3 blocks)": "Artık ÇKA",
+    "Attention MLP": "Dikkat ÇKA",
+}
+
+GENERATOR_TR = {
+    "aime": "AImE",
+    "deepfake_audio": "Deepfake Seti",
+    "deepfake_audio_dataset": "Deepfake Veri Kümesi",
+    "echoes": "Echoes",
+    "suno_audio": "Suno",
+}
+
+
+def _read_csv(name: str) -> list[dict]:
+    with open(TABLES / name, "r", encoding="utf-8") as f:
+        return list(csv.DictReader(f))
+
+
+def _save(fig, name: str):
+    for ext in ("png", "pdf"):
+        p = OUT / f"{name}.{ext}"
+        fig.savefig(p)
+    plt.close(fig)
+    print(f"  {name}.png / .pdf")
+
+
+def fig_logo() -> None:
+    rows = _read_csv("logo_results.csv")
+    names = [GENERATOR_TR.get(r["held_out_generator"], r["held_out_generator"]) for r in rows]
+    recalls = [float(r["recall"]) for r in rows]
+    f1s = [float(r["f1"]) for r in rows]
+    bal_accs = [float(r["balanced_accuracy"]) for r in rows]
+
+    x = np.arange(len(names))
+    width = 0.25
+    fig, ax = plt.subplots(figsize=(8, 4.5))
+    fig.patch.set_facecolor(BG)
+    ax.set_facecolor(BG)
+    ax.bar(x - width, recalls, width, label="Duyarlılık (YZ sınıfı)", color=BLUE)
+    ax.bar(x, f1s, width, label="F1", color=ORANGE)
+    ax.bar(x + width, bal_accs, width, label="Dengeli Doğruluk", color=HUMAN)
+    ax.axhline(0.9548, color="#666666", linestyle="--", linewidth=1, label="Aynı Dağılım İçi ÇD AUC (referans)")
+    ax.set_xticks(x)
+    ax.set_xticklabels(names, rotation=20, ha="right")
+    ax.set_ylabel("Skor")
+    ax.set_title("Leave-One-Generator-Out: görülmemiş üreticilerdeki performans")
+    ax.set_ylim(0, 1.05)
+    ax.legend(loc="upper right", framealpha=0.9)
+    fig.tight_layout()
+    _save(fig, "revision_logo_results")
+
+
+def fig_flatness_ablation() -> None:
+    rows = _read_csv("flatness_ablation.csv")
+    labels = ["47 Öznitelik\n(Tümü)", "46 Öznitelik\n(Flatness Std Hariç)", "1 Öznitelik\n(Yalnızca Flatness Std)"]
+    aucs = [float(r["roc_auc"]) for r in rows]
+    f1s = [float(r["f1"]) for r in rows]
+    bal_accs = [float(r["balanced_accuracy"]) for r in rows]
+
+    x = np.arange(len(labels))
+    width = 0.25
+    fig, ax = plt.subplots(figsize=(7, 4.5))
+    fig.patch.set_facecolor(BG)
+    ax.set_facecolor(BG)
+    ax.bar(x - width, aucs, width, label="ROC-AUC", color=GOLD)
+    ax.bar(x, f1s, width, label="F1", color=HUMAN)
+    ax.bar(x + width, bal_accs, width, label="Dengeli Doğruluk", color=AIRED)
+    ax.set_xticks(x)
+    ax.set_xticklabels(labels)
+    ax.set_ylabel("Skor")
+    ax.set_title("Spectral flatness ablasyonu (LightGBM, 5-katlı ÇD)")
+    ax.set_ylim(0, 1.05)
+    ax.legend(loc="upper right", framealpha=0.9)
+    fig.tight_layout()
+    _save(fig, "revision_flatness_ablation")
+
+
+def fig_ensemble_comparison() -> None:
+    rows = _read_csv("ensemble_comparison.csv")
+    rows_sorted = sorted(rows, key=lambda r: -float(r["roc_auc"]))
+    names = []
+    for r in rows_sorted:
+        m = r["model"]
+        if "stacking" in m.lower():
+            names.append("Yığınlama Topluluğu")
+        elif "voting" in m.lower():
+            names.append("Ağırlıksız Ortalama Topluluğu")
+        else:
+            names.append(MODEL_TR.get(m, m))
+    aucs = [float(r["roc_auc"]) for r in rows_sorted]
+    colors = [AIRED if "Topluluğu" in n else GOLD for n in names]
+
+    fig, ax = plt.subplots(figsize=(8, 5.5))
+    fig.patch.set_facecolor(BG)
+    ax.set_facecolor(BG)
+    y_pos = np.arange(len(names))
+    ax.barh(y_pos, aucs, color=colors)
+    ax.set_yticks(y_pos)
+    ax.set_yticklabels(names, fontsize=9)
+    ax.invert_yaxis()
+    ax.set_xlabel("ROC-AUC")
+    ax.set_xlim(0.8, 1.0)
+    ax.set_title("Topluluk yöntemleri ile tekil modellerin karşılaştırılması (5-katlı OOF)")
+    for i, v in enumerate(aucs):
+        ax.text(v + 0.002, i, f"{v:.4f}".replace(".", ","), va="center", fontsize=8)
+    fig.tight_layout()
+    _save(fig, "revision_ensemble_comparison")
+
+
+def fig_calibration_reliability() -> None:
+    rows = []
+    with open(TABLES / "calibration_diagnostics.csv", "r", encoding="utf-8") as f:
+        lines = f.read().splitlines()
+    bin_start = lines.index("bin_mean_predicted,bin_fraction_positive") + 1
+    for line in lines[bin_start:]:
+        if not line.strip():
+            continue
+        pred, actual = line.split(",")
+        rows.append((float(pred), float(actual)))
+
+    fig, ax = plt.subplots(figsize=(5.5, 5.5))
+    fig.patch.set_facecolor(BG)
+    ax.set_facecolor(BG)
+    ax.plot([0, 1], [0, 1], color="#333333", linestyle="--", linewidth=1, label="Mükemmel kalibrasyon")
+    preds = [r[0] for r in rows]
+    actuals = [r[1] for r in rows]
+    ax.plot(preds, actuals, "o-", color=GOLD, linewidth=2.5, markersize=8, label="LightGBM (OOF)")
+    ax.set_xlabel("Ortalama Tahmin Olasılığı")
+    ax.set_ylabel("Pozitif Oranı")
+    ax.set_title("Güvenilirlik Diyagramı (10 Kutu)")
+    ax.set_xlim(0, 1)
+    ax.set_ylim(0, 1)
+    ax.legend(loc="upper left")
+    fig.tight_layout()
+    _save(fig, "revision_reliability_diagram")
+
+
+def fig_nested_cv_boxplot() -> None:
+    rows = _read_csv("nested_cv_results.csv")
+    nested_aucs = [float(r["roc_auc"]) for r in rows]
+    plain_cv_auc = 0.9548
+
+    fig, ax = plt.subplots(figsize=(5, 4.5))
+    fig.patch.set_facecolor(BG)
+    ax.set_facecolor(BG)
+    bp = ax.boxplot([nested_aucs], positions=[1], widths=0.5, tick_labels=["Nested ÇD\n(dış=5, iç=3)"],
+                     patch_artist=True)
+    for box in bp["boxes"]:
+        box.set_facecolor(GOLD)
+        box.set_alpha(0.5)
+    ax.axhline(plain_cv_auc, color=AIRED, linestyle="--", linewidth=1.2,
+               label=f"Düz 5-katlı ÇD (referans) = {plain_cv_auc:.4f}".replace(".", ","))
+    ax.scatter([1] * len(nested_aucs), nested_aucs, color=BLUE, alpha=0.8, zorder=3)
+    ax.set_ylabel("ROC-AUC")
+    ax.set_title("Nested ÇD ile düz ÇD karşılaştırması (LightGBM)")
+    y_min, y_max = ax.get_ylim()
+    ax.set_ylim(y_min - (y_max - y_min) * 0.18, y_max)
+    ax.legend(loc="lower center", fontsize=8, framealpha=0.95)
+    fig.tight_layout()
+    _save(fig, "revision_nested_cv")
+
+
+def fig_dataset_bias() -> None:
+    rows = _read_csv("dataset_bias_table.csv")
+    names = [GENERATOR_TR.get(r["source"], r["source"]) if r["source"] not in ("fma", "gtzan", "sleepyjesse", "archive_org")
+              else {"fma": "FMA Small", "gtzan": "GTZAN", "sleepyjesse": "SleepyJesse", "archive_org": "Diğer İnsan"}[r["source"]]
+              for r in rows]
+    ns = [int(r["n"]) for r in rows]
+    colors = [AIRED if r["label"] == "AI" else HUMAN for r in rows]
+
+    fig, ax = plt.subplots(figsize=(8, 4.5))
+    fig.patch.set_facecolor(BG)
+    ax.set_facecolor(BG)
+    ax.bar(names, ns, color=colors)
+    ax.set_xticks(range(len(names)))
+    ax.set_xticklabels(names, rotation=30, ha="right")
+    ax.set_ylabel("Örnek Sayısı")
+    ax.set_title("Kaynak bazlı veri kümesi kompozisyonu (yeşil=İnsan, kırmızı=YZ)")
+    fig.tight_layout()
+    _save(fig, "revision_dataset_bias")
+
+
+def run() -> None:
+    print("Revizyon figürleri üretiliyor (Türkçe, saydam arka plan, Times New Roman)...")
+    fig_logo()
+    fig_flatness_ablation()
+    fig_ensemble_comparison()
+    fig_calibration_reliability()
+    fig_nested_cv_boxplot()
+    fig_dataset_bias()
+    print(f"\nTüm figürler yazıldı: {OUT}")
+
+
+if __name__ == "__main__":
+    run()

app/training/hyperparameter_table.py

@@ -0,0 +1,125 @@
+"""
+Hyperparameter search table (reviewer priority #6 supporting evidence).
+
+The reviewer questions whether all 11 models received equally disciplined
+hyperparameter optimization, or whether some were left at library defaults
+while others (implicitly LightGBM) were tuned. train_classifier.py's
+_build_candidate_families() already runs a per-family holdout search for
+every ML model family (see _select_best_candidates), and
+training_results.json records each model's selected_params and
+validation_auc. This script turns that into a single audit table showing,
+per model: how many candidates were tried, what the search space was, and
+which configuration was selected — so the answer is verifiable rather than
+asserted.
+
+Usage:
+    python -m app.training.hyperparameter_table
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+from pathlib import Path
+
+MODELS_DIR = Path(__file__).resolve().parents[2] / "models"
+TABLES_DIR = Path(__file__).resolve().parents[3] / "docs/academic/paper/real_tables"
+
+# Number of hand-specified candidates per family, taken directly from
+# _build_candidate_families() in train_classifier.py.
+_N_CANDIDATES = {
+    "Logistic Regression": 4,   # C in (0.25, 0.5, 1.0, 2.0)
+    "Random Forest": 3,
+    "Gradient Boosting": 3,
+    "SVM (RBF)": 4,
+    "MLP Neural Network": 3,
+    "XGBoost": 3,
+    "LightGBM": 3,
+}
+
+_SEARCH_SPACE_SUMMARY = {
+    "Logistic Regression": "C in {0.25, 0.5, 1.0, 2.0}; class_weight=balanced fixed",
+    "Random Forest": "n_estimators in {300,450,500}; max_depth in {12,18,None}; "
+                      "max_features in {sqrt,sqrt,log2}",
+    "Gradient Boosting": "n_estimators in {180,200,260}; max_depth in {2,3,4}; "
+                          "learning_rate in {0.04,0.05,0.07}",
+    "SVM (RBF)": "C in {1.0,3.0,6.0,10.0}; gamma in {scale,scale,0.02,0.05}",
+    "MLP Neural Network": "hidden_layer_sizes in {(128,64),(192,96,32),(256,128)}; "
+                           "alpha in {5e-4,1e-3,2e-3}",
+    "XGBoost": "n_estimators in {240,300,500}; max_depth in {3,4,5}; "
+               "learning_rate in {0.03,0.05,0.06}",
+    "LightGBM": "n_estimators in {220,300,500}; max_depth in {-1,6,8}; "
+                "num_leaves in {18,24,31}; learning_rate in {0.03,0.05,0.07}",
+}
+
+# DL models: architecture is fixed per model (no per-family candidate search
+# like the ML side), but all four share identical training hyperparameters
+# (optimizer, LR, epochs, early stopping) — recorded here for the same
+# fairness audit.
+_DL_SHARED_TRAINING = {
+    "optimizer": "AdamW",
+    "learning_rate": "1e-3",
+    "weight_decay": "1e-4",
+    "lr_scheduler": "ReduceLROnPlateau(mode=max, factor=0.5, patience=5)",
+    "loss": "BCEWithLogitsLoss(pos_weight=n_neg/n_pos)",
+    "epochs_max": "100",
+    "early_stopping_patience": "10",
+    "batch_size": "64",
+    "seed": "42 (+fold index)",
+}
+
+
+def run() -> None:
+    with open(MODELS_DIR / "training_results.json", "r", encoding="utf-8") as f:
+        results = json.load(f)
+
+    rows = []
+    for name in _N_CANDIDATES:
+        entry = results.get(name, {})
+        rows.append({
+            "model": name,
+            "n_candidates_evaluated": _N_CANDIDATES[name],
+            "selection_method": "stratified holdout (80/20), best validation AUC wins",
+            "validation_auc": entry.get("validation_auc"),
+            "selected_params": json.dumps(entry.get("selected_params", {}), ensure_ascii=False),
+            "selection_time_sec": entry.get("selection_time_sec"),
+            "search_space": _SEARCH_SPACE_SUMMARY[name],
+        })
+
+    TABLES_DIR.mkdir(parents=True, exist_ok=True)
+    ml_path = TABLES_DIR / "hyperparameter_search_table.csv"
+    with open(ml_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
+        writer.writeheader()
+        writer.writerows(rows)
+
+    print(f"ML hyperparameter search table written: {ml_path}")
+    for r in rows:
+        print(f"  {r['model']:22s} candidates={r['n_candidates_evaluated']} "
+              f"val_auc={r['validation_auc']} time={r['selection_time_sec']}s")
+
+    dl_path = TABLES_DIR / "dl_training_hyperparameters.csv"
+    with open(dl_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.writer(f)
+        writer.writerow(["hyperparameter", "value"])
+        writer.writerow(["note", "shared across all 4 DL architectures (Deep MLP, 1D-CNN, "
+                                  "Residual MLP, Attention MLP) — architecture differs, "
+                                  "training recipe does not"])
+        for k, v in _DL_SHARED_TRAINING.items():
+            writer.writerow([k, v])
+
+    print(f"\nDL shared training hyperparameters written: {dl_path}")
+    print(
+        "\nNOTE: DL models do not go through a per-family candidate search "
+        "like the ML models — each of the 4 DL architectures is trained "
+        "once with an identical, fixed training recipe (see dl_training_"
+        "hyperparameters.csv). This is architecturally different fairness "
+        "(same recipe, different architecture) vs. the ML side (same "
+        "architecture family, tuned hyperparameters) — both are internally "
+        "consistent, but the paper should state this distinction explicitly "
+        "rather than implying all 11 models went through identical tuning."
+    )
+
+
+if __name__ == "__main__":
+    run()

app/training/logo_eval.py

@@ -0,0 +1,203 @@
+"""
+Leave-One-Generator-Out (LOGO) evaluation for AURIS.
+
+The reviewer's #1 priority: standard 5-fold CV shuffles all samples together,
+so it never tests whether the model generalizes to an AI generator it has
+never seen during training. This script holds out one AI generator at a time,
+trains on everything else (all human sources + all remaining AI generators),
+and reports how the model performs on the unseen generator.
+
+Human samples are never held out — they remain in the training set for every
+fold, since the reviewer's concern is specifically about generator-level
+generalization on the AI side.
+
+Usage:
+    python -m app.training.logo_eval
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+import sys
+import warnings
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+import lightgbm as lgb
+from sklearn.exceptions import ConvergenceWarning
+from sklearn.metrics import (
+    accuracy_score,
+    average_precision_score,
+    balanced_accuracy_score,
+    f1_score,
+    matthews_corrcoef,
+    precision_score,
+    recall_score,
+    roc_auc_score,
+    roc_curve,
+)
+from sklearn.preprocessing import StandardScaler
+
+DATASET_DIR = Path("D:/CrownCode/DataSet")
+FEATURES_WITH_META = DATASET_DIR / "features_with_meta.csv"
+OUTPUT_CSV = Path(__file__).resolve().parents[3] / "docs/academic/paper/real_tables/logo_results.csv"
+
+_EXCLUDED_COLUMNS = {
+    "file_path", "label_int", "duration_sec", "sample_rate",
+    "genre", "generator", "ai_model", "meta_sample_rate", "meta_duration_sec",
+    "match_source",
+}
+
+# LightGBM config matching the best-performing candidate from train_classifier.py
+_LGBM_PARAMS = dict(
+    n_estimators=300,
+    max_depth=-1,
+    learning_rate=0.05,
+    num_leaves=31,
+    subsample=0.8,
+    colsample_bytree=0.8,
+    min_child_samples=20,
+    reg_alpha=0.1,
+    reg_lambda=1.0,
+    class_weight="balanced",
+    random_state=42,
+    verbose=-1,
+)
+
+
+def _load() -> tuple[np.ndarray, np.ndarray, list[str], list[str]]:
+    with open(FEATURES_WITH_META, "r", encoding="utf-8") as f:
+        reader = csv.DictReader(f)
+        fieldnames = reader.fieldnames or []
+        feature_cols = [c for c in fieldnames if c not in _EXCLUDED_COLUMNS]
+        rows = list(reader)
+
+    X = np.array([[float(r[c]) for c in feature_cols] for r in rows], dtype=np.float32)
+    X = np.nan_to_num(X, nan=0.0, posinf=1.0, neginf=-1.0)
+    y = np.array([int(r["label_int"]) for r in rows], dtype=np.int32)
+    generators = [r["generator"] for r in rows]
+
+    return X, y, generators, feature_cols
+
+
+def _optimal_threshold(y_true: np.ndarray, y_prob: np.ndarray) -> float:
+    fpr, tpr, thresholds = roc_curve(y_true, y_prob)
+    j_scores = tpr - fpr
+    return float(thresholds[np.argmax(j_scores)])
+
+
+def _metrics(y_true: np.ndarray, y_prob: np.ndarray, threshold: float) -> dict[str, float]:
+    y_pred = (y_prob >= threshold).astype(int)
+    out: dict[str, float] = {
+        "n_test": int(len(y_true)),
+        "n_ai_test": int(np.sum(y_true == 1)),
+        "n_human_test": int(np.sum(y_true == 0)),
+        "accuracy": round(float(accuracy_score(y_true, y_pred)), 4),
+        "precision": round(float(precision_score(y_true, y_pred, zero_division=0)), 4),
+        "recall": round(float(recall_score(y_true, y_pred, zero_division=0)), 4),
+        "f1": round(float(f1_score(y_true, y_pred, zero_division=0)), 4),
+        "balanced_accuracy": round(float(balanced_accuracy_score(y_true, y_pred)), 4),
+        "mcc": round(float(matthews_corrcoef(y_true, y_pred)), 4) if len(set(y_pred)) > 1 else 0.0,
+    }
+    # ROC-AUC / PR-AUC require both classes present in the held-out generator's
+    # test fold; a single-class generator fold cannot report them.
+    if len(set(y_true.tolist())) > 1:
+        out["roc_auc"] = round(float(roc_auc_score(y_true, y_prob)), 4)
+        out["pr_auc"] = round(float(average_precision_score(y_true, y_prob)), 4)
+    else:
+        out["roc_auc"] = None
+        out["pr_auc"] = None
+    return out
+
+
+def run() -> dict[str, Any]:
+    X, y, generators, feature_cols = _load()
+    generators_arr = np.array(generators)
+
+    ai_generators = sorted(set(g for g, label in zip(generators, y) if label == 1))
+    print(f"AI generators found: {ai_generators}")
+    print(f"Total samples: {len(y)} (AI={int(np.sum(y == 1))}, Human={int(np.sum(y == 0))})")
+
+    results: dict[str, dict] = {}
+
+    for held_out in ai_generators:
+        test_mask = generators_arr == held_out
+        train_mask = ~test_mask
+
+        X_train, y_train = X[train_mask], y[train_mask]
+        X_test, y_test = X[test_mask], y[test_mask]
+
+        if len(set(y_train.tolist())) < 2:
+            print(f"  Skipping {held_out}: training set has only one class after holdout")
+            continue
+
+        scaler = StandardScaler()
+        X_train_scaled = scaler.fit_transform(X_train)
+        X_test_scaled = scaler.transform(X_test)
+
+        model = lgb.LGBMClassifier(**_LGBM_PARAMS)
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", category=ConvergenceWarning)
+            model.fit(X_train_scaled, y_train)
+
+        y_prob_train = model.predict_proba(X_train_scaled)[:, 1]
+        threshold = _optimal_threshold(y_train, y_prob_train)
+
+        y_prob_test = model.predict_proba(X_test_scaled)[:, 1]
+        metrics = _metrics(y_test, y_prob_test, threshold)
+        metrics["held_out_generator"] = held_out
+        metrics["threshold_from_train"] = round(threshold, 4)
+        metrics["n_train"] = int(len(y_train))
+
+        results[held_out] = metrics
+
+        auc_str = f"{metrics['roc_auc']:.4f}" if metrics["roc_auc"] is not None else "N/A (single class)"
+        print(
+            f"  Held out: {held_out:25s} n_test={metrics['n_test']:4d} "
+            f"AUC={auc_str} F1={metrics['f1']:.4f} "
+            f"BalAcc={metrics['balanced_accuracy']:.4f} MCC={metrics['mcc']:.4f} "
+            f"Recall={metrics['recall']:.4f}"
+        )
+
+    # ── Aggregate: mean/std across generators with a valid AUC ──
+    valid_aucs = [r["roc_auc"] for r in results.values() if r["roc_auc"] is not None]
+    summary = {
+        "mean_roc_auc": round(float(np.mean(valid_aucs)), 4) if valid_aucs else None,
+        "std_roc_auc": round(float(np.std(valid_aucs)), 4) if valid_aucs else None,
+        "n_generators_evaluated": len(results),
+        "reference_5fold_cv_auc": 0.9548,  # from training_results.json, same-distribution CV
+    }
+
+    print("\n" + "=" * 70)
+    print("LOGO SUMMARY")
+    print("=" * 70)
+    print(f"  Mean ROC-AUC across held-out generators: {summary['mean_roc_auc']}")
+    print(f"  Std  ROC-AUC across held-out generators: {summary['std_roc_auc']}")
+    print(f"  Reference (in-distribution 5-fold CV):   {summary['reference_5fold_cv_auc']}")
+
+    # ── Write CSV ──
+    OUTPUT_CSV.parent.mkdir(parents=True, exist_ok=True)
+    fieldnames = [
+        "held_out_generator", "n_train", "n_test", "n_ai_test", "n_human_test",
+        "threshold_from_train", "accuracy", "precision", "recall", "f1",
+        "balanced_accuracy", "mcc", "roc_auc", "pr_auc",
+    ]
+    with open(OUTPUT_CSV, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        for gen in ai_generators:
+            if gen in results:
+                writer.writerow({k: results[gen].get(k) for k in fieldnames})
+
+    print(f"\nOutput: {OUTPUT_CSV}")
+
+    return {"per_generator": results, "summary": summary}
+
+
+if __name__ == "__main__":
+    run()

app/training/nested_cv.py

@@ -0,0 +1,237 @@
+"""
+Nested cross-validation and an independent held-out test set (reviewer
+priority #4).
+
+Two separate concerns the reviewer raised:
+
+1. The reported hyperparameters were selected on a single stratified
+   holdout, then the SAME hyperparameters were evaluated with plain 5-fold
+   CV. That lets a lucky hyperparameter choice leak into the reported CV
+   score. Nested CV (outer loop for evaluation, inner loop for hyperparameter
+   selection, refit per outer fold) removes that leakage.
+
+2. The reviewer explicitly warns that if the Youden J threshold is tuned on
+   the same data used to report performance, "test information" has
+   implicitly entered the model. This script fixes that: split
+   train/val/test 70/15/15 (stratified), select the threshold on train+val
+   only, then apply that FIXED threshold to the untouched test set and report
+   test-set metrics.
+
+Usage:
+    python -m app.training.nested_cv
+"""
+
+from __future__ import annotations
+
+import csv
+import sys
+import warnings
+from pathlib import Path
+
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+import lightgbm as lgb
+from sklearn.exceptions import ConvergenceWarning
+from sklearn.metrics import (
+    accuracy_score,
+    average_precision_score,
+    balanced_accuracy_score,
+    f1_score,
+    matthews_corrcoef,
+    precision_score,
+    recall_score,
+    roc_auc_score,
+    roc_curve,
+)
+from sklearn.model_selection import StratifiedKFold, train_test_split
+from sklearn.preprocessing import StandardScaler
+
+from app.training.evaluate import load_features_csv
+
+FEATURES_CSV = Path("D:/CrownCode/DataSet/features.csv")
+TABLES_DIR = Path(__file__).resolve().parents[3] / "docs/academic/paper/real_tables"
+
+# Same search space as train_classifier.py's LightGBM candidates, used here
+# for the inner-loop hyperparameter search.
+_LGBM_CANDIDATES = [
+    dict(n_estimators=300, max_depth=-1, learning_rate=0.05, num_leaves=31,
+         subsample=0.8, colsample_bytree=0.8, min_child_samples=20,
+         reg_alpha=0.1, reg_lambda=1.0),
+    dict(n_estimators=500, max_depth=8, learning_rate=0.03, num_leaves=24,
+         subsample=0.9, colsample_bytree=0.8, min_child_samples=30,
+         reg_alpha=0.2, reg_lambda=1.2),
+    dict(n_estimators=220, max_depth=6, learning_rate=0.07, num_leaves=18,
+         subsample=0.75, colsample_bytree=0.75, min_child_samples=24,
+         reg_alpha=0.3, reg_lambda=1.5),
+]
+
+
+def _fit_lgbm(params: dict, X: np.ndarray, y: np.ndarray) -> lgb.LGBMClassifier:
+    model = lgb.LGBMClassifier(**params, class_weight="balanced", random_state=42, verbose=-1)
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", category=ConvergenceWarning)
+        model.fit(X, y)
+    return model
+
+
+def _optimal_threshold(y_true: np.ndarray, y_prob: np.ndarray) -> float:
+    fpr, tpr, thresholds = roc_curve(y_true, y_prob)
+    return float(thresholds[np.argmax(tpr - fpr)])
+
+
+def _full_metrics(y_true: np.ndarray, y_prob: np.ndarray, threshold: float) -> dict:
+    y_pred = (y_prob >= threshold).astype(int)
+    return {
+        "accuracy": round(float(accuracy_score(y_true, y_pred)), 4),
+        "precision": round(float(precision_score(y_true, y_pred, zero_division=0)), 4),
+        "recall": round(float(recall_score(y_true, y_pred, zero_division=0)), 4),
+        "f1": round(float(f1_score(y_true, y_pred, zero_division=0)), 4),
+        "balanced_accuracy": round(float(balanced_accuracy_score(y_true, y_pred)), 4),
+        "mcc": round(float(matthews_corrcoef(y_true, y_pred)), 4),
+        "roc_auc": round(float(roc_auc_score(y_true, y_prob)), 4),
+        "pr_auc": round(float(average_precision_score(y_true, y_prob)), 4),
+    }
+
+
+def nested_cv(X: np.ndarray, y: np.ndarray, outer_folds: int = 5, inner_folds: int = 3) -> list[dict]:
+    """Outer loop evaluates; inner loop selects hyperparameters per outer fold."""
+    outer_cv = StratifiedKFold(n_splits=outer_folds, shuffle=True, random_state=42)
+    results = []
+
+    for fold_idx, (train_idx, test_idx) in enumerate(outer_cv.split(X, y), start=1):
+        X_train_outer, y_train_outer = X[train_idx], y[train_idx]
+        X_test_outer, y_test_outer = X[test_idx], y[test_idx]
+
+        scaler_outer = StandardScaler()
+        X_train_outer_scaled = scaler_outer.fit_transform(X_train_outer)
+        X_test_outer_scaled = scaler_outer.transform(X_test_outer)
+
+        # Inner loop: pick the best candidate params by mean inner-fold AUC
+        inner_cv = StratifiedKFold(n_splits=inner_folds, shuffle=True, random_state=fold_idx)
+        best_params = None
+        best_inner_auc = -1.0
+
+        for params in _LGBM_CANDIDATES:
+            inner_aucs = []
+            for inner_train_idx, inner_val_idx in inner_cv.split(X_train_outer_scaled, y_train_outer):
+                model = _fit_lgbm(
+                    params,
+                    X_train_outer_scaled[inner_train_idx],
+                    y_train_outer[inner_train_idx],
+                )
+                y_prob_inner = model.predict_proba(X_train_outer_scaled[inner_val_idx])[:, 1]
+                inner_aucs.append(roc_auc_score(y_train_outer[inner_val_idx], y_prob_inner))
+            mean_inner_auc = float(np.mean(inner_aucs))
+            if mean_inner_auc > best_inner_auc:
+                best_inner_auc = mean_inner_auc
+                best_params = params
+
+        # Refit on the full outer-train split with the winning params
+        final_model = _fit_lgbm(best_params, X_train_outer_scaled, y_train_outer)
+        y_prob_train = final_model.predict_proba(X_train_outer_scaled)[:, 1]
+        threshold = _optimal_threshold(y_train_outer, y_prob_train)
+
+        y_prob_test = final_model.predict_proba(X_test_outer_scaled)[:, 1]
+        metrics = _full_metrics(y_test_outer, y_prob_test, threshold)
+        metrics["outer_fold"] = fold_idx
+        metrics["inner_selected_n_estimators"] = best_params["n_estimators"]
+        metrics["inner_selected_max_depth"] = best_params["max_depth"]
+        metrics["inner_val_auc"] = round(best_inner_auc, 4)
+        results.append(metrics)
+
+        print(
+            f"  Outer fold {fold_idx}: inner-selected params -> "
+            f"n_estimators={best_params['n_estimators']}, max_depth={best_params['max_depth']} "
+            f"| outer-test AUC={metrics['roc_auc']:.4f} F1={metrics['f1']:.4f}"
+        )
+
+    return results
+
+
+def independent_test_split(X: np.ndarray, y: np.ndarray) -> dict:
+    """
+    70/15/15 stratified train/val/test split. Threshold is selected on
+    train+val ONLY, then applied as a fixed value to the untouched test set.
+    """
+    X_temp, X_test, y_temp, y_test = train_test_split(
+        X, y, test_size=0.15, stratify=y, random_state=42,
+    )
+    X_train, X_val, y_train, y_val = train_test_split(
+        X_temp, y_temp, test_size=0.15 / 0.85, stratify=y_temp, random_state=42,
+    )
+
+    print(f"\n  Split sizes: train={len(y_train)} val={len(y_val)} test={len(y_test)}")
+
+    scaler = StandardScaler()
+    X_train_scaled = scaler.fit_transform(X_train)
+    X_val_scaled = scaler.transform(X_val)
+    X_test_scaled = scaler.transform(X_test)
+
+    # Train on train split, select threshold on val split only
+    model = _fit_lgbm(_LGBM_CANDIDATES[0], X_train_scaled, y_train)
+    y_prob_val = model.predict_proba(X_val_scaled)[:, 1]
+    threshold = _optimal_threshold(y_val, y_prob_val)
+    print(f"  Threshold selected on VAL split only: theta* = {threshold:.4f}")
+
+    # Refit on train+val (standard practice once threshold is frozen),
+    # evaluate once on the untouched test split with the frozen threshold.
+    X_trainval_scaled = np.vstack([X_train_scaled, X_val_scaled])
+    y_trainval = np.concatenate([y_train, y_val])
+    final_model = _fit_lgbm(_LGBM_CANDIDATES[0], X_trainval_scaled, y_trainval)
+
+    y_prob_test = final_model.predict_proba(X_test_scaled)[:, 1]
+    metrics = _full_metrics(y_test, y_prob_test, threshold)
+    metrics["threshold_source"] = "train+val only (frozen before touching test)"
+    metrics["n_train"] = len(y_train)
+    metrics["n_val"] = len(y_val)
+    metrics["n_test"] = len(y_test)
+    metrics["threshold"] = round(threshold, 4)
+
+    print(f"  Independent test-set metrics: {metrics}")
+    return metrics
+
+
+def run() -> None:
+    X, y = load_features_csv(FEATURES_CSV)
+    X = np.nan_to_num(X, nan=0.0, posinf=1.0, neginf=-1.0)
+    TABLES_DIR.mkdir(parents=True, exist_ok=True)
+
+    print("=" * 70)
+    print("STEP 1/2 — Nested cross-validation (outer=5, inner=3)")
+    print("=" * 70)
+    nested_results = nested_cv(X, y)
+
+    nested_fieldnames = [
+        "outer_fold", "inner_selected_n_estimators", "inner_selected_max_depth",
+        "inner_val_auc", "accuracy", "precision", "recall", "f1",
+        "balanced_accuracy", "mcc", "roc_auc", "pr_auc",
+    ]
+    nested_path = TABLES_DIR / "nested_cv_results.csv"
+    with open(nested_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=nested_fieldnames)
+        writer.writeheader()
+        for r in nested_results:
+            writer.writerow({k: r[k] for k in nested_fieldnames})
+
+    aucs = [r["roc_auc"] for r in nested_results]
+    print(f"\n  Nested CV mean AUC: {np.mean(aucs):.4f} +/- {np.std(aucs):.4f}")
+    print(f"  (reference: plain 5-fold CV AUC = 0.9548)")
+    print(f"  Output: {nested_path}")
+
+    print("\n" + "=" * 70)
+    print("STEP 2/2 — Independent 70/15/15 train/val/test split")
+    print("=" * 70)
+    independent_metrics = independent_test_split(X, y)
+
+    independent_path = TABLES_DIR / "independent_test_results.csv"
+    with open(independent_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=list(independent_metrics.keys()))
+        writer.writeheader()
+        writer.writerow(independent_metrics)
+    print(f"\n  Output: {independent_path}")
+
+
+if __name__ == "__main__":
+    run()