#!/usr/bin/env python3 """Development-only classifier-chain experiments for v801.""" from __future__ import annotations import argparse import csv import itertools import json import time from pathlib import Path import numpy as np from iterstrat.ml_stratifiers import MultilabelStratifiedKFold from sklearn.linear_model import LogisticRegression from sklearn.metrics import accuracy_score from sklearn.multioutput import ClassifierChain from sklearn.preprocessing import StandardScaler from train_v801 import ( Spec, crossfit_threshold_predictions, make_variant, oof_probabilities, report_metrics, tune_structured_decoder, ) def chain_oof( x: np.ndarray, y: np.ndarray, anchor_idx: np.ndarray, target_idx: np.ndarray, folds: list[tuple[np.ndarray, np.ndarray]], c: float, class_weight: str | None, orders: list[np.ndarray], ) -> np.ndarray: result = np.zeros((len(target_idx), y.shape[1]), dtype=np.float64) for train_local, valid_local in folds: train_idx = np.concatenate((anchor_idx, target_idx[train_local])) valid_idx = target_idx[valid_local] scaler = StandardScaler().fit(x[train_idx]) train_x = scaler.transform(x[train_idx]) valid_x = scaler.transform(x[valid_idx]) fold_probabilities = [] for order in orders: estimator = LogisticRegression( C=c, solver="liblinear", dual=True, max_iter=3000, class_weight=class_weight, random_state=801, ) chain = ClassifierChain( estimator=estimator, order=order, chain_method="predict_proba", random_state=801, ).fit(train_x, y[train_idx]) fold_probabilities.append(chain.predict_proba(valid_x)) result[valid_local] = np.mean(fold_probabilities, axis=0) return result def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--features", type=Path, required=True) parser.add_argument("--development", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--seed", type=int, default=801208) args = parser.parse_args() started = time.time() bundle = np.load(args.features) names = bundle["file_names"].astype(str) y = bundle["labels"].astype(np.uint8) lookup = {name: index for index, name in enumerate(names)} with args.development.open(newline="", encoding="utf-8-sig") as handle: rows = list(csv.DictReader(handle)) development_idx = np.asarray([lookup[row["file_name"]] for row in rows]) origins = np.full(len(names), "excluded", dtype="U32") for row in rows: origins[lookup[row["file_name"]]] = row["origin"] anchor_idx = development_idx[origins[development_idx] == "old440"] target_idx = development_idx[origins[development_idx] != "old440"] folds = list( MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=args.seed).split( np.zeros(len(target_idx)), y[target_idx] ) ) recent = origins[target_idx] == "incremental100" x = make_variant(bundle, "meanmax_clspatchmax") orders = [ np.arange(y.shape[1]), np.arange(y.shape[1])[::-1], np.random.default_rng(801).permutation(y.shape[1]), ] base_specs = { "legacy": Spec("legacy", 0.003, class_weight=None), "patch": Spec("meanmax_clspatchmax", 0.003, class_weight=None), } base_probabilities = { name: oof_probabilities( make_variant(bundle, spec.variant), y, origins, anchor_idx, target_idx, folds, spec, ) for name, spec in base_specs.items() } reports: list[dict[str, object]] = [] def evaluate(name: str, probabilities: np.ndarray) -> None: threshold_predictions = crossfit_threshold_predictions( y[target_idx], probabilities, folds ) threshold_metrics = report_metrics( y[target_idx], probabilities, threshold_predictions ) threshold_metrics["incremental100_exact_match_accuracy"] = float( accuracy_score(y[target_idx][recent], threshold_predictions[recent]) ) decoder, _, structured_metrics = tune_structured_decoder( y[target_idx], probabilities, folds, origins[target_idx] ) reports.append( { "name": name, "threshold_metrics": threshold_metrics, "decoder": decoder, "structured_metrics": structured_metrics, } ) print( f"{name}: threshold={threshold_metrics['exact_match_accuracy']:.4f} " f"structured={structured_metrics['exact_match_accuracy']:.4f} " f"recent={structured_metrics['incremental100_exact_match_accuracy']:.4f}", flush=True, ) evaluate("base-legacy", base_probabilities["legacy"]) evaluate("base-patch", base_probabilities["patch"]) for c, class_weight in itertools.product((0.003, 0.01, 0.03), (None, "balanced")): balance = "plain" if class_weight is None else "balanced" chain_probabilities = chain_oof( x, y, anchor_idx, target_idx, folds, c, class_weight, orders ) evaluate(f"chain3-C{c:g}-{balance}", chain_probabilities) for base_name, weight in itertools.product(("legacy", "patch"), (0.25, 0.5, 0.75)): probabilities = ( weight * chain_probabilities + (1.0 - weight) * base_probabilities[base_name] ) evaluate( f"chain3-C{c:g}-{balance}-w{weight:g}+{base_name}", probabilities ) reports.sort( key=lambda report: ( float(report["structured_metrics"]["exact_match_accuracy"]), float(report["structured_metrics"]["incremental100_exact_match_accuracy"]), float(report["structured_metrics"]["macro_f1"]), ), reverse=True, ) output = { "protocol": "development736 only; locked manifests are not read", "seed": args.seed, "orders": [order.tolist() for order in orders], "candidates": reports, "seconds": time.time() - started, } args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8") print(json.dumps(reports[0], indent=2), flush=True) if __name__ == "__main__": main()