v802-classifier / src /experiment_v801_dependencies.py
sneo's picture
Add v802 classifier head and source code
59839e3 verified
Raw History Blame Contribute Delete
6.64 kB
#!/usr/bin/env python3
"""Development-only classifier-chain experiments for v801."""
from __future__ import annotations
import argparse
import csv
import itertools
import json
import time
from pathlib import Path
import numpy as np
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from sklearn.multioutput import ClassifierChain
from sklearn.preprocessing import StandardScaler
from train_v801 import (
Spec,
crossfit_threshold_predictions,
make_variant,
oof_probabilities,
report_metrics,
tune_structured_decoder,
)
def chain_oof(
x: np.ndarray,
y: np.ndarray,
anchor_idx: np.ndarray,
target_idx: np.ndarray,
folds: list[tuple[np.ndarray, np.ndarray]],
c: float,
class_weight: str | None,
orders: list[np.ndarray],
) -> np.ndarray:
result = np.zeros((len(target_idx), y.shape[1]), dtype=np.float64)
for train_local, valid_local in folds:
train_idx = np.concatenate((anchor_idx, target_idx[train_local]))
valid_idx = target_idx[valid_local]
scaler = StandardScaler().fit(x[train_idx])
train_x = scaler.transform(x[train_idx])
valid_x = scaler.transform(x[valid_idx])
fold_probabilities = []
for order in orders:
estimator = LogisticRegression(
C=c,
solver="liblinear",
dual=True,
max_iter=3000,
class_weight=class_weight,
random_state=801,
)
chain = ClassifierChain(
estimator=estimator,
order=order,
chain_method="predict_proba",
random_state=801,
).fit(train_x, y[train_idx])
fold_probabilities.append(chain.predict_proba(valid_x))
result[valid_local] = np.mean(fold_probabilities, axis=0)
return result
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--features", type=Path, required=True)
parser.add_argument("--development", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--seed", type=int, default=801208)
args = parser.parse_args()
started = time.time()
bundle = np.load(args.features)
names = bundle["file_names"].astype(str)
y = bundle["labels"].astype(np.uint8)
lookup = {name: index for index, name in enumerate(names)}
with args.development.open(newline="", encoding="utf-8-sig") as handle:
rows = list(csv.DictReader(handle))
development_idx = np.asarray([lookup[row["file_name"]] for row in rows])
origins = np.full(len(names), "excluded", dtype="U32")
for row in rows:
origins[lookup[row["file_name"]]] = row["origin"]
anchor_idx = development_idx[origins[development_idx] == "old440"]
target_idx = development_idx[origins[development_idx] != "old440"]
folds = list(
MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=args.seed).split(
np.zeros(len(target_idx)), y[target_idx]
)
)
recent = origins[target_idx] == "incremental100"
x = make_variant(bundle, "meanmax_clspatchmax")
orders = [
np.arange(y.shape[1]),
np.arange(y.shape[1])[::-1],
np.random.default_rng(801).permutation(y.shape[1]),
]
base_specs = {
"legacy": Spec("legacy", 0.003, class_weight=None),
"patch": Spec("meanmax_clspatchmax", 0.003, class_weight=None),
}
base_probabilities = {
name: oof_probabilities(
make_variant(bundle, spec.variant), y, origins,
anchor_idx, target_idx, folds, spec,
)
for name, spec in base_specs.items()
}
reports: list[dict[str, object]] = []
def evaluate(name: str, probabilities: np.ndarray) -> None:
threshold_predictions = crossfit_threshold_predictions(
y[target_idx], probabilities, folds
)
threshold_metrics = report_metrics(
y[target_idx], probabilities, threshold_predictions
)
threshold_metrics["incremental100_exact_match_accuracy"] = float(
accuracy_score(y[target_idx][recent], threshold_predictions[recent])
)
decoder, _, structured_metrics = tune_structured_decoder(
y[target_idx], probabilities, folds, origins[target_idx]
)
reports.append(
{
"name": name,
"threshold_metrics": threshold_metrics,
"decoder": decoder,
"structured_metrics": structured_metrics,
}
)
print(
f"{name}: threshold={threshold_metrics['exact_match_accuracy']:.4f} "
f"structured={structured_metrics['exact_match_accuracy']:.4f} "
f"recent={structured_metrics['incremental100_exact_match_accuracy']:.4f}",
flush=True,
)
evaluate("base-legacy", base_probabilities["legacy"])
evaluate("base-patch", base_probabilities["patch"])
for c, class_weight in itertools.product((0.003, 0.01, 0.03), (None, "balanced")):
balance = "plain" if class_weight is None else "balanced"
chain_probabilities = chain_oof(
x, y, anchor_idx, target_idx, folds, c, class_weight, orders
)
evaluate(f"chain3-C{c:g}-{balance}", chain_probabilities)
for base_name, weight in itertools.product(("legacy", "patch"), (0.25, 0.5, 0.75)):
probabilities = (
weight * chain_probabilities
+ (1.0 - weight) * base_probabilities[base_name]
)
evaluate(
f"chain3-C{c:g}-{balance}-w{weight:g}+{base_name}", probabilities
)
reports.sort(
key=lambda report: (
float(report["structured_metrics"]["exact_match_accuracy"]),
float(report["structured_metrics"]["incremental100_exact_match_accuracy"]),
float(report["structured_metrics"]["macro_f1"]),
),
reverse=True,
)
output = {
"protocol": "development736 only; locked manifests are not read",
"seed": args.seed,
"orders": [order.tolist() for order in orders],
"candidates": reports,
"seconds": time.time() - started,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8")
print(json.dumps(reports[0], indent=2), flush=True)
if __name__ == "__main__":
main()