Download src/experiment_v801_dependencies.py from sneo/v802-classifier: direct link, hf CLI and curl.
- Browser
- Download file 6.64 kB
-
https://huggingface.co/sneo/v802-classifier/resolve/main/src/experiment_v801_dependencies.py
- Command line
-
hf download hf://sneo/v802-classifier/src/experiment_v801_dependencies.py
-
curl -L -o experiment_v801_dependencies.py https://huggingface.co/sneo/v802-classifier/resolve/main/src/experiment_v801_dependencies.py
6.64 kB
| #!/usr/bin/env python3 | |
| """Development-only classifier-chain experiments for v801.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import itertools | |
| import json | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| from iterstrat.ml_stratifiers import MultilabelStratifiedKFold | |
| from sklearn.linear_model import LogisticRegression | |
| from sklearn.metrics import accuracy_score | |
| from sklearn.multioutput import ClassifierChain | |
| from sklearn.preprocessing import StandardScaler | |
| from train_v801 import ( | |
| Spec, | |
| crossfit_threshold_predictions, | |
| make_variant, | |
| oof_probabilities, | |
| report_metrics, | |
| tune_structured_decoder, | |
| ) | |
| def chain_oof( | |
| x: np.ndarray, | |
| y: np.ndarray, | |
| anchor_idx: np.ndarray, | |
| target_idx: np.ndarray, | |
| folds: list[tuple[np.ndarray, np.ndarray]], | |
| c: float, | |
| class_weight: str | None, | |
| orders: list[np.ndarray], | |
| ) -> np.ndarray: | |
| result = np.zeros((len(target_idx), y.shape[1]), dtype=np.float64) | |
| for train_local, valid_local in folds: | |
| train_idx = np.concatenate((anchor_idx, target_idx[train_local])) | |
| valid_idx = target_idx[valid_local] | |
| scaler = StandardScaler().fit(x[train_idx]) | |
| train_x = scaler.transform(x[train_idx]) | |
| valid_x = scaler.transform(x[valid_idx]) | |
| fold_probabilities = [] | |
| for order in orders: | |
| estimator = LogisticRegression( | |
| C=c, | |
| solver="liblinear", | |
| dual=True, | |
| max_iter=3000, | |
| class_weight=class_weight, | |
| random_state=801, | |
| ) | |
| chain = ClassifierChain( | |
| estimator=estimator, | |
| order=order, | |
| chain_method="predict_proba", | |
| random_state=801, | |
| ).fit(train_x, y[train_idx]) | |
| fold_probabilities.append(chain.predict_proba(valid_x)) | |
| result[valid_local] = np.mean(fold_probabilities, axis=0) | |
| return result | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--features", type=Path, required=True) | |
| parser.add_argument("--development", type=Path, required=True) | |
| parser.add_argument("--output", type=Path, required=True) | |
| parser.add_argument("--seed", type=int, default=801208) | |
| args = parser.parse_args() | |
| started = time.time() | |
| bundle = np.load(args.features) | |
| names = bundle["file_names"].astype(str) | |
| y = bundle["labels"].astype(np.uint8) | |
| lookup = {name: index for index, name in enumerate(names)} | |
| with args.development.open(newline="", encoding="utf-8-sig") as handle: | |
| rows = list(csv.DictReader(handle)) | |
| development_idx = np.asarray([lookup[row["file_name"]] for row in rows]) | |
| origins = np.full(len(names), "excluded", dtype="U32") | |
| for row in rows: | |
| origins[lookup[row["file_name"]]] = row["origin"] | |
| anchor_idx = development_idx[origins[development_idx] == "old440"] | |
| target_idx = development_idx[origins[development_idx] != "old440"] | |
| folds = list( | |
| MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=args.seed).split( | |
| np.zeros(len(target_idx)), y[target_idx] | |
| ) | |
| ) | |
| recent = origins[target_idx] == "incremental100" | |
| x = make_variant(bundle, "meanmax_clspatchmax") | |
| orders = [ | |
| np.arange(y.shape[1]), | |
| np.arange(y.shape[1])[::-1], | |
| np.random.default_rng(801).permutation(y.shape[1]), | |
| ] | |
| base_specs = { | |
| "legacy": Spec("legacy", 0.003, class_weight=None), | |
| "patch": Spec("meanmax_clspatchmax", 0.003, class_weight=None), | |
| } | |
| base_probabilities = { | |
| name: oof_probabilities( | |
| make_variant(bundle, spec.variant), y, origins, | |
| anchor_idx, target_idx, folds, spec, | |
| ) | |
| for name, spec in base_specs.items() | |
| } | |
| reports: list[dict[str, object]] = [] | |
| def evaluate(name: str, probabilities: np.ndarray) -> None: | |
| threshold_predictions = crossfit_threshold_predictions( | |
| y[target_idx], probabilities, folds | |
| ) | |
| threshold_metrics = report_metrics( | |
| y[target_idx], probabilities, threshold_predictions | |
| ) | |
| threshold_metrics["incremental100_exact_match_accuracy"] = float( | |
| accuracy_score(y[target_idx][recent], threshold_predictions[recent]) | |
| ) | |
| decoder, _, structured_metrics = tune_structured_decoder( | |
| y[target_idx], probabilities, folds, origins[target_idx] | |
| ) | |
| reports.append( | |
| { | |
| "name": name, | |
| "threshold_metrics": threshold_metrics, | |
| "decoder": decoder, | |
| "structured_metrics": structured_metrics, | |
| } | |
| ) | |
| print( | |
| f"{name}: threshold={threshold_metrics['exact_match_accuracy']:.4f} " | |
| f"structured={structured_metrics['exact_match_accuracy']:.4f} " | |
| f"recent={structured_metrics['incremental100_exact_match_accuracy']:.4f}", | |
| flush=True, | |
| ) | |
| evaluate("base-legacy", base_probabilities["legacy"]) | |
| evaluate("base-patch", base_probabilities["patch"]) | |
| for c, class_weight in itertools.product((0.003, 0.01, 0.03), (None, "balanced")): | |
| balance = "plain" if class_weight is None else "balanced" | |
| chain_probabilities = chain_oof( | |
| x, y, anchor_idx, target_idx, folds, c, class_weight, orders | |
| ) | |
| evaluate(f"chain3-C{c:g}-{balance}", chain_probabilities) | |
| for base_name, weight in itertools.product(("legacy", "patch"), (0.25, 0.5, 0.75)): | |
| probabilities = ( | |
| weight * chain_probabilities | |
| + (1.0 - weight) * base_probabilities[base_name] | |
| ) | |
| evaluate( | |
| f"chain3-C{c:g}-{balance}-w{weight:g}+{base_name}", probabilities | |
| ) | |
| reports.sort( | |
| key=lambda report: ( | |
| float(report["structured_metrics"]["exact_match_accuracy"]), | |
| float(report["structured_metrics"]["incremental100_exact_match_accuracy"]), | |
| float(report["structured_metrics"]["macro_f1"]), | |
| ), | |
| reverse=True, | |
| ) | |
| output = { | |
| "protocol": "development736 only; locked manifests are not read", | |
| "seed": args.seed, | |
| "orders": [order.tolist() for order in orders], | |
| "candidates": reports, | |
| "seconds": time.time() - started, | |
| } | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| args.output.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8") | |
| print(json.dumps(reports[0], indent=2), flush=True) | |
| if __name__ == "__main__": | |
| main() | |