Download src/experiment_v801_historical_reference.py from sneo/v802-classifier: direct link, hf CLI and curl.
- Browser
- Download file 7.75 kB
-
https://huggingface.co/sneo/v802-classifier/resolve/main/src/experiment_v801_historical_reference.py
- Command line
-
hf download hf://sneo/v802-classifier/src/experiment_v801_historical_reference.py
-
curl -L -o experiment_v801_historical_reference.py https://huggingface.co/sneo/v802-classifier/resolve/main/src/experiment_v801_historical_reference.py
7.75 kB
| #!/usr/bin/env python3 | |
| """Historical-reference check for fixed v801 local decoder candidates.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import json | |
| from pathlib import Path | |
| import joblib | |
| import numpy as np | |
| from iterstrat.ml_stratifiers import MultilabelStratifiedKFold | |
| from sklearn.linear_model import LogisticRegression | |
| from experiment_v801_knn import knn_oof | |
| from train_probe import apply_thresholds, predict as legacy_predict | |
| from train_v801 import ( | |
| Spec, | |
| all_labelsets, | |
| artifact_probabilities, | |
| binary_probabilities, | |
| choose_thresholds, | |
| fit_binary, | |
| labelset_ids, | |
| make_variant, | |
| meta_features, | |
| oof_probabilities, | |
| report_metrics, | |
| structured_predictions, | |
| threshold_predictions, | |
| ) | |
| def knn_probabilities( | |
| query: np.ndarray, | |
| references: np.ndarray, | |
| labels: np.ndarray, | |
| k: int = 11, | |
| temperature: float = 0.1, | |
| ) -> np.ndarray: | |
| similarity = query @ references.T | |
| neighbours = np.argpartition(similarity, -k, axis=1)[:, -k:] | |
| neighbour_similarity = np.take_along_axis(similarity, neighbours, axis=1) | |
| neighbour_similarity -= neighbour_similarity.max(axis=1, keepdims=True) | |
| weights = np.exp(neighbour_similarity / temperature) | |
| weights /= weights.sum(axis=1, keepdims=True) | |
| return np.einsum("nk,nkc->nc", weights, labels[neighbours]) | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--features", type=Path, required=True) | |
| parser.add_argument("--legacy-features", type=Path, required=True) | |
| parser.add_argument("--development", type=Path, required=True) | |
| parser.add_argument("--historical", type=Path, required=True) | |
| parser.add_argument("--v800", type=Path, required=True) | |
| parser.add_argument("--stage1-v801", type=Path, required=True) | |
| parser.add_argument("--robust-pattern", required=True) | |
| parser.add_argument("--output", type=Path, required=True) | |
| parser.add_argument("--seed", type=int, default=801208) | |
| args = parser.parse_args() | |
| bundle = np.load(args.features) | |
| legacy_bundle = np.load(args.legacy_features) | |
| names = bundle["file_names"].astype(str) | |
| y = bundle["labels"].astype(np.uint8) | |
| lookup = {name: index for index, name in enumerate(names)} | |
| with args.development.open(newline="", encoding="utf-8-sig") as handle: | |
| development_rows = list(csv.DictReader(handle)) | |
| with args.historical.open(newline="", encoding="utf-8-sig") as handle: | |
| historical_rows = list(csv.DictReader(handle)) | |
| development_idx = np.asarray([lookup[row["file_name"]] for row in development_rows]) | |
| historical_idx = np.asarray([lookup[row["file_name"]] for row in historical_rows]) | |
| origins = np.full(len(names), "excluded", dtype="U32") | |
| for row in development_rows: | |
| origins[lookup[row["file_name"]]] = row["origin"] | |
| anchor_idx = development_idx[origins[development_idx] == "old440"] | |
| target_idx = development_idx[origins[development_idx] != "old440"] | |
| folds = list( | |
| MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=args.seed).split( | |
| np.zeros(len(target_idx)), y[target_idx] | |
| ) | |
| ) | |
| legacy_x = make_variant(bundle, "legacy") | |
| patch_x = make_variant(bundle, "meanmax_clspatchmax") | |
| spec = Spec("legacy", 0.003, class_weight=None) | |
| legacy_oof = oof_probabilities( | |
| legacy_x, y, origins, anchor_idx, target_idx, folds, spec | |
| ) | |
| neighbour_oof = knn_oof( | |
| patch_x, y, anchor_idx, target_idx, folds, k=11, temperature=0.1 | |
| ) | |
| mixed_oof = 0.8 * legacy_oof + 0.2 * neighbour_oof | |
| thresholds = choose_thresholds(y[target_idx], mixed_oof) | |
| labelsets = all_labelsets(y.shape[1]) | |
| set_ids = labelset_ids(y[target_idx], labelsets) | |
| counts = np.bincount(set_ids, minlength=len(labelsets)).astype(np.float64) | |
| labelset_log_prior = np.log((counts + 1.0) / (len(target_idx) + len(labelsets))) | |
| cardinality_model = LogisticRegression( | |
| C=1.0, max_iter=2000, class_weight="balanced", random_state=801 | |
| ).fit(meta_features(mixed_oof), y[target_idx].sum(axis=1)) | |
| probe = fit_binary( | |
| legacy_x[development_idx], y[development_idx], spec, | |
| np.ones(len(development_idx), dtype=np.float64), | |
| ) | |
| legacy_probability = binary_probabilities(probe, legacy_x[historical_idx]) | |
| neighbour_probability = knn_probabilities( | |
| patch_x[historical_idx], patch_x[development_idx], y[development_idx] | |
| ) | |
| mixed_probability = 0.8 * legacy_probability + 0.2 * neighbour_probability | |
| targets = y[historical_idx] | |
| decoder_candidates = { | |
| "q0-prior0.5-card0.25": (0.0, 0.5, 0.25), | |
| "q0.25-prior0.5-card0.5": (0.25, 0.5, 0.5), | |
| "q0.25-prior0.75-card1": (0.25, 0.75, 1.0), | |
| "q0.5-prior0.75-card1": (0.5, 0.75, 1.0), | |
| "q0.5-prior0.75-card1.5": (0.5, 0.75, 1.5), | |
| } | |
| robust_paths = sorted(Path().glob(args.robust_pattern)) | |
| robust = {} | |
| for name in decoder_candidates: | |
| values = [] | |
| recent_values = [] | |
| for path in robust_paths: | |
| report = json.loads(path.read_text()) | |
| metrics = report["fixed_decoders"][name]["metrics"] | |
| values.append(metrics["exact_match_accuracy"]) | |
| recent_values.append(metrics["incremental100_exact_match_accuracy"]) | |
| robust[name] = { | |
| "seeds": len(values), | |
| "mean_exact_match_accuracy": float(np.mean(values)), | |
| "min_exact_match_accuracy": float(np.min(values)), | |
| "mean_incremental100_exact_match_accuracy": float(np.mean(recent_values)), | |
| } | |
| candidates = {} | |
| for name, (q, prior, cardinality) in decoder_candidates.items(): | |
| artifact = { | |
| "thresholds": thresholds, | |
| "decoder": { | |
| "threshold_bias_scale": q, | |
| "labelset_prior_weight": prior, | |
| "cardinality_weight": cardinality, | |
| "prior_smoothing": 1.0, | |
| }, | |
| "labelsets": labelsets, | |
| "labelset_log_prior": labelset_log_prior, | |
| "cardinality_model": cardinality_model, | |
| } | |
| predictions = structured_predictions(artifact, mixed_probability) | |
| candidates[name] = { | |
| "decoder": artifact["decoder"], | |
| "robust_development": robust[name], | |
| "historical_metrics": report_metrics(targets, mixed_probability, predictions), | |
| } | |
| threshold_prediction = threshold_predictions(mixed_probability, thresholds) | |
| v800 = joblib.load(args.v800) | |
| v800_probability = legacy_predict(v800, legacy_bundle["features"][historical_idx]) | |
| v800_prediction = apply_thresholds(v800_probability, np.asarray(v800["thresholds"])) | |
| stage1 = joblib.load(args.stage1_v801) | |
| stage1_probability = artifact_probabilities( | |
| stage1, bundle["view_descriptors"][historical_idx], bundle["features"][historical_idx] | |
| ) | |
| stage1_prediction = structured_predictions(stage1, stage1_probability) | |
| output = { | |
| "protocol": "historical32 is a non-independent reference; new_holdout32 is not read", | |
| "robust_files": [str(path) for path in robust_paths], | |
| "baselines": { | |
| "v800": report_metrics(targets, v800_probability, v800_prediction), | |
| "stage1_v801": report_metrics(targets, stage1_probability, stage1_prediction), | |
| }, | |
| "local_candidates": candidates, | |
| "local_crossfit_threshold_historical": report_metrics( | |
| targets, mixed_probability, threshold_prediction | |
| ), | |
| } | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| args.output.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8") | |
| print(json.dumps(output, indent=2), flush=True) | |
| if __name__ == "__main__": | |
| main() | |