| |
| """Evaluate the best saved checkpoint of a run that was stopped early by the time cap. |
| Usage: eval_checkpoint.py <label> <model_dir> <epochs_run> <best_val_macro_f1> <train_seconds_wall> <max_epochs> <patience>""" |
| import json, os, sys |
| import numpy as np, pandas as pd, torch |
| from sklearn.metrics import accuracy_score, f1_score |
| from transformers import AutoTokenizer, AutoModelForSequenceClassification |
| sys.path.insert(0, os.path.dirname(__file__)); from common import * |
| label, d, ep, bv, secs, mx, pat = sys.argv[1], sys.argv[2], int(sys.argv[3]), float(sys.argv[4]), float(sys.argv[5]), int(sys.argv[6]), int(sys.argv[7]) |
| te = pd.read_csv(f"{ROOT}/data/published/test.csv", dtype=str).fillna("") |
| tok = AutoTokenizer.from_pretrained(d); mod = AutoModelForSequenceClassification.from_pretrained(d).eval() |
| dev = "mps" if torch.backends.mps.is_available() else "cpu"; mod.to(dev) |
| X = [make_input(t, a) for t, a in zip(te.title, te.agency)]; P, T = [], [] |
| for i in range(0, len(X), 64): |
| b = tok(X[i:i+64], truncation=True, max_length=128, padding=True, return_tensors="pt").to(dev) |
| with torch.no_grad(): lg = mod(**b).logits |
| P += lg.argmax(-1).tolist(); T += lg.topk(3, -1).indices.tolist() |
| id2 = mod.config.id2label; pred = [id2[i] for i in P]; true = list(te.activity_label if label == "activity" else te[label]) |
| top3 = float(np.mean([y in [id2[k] for k in t] for y, t in zip(true, T)])) |
| res = {"label": label, "base_model": json.load(open(f"{d}/config.json")).get("_name_or_path", ""), "device": dev, "accuracy": accuracy_score(true, pred), |
| "macro_f1": f1_score(true, pred, average="macro"), "top3_accuracy": top3, "best_val_macro_f1": bv, "epochs_run": ep, "max_epochs": mx, |
| "patience": pat, "train_seconds": secs, "n_classes": len(id2), "stopped_by": "time cap", |
| "size_mb": round(sum(os.path.getsize(os.path.join(d, f)) for f in os.listdir(d)) / 1e6, 1)} |
| json.dump(res, open(f"{ROOT}/outputs/encoder_{label}_metrics.json", "w"), indent=1) |
| pd.DataFrame({"ref": te.ref, "true": true, "pred": pred}).to_csv(f"{ROOT}/outputs/pred_encoder_{label}.csv", index=False); print(res) |
|
|
