"""Shared data loading and evaluation for old-vs-new comparison (same code for both models).""" import csv, zipfile, json, sys, re import numpy as np, torch import pyarrow.parquet as pq from huggingface_hub import hf_hub_download from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score, f1_score, precision_recall_fscore_support LABELS = ["negative", "neutral", "positive"] L2I = {l: i for i, l in enumerate(LABELS)} TW_REV = "ccbe24de388e287beb92dd393a335c376b350ac3" JB_REV = "547fa98342d2c80734a1c841dbf3de2cfcaaab95" TW_MAP = {"0": "negative", "1": "positive", "2": "neutral"} # Bearish, Bullish, Neutral JB_MAP = {0: "negative", 1: "neutral", 2: "positive"} # ClassLabel names in dataset_infos.json PB_REV = "8d3fe0c36d5feec6b3cc5e455b0fcb4820fb9964" SF_REV = "d6294b768091ae7198051294c7fdeee2fbfbd224" NS_REV = "9bd8206a729ccd64fa259f0368f0b4025bb472af" def norm(t): return " ".join(t.lower().split()) def dnorm(t): """Dedupe key: lower-case, URLs removed, punctuation removed, whitespace collapsed.""" t = re.sub(r"(https?://\S+|www\.\S+)", " ", t.lower()) return " ".join(re.sub(r"[^\w\s]|_", " ", t).split()) def carve(X, y): """90/10 stratified train/validation slice of a training file (random_state=42).""" return train_test_split(X, y, test_size=0.10, stratify=y, random_state=42) def phrasebank(agree="AllAgree"): # AllAgree, 75Agree, 66Agree or 50Agree z = zipfile.ZipFile(hf_hub_download("takala/financial_phrasebank", "data/FinancialPhraseBank-v1.0.zip", repo_type="dataset", revision=PB_REV)) name = [x for x in z.namelist() if x.endswith(f"Sentences_{agree}.txt") and not x.startswith("__MACOSX")][0] rows = [l.rsplit("@", 1) for l in z.read(name).decode("latin-1").splitlines() if "@" in l] return [t.strip() for t, _ in rows], [s.strip() for _, s in rows] def phrasebank_split(): """Stratified 70/15/15, random_state=42: test first, then validation from the rest.""" X, y = phrasebank() Xtv, Xte, ytv, yte = train_test_split(X, y, test_size=0.15, stratify=y, random_state=42) Xtr, Xva, ytr, yva = train_test_split(Xtv, ytv, test_size=0.15 / 0.85, stratify=ytv, random_state=42) return (Xtr, ytr), (Xva, yva), (Xte, yte) def twitter(split): # "train" or "valid" p = hf_hub_download("zeroshot/twitter-financial-news-sentiment", f"sent_{split}.csv", repo_type="dataset", revision=TW_REV) rows = list(csv.DictReader(open(p, newline="", encoding="utf-8"))) return [r["text"] for r in rows], [TW_MAP[r["label"]] for r in rows] def jb(split): # "train" or "test"; text = headline (title) f = {"train": "data/train-00000-of-00001-f6c9e78174b9882e.parquet", "test": "data/test-00000-of-00001-f8c02546a74f96bb.parquet"}[split] rows = pq.read_table(hf_hub_download("Jean-Baptiste/financial_news_sentiment", f, repo_type="dataset", revision=JB_REV)).to_pylist() return [r["title"] for r in rows], [JB_MAP[int(r["labels"])] for r in rows] def phrasebank_extra(agree): """Sentences in a lower-agreement PhraseBank file that are not in AllAgree at all (so none of the AllAgree test or validation sentences can leak in); label = the file's majority label.""" allagree = {dnorm(t) for t in phrasebank()[0]} X, y = phrasebank(agree) keep = [(t, l) for t, l in zip(X, y) if dnorm(t) not in allagree] return [t for t, _ in keep], [l for _, l in keep] def sentfin(split="test"): """SEntFiN headlines (tasksource/sentfin repackaging, 80/10/10 split grouped by headline). The source labels each entity in a headline; a headline is kept only when every entity has the same label.""" rows = pq.read_table(hf_hub_download("tasksource/sentfin", f"data/{split}-00000-of-00001.parquet", repo_type="dataset", revision=SF_REV)).to_pylist() by = {} for r in rows: hid = json.loads(r["metadata"])["headline_id"] by.setdefault(hid, (r["sentence1"], set()))[1].add(r["labels"]) keep = [(t, next(iter(ls))) for t, ls in by.values() if len(ls) == 1] return [t for t, _ in keep], [l for _, l in keep] def nosible(n, max_words=60, seed=42): """n random short snippets (<= max_words words) from NOSIBLE/financial-sentiment; its labels come from an automated multi-annotator pipeline (see that dataset card), not from human annotators.""" import random t = pq.read_table(hf_hub_download("NOSIBLE/financial-sentiment", "data.parquet", repo_type="dataset", revision=NS_REV), columns=["text", "label"]).to_pylist() t = [r for r in t if len(r["text"].split()) <= max_words and r["label"] in L2I] t = random.Random(seed).sample(t, min(n, len(t))) return [r["text"] for r in t], [r["label"] for r in t] def device(): import os return torch.device("mps" if torch.backends.mps.is_available() and not os.environ.get("FORCE_CPU") else "cpu") @torch.no_grad() def predict(model, tok, texts, max_len=96, bs=64, dev=None): dev = dev or next(model.parameters()).device model.eval() id2l = {int(k): v.lower() for k, v in model.config.id2label.items()} out = [] for i in range(0, len(texts), bs): # fixed padding buckets: each new batch shape makes the MPS allocator keep another set of buffers enc = tok(texts[i:i + bs], padding=True, truncation=True, max_length=max_len, pad_to_multiple_of=32, return_tensors="pt").to(dev) out += [id2l[int(j)] for j in model(**enc).logits.argmax(-1).cpu()] if dev.type == "mps": torch.mps.empty_cache() return out def report(gold, pred): p, r, f, s = precision_recall_fscore_support(gold, pred, labels=LABELS, zero_division=0) return {"n": len(gold), "accuracy": accuracy_score(gold, pred), "macro_f1": f1_score(gold, pred, labels=LABELS, average="macro"), "per_class": {l: {"precision": float(p[i]), "recall": float(r[i]), "f1": float(f[i]), "support": int(s[i])} for i, l in enumerate(LABELS)}} def heldout_sets(): _, _, (Xte, yte) = phrasebank_split() return {"phrasebank_test": (Xte, yte), "twitter_valid": twitter("valid"), "jb_test": jb("test"), "sentfin_test": sentfin("test")} def bootstrap(gold, pred, n=2000, seed=0): """95% percentile intervals for macro-F1 and negative recall, resampling rows with replacement.""" from sklearn.metrics import recall_score rng = np.random.default_rng(seed) g, p = np.array(gold), np.array(pred) mf, nr = [], [] for _ in range(n): i = rng.integers(0, len(g), len(g)) mf.append(f1_score(g[i], p[i], labels=LABELS, average="macro", zero_division=0)) nr.append(recall_score(g[i], p[i], labels=["negative"], average="macro", zero_division=0)) q = lambda a: [float(np.percentile(a, 2.5)), float(np.percentile(a, 97.5))] return {"macro_f1_ci": q(mf), "neg_recall_ci": q(nr)} def evaluate(model, tok): out = {} for k, (X, y) in heldout_sets().items(): pred = predict(model, tok, X) out[k] = {**report(y, pred), **bootstrap(y, pred), "pred": pred} return out def fmt(res): lines = [] for k, v in res.items(): pc = " ".join(f"{l[:3]} P{c['precision']:.3f} R{c['recall']:.3f} (n={c['support']})" for l, c in v["per_class"].items()) ci = f" [{v['macro_f1_ci'][0]:.3f},{v['macro_f1_ci'][1]:.3f}]" if "macro_f1_ci" in v else "" lines.append(f"{k:16s} n={v['n']:5d} acc={v['accuracy']:.4f} mF1={v['macro_f1']:.4f}{ci} negF1={v['per_class']['negative']['f1']:.3f} | {pc}") return "\n".join(lines)