9mark9's picture
Add DeBERTa-v3-large financial sentiment model, card, training and scoring scripts
5e7bf77 verified
Raw History Blame Contribute Delete
7.51 kB
"""Shared data loading and evaluation for old-vs-new comparison (same code for both models)."""
import csv, zipfile, json, sys, re
import numpy as np, torch
import pyarrow.parquet as pq
from huggingface_hub import hf_hub_download
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, f1_score, precision_recall_fscore_support
LABELS = ["negative", "neutral", "positive"]
L2I = {l: i for i, l in enumerate(LABELS)}
TW_REV = "ccbe24de388e287beb92dd393a335c376b350ac3"
JB_REV = "547fa98342d2c80734a1c841dbf3de2cfcaaab95"
TW_MAP = {"0": "negative", "1": "positive", "2": "neutral"} # Bearish, Bullish, Neutral
JB_MAP = {0: "negative", 1: "neutral", 2: "positive"} # ClassLabel names in dataset_infos.json
PB_REV = "8d3fe0c36d5feec6b3cc5e455b0fcb4820fb9964"
SF_REV = "d6294b768091ae7198051294c7fdeee2fbfbd224"
NS_REV = "9bd8206a729ccd64fa259f0368f0b4025bb472af"
def norm(t):
return " ".join(t.lower().split())
def dnorm(t):
"""Dedupe key: lower-case, URLs removed, punctuation removed, whitespace collapsed."""
t = re.sub(r"(https?://\S+|www\.\S+)", " ", t.lower())
return " ".join(re.sub(r"[^\w\s]|_", " ", t).split())
def carve(X, y):
"""90/10 stratified train/validation slice of a training file (random_state=42)."""
return train_test_split(X, y, test_size=0.10, stratify=y, random_state=42)
def phrasebank(agree="AllAgree"): # AllAgree, 75Agree, 66Agree or 50Agree
z = zipfile.ZipFile(hf_hub_download("takala/financial_phrasebank", "data/FinancialPhraseBank-v1.0.zip", repo_type="dataset", revision=PB_REV))
name = [x for x in z.namelist() if x.endswith(f"Sentences_{agree}.txt") and not x.startswith("__MACOSX")][0]
rows = [l.rsplit("@", 1) for l in z.read(name).decode("latin-1").splitlines() if "@" in l]
return [t.strip() for t, _ in rows], [s.strip() for _, s in rows]
def phrasebank_split():
"""Stratified 70/15/15, random_state=42: test first, then validation from the rest."""
X, y = phrasebank()
Xtv, Xte, ytv, yte = train_test_split(X, y, test_size=0.15, stratify=y, random_state=42)
Xtr, Xva, ytr, yva = train_test_split(Xtv, ytv, test_size=0.15 / 0.85, stratify=ytv, random_state=42)
return (Xtr, ytr), (Xva, yva), (Xte, yte)
def twitter(split): # "train" or "valid"
p = hf_hub_download("zeroshot/twitter-financial-news-sentiment", f"sent_{split}.csv", repo_type="dataset", revision=TW_REV)
rows = list(csv.DictReader(open(p, newline="", encoding="utf-8")))
return [r["text"] for r in rows], [TW_MAP[r["label"]] for r in rows]
def jb(split): # "train" or "test"; text = headline (title)
f = {"train": "data/train-00000-of-00001-f6c9e78174b9882e.parquet", "test": "data/test-00000-of-00001-f8c02546a74f96bb.parquet"}[split]
rows = pq.read_table(hf_hub_download("Jean-Baptiste/financial_news_sentiment", f, repo_type="dataset", revision=JB_REV)).to_pylist()
return [r["title"] for r in rows], [JB_MAP[int(r["labels"])] for r in rows]
def phrasebank_extra(agree):
"""Sentences in a lower-agreement PhraseBank file that are not in AllAgree at all (so none of the
AllAgree test or validation sentences can leak in); label = the file's majority label."""
allagree = {dnorm(t) for t in phrasebank()[0]}
X, y = phrasebank(agree)
keep = [(t, l) for t, l in zip(X, y) if dnorm(t) not in allagree]
return [t for t, _ in keep], [l for _, l in keep]
def sentfin(split="test"):
"""SEntFiN headlines (tasksource/sentfin repackaging, 80/10/10 split grouped by headline). The source
labels each entity in a headline; a headline is kept only when every entity has the same label."""
rows = pq.read_table(hf_hub_download("tasksource/sentfin", f"data/{split}-00000-of-00001.parquet", repo_type="dataset", revision=SF_REV)).to_pylist()
by = {}
for r in rows:
hid = json.loads(r["metadata"])["headline_id"]
by.setdefault(hid, (r["sentence1"], set()))[1].add(r["labels"])
keep = [(t, next(iter(ls))) for t, ls in by.values() if len(ls) == 1]
return [t for t, _ in keep], [l for _, l in keep]
def nosible(n, max_words=60, seed=42):
"""n random short snippets (<= max_words words) from NOSIBLE/financial-sentiment; its labels come from an
automated multi-annotator pipeline (see that dataset card), not from human annotators."""
import random
t = pq.read_table(hf_hub_download("NOSIBLE/financial-sentiment", "data.parquet", repo_type="dataset", revision=NS_REV), columns=["text", "label"]).to_pylist()
t = [r for r in t if len(r["text"].split()) <= max_words and r["label"] in L2I]
t = random.Random(seed).sample(t, min(n, len(t)))
return [r["text"] for r in t], [r["label"] for r in t]
def device():
import os
return torch.device("mps" if torch.backends.mps.is_available() and not os.environ.get("FORCE_CPU") else "cpu")
@torch.no_grad()
def predict(model, tok, texts, max_len=96, bs=64, dev=None):
dev = dev or next(model.parameters()).device
model.eval()
id2l = {int(k): v.lower() for k, v in model.config.id2label.items()}
out = []
for i in range(0, len(texts), bs):
# fixed padding buckets: each new batch shape makes the MPS allocator keep another set of buffers
enc = tok(texts[i:i + bs], padding=True, truncation=True, max_length=max_len, pad_to_multiple_of=32, return_tensors="pt").to(dev)
out += [id2l[int(j)] for j in model(**enc).logits.argmax(-1).cpu()]
if dev.type == "mps":
torch.mps.empty_cache()
return out
def report(gold, pred):
p, r, f, s = precision_recall_fscore_support(gold, pred, labels=LABELS, zero_division=0)
return {"n": len(gold), "accuracy": accuracy_score(gold, pred), "macro_f1": f1_score(gold, pred, labels=LABELS, average="macro"),
"per_class": {l: {"precision": float(p[i]), "recall": float(r[i]), "f1": float(f[i]), "support": int(s[i])} for i, l in enumerate(LABELS)}}
def heldout_sets():
_, _, (Xte, yte) = phrasebank_split()
return {"phrasebank_test": (Xte, yte), "twitter_valid": twitter("valid"), "jb_test": jb("test"), "sentfin_test": sentfin("test")}
def bootstrap(gold, pred, n=2000, seed=0):
"""95% percentile intervals for macro-F1 and negative recall, resampling rows with replacement."""
from sklearn.metrics import recall_score
rng = np.random.default_rng(seed)
g, p = np.array(gold), np.array(pred)
mf, nr = [], []
for _ in range(n):
i = rng.integers(0, len(g), len(g))
mf.append(f1_score(g[i], p[i], labels=LABELS, average="macro", zero_division=0))
nr.append(recall_score(g[i], p[i], labels=["negative"], average="macro", zero_division=0))
q = lambda a: [float(np.percentile(a, 2.5)), float(np.percentile(a, 97.5))]
return {"macro_f1_ci": q(mf), "neg_recall_ci": q(nr)}
def evaluate(model, tok):
out = {}
for k, (X, y) in heldout_sets().items():
pred = predict(model, tok, X)
out[k] = {**report(y, pred), **bootstrap(y, pred), "pred": pred}
return out
def fmt(res):
lines = []
for k, v in res.items():
pc = " ".join(f"{l[:3]} P{c['precision']:.3f} R{c['recall']:.3f} (n={c['support']})" for l, c in v["per_class"].items())
ci = f" [{v['macro_f1_ci'][0]:.3f},{v['macro_f1_ci'][1]:.3f}]" if "macro_f1_ci" in v else ""
lines.append(f"{k:16s} n={v['n']:5d} acc={v['accuracy']:.4f} mF1={v['macro_f1']:.4f}{ci} negF1={v['per_class']['negative']['f1']:.3f} | {pc}")
return "\n".join(lines)