researchbee / train_asjc.py
nikeshn's picture
Upload 47 files
1916d8e verified
Raw History Blame Contribute Delete
15.5 kB
"""
Train the abstract -> ASJC multi-label classifier.
# interim model, no network needed β€” trains on journal titles in the index
python train_asjc.py --corpus titles --db journals.db --out asjc_model.joblib
# full model, after harvest_openalex.py has produced labelled abstracts
python train_asjc.py --corpus abstracts --train-jsonl asjc_train.jsonl \
--db journals.db --out asjc_model.joblib
Why TF-IDF + linear and not a transformer
-----------------------------------------
The task is topical multi-label text classification over 334 labels with a large
vocabulary signal β€” exactly where a linear model over character and word n-grams
is hard to beat per unit of cost. It trains in seconds on CPU, the artifact is
tens of MB, and inference is sub-millisecond with no GPU and no model download at
Space startup. A fine-tuned transformer might add a few points of F1 for ~400MB
and a cold-start penalty on every Space restart. Take the linear model; spend the
complexity budget on the LLM fallback for the cases it actually gets wrong.
Two corpora, deliberately
-------------------------
`titles` trains on journal titles (48,888 labelled examples, already in the
index, zero network). Ships immediately. But it trains on titles and
is applied to abstracts, and that domain gap is real β€” treat its
numbers as an upper bound on what it will do in production.
`abstracts` trains on article abstracts labelled with their journal's ASJC codes
(weak supervision). Far better matched to inference, and the only
version whose held-out score means what it appears to mean.
Label noise in the `abstracts` corpus
-------------------------------------
Giving every article its journal's codes is weak supervision: a physics paper in
a multidisciplinary journal inherits 'Medicine' too. `--max-labels` drops journals
carrying more than N ASJC codes from training, which removes the worst offenders
(megajournals and general-science titles) at the cost of some coverage. Their
articles are still *classifiable* at inference β€” they are just poor teachers.
"""
from __future__ import annotations
import argparse
import json
import re
import sqlite3
import sys
from pathlib import Path
import numpy as np
from scipy import sparse
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import train_test_split
from sklearn.multiclass import OneVsRestClassifier
from sklearn.preprocessing import MultiLabelBinarizer
import joblib
# ---------------------------------------------------------------------------
# Corpora
# ---------------------------------------------------------------------------
def load_titles(db: str, max_labels: int) -> tuple[list[str], list[list[str]]]:
"""Journal title + subject names -> its ASJC codes."""
con = sqlite3.connect(db)
con.row_factory = sqlite3.Row
rows = con.execute(
"""SELECT j.scopus_id, j.title, j.publisher_group,
group_concat(ja.asjc) codes, count(*) n
FROM journals j JOIN journal_asjc ja ON ja.scopus_id = j.scopus_id
WHERE j.title IS NOT NULL
GROUP BY j.scopus_id"""
).fetchall()
X, y = [], []
for r in rows:
if r["n"] > max_labels:
continue
X.append(r["title"])
y.append(r["codes"].split(","))
con.close()
return X, y
def load_abstracts(path: str, db: str, max_labels: int) -> tuple[list[str], list[list[str]]]:
"""
JSONL from harvest_openalex.py: {"text": ..., "scopus_id": ...}.
Labels come from the index, so the harvest never has to carry them.
"""
con = sqlite3.connect(db)
codes: dict[str, list[str]] = {}
for sid, c in con.execute(
"SELECT scopus_id, group_concat(asjc) FROM journal_asjc GROUP BY scopus_id"
):
codes[sid] = c.split(",")
con.close()
X, y = [], []
with open(path, encoding="utf-8") as f:
for line in f:
rec = json.loads(line)
c = codes.get(rec.get("scopus_id"))
if not c or len(c) > max_labels:
continue
t = (rec.get("text") or "").strip()
if len(t) < 120: # a bare title is not an abstract
continue
X.append(t)
y.append(c)
return X, y
# ---------------------------------------------------------------------------
# Evaluation
# ---------------------------------------------------------------------------
def evaluate(clf, Xv, Y_true, classes: np.ndarray, parent_of: dict[str, str]) -> dict:
"""
Ranking metrics, not threshold metrics.
The classifier feeds a *filter*, and the filter takes the top few codes β€” so
what matters is whether a correct code is ranked highly, not whether some
probability cleared 0.5. Reporting only micro-F1 at a fixed threshold would
flatter a model that is well-calibrated but badly ordered, and vice versa.
Parent accuracy (2-digit ASJC) is reported alongside because confusing
'Information Systems' with 'Computer Science Applications' barely harms the
candidate set, while confusing Computer Science with Dentistry destroys it.
"""
P = clf.predict_proba(Xv)
order = np.argsort(-P, axis=1)
res: dict[str, float] = {}
for k in (1, 3, 5):
topk = order[:, :k]
hit = sum(
bool(set(classes[topk[i]]) & set(Y_true[i])) for i in range(len(Y_true))
)
res[f"top{k}_hit"] = hit / len(Y_true)
# Recall of the full true label set within the top 5 predictions.
top5 = order[:, :5]
res["top5_label_recall"] = float(np.mean([
len(set(classes[top5[i]]) & set(Y_true[i])) / len(Y_true[i])
for i in range(len(Y_true))
]))
top1_parent = [parent_of.get(classes[order[i, 0]]) for i in range(len(Y_true))]
res["top1_parent_hit"] = float(np.mean([
top1_parent[i] in {parent_of.get(c) for c in Y_true[i]}
for i in range(len(Y_true))
]))
return res
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--corpus", choices=["titles", "abstracts"], default="titles")
ap.add_argument("--db", default="journals.db")
ap.add_argument("--train-jsonl", default="asjc_train.jsonl")
ap.add_argument("--out", default="asjc_model.joblib")
ap.add_argument("--max-labels", type=int, default=6,
help="drop journals carrying more ASJC codes than this from training")
ap.add_argument("--min-per-label", type=int, default=8,
help="drop labels with fewer training examples than this")
ap.add_argument("--eval-jsonl", default="asjc_eval.jsonl")
ap.add_argument("--max-features", type=int, default=120_000)
ap.add_argument("--n-jobs", type=int, default=2,
help="parallel per-label fits. 334 estimators over a large sparse "
"matrix can exhaust memory at -1; 4 is safe on a 16GB Space.")
args = ap.parse_args()
if args.corpus == "titles":
X, y = load_titles(args.db, args.max_labels)
else:
if not Path(args.train_jsonl).exists():
sys.exit(f"{args.train_jsonl} not found β€” run harvest_openalex.py first")
X, y = load_abstracts(args.train_jsonl, args.db, args.max_labels)
print(f"corpus={args.corpus} examples={len(X)} "
f"mean labels/example={np.mean([len(c) for c in y]):.2f}")
# Drop labels too rare to learn. They stay resolvable through the LLM
# fallback, which does not need training examples.
from collections import Counter
freq = Counter(c for cs in y for c in cs)
keep = {c for c, n in freq.items() if n >= args.min_per_label}
y = [[c for c in cs if c in keep] for cs in y]
mask = [i for i, cs in enumerate(y) if cs]
X, y = [X[i] for i in mask], [y[i] for i in mask]
print(f"labels kept: {len(keep)} of {len(freq)} examples after filtering: {len(X)}")
mlb = MultiLabelBinarizer()
Y = mlb.fit_transform(y)
Xtr, Xte, Ytr, Yte, ytr_raw, yte_raw = train_test_split(
X, Y, y, test_size=0.2, random_state=42
)
# Word n-grams carry the topical signal; character n-grams cover
# morphology, non-English titles and the domain shift between titles and
# abstracts. Union of both, sublinear tf to damp long abstracts.
word = TfidfVectorizer(
sublinear_tf=True, strip_accents="unicode", lowercase=True,
ngram_range=(1, 2), min_df=2, max_features=args.max_features,
stop_words="english", dtype=np.float32,
)
char = TfidfVectorizer(
sublinear_tf=True, strip_accents="unicode", lowercase=True,
analyzer="char_wb", ngram_range=(3, 5), min_df=3,
max_features=args.max_features // 3, dtype=np.float32,
)
Xtr_v = sparse.hstack([word.fit_transform(Xtr), char.fit_transform(Xtr)]).tocsr()
Xte_v = sparse.hstack([word.transform(Xte), char.transform(Xte)]).tocsr()
print(f"features: {Xtr_v.shape[1]}")
# SGD with modified_huber rather than liblinear logistic regression.
# 334 one-vs-rest fits over ~450k sparse features takes liblinear many
# minutes and it trips joblib's read-only memmap under parallel fan-out;
# SGD fits the same problem in seconds, scales to the ~1M-abstract corpus
# the harvest will produce, and modified_huber still exposes predict_proba,
# which the LLM fallback needs to decide when to escalate.
# log_loss, not modified_huber. modified_huber's predict_proba clips the
# margin and normalises, so it returns a saturated 1.0/0.0 with nothing in
# between β€” fine for ranking, useless for deciding when to escalate to the
# LLM. log_loss gives genuine logistic probabilities that a threshold can
# act on.
#
# class_weight is left unbalanced on purpose. Balancing inflates scores for
# rare labels, which is right for per-label F1 and wrong here: it pushed
# 'Rehabilitation' to the top of a cardiology abstract. The output feeds a
# ranked filter, so ordering matters more than per-label parity.
clf = OneVsRestClassifier(
SGDClassifier(
loss="log_loss", alpha=1e-6, max_iter=30, tol=1e-4,
random_state=42,
),
n_jobs=args.n_jobs,
)
with joblib.parallel_config(backend="loky", max_nbytes=None):
clf.fit(Xtr_v, Ytr)
con = sqlite3.connect(args.db)
parent_of = {c: p for c, p in con.execute("SELECT code, parent_code FROM asjc")}
names = {c: (n or pn) for c, n, pn in
con.execute("SELECT code, name, parent_name FROM asjc")}
con.close()
res = evaluate(clf, Xte_v, yte_raw, mlb.classes_, parent_of)
print(f"\nheld-out performance (same distribution as training: {args.corpus})")
for k, v in res.items():
print(f" {k:<20} {v:.3f}")
# The number that actually matters. When training on `titles`, the held-out
# score above is measured on 5-word journal titles while production input is
# a 200-word abstract β€” a different and much easier problem. This second
# evaluation runs on hand-labelled abstracts across 20 fields, which is the
# distribution the classifier is really deployed against.
if Path(args.eval_jsonl).exists():
ev = [json.loads(l) for l in open(args.eval_jsonl, encoding="utf-8")]
Xe = sparse.hstack([word.transform([e["text"] for e in ev]),
char.transform([e["text"] for e in ev])]).tocsr()
Pe = clf.predict_proba(Xe)
order = np.argsort(-Pe, axis=1)
# Two standards, reported separately and deliberately.
#
# LENIENT counts a hit when *any* acceptable code appears. It overstates
# usefulness: for a seismic-engineering abstract, 'Engineering
# (miscellaneous)' is in `accept` and scores a hit, but filtering on it
# returns nonlinear-analysis maths journals, while the preferred code
# 2205 returns concrete-structures journals. Same "hit", different
# recommendation quality.
#
# STRICT requires the single best code β€” accept[0], the one a subject
# librarian would choose. This is the number to watch when retraining on
# the abstract corpus, because it is the one that tracks whether the
# recommendations are actually good.
n = len(ev)
hits1 = hits3 = parent1 = 0
s5 = s3 = 0
print(f"\nabstract eval ({n} hand-labelled abstracts)")
for i, e in enumerate(ev):
top5 = [mlb.classes_[j] for j in order[i, :5]]
top3, top1 = top5[:3], top5[0]
pref = e["accept"][0]
ok1 = top1 in e["accept"]
ok3 = bool(set(top3) & set(e["accept"]))
okp = parent_of.get(top1) in {parent_of.get(c) for c in e["accept"]}
hits1 += ok1; hits3 += ok3; parent1 += okp
s5 += pref in top5
s3 += pref in top3
flag = "OK " if pref in top5 else ("weak" if ok3 else "MISS")
print(f" {flag} {e['field'][:28]:<28} pref {pref} "
f"{'in top5' if pref in top5 else 'ABSENT':<8} -> "
f"{', '.join(f'{c}' for c in top3)}")
print(f" lenient: top1 {hits1/n:.2f} top3 {hits3/n:.2f} parent {parent1/n:.2f}")
print(f" STRICT : preferred-in-top3 {s3/n:.2f} preferred-in-top5 {s5/n:.2f}")
print(" strict preferred-in-top5 is the number that gates shipping β€” it "
"bounds what the LLM stage can possibly pick.")
res.update({"abstract_top1": hits1 / n, "abstract_top3": hits3 / n,
"abstract_top1_parent": parent1 / n,
"abstract_preferred_top3": s3 / n,
"abstract_preferred_top5": s5 / n})
# Compact the artifact before saving. The bulk is not the vocabulary β€” it is
# 334 dense float64 coefficient vectors (334 x ~160k x 8 bytes β‰ˆ 427 MB
# before compression). Casting to float32 halves that with no measurable
# effect on ranking, and sparsify() drops the near-zero majority. This
# matters operationally: the Space pulls this file on every cold start.
for est in clf.estimators_:
est.coef_ = est.coef_.astype("float32")
est.sparsify()
joblib.dump(
{"word": word, "char": char, "clf": clf, "classes": mlb.classes_,
"names": names, "parent_of": parent_of, "corpus": args.corpus,
"metrics": res},
args.out, compress=3,
)
print(f"\nwrote {args.out} ({Path(args.out).stat().st_size/1e6:.1f} MB)")
# Sparsified coefficients change the predict path, so re-check the eval
# rather than assuming the compaction was lossless.
chk = joblib.load(args.out)
Xc = sparse.hstack([chk["word"].transform([X[0]]),
chk["char"].transform([X[0]])]).tocsr()
assert chk["clf"].predict_proba(Xc).shape[1] == len(mlb.classes_)
print("compacted model reloads and predicts correctly")
if __name__ == "__main__":
main()