Spaces:
Running
Running
Download train_asjc.py from nikeshn/researchbee: direct link, hf CLI and curl.
- Browser
- Download file 15.5 kB
-
https://huggingface.co/spaces/nikeshn/researchbee/resolve/main/train_asjc.py
- Command line
-
hf download hf://spaces/nikeshn/researchbee/train_asjc.py
-
curl -L -o train_asjc.py https://huggingface.co/spaces/nikeshn/researchbee/resolve/main/train_asjc.py
15.5 kB
| """ | |
| Train the abstract -> ASJC multi-label classifier. | |
| # interim model, no network needed β trains on journal titles in the index | |
| python train_asjc.py --corpus titles --db journals.db --out asjc_model.joblib | |
| # full model, after harvest_openalex.py has produced labelled abstracts | |
| python train_asjc.py --corpus abstracts --train-jsonl asjc_train.jsonl \ | |
| --db journals.db --out asjc_model.joblib | |
| Why TF-IDF + linear and not a transformer | |
| ----------------------------------------- | |
| The task is topical multi-label text classification over 334 labels with a large | |
| vocabulary signal β exactly where a linear model over character and word n-grams | |
| is hard to beat per unit of cost. It trains in seconds on CPU, the artifact is | |
| tens of MB, and inference is sub-millisecond with no GPU and no model download at | |
| Space startup. A fine-tuned transformer might add a few points of F1 for ~400MB | |
| and a cold-start penalty on every Space restart. Take the linear model; spend the | |
| complexity budget on the LLM fallback for the cases it actually gets wrong. | |
| Two corpora, deliberately | |
| ------------------------- | |
| `titles` trains on journal titles (48,888 labelled examples, already in the | |
| index, zero network). Ships immediately. But it trains on titles and | |
| is applied to abstracts, and that domain gap is real β treat its | |
| numbers as an upper bound on what it will do in production. | |
| `abstracts` trains on article abstracts labelled with their journal's ASJC codes | |
| (weak supervision). Far better matched to inference, and the only | |
| version whose held-out score means what it appears to mean. | |
| Label noise in the `abstracts` corpus | |
| ------------------------------------- | |
| Giving every article its journal's codes is weak supervision: a physics paper in | |
| a multidisciplinary journal inherits 'Medicine' too. `--max-labels` drops journals | |
| carrying more than N ASJC codes from training, which removes the worst offenders | |
| (megajournals and general-science titles) at the cost of some coverage. Their | |
| articles are still *classifiable* at inference β they are just poor teachers. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import re | |
| import sqlite3 | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| from scipy import sparse | |
| from sklearn.feature_extraction.text import TfidfVectorizer | |
| from sklearn.linear_model import SGDClassifier | |
| from sklearn.model_selection import train_test_split | |
| from sklearn.multiclass import OneVsRestClassifier | |
| from sklearn.preprocessing import MultiLabelBinarizer | |
| import joblib | |
| # --------------------------------------------------------------------------- | |
| # Corpora | |
| # --------------------------------------------------------------------------- | |
| def load_titles(db: str, max_labels: int) -> tuple[list[str], list[list[str]]]: | |
| """Journal title + subject names -> its ASJC codes.""" | |
| con = sqlite3.connect(db) | |
| con.row_factory = sqlite3.Row | |
| rows = con.execute( | |
| """SELECT j.scopus_id, j.title, j.publisher_group, | |
| group_concat(ja.asjc) codes, count(*) n | |
| FROM journals j JOIN journal_asjc ja ON ja.scopus_id = j.scopus_id | |
| WHERE j.title IS NOT NULL | |
| GROUP BY j.scopus_id""" | |
| ).fetchall() | |
| X, y = [], [] | |
| for r in rows: | |
| if r["n"] > max_labels: | |
| continue | |
| X.append(r["title"]) | |
| y.append(r["codes"].split(",")) | |
| con.close() | |
| return X, y | |
| def load_abstracts(path: str, db: str, max_labels: int) -> tuple[list[str], list[list[str]]]: | |
| """ | |
| JSONL from harvest_openalex.py: {"text": ..., "scopus_id": ...}. | |
| Labels come from the index, so the harvest never has to carry them. | |
| """ | |
| con = sqlite3.connect(db) | |
| codes: dict[str, list[str]] = {} | |
| for sid, c in con.execute( | |
| "SELECT scopus_id, group_concat(asjc) FROM journal_asjc GROUP BY scopus_id" | |
| ): | |
| codes[sid] = c.split(",") | |
| con.close() | |
| X, y = [], [] | |
| with open(path, encoding="utf-8") as f: | |
| for line in f: | |
| rec = json.loads(line) | |
| c = codes.get(rec.get("scopus_id")) | |
| if not c or len(c) > max_labels: | |
| continue | |
| t = (rec.get("text") or "").strip() | |
| if len(t) < 120: # a bare title is not an abstract | |
| continue | |
| X.append(t) | |
| y.append(c) | |
| return X, y | |
| # --------------------------------------------------------------------------- | |
| # Evaluation | |
| # --------------------------------------------------------------------------- | |
| def evaluate(clf, Xv, Y_true, classes: np.ndarray, parent_of: dict[str, str]) -> dict: | |
| """ | |
| Ranking metrics, not threshold metrics. | |
| The classifier feeds a *filter*, and the filter takes the top few codes β so | |
| what matters is whether a correct code is ranked highly, not whether some | |
| probability cleared 0.5. Reporting only micro-F1 at a fixed threshold would | |
| flatter a model that is well-calibrated but badly ordered, and vice versa. | |
| Parent accuracy (2-digit ASJC) is reported alongside because confusing | |
| 'Information Systems' with 'Computer Science Applications' barely harms the | |
| candidate set, while confusing Computer Science with Dentistry destroys it. | |
| """ | |
| P = clf.predict_proba(Xv) | |
| order = np.argsort(-P, axis=1) | |
| res: dict[str, float] = {} | |
| for k in (1, 3, 5): | |
| topk = order[:, :k] | |
| hit = sum( | |
| bool(set(classes[topk[i]]) & set(Y_true[i])) for i in range(len(Y_true)) | |
| ) | |
| res[f"top{k}_hit"] = hit / len(Y_true) | |
| # Recall of the full true label set within the top 5 predictions. | |
| top5 = order[:, :5] | |
| res["top5_label_recall"] = float(np.mean([ | |
| len(set(classes[top5[i]]) & set(Y_true[i])) / len(Y_true[i]) | |
| for i in range(len(Y_true)) | |
| ])) | |
| top1_parent = [parent_of.get(classes[order[i, 0]]) for i in range(len(Y_true))] | |
| res["top1_parent_hit"] = float(np.mean([ | |
| top1_parent[i] in {parent_of.get(c) for c in Y_true[i]} | |
| for i in range(len(Y_true)) | |
| ])) | |
| return res | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--corpus", choices=["titles", "abstracts"], default="titles") | |
| ap.add_argument("--db", default="journals.db") | |
| ap.add_argument("--train-jsonl", default="asjc_train.jsonl") | |
| ap.add_argument("--out", default="asjc_model.joblib") | |
| ap.add_argument("--max-labels", type=int, default=6, | |
| help="drop journals carrying more ASJC codes than this from training") | |
| ap.add_argument("--min-per-label", type=int, default=8, | |
| help="drop labels with fewer training examples than this") | |
| ap.add_argument("--eval-jsonl", default="asjc_eval.jsonl") | |
| ap.add_argument("--max-features", type=int, default=120_000) | |
| ap.add_argument("--n-jobs", type=int, default=2, | |
| help="parallel per-label fits. 334 estimators over a large sparse " | |
| "matrix can exhaust memory at -1; 4 is safe on a 16GB Space.") | |
| args = ap.parse_args() | |
| if args.corpus == "titles": | |
| X, y = load_titles(args.db, args.max_labels) | |
| else: | |
| if not Path(args.train_jsonl).exists(): | |
| sys.exit(f"{args.train_jsonl} not found β run harvest_openalex.py first") | |
| X, y = load_abstracts(args.train_jsonl, args.db, args.max_labels) | |
| print(f"corpus={args.corpus} examples={len(X)} " | |
| f"mean labels/example={np.mean([len(c) for c in y]):.2f}") | |
| # Drop labels too rare to learn. They stay resolvable through the LLM | |
| # fallback, which does not need training examples. | |
| from collections import Counter | |
| freq = Counter(c for cs in y for c in cs) | |
| keep = {c for c, n in freq.items() if n >= args.min_per_label} | |
| y = [[c for c in cs if c in keep] for cs in y] | |
| mask = [i for i, cs in enumerate(y) if cs] | |
| X, y = [X[i] for i in mask], [y[i] for i in mask] | |
| print(f"labels kept: {len(keep)} of {len(freq)} examples after filtering: {len(X)}") | |
| mlb = MultiLabelBinarizer() | |
| Y = mlb.fit_transform(y) | |
| Xtr, Xte, Ytr, Yte, ytr_raw, yte_raw = train_test_split( | |
| X, Y, y, test_size=0.2, random_state=42 | |
| ) | |
| # Word n-grams carry the topical signal; character n-grams cover | |
| # morphology, non-English titles and the domain shift between titles and | |
| # abstracts. Union of both, sublinear tf to damp long abstracts. | |
| word = TfidfVectorizer( | |
| sublinear_tf=True, strip_accents="unicode", lowercase=True, | |
| ngram_range=(1, 2), min_df=2, max_features=args.max_features, | |
| stop_words="english", dtype=np.float32, | |
| ) | |
| char = TfidfVectorizer( | |
| sublinear_tf=True, strip_accents="unicode", lowercase=True, | |
| analyzer="char_wb", ngram_range=(3, 5), min_df=3, | |
| max_features=args.max_features // 3, dtype=np.float32, | |
| ) | |
| Xtr_v = sparse.hstack([word.fit_transform(Xtr), char.fit_transform(Xtr)]).tocsr() | |
| Xte_v = sparse.hstack([word.transform(Xte), char.transform(Xte)]).tocsr() | |
| print(f"features: {Xtr_v.shape[1]}") | |
| # SGD with modified_huber rather than liblinear logistic regression. | |
| # 334 one-vs-rest fits over ~450k sparse features takes liblinear many | |
| # minutes and it trips joblib's read-only memmap under parallel fan-out; | |
| # SGD fits the same problem in seconds, scales to the ~1M-abstract corpus | |
| # the harvest will produce, and modified_huber still exposes predict_proba, | |
| # which the LLM fallback needs to decide when to escalate. | |
| # log_loss, not modified_huber. modified_huber's predict_proba clips the | |
| # margin and normalises, so it returns a saturated 1.0/0.0 with nothing in | |
| # between β fine for ranking, useless for deciding when to escalate to the | |
| # LLM. log_loss gives genuine logistic probabilities that a threshold can | |
| # act on. | |
| # | |
| # class_weight is left unbalanced on purpose. Balancing inflates scores for | |
| # rare labels, which is right for per-label F1 and wrong here: it pushed | |
| # 'Rehabilitation' to the top of a cardiology abstract. The output feeds a | |
| # ranked filter, so ordering matters more than per-label parity. | |
| clf = OneVsRestClassifier( | |
| SGDClassifier( | |
| loss="log_loss", alpha=1e-6, max_iter=30, tol=1e-4, | |
| random_state=42, | |
| ), | |
| n_jobs=args.n_jobs, | |
| ) | |
| with joblib.parallel_config(backend="loky", max_nbytes=None): | |
| clf.fit(Xtr_v, Ytr) | |
| con = sqlite3.connect(args.db) | |
| parent_of = {c: p for c, p in con.execute("SELECT code, parent_code FROM asjc")} | |
| names = {c: (n or pn) for c, n, pn in | |
| con.execute("SELECT code, name, parent_name FROM asjc")} | |
| con.close() | |
| res = evaluate(clf, Xte_v, yte_raw, mlb.classes_, parent_of) | |
| print(f"\nheld-out performance (same distribution as training: {args.corpus})") | |
| for k, v in res.items(): | |
| print(f" {k:<20} {v:.3f}") | |
| # The number that actually matters. When training on `titles`, the held-out | |
| # score above is measured on 5-word journal titles while production input is | |
| # a 200-word abstract β a different and much easier problem. This second | |
| # evaluation runs on hand-labelled abstracts across 20 fields, which is the | |
| # distribution the classifier is really deployed against. | |
| if Path(args.eval_jsonl).exists(): | |
| ev = [json.loads(l) for l in open(args.eval_jsonl, encoding="utf-8")] | |
| Xe = sparse.hstack([word.transform([e["text"] for e in ev]), | |
| char.transform([e["text"] for e in ev])]).tocsr() | |
| Pe = clf.predict_proba(Xe) | |
| order = np.argsort(-Pe, axis=1) | |
| # Two standards, reported separately and deliberately. | |
| # | |
| # LENIENT counts a hit when *any* acceptable code appears. It overstates | |
| # usefulness: for a seismic-engineering abstract, 'Engineering | |
| # (miscellaneous)' is in `accept` and scores a hit, but filtering on it | |
| # returns nonlinear-analysis maths journals, while the preferred code | |
| # 2205 returns concrete-structures journals. Same "hit", different | |
| # recommendation quality. | |
| # | |
| # STRICT requires the single best code β accept[0], the one a subject | |
| # librarian would choose. This is the number to watch when retraining on | |
| # the abstract corpus, because it is the one that tracks whether the | |
| # recommendations are actually good. | |
| n = len(ev) | |
| hits1 = hits3 = parent1 = 0 | |
| s5 = s3 = 0 | |
| print(f"\nabstract eval ({n} hand-labelled abstracts)") | |
| for i, e in enumerate(ev): | |
| top5 = [mlb.classes_[j] for j in order[i, :5]] | |
| top3, top1 = top5[:3], top5[0] | |
| pref = e["accept"][0] | |
| ok1 = top1 in e["accept"] | |
| ok3 = bool(set(top3) & set(e["accept"])) | |
| okp = parent_of.get(top1) in {parent_of.get(c) for c in e["accept"]} | |
| hits1 += ok1; hits3 += ok3; parent1 += okp | |
| s5 += pref in top5 | |
| s3 += pref in top3 | |
| flag = "OK " if pref in top5 else ("weak" if ok3 else "MISS") | |
| print(f" {flag} {e['field'][:28]:<28} pref {pref} " | |
| f"{'in top5' if pref in top5 else 'ABSENT':<8} -> " | |
| f"{', '.join(f'{c}' for c in top3)}") | |
| print(f" lenient: top1 {hits1/n:.2f} top3 {hits3/n:.2f} parent {parent1/n:.2f}") | |
| print(f" STRICT : preferred-in-top3 {s3/n:.2f} preferred-in-top5 {s5/n:.2f}") | |
| print(" strict preferred-in-top5 is the number that gates shipping β it " | |
| "bounds what the LLM stage can possibly pick.") | |
| res.update({"abstract_top1": hits1 / n, "abstract_top3": hits3 / n, | |
| "abstract_top1_parent": parent1 / n, | |
| "abstract_preferred_top3": s3 / n, | |
| "abstract_preferred_top5": s5 / n}) | |
| # Compact the artifact before saving. The bulk is not the vocabulary β it is | |
| # 334 dense float64 coefficient vectors (334 x ~160k x 8 bytes β 427 MB | |
| # before compression). Casting to float32 halves that with no measurable | |
| # effect on ranking, and sparsify() drops the near-zero majority. This | |
| # matters operationally: the Space pulls this file on every cold start. | |
| for est in clf.estimators_: | |
| est.coef_ = est.coef_.astype("float32") | |
| est.sparsify() | |
| joblib.dump( | |
| {"word": word, "char": char, "clf": clf, "classes": mlb.classes_, | |
| "names": names, "parent_of": parent_of, "corpus": args.corpus, | |
| "metrics": res}, | |
| args.out, compress=3, | |
| ) | |
| print(f"\nwrote {args.out} ({Path(args.out).stat().st_size/1e6:.1f} MB)") | |
| # Sparsified coefficients change the predict path, so re-check the eval | |
| # rather than assuming the compaction was lossless. | |
| chk = joblib.load(args.out) | |
| Xc = sparse.hstack([chk["word"].transform([X[0]]), | |
| chk["char"].transform([X[0]])]).tocsr() | |
| assert chk["clf"].predict_proba(Xc).shape[1] == len(mlb.classes_) | |
| print("compacted model reloads and predicts correctly") | |
| if __name__ == "__main__": | |
| main() | |