Spaces:
Running on Zero
Running on Zero
Download train_export.py from Akash212/Highlight_Generator: direct link, hf CLI and curl.
- Browser
- Download file 10.8 kB
-
https://huggingface.co/spaces/Akash212/Highlight_Generator/resolve/main/train_export.py
- Command line
-
hf download hf://spaces/Akash212/Highlight_Generator/train_export.py
-
curl -L -o train_export.py https://huggingface.co/spaces/Akash212/Highlight_Generator/resolve/main/train_export.py
10.8 kB
| """ | |
| Re-trains the submitted SciHigh 2026 ensemble and exports it into this folder. | |
| This is SciHigh_2026_CPU_Hybrid.ipynb with the model search removed: the ensemble members and the | |
| decoder settings are fixed to the ones the notebook selected on validation (see run_summary.csv), | |
| and the fitted objects are saved so they can be uploaded to the Hugging Face Hub. | |
| python train_export.py --data <folder with the three MixSub CSVs> --encoder <all-MiniLM-L6-v2 path or id> | |
| """ | |
| import os, re, sys, time, json, bisect, argparse, collections, warnings | |
| import numpy as np, pandas as pd, joblib | |
| warnings.filterwarnings("ignore") | |
| np.random.seed(42) | |
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) | |
| from scihigh_model import (clean, sent_split, toks, content, featurize, offsets, cent, emb_feats, | |
| load_encoder, decode, prob, fuse) | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--data", default=".") | |
| ap.add_argument("--encoder", default="sentence-transformers/all-MiniLM-L6-v2") | |
| ap.add_argument("--out", default=os.path.dirname(os.path.abspath(__file__))) | |
| ap.add_argument("--cache", default=None, help="optional folder for intermediate results") | |
| args = ap.parse_args() | |
| RANDOM = 42 | |
| SEL = ["LinearSVC", "Ridge", "LogReg", "MLP", "ExtraTrees"] # "top5-overall", chosen in the notebook | |
| CFG = dict(budget=110, maxs=3, mmr=0.85, trim=0) # tuned by coordinate descent in the notebook | |
| t0 = time.time() | |
| def log(*a): print(f"[{time.time()-t0:6.1f}s]", *a, flush=True) | |
| def cached(name, fn): | |
| if not args.cache: return fn() | |
| p = os.path.join(args.cache, name + ".joblib") | |
| if os.path.exists(p): return joblib.load(p) | |
| os.makedirs(args.cache, exist_ok=True); v = fn(); joblib.dump(v, p); return v | |
| # ---------------------------------------------------------------- data | |
| train = pd.read_csv(os.path.join(args.data, "MixSub-SciHigh_train_FIRE.csv")) | |
| val = pd.read_csv(os.path.join(args.data, "MixSub-SciHigh_val_FIRE.csv")) | |
| test = pd.read_csv(os.path.join(args.data, "MixSub-SciHigh_test_FIRE(Masked).csv")) | |
| for df in (train, val, test): df["A"] = df["Abstract"].map(clean) | |
| for df in (train, val): df["H"] = df["Highlights"].map(clean) | |
| log("train", train.shape, "| val", val.shape, "| test", test.shape) | |
| # ---------------------------------------------------------------- oracle labels (notebook sections 4-5) | |
| from functools import lru_cache | |
| from nltk.stem.porter import PorterStemmer | |
| _PS = PorterStemmer() | |
| TOK = re.compile(r"[a-z0-9]+") | |
| def _stem(w): return _PS.stem(w) if len(w) > 3 else w | |
| def stoks(t): return [_stem(w) for w in TOK.findall(str(t).lower())] | |
| def ngrams(ws, n): | |
| return collections.Counter(zip(*[ws[i:] for i in range(n)])) if len(ws) >= n else collections.Counter() | |
| def _f(t, nc, nr): | |
| if not t: return 0.0 | |
| p, r = t / max(nc, 1), t / max(nr, 1) | |
| return 2 * p * r / (p + r) | |
| def r_gain(cand_ws, ref1, ref2): | |
| c1, c2 = ngrams(cand_ws, 1), ngrams(cand_ws, 2) | |
| t1 = sum(min(v, c1[k]) for k, v in ref1.items()) | |
| t2 = sum(min(v, c2[k]) for k, v in ref2.items()) | |
| return _f(t1, sum(c1.values()), sum(ref1.values())) + _f(t2, sum(c2.values()), sum(ref2.values())) | |
| def r12f(ws, ref_ws): | |
| return r_gain(ws, ngrams(ref_ws, 1), ngrams(ref_ws, 2)) / 2 | |
| def lcs_len(a, b): | |
| pos = collections.defaultdict(list) | |
| for j, x in enumerate(b): pos[x].append(j) | |
| tails = [] | |
| for x in a: | |
| for j in reversed(pos.get(x, ())): | |
| k = bisect.bisect_left(tails, j) | |
| if k == len(tails): tails.append(j) | |
| else: tails[k] = j | |
| return len(tails) | |
| def fast_rl(pred, ref_stems): | |
| p = stoks(pred) | |
| if not p or not ref_stems: return 0.0 | |
| l = lcs_len(p, ref_stems) | |
| if not l: return 0.0 | |
| pr, rc = l / len(p), l / len(ref_stems) | |
| return 2 * pr * rc / (pr + rc) | |
| def make_docs(df, labelled, maxsel=6): | |
| docs = [] | |
| for rec in df.itertuples(): | |
| S = sent_split(rec.A); n = len(S) | |
| d = dict(S=S, n=n, tk=[toks(s) for s in S], cw=[set(content(s)) for s in S]) | |
| if labelled: | |
| rw = toks(rec.H); r1, r2 = ngrams(rw, 1), ngrams(rw, 2) | |
| chosen, cur, best = [], [], 0.0 | |
| for _ in range(min(maxsel, n)): | |
| cand, cbest = None, best | |
| for i in range(n): | |
| if i in chosen: continue | |
| sc = r_gain(cur + d["tk"][i], r1, r2) | |
| if sc > cbest + 1e-9: cbest, cand = sc, i | |
| if cand is None: break | |
| chosen.append(cand); cur += d["tk"][cand]; best = cbest | |
| ch = set(chosen) | |
| d["y"] = [int(i in ch) for i in range(n)] | |
| d["soft"] = [r12f(d["tk"][i], rw) for i in range(n)] | |
| d["ref"] = rec.H | |
| d["refs"] = stoks(rec.H) | |
| docs.append(d) | |
| return docs | |
| log("labelling ...") | |
| Dtr, Dva, Dte = cached("docs", lambda: (make_docs(train, True), make_docs(val, True), make_docs(test, False))) | |
| flat = lambda ds, k: np.concatenate([d[k] for d in ds]) | |
| log("train sentences:", sum(d["n"] for d in Dtr), "| positive rate: %.3f" % flat(Dtr, "y").mean()) | |
| # ---------------------------------------------------------------- features (sections 7-8) | |
| from sklearn.feature_extraction.text import TfidfVectorizer | |
| from sklearn.decomposition import TruncatedSVD | |
| from sklearn.preprocessing import normalize | |
| allsent = [s for d in Dtr for s in d["S"]] | |
| tv = TfidfVectorizer(sublinear_tf=True, stop_words="english", min_df=2).fit(allsent) | |
| Ftr, Fva, Fte = cached("feats", lambda: (featurize(Dtr, tv), featurize(Dva, tv), featurize(Dte, tv))) | |
| log("feature block:", Ftr.shape) | |
| def encode_all(): | |
| enc = load_encoder(args.encoder, "cpu") | |
| emb = lambda ts: enc.encode(list(ts), batch_size=256, convert_to_numpy=True, | |
| normalize_embeddings=True, show_progress_bar=False) | |
| log("encoding sentences on CPU ...") | |
| Etr = emb(allsent); log(" train done") | |
| Eva = emb([s for d in Dva for s in d["S"]]) | |
| Ete = emb([s for d in Dte for s in d["S"]]) | |
| hl = [x for h in train.H for x in re.split(r"(?<=[a-z0-9\)\%])\.\s+", h) if len(x.split()) > 3] | |
| proto = normalize(emb(hl[:4000]).mean(0).reshape(1, -1)) | |
| Hemb = emb(train.H.tolist()) | |
| return Etr, Eva, Ete, proto, Hemb | |
| Etr, Eva, Ete, proto, Hemb = cached("emb", encode_all) | |
| Otr, Ova, Ote = offsets(Dtr), offsets(Dva), offsets(Dte) | |
| Ctr = cent(Etr, Otr) | |
| svd = TruncatedSVD(32, random_state=RANDOM).fit(Etr) | |
| proj = lambda M: pd.DataFrame(svd.transform(M), columns=[f"sv{i}" for i in range(32)]) | |
| Ftr = pd.concat([Ftr, emb_feats(Etr, Otr, Ctr, Hemb, proto, True), proj(Etr)], axis=1) | |
| Fva = pd.concat([Fva, emb_feats(Eva, Ova, Ctr, Hemb, proto, False), proj(Eva)], axis=1) | |
| Fte = pd.concat([Fte, emb_feats(Ete, Ote, Ctr, Hemb, proto, False), proj(Ete)], axis=1) | |
| Xtr, Xva, Xte = [f.astype(np.float32).fillna(0) for f in (Ftr, Fva, Fte)] | |
| ytr, yva = flat(Dtr, "y"), flat(Dva, "y") | |
| log("final design matrix:", Xtr.shape) | |
| # ---------------------------------------------------------------- the five ensemble members (section 10) | |
| from sklearn.linear_model import LogisticRegression, RidgeClassifier | |
| from sklearn.svm import LinearSVC | |
| from sklearn.ensemble import ExtraTreesClassifier | |
| from sklearn.neural_network import MLPClassifier | |
| from sklearn.pipeline import make_pipeline | |
| from sklearn.preprocessing import StandardScaler | |
| SC = lambda m: make_pipeline(StandardScaler(), m) | |
| MODELS = { | |
| "LogReg": SC(LogisticRegression(max_iter=1000)), | |
| "Ridge": SC(RidgeClassifier()), | |
| "LinearSVC": SC(LinearSVC(C=0.1, max_iter=3000)), | |
| "ExtraTrees": ExtraTreesClassifier(150, min_samples_leaf=20, max_features=0.5, n_jobs=-1, random_state=RANDOM), | |
| "MLP": SC(MLPClassifier((96, 48), max_iter=25, early_stopping=True, random_state=RANDOM)), | |
| } | |
| SV, ST = {}, {} | |
| for name, m in MODELS.items(): | |
| t = time.time(); m.fit(Xtr, ytr) | |
| SV[name], ST[name] = prob(m, Xva), prob(m, Xte) | |
| log(f" {name:11s} fitted in {time.time()-t:.1f}s") | |
| # ---------------------------------------------------------------- evaluation (sections 11, 14, 16) | |
| from rouge_score import rouge_scorer | |
| _RS = rouge_scorer.RougeScorer(["rouge1", "rouge2", "rougeL"], use_stemmer=True) | |
| REFS, REFST = [d["ref"] for d in Dva], [d["refs"] for d in Dva] | |
| def eval_all(preds, refs): | |
| o = {k: [] for k in ["rouge1", "rouge2", "rougeL"]} | |
| for p, r in zip(preds, refs): | |
| s = _RS.score(r, p) | |
| for k in o: o[k].append(s[k].fmeasure) | |
| return {k: round(100 * float(np.mean(v)), 2) for k, v in o.items()} | |
| qrl = lambda preds: 100 * float(np.mean([fast_rl(p, r) for p, r in zip(preds, REFST)])) | |
| single = {n: round(qrl(decode(Dva, SV[n], 110, 3, .55, 0, Eva)), 4) for n in SEL} | |
| log("single-model decoded ROUGE-L (notebook settings):", single) | |
| FZ_va, FZ_te = fuse(Dva, SV, SEL), fuse(Dte, ST, SEL) | |
| pred_va = decode(Dva, FZ_va, **CFG, E=Eva) | |
| pred_te = decode(Dte, FZ_te, **CFG, E=Ete) | |
| scores = eval_all(pred_va, REFS) | |
| log("validation (fused):", scores) | |
| # ---------------------------------------------------------------- export | |
| import io, sklearn | |
| from scihigh_model import save_bytes | |
| out = args.out | |
| os.makedirs(out, exist_ok=True) | |
| def dump(obj, name, compress=3): | |
| b = io.BytesIO(); joblib.dump(obj, b, compress=compress); save_bytes(b.getvalue(), out, name) | |
| def npsave(arr, name): | |
| b = io.BytesIO(); np.save(b, np.asarray(arr, np.float32)); save_bytes(b.getvalue(), out, name) | |
| for n in SEL: dump(MODELS[n], f"clf_{n}.joblib") | |
| dump(tv, "tfidf.joblib"); dump(svd, "svd.joblib") | |
| npsave(proto, "proto.npy"); npsave(Ctr, "train_centroids.npy"); npsave(Hemb, "train_highlight_emb.npy") | |
| config = { | |
| "model_type": "scihigh-extractive-ranker", | |
| "task": "FIRE 2026 SciHigh Subtask 1 - research highlight generation", | |
| "ensemble": SEL, "fusion": "reciprocal_rank_fusion", "rrf_k": 60, | |
| "decoder": CFG, | |
| "embeddings_used": True, | |
| "encoder": "sentence-transformers/all-MiniLM-L6-v2", | |
| "retrieval_neighbours": 10, | |
| "feature_columns": list(Xtr.columns), | |
| "n_features": int(Xtr.shape[1]), | |
| "train_rows": int(len(train)), "train_sentences": int(len(Xtr)), "validation_docs": int(len(val)), | |
| "validation_scores": scores, | |
| "validation_single_model_rougeL": single, | |
| "library_versions": {"python": sys.version.split()[0], "scikit-learn": sklearn.__version__, | |
| "numpy": np.__version__, "pandas": pd.__version__, "joblib": joblib.__version__}, | |
| } | |
| json.dump(config, open(os.path.join(out, "config.json"), "w"), indent=2) | |
| pd.DataFrame({"Filename": test["Filename"].values, "Highlights": pred_te}).to_csv( | |
| os.path.join(out, "test_predictions.csv"), index=False) | |
| pd.DataFrame({"Filename": val["Filename"].values, "Highlights": pred_va}).to_csv( | |
| os.path.join(out, "val_predictions.csv"), index=False) | |
| log("exported to", out) | |