Spaces:
Running on Zero
Running on Zero
Download scihigh_model.py from Akash212/Highlight_Generator: direct link, hf CLI and curl.
- Browser
- Download file 21.3 kB
-
https://huggingface.co/spaces/Akash212/Highlight_Generator/resolve/main/scihigh_model.py
- Command line
-
hf download hf://spaces/Akash212/Highlight_Generator/scihigh_model.py
-
curl -L -o scihigh_model.py https://huggingface.co/spaces/Akash212/Highlight_Generator/resolve/main/scihigh_model.py
21.3 kB
| """ | |
| SciHigh 2026 - Supervised extractive research-highlight generator (CPU-only). | |
| FIRE 2026 SciHigh, Subtask 1: research highlight generation from MixSub abstracts. | |
| Pipeline for one abstract | |
| 1. clean the text and split it into sentences | |
| 2. 31 hand-built features per sentence (position, length, lexical, discourse cues, TF-IDF graph) | |
| 3. 36 embedding features from all-MiniLM-L6-v2 (4 similarity features + 32-dim SVD projection), | |
| including a retrieval feature built from the 10 nearest training abstracts | |
| 4. five classifiers score every sentence; their rankings are merged by Reciprocal Rank Fusion | |
| 5. a decoder picks sentences under a word budget with an MMR redundancy filter, restores | |
| reading order and rewrites each sentence into highlight style | |
| Usage | |
| from scihigh_model import SciHighExtractor | |
| model = SciHighExtractor.from_pretrained(".") # local folder or a Hugging Face repo id | |
| print(model.predict(["<abstract text>"])[0]) | |
| The text-processing functions below are copied unchanged from SciHigh_2026_CPU_Hybrid.ipynb, so | |
| the saved models see exactly the features they were trained on. | |
| """ | |
| import os, re, io, json, glob, collections | |
| import numpy as np, pandas as pd | |
| from sklearn.preprocessing import normalize | |
| # ---------------------------------------------------------------- file helpers | |
| # Files larger than PART_LIMIT are stored as byte chunks named <stem>.part00<ext>, <stem>.part01<ext>, ... | |
| # so every file stays under 10 MB (browser upload limit). The loader joins them back byte for byte. | |
| PART_LIMIT = 9_500_000 | |
| def save_bytes(data, folder, name, limit=PART_LIMIT): | |
| stem, ext = os.path.splitext(name) | |
| for old in glob.glob(os.path.join(folder, f"{stem}.part*{ext}")): os.remove(old) | |
| if len(data) <= limit: | |
| open(os.path.join(folder, name), "wb").write(data); return [name] | |
| names = [] | |
| for i in range(0, len(data), limit): | |
| nm = f"{stem}.part{i // limit:02d}{ext}" | |
| open(os.path.join(folder, nm), "wb").write(data[i:i + limit]); names.append(nm) | |
| return names | |
| def read_bytes(folder, name): | |
| p = os.path.join(folder, name) | |
| if os.path.exists(p): return open(p, "rb").read() | |
| stem, ext = os.path.splitext(name) | |
| parts = sorted(glob.glob(os.path.join(folder, f"{stem}.part*{ext}"))) | |
| if not parts: raise FileNotFoundError(p) | |
| return b"".join(open(q, "rb").read() for q in parts) | |
| def load_joblib(folder, name): | |
| import joblib | |
| return joblib.load(io.BytesIO(read_bytes(folder, name))) | |
| def load_npy(folder, name): | |
| return np.load(io.BytesIO(read_bytes(folder, name))) | |
| # ---------------------------------------------------------------- cleaning and sentence splitting | |
| def clean(t): | |
| t = str(t) | |
| t = re.sub(r"\.\s*None\.\s*", " ", t) # "value . None . unit" artifact | |
| t = re.sub(r"\bNone\b", " ", t) | |
| t = re.sub(r"\s*\.\s*\.", ".", t) # collapse double periods left behind | |
| return re.sub(r"\s+", " ", t).strip() | |
| def sent_split(t): | |
| # split on " . " (MixSub style) or on ". " followed by a capital | |
| parts = re.split(r"(?<=[A-Za-z0-9\)\]\%])\s+\.\s+|(?<=[a-z0-9\)\]\%])\.\s+(?=[A-Z])", str(t).strip()) | |
| out = [] | |
| for p in parts: | |
| if not p: continue | |
| p = p.strip().rstrip(".").strip() | |
| if len(p.split()) >= 3: | |
| out.append(p) | |
| elif out: | |
| out[-1] += " " + p # glue fragments onto the previous sentence | |
| return out or [str(t).rstrip(".").strip()] | |
| STOP = set('''a about above after again against all also am among an and any are as at be because been before | |
| being below between both but by can cannot could did do does doing during each few for from further had has | |
| have having he her here hers herself him himself his how i if in into is it its itself me more most my myself | |
| no nor not now of off on once only or other others our ours out over own same she should so some such than | |
| that the their theirs them themselves then there these they this those through to too under until up very | |
| was we were what when where which while who whom why will with would you your yours'''.split()) | |
| TOK = re.compile(r"[a-z0-9]+") | |
| def toks(t): return TOK.findall(str(t).lower()) | |
| def content(t): return [w for w in toks(t) if w not in STOP and len(w) > 2] | |
| def make_doc(abstract): | |
| """One cleaned abstract -> the document dict used by featurize() and decode().""" | |
| S = sent_split(clean(abstract)); n = len(S) | |
| return dict(S=S, n=n, tk=[toks(s) for s in S], cw=[set(content(s)) for s in S]) | |
| # ---------------------------------------------------------------- hand-built sentence features | |
| CUES = { | |
| "res": r"\b(result|show|showed|shown|found|find|reveal|demonstrat|indicat|observ|obtain|achiev|improv|outperform|increas|decreas|significant)", | |
| "meth": r"\b(propos|present|develop|design|introduc|novel|new |method|approach|model|framework|algorithm|technique|scheme|system|strateg)", | |
| "purp": r"\b(aim|objective|purpose|goal|investigat|examin|explor|evaluat|assess|analys|analyz)", | |
| "bkg": r"\b(previous|literature|recently|recent years|has been|have been|widely|however|although|despite|remains|little is known|few studies)", | |
| "conc": r"\b(conclu|suggest|impl|therefore|thus|hence|overall|consequently|potential|promis|can be used|may be)", | |
| "quant": r"(\d+\.?\d*\s?%|p\s?[<=]|\bn\s?=|\bmean\b|\bmedian\b|\bratio\b|\brate\b|fold)", | |
| } | |
| LEAD_RE = (r"^(in this (study|paper|work|article|research)|in the present|the (aim|purpose|objective|goal) of" | |
| r"|here we|in addition|moreover|furthermore|therefore|thus|however|finally|overall|first|second" | |
| r"|third|also|based on)") | |
| def featurize(docs, tv): | |
| rows = [] | |
| for d in docs: | |
| S, n, TKs, CWs = d["S"], d["n"], d["tk"], d["cw"] | |
| tw = max(sum(len(t) for t in TKs), 1) | |
| X = normalize(tv.transform(S)) | |
| cen = (X @ normalize(np.asarray(X.mean(0))).T).ravel() # cosine to abstract centroid | |
| M = (X @ X.T).toarray(); np.fill_diagonal(M, 0); deg = M.sum(1) | |
| P = M / np.maximum(deg[:, None], 1e-9); pr = np.ones(n) / n | |
| for _ in range(20): pr = 0.15 / n + 0.85 * (P.T @ pr) # TextRank power iteration | |
| tsum = np.asarray(X.sum(1)).ravel() | |
| mx = M.max(1) if n > 1 else np.zeros(n) | |
| for i, s in enumerate(S): | |
| sw, cw, low = TKs[i], CWs[i], s.lower() | |
| nov = max((len(cw & CWs[j]) / max(len(cw | CWs[j]), 1) for j in range(i)), default=0.0) | |
| f = {"sid": i, "rel_pos": i / max(n-1, 1), "rel_end": (n-1-i) / max(n-1, 1), | |
| "is_first": int(i == 0), "is_last": int(i == n-1), "is_last2": int(i >= n-2), | |
| "is_first2": int(i <= 1), "nsent": n, "nw": len(sw), "nchar": len(s), | |
| "rel_len": len(sw) / tw, | |
| "awlen": float(np.mean([len(w) for w in sw])) if sw else 0.0, | |
| "stop_ratio": sum(1 for w in sw if w in STOP) / max(len(sw), 1), | |
| "n_content": len(cw), "content_dens": len(cw) / max(len(sw), 1), | |
| "n_digit": sum(c.isdigit() for c in s), | |
| "n_acro": len(re.findall(r"\b[A-Z]{2,}\b", s)), "has_pct": int("%" in s), | |
| "lead": int(bool(re.match(LEAD_RE, low))), "redund": nov, | |
| "tf_cent": cen[i], "tf_deg": deg[i], "textrank": pr[i], | |
| "tf_sum": tsum[i], "tf_maxsim": mx[i]} | |
| for k, pat in CUES.items(): | |
| f["cue_" + k] = len(re.findall(pat, low)) | |
| rows.append(f) | |
| return pd.DataFrame(rows) | |
| # ---------------------------------------------------------------- embedding features | |
| def offsets(docs): | |
| o, c = [], 0 | |
| for d in docs: o.append((c, c + d["n"])); c += d["n"] | |
| return o | |
| def cent(E, O): | |
| return np.vstack([normalize(E[a:b].mean(0).reshape(1, -1)) for a, b in O]) | |
| def emb_feats(E, O, Ctr, Hemb, proto, is_train=False, k=10): | |
| out = np.zeros((len(E), 4), np.float32) | |
| sim = cent(E, O) @ Ctr.T | |
| if is_train: np.fill_diagonal(sim, -1) # never retrieve yourself | |
| nn = np.argsort(-sim, 1)[:, :k] | |
| for q, (a, b) in enumerate(O): | |
| X = E[a:b] | |
| out[a:b, 0] = (X @ normalize(X.mean(0).reshape(1, -1)).T).ravel() | |
| out[a:b, 1] = (X @ proto.T).ravel() | |
| M = X @ X.T; np.fill_diagonal(M, 0) | |
| out[a:b, 2] = M.max(1) if b - a > 1 else 0 | |
| out[a:b, 3] = (X @ normalize(Hemb[nn[q]].mean(0).reshape(1, -1)).T).ravel() | |
| return pd.DataFrame(out, columns=["e_cent", "e_proto", "e_maxsim", "e_rag"]) | |
| def load_encoder(name_or_path="sentence-transformers/all-MiniLM-L6-v2", device="cpu"): | |
| """all-MiniLM-L6-v2 as a SentenceTransformer: Transformer(max_seq_length=256) -> mean pooling -> L2 norm. | |
| Accepts a Hugging Face id, a full sentence-transformers folder, or a folder that holds only | |
| config.json + model.safetensors + tokenizer.json (the same three modules are then assembled here).""" | |
| from sentence_transformers import SentenceTransformer, models | |
| p = name_or_path | |
| if os.path.isdir(p) and not os.path.exists(os.path.join(p, "modules.json")): | |
| tr = models.Transformer(p, max_seq_length=256, do_lower_case=False) | |
| dim = (tr.get_embedding_dimension() if hasattr(tr, "get_embedding_dimension") | |
| else tr.get_word_embedding_dimension()) | |
| pool = models.Pooling(dim, pooling_mode="mean") | |
| return SentenceTransformer(modules=[tr, pool, models.Normalize()], device=device) | |
| return SentenceTransformer(p, device=device) | |
| # ---------------------------------------------------------------- decoder and restyling | |
| DROP = re.compile( | |
| r"^(in this (study|paper|work|article|research|comment|letter|note|review|report|contribution)" | |
| r"|in the (present|current) (study|work|paper|article|research)|in the following" | |
| r"|the (aim|aims|purpose|objective|objectives|goal|goals) of (this|the present|the current) " | |
| r"(study|paper|work|research|article) (was|were|is|are) to" | |
| r"|the (present|current) (study|work|paper) (aims?|aimed|sought|seeks?)( to)?" | |
| r"|here (we|the authors)|in addition|additionally|moreover|furthermore|therefore|thus|hence|however" | |
| r"|nevertheless|nonetheless|meanwhile|finally|overall|consequently|accordingly|notably|importantly" | |
| r"|first(ly)?|second(ly)?|third(ly)?|also|in conclusion|in summary|to summari[sz]e|taken together" | |
| r"|it (was|is|has been|had been) (found|shown|observed|concluded|demonstrated|reported|noted|revealed)( that)?" | |
| r"|we (found|find|show|showed|observe|observed|conclude|concluded|report|reported|note|noted" | |
| r"|demonstrate|demonstrated|argue|suggest|propose)( that| with| in)?" | |
| r"|(the|our|these|this) (results?|findings?|data|analysis|study|work) (show|shows|showed|indicate" | |
| r"|indicated|suggest|suggests|suggested|reveal|revealed|demonstrate|demonstrated|confirm|confirmed)( that)?" | |
| r"|(this|these) (results?|findings?) (suggest|suggests|indicate|indicates|impl(y|ies))( that)?" | |
| r"|as (a|an) (result|consequence)|on the (other hand|contrary)|in contrast|by contrast" | |
| r"|based on (this|these|the)( results?| findings?)?" | |
| r"|the (results?|findings?) (of (this|the present) (study )?)?(show|showed|indicate|indicated" | |
| r"|suggest|suggested|revealed)( that)?)\s*[,:]?\s*", re.I) | |
| CONN = re.compile(r"\s+(?:and|but|whereas|while|although|though|because|since|which|that)\s+") | |
| def restyle(s, trim=0): | |
| prev = None | |
| while prev != s: # rules can cascade: "However, we found that X" -> "X" | |
| prev = s; s = DROP.sub("", s).strip() | |
| s = re.sub(r"^(and|but|that|which|to|of|in|for|with|as)\s+", "", s, flags=re.I).strip() | |
| if trim and len(s.split()) > trim: # optional clause-level truncation | |
| parts = CONN.split(s) | |
| if len(parts) > 1: | |
| acc, keep = 0, [] | |
| for p in parts: | |
| if keep and acc + len(p.split()) > trim: break | |
| keep.append(p); acc += len(p.split()) | |
| if keep: s = " and ".join(keep) | |
| s = re.sub(r"\s+", " ", s).strip() | |
| return (s[0].upper() + s[1:]) if s else s | |
| def decode(docs, sc, budget=110, maxs=3, mmr=0.55, trim=0, E=None): | |
| out, c = [], 0 | |
| for d in docs: | |
| n = d["n"]; s = sc[c:c+n]; base = c; c += n | |
| order = list(np.argsort(-s)); pick, wc = [], 0 | |
| for i in order: | |
| if len(pick) >= maxs: break | |
| w = len(d["tk"][i]) | |
| if wc + w > budget and pick: continue | |
| if mmr < 1.0 and pick: | |
| if E is not None: | |
| sim = max(float(E[base+i] @ E[base+j]) for j in pick) | |
| else: | |
| a = d["cw"][i] | |
| sim = max(len(a & d["cw"][j]) / max(len(a | d["cw"][j]), 1) for j in pick) | |
| if sim > mmr: continue | |
| pick.append(i); wc += w | |
| if not pick: pick = [order[0]] | |
| pick.sort() # restore reading order | |
| segs = [x for x in (restyle(d["S"][i], trim) for i in pick) if len(x.split()) >= 3] | |
| out.append((". ".join(segs) + ".") if segs else d["S"][pick[0]] + ".") | |
| return out | |
| # ---------------------------------------------------------------- scoring and rank fusion | |
| def prob(m, X): | |
| return (m.predict_proba(X)[:, 1] if hasattr(m, "predict_proba") | |
| else 1 / (1 + np.exp(-m.decision_function(X)))) | |
| def fuse(docs, D, names, k=60): | |
| z = np.zeros(sum(d["n"] for d in docs)) | |
| for n in names: | |
| s = D[n]; c = 0 | |
| for d in docs: | |
| m = d["n"] | |
| o = np.argsort(-s[c:c+m]); rr = np.empty(m, int); rr[o] = np.arange(m) | |
| z[c:c+m] += 1.0 / (k + rr + 1); c += m | |
| return z | |
| # ---------------------------------------------------------------- the model | |
| class SciHighExtractor: | |
| """Loads the saved artifacts and turns raw abstracts into highlights.""" | |
| def __init__(self, folder, encoder=None, device="cpu"): | |
| self.folder = folder | |
| self.config = json.load(open(os.path.join(folder, "config.json"))) | |
| self.tv = load_joblib(folder, "tfidf.joblib") | |
| self.models = {n: load_joblib(folder, f"clf_{n}.joblib") for n in self.config["ensemble"]} | |
| self.use_emb = self.config["embeddings_used"] | |
| if self.use_emb: | |
| self.svd = load_joblib(folder, "svd.joblib") | |
| self.proto = load_npy(folder, "proto.npy") | |
| self.Ctr = load_npy(folder, "train_centroids.npy") | |
| self.Hemb = load_npy(folder, "train_highlight_emb.npy") | |
| self._enc_src, self._enc, self._device = encoder or self.config["encoder"], None, device | |
| lex = os.path.join(folder, "highlight_lexicon.json") | |
| self.lexicon = json.load(open(lex)) if os.path.exists(lex) else {} | |
| self._voc, self._idf = self.tv.vocabulary_, self.tv.idf_ | |
| self._idf_oov = float(self._idf.max()) | |
| def from_pretrained(cls, path_or_repo_id, encoder=None, device="cpu", **hub_kwargs): | |
| """path_or_repo_id: a local folder, or a Hugging Face repo id such as "user/scihigh2026-extractive-ranker". | |
| encoder: optional local path to all-MiniLM-L6-v2 (default: downloaded from the Hub by name).""" | |
| folder = path_or_repo_id | |
| if not os.path.isdir(folder): | |
| from huggingface_hub import snapshot_download | |
| folder = snapshot_download(path_or_repo_id, **hub_kwargs) | |
| return cls(folder, encoder=encoder, device=device) | |
| def _embed(self, texts): | |
| if self._enc is None: | |
| self._enc = load_encoder(self._enc_src, self._device) | |
| return self._enc.encode(list(texts), batch_size=256, convert_to_numpy=True, | |
| normalize_embeddings=True, show_progress_bar=False) | |
| def features(self, docs): | |
| """Design matrix (one row per sentence) and the sentence embeddings used by the MMR filter.""" | |
| F, E = featurize(docs, self.tv), None | |
| if self.use_emb: | |
| E = self._embed([s for d in docs for s in d["S"]]) | |
| proj = pd.DataFrame(self.svd.transform(E), columns=[f"sv{i}" for i in range(32)]) | |
| F = pd.concat([F, emb_feats(E, offsets(docs), self.Ctr, self.Hemb, self.proto), proj], axis=1) | |
| X = F[self.config["feature_columns"]].astype(np.float32).fillna(0) | |
| return X, E | |
| def sentence_scores(self, docs, per_model=False): | |
| X, E = self.features(docs) | |
| S = {n: prob(m, X) for n, m in self.models.items()} | |
| z = fuse(docs, S, self.config["ensemble"], k=self.config["rrf_k"]) | |
| return (z, E, S) if per_model else (z, E) | |
| def _cfg(self, overrides): | |
| cfg = dict(self.config["decoder"]) | |
| cfg.update({k: v for k, v in overrides.items() if v is not None and k in cfg}) | |
| return cfg | |
| # ---- word importance (TF-IDF x IDF within the abstract + highlight log-odds lexicon) | |
| def idf_of(self, w): | |
| parts = [p for p in w.lower().replace("'", "").split("-") if p] | |
| v = [self._idf[self._voc[p]] if p in self._voc else self._idf_oov for p in parts] or [self._idf_oov] | |
| return float(np.mean(v)) | |
| def word_weights(self, doc): | |
| from concise import doc_weights | |
| return doc_weights(doc["S"], self.idf_of, lambda w: self.lexicon.get(w.lower(), 0.0)) | |
| # ---- output modes | |
| def predict(self, abstracts, **decoder): | |
| """FIRE mode: list of abstract strings -> list of highlight strings (tuned extractive decoder). | |
| Optional decoder overrides: budget (words), maxs (sentences), mmr (redundancy threshold), trim.""" | |
| if isinstance(abstracts, str): abstracts = [abstracts] | |
| docs = [make_doc(a) for a in abstracts] | |
| z, E = self.sentence_scores(docs) | |
| return decode(docs, z, **self._cfg(decoder), E=E) | |
| def concise(self, abstracts, target=18, **decoder): | |
| """Concise mode: the same sentence choice, each sentence compressed to about `target` words.""" | |
| from concise import compress | |
| if isinstance(abstracts, str): abstracts = [abstracts] | |
| docs = [make_doc(a) for a in abstracts] | |
| z, E = self.sentence_scores(docs) | |
| cfg, out, c = self._cfg(decoder), [], 0 | |
| for d in docs: | |
| n = d["n"]; picks = pick_indices(d, z[c:c + n], None if E is None else E[c:c + n], **cfg); c += n | |
| w = self.word_weights(d) | |
| hs = [compress(restyle(d["S"][i]), w, target) for i in picks] | |
| hs = [h for h in hs if len(h.split()) >= 3] or [d["S"][picks[0]]] | |
| out.append(". ".join(hs) + ".") | |
| return out | |
| def analyse(self, abstract, target=18, n_keywords=4, **decoder): | |
| """One abstract -> dict with the FIRE output, the Concise output and a per-sentence table | |
| (fused rank, each model's probability, a compressed version of every sentence and its keywords).""" | |
| from concise import compress, WORD, FUNC | |
| docs = [make_doc(abstract)]; d = docs[0] | |
| z, E, S = self.sentence_scores(docs, per_model=True) | |
| cfg = self._cfg(decoder) | |
| picks = pick_indices(d, z, E, **cfg) | |
| w = self.word_weights(d) | |
| summaries = [compress(restyle(s), w, target) for s in d["S"]] | |
| def keywords(s): | |
| ws = [] | |
| for t in WORD.findall(s): | |
| t = t.lower().strip(".") | |
| if t not in FUNC and len(t) > 2 and t not in ws: ws.append(t) | |
| return ", ".join(sorted(ws, key=lambda t: -w.get(t, 0.0))[:n_keywords]) | |
| order = np.argsort(-z); rank = np.empty(len(z), int); rank[order] = np.arange(1, len(z) + 1) | |
| table = pd.DataFrame({"#": np.arange(1, len(z) + 1), "picked": ["yes" if i in picks else "" for i in range(len(z))], | |
| "rank": rank, "sentence": d["S"], "summary": summaries, | |
| "keywords": [keywords(s) for s in d["S"]], "RRF": np.round(z.astype(float), 4)}) | |
| for n in self.config["ensemble"]: table[n] = np.round(np.asarray(S[n], dtype=float), 3) | |
| concise_hl = [summaries[i] for i in picks if len(summaries[i].split()) >= 3] or [d["S"][picks[0]]] | |
| return {"fire": decode(docs, z, **cfg, E=E)[0], "concise": concise_hl, "picks": picks, | |
| "sentences": d["S"], "table": table} | |
| def explain(self, abstract, **decoder): | |
| """One abstract -> (FIRE highlights, per-sentence table). Kept for backward compatibility.""" | |
| r = self.analyse(abstract, **decoder) | |
| return r["fire"], r["table"] | |
| def pick_indices(d, s, E=None, budget=110, maxs=3, mmr=0.55, trim=0): | |
| """The sentence-selection half of decode() for a single document, returning indices (for display).""" | |
| order = list(np.argsort(-s)); pick, wc = [], 0 | |
| for i in order: | |
| if len(pick) >= maxs: break | |
| w = len(d["tk"][i]) | |
| if wc + w > budget and pick: continue | |
| if mmr < 1.0 and pick: | |
| if E is not None: | |
| sim = max(float(E[i] @ E[j]) for j in pick) | |
| else: | |
| a = d["cw"][i] | |
| sim = max(len(a & d["cw"][j]) / max(len(a | d["cw"][j]), 1) for j in pick) | |
| if sim > mmr: continue | |
| pick.append(i); wc += w | |
| return sorted(pick) if pick else [order[0]] | |
| if __name__ == "__main__": | |
| import sys | |
| m = SciHighExtractor.from_pretrained(os.path.dirname(os.path.abspath(__file__))) | |
| text = " ".join(sys.argv[1:]) or sys.stdin.read() | |
| print(m.predict([text])[0]) | |