""" SciHigh 2026 - Supervised extractive research-highlight generator (CPU-only). FIRE 2026 SciHigh, Subtask 1: research highlight generation from MixSub abstracts. Pipeline for one abstract 1. clean the text and split it into sentences 2. 31 hand-built features per sentence (position, length, lexical, discourse cues, TF-IDF graph) 3. 36 embedding features from all-MiniLM-L6-v2 (4 similarity features + 32-dim SVD projection), including a retrieval feature built from the 10 nearest training abstracts 4. five classifiers score every sentence; their rankings are merged by Reciprocal Rank Fusion 5. a decoder picks sentences under a word budget with an MMR redundancy filter, restores reading order and rewrites each sentence into highlight style Usage from scihigh_model import SciHighExtractor model = SciHighExtractor.from_pretrained(".") # local folder or a Hugging Face repo id print(model.predict([""])[0]) The text-processing functions below are copied unchanged from SciHigh_2026_CPU_Hybrid.ipynb, so the saved models see exactly the features they were trained on. """ import os, re, io, json, glob, collections import numpy as np, pandas as pd from sklearn.preprocessing import normalize # ---------------------------------------------------------------- file helpers # Files larger than PART_LIMIT are stored as byte chunks named .part00, .part01, ... # so every file stays under 10 MB (browser upload limit). The loader joins them back byte for byte. PART_LIMIT = 9_500_000 def save_bytes(data, folder, name, limit=PART_LIMIT): stem, ext = os.path.splitext(name) for old in glob.glob(os.path.join(folder, f"{stem}.part*{ext}")): os.remove(old) if len(data) <= limit: open(os.path.join(folder, name), "wb").write(data); return [name] names = [] for i in range(0, len(data), limit): nm = f"{stem}.part{i // limit:02d}{ext}" open(os.path.join(folder, nm), "wb").write(data[i:i + limit]); names.append(nm) return names def read_bytes(folder, name): p = os.path.join(folder, name) if os.path.exists(p): return open(p, "rb").read() stem, ext = os.path.splitext(name) parts = sorted(glob.glob(os.path.join(folder, f"{stem}.part*{ext}"))) if not parts: raise FileNotFoundError(p) return b"".join(open(q, "rb").read() for q in parts) def load_joblib(folder, name): import joblib return joblib.load(io.BytesIO(read_bytes(folder, name))) def load_npy(folder, name): return np.load(io.BytesIO(read_bytes(folder, name))) # ---------------------------------------------------------------- cleaning and sentence splitting def clean(t): t = str(t) t = re.sub(r"\.\s*None\.\s*", " ", t) # "value . None . unit" artifact t = re.sub(r"\bNone\b", " ", t) t = re.sub(r"\s*\.\s*\.", ".", t) # collapse double periods left behind return re.sub(r"\s+", " ", t).strip() def sent_split(t): # split on " . " (MixSub style) or on ". " followed by a capital parts = re.split(r"(?<=[A-Za-z0-9\)\]\%])\s+\.\s+|(?<=[a-z0-9\)\]\%])\.\s+(?=[A-Z])", str(t).strip()) out = [] for p in parts: if not p: continue p = p.strip().rstrip(".").strip() if len(p.split()) >= 3: out.append(p) elif out: out[-1] += " " + p # glue fragments onto the previous sentence return out or [str(t).rstrip(".").strip()] STOP = set('''a about above after again against all also am among an and any are as at be because been before being below between both but by can cannot could did do does doing during each few for from further had has have having he her here hers herself him himself his how i if in into is it its itself me more most my myself no nor not now of off on once only or other others our ours out over own same she should so some such than that the their theirs them themselves then there these they this those through to too under until up very was we were what when where which while who whom why will with would you your yours'''.split()) TOK = re.compile(r"[a-z0-9]+") def toks(t): return TOK.findall(str(t).lower()) def content(t): return [w for w in toks(t) if w not in STOP and len(w) > 2] def make_doc(abstract): """One cleaned abstract -> the document dict used by featurize() and decode().""" S = sent_split(clean(abstract)); n = len(S) return dict(S=S, n=n, tk=[toks(s) for s in S], cw=[set(content(s)) for s in S]) # ---------------------------------------------------------------- hand-built sentence features CUES = { "res": r"\b(result|show|showed|shown|found|find|reveal|demonstrat|indicat|observ|obtain|achiev|improv|outperform|increas|decreas|significant)", "meth": r"\b(propos|present|develop|design|introduc|novel|new |method|approach|model|framework|algorithm|technique|scheme|system|strateg)", "purp": r"\b(aim|objective|purpose|goal|investigat|examin|explor|evaluat|assess|analys|analyz)", "bkg": r"\b(previous|literature|recently|recent years|has been|have been|widely|however|although|despite|remains|little is known|few studies)", "conc": r"\b(conclu|suggest|impl|therefore|thus|hence|overall|consequently|potential|promis|can be used|may be)", "quant": r"(\d+\.?\d*\s?%|p\s?[<=]|\bn\s?=|\bmean\b|\bmedian\b|\bratio\b|\brate\b|fold)", } LEAD_RE = (r"^(in this (study|paper|work|article|research)|in the present|the (aim|purpose|objective|goal) of" r"|here we|in addition|moreover|furthermore|therefore|thus|however|finally|overall|first|second" r"|third|also|based on)") def featurize(docs, tv): rows = [] for d in docs: S, n, TKs, CWs = d["S"], d["n"], d["tk"], d["cw"] tw = max(sum(len(t) for t in TKs), 1) X = normalize(tv.transform(S)) cen = (X @ normalize(np.asarray(X.mean(0))).T).ravel() # cosine to abstract centroid M = (X @ X.T).toarray(); np.fill_diagonal(M, 0); deg = M.sum(1) P = M / np.maximum(deg[:, None], 1e-9); pr = np.ones(n) / n for _ in range(20): pr = 0.15 / n + 0.85 * (P.T @ pr) # TextRank power iteration tsum = np.asarray(X.sum(1)).ravel() mx = M.max(1) if n > 1 else np.zeros(n) for i, s in enumerate(S): sw, cw, low = TKs[i], CWs[i], s.lower() nov = max((len(cw & CWs[j]) / max(len(cw | CWs[j]), 1) for j in range(i)), default=0.0) f = {"sid": i, "rel_pos": i / max(n-1, 1), "rel_end": (n-1-i) / max(n-1, 1), "is_first": int(i == 0), "is_last": int(i == n-1), "is_last2": int(i >= n-2), "is_first2": int(i <= 1), "nsent": n, "nw": len(sw), "nchar": len(s), "rel_len": len(sw) / tw, "awlen": float(np.mean([len(w) for w in sw])) if sw else 0.0, "stop_ratio": sum(1 for w in sw if w in STOP) / max(len(sw), 1), "n_content": len(cw), "content_dens": len(cw) / max(len(sw), 1), "n_digit": sum(c.isdigit() for c in s), "n_acro": len(re.findall(r"\b[A-Z]{2,}\b", s)), "has_pct": int("%" in s), "lead": int(bool(re.match(LEAD_RE, low))), "redund": nov, "tf_cent": cen[i], "tf_deg": deg[i], "textrank": pr[i], "tf_sum": tsum[i], "tf_maxsim": mx[i]} for k, pat in CUES.items(): f["cue_" + k] = len(re.findall(pat, low)) rows.append(f) return pd.DataFrame(rows) # ---------------------------------------------------------------- embedding features def offsets(docs): o, c = [], 0 for d in docs: o.append((c, c + d["n"])); c += d["n"] return o def cent(E, O): return np.vstack([normalize(E[a:b].mean(0).reshape(1, -1)) for a, b in O]) def emb_feats(E, O, Ctr, Hemb, proto, is_train=False, k=10): out = np.zeros((len(E), 4), np.float32) sim = cent(E, O) @ Ctr.T if is_train: np.fill_diagonal(sim, -1) # never retrieve yourself nn = np.argsort(-sim, 1)[:, :k] for q, (a, b) in enumerate(O): X = E[a:b] out[a:b, 0] = (X @ normalize(X.mean(0).reshape(1, -1)).T).ravel() out[a:b, 1] = (X @ proto.T).ravel() M = X @ X.T; np.fill_diagonal(M, 0) out[a:b, 2] = M.max(1) if b - a > 1 else 0 out[a:b, 3] = (X @ normalize(Hemb[nn[q]].mean(0).reshape(1, -1)).T).ravel() return pd.DataFrame(out, columns=["e_cent", "e_proto", "e_maxsim", "e_rag"]) def load_encoder(name_or_path="sentence-transformers/all-MiniLM-L6-v2", device="cpu"): """all-MiniLM-L6-v2 as a SentenceTransformer: Transformer(max_seq_length=256) -> mean pooling -> L2 norm. Accepts a Hugging Face id, a full sentence-transformers folder, or a folder that holds only config.json + model.safetensors + tokenizer.json (the same three modules are then assembled here).""" from sentence_transformers import SentenceTransformer, models p = name_or_path if os.path.isdir(p) and not os.path.exists(os.path.join(p, "modules.json")): tr = models.Transformer(p, max_seq_length=256, do_lower_case=False) dim = (tr.get_embedding_dimension() if hasattr(tr, "get_embedding_dimension") else tr.get_word_embedding_dimension()) pool = models.Pooling(dim, pooling_mode="mean") return SentenceTransformer(modules=[tr, pool, models.Normalize()], device=device) return SentenceTransformer(p, device=device) # ---------------------------------------------------------------- decoder and restyling DROP = re.compile( r"^(in this (study|paper|work|article|research|comment|letter|note|review|report|contribution)" r"|in the (present|current) (study|work|paper|article|research)|in the following" r"|the (aim|aims|purpose|objective|objectives|goal|goals) of (this|the present|the current) " r"(study|paper|work|research|article) (was|were|is|are) to" r"|the (present|current) (study|work|paper) (aims?|aimed|sought|seeks?)( to)?" r"|here (we|the authors)|in addition|additionally|moreover|furthermore|therefore|thus|hence|however" r"|nevertheless|nonetheless|meanwhile|finally|overall|consequently|accordingly|notably|importantly" r"|first(ly)?|second(ly)?|third(ly)?|also|in conclusion|in summary|to summari[sz]e|taken together" r"|it (was|is|has been|had been) (found|shown|observed|concluded|demonstrated|reported|noted|revealed)( that)?" r"|we (found|find|show|showed|observe|observed|conclude|concluded|report|reported|note|noted" r"|demonstrate|demonstrated|argue|suggest|propose)( that| with| in)?" r"|(the|our|these|this) (results?|findings?|data|analysis|study|work) (show|shows|showed|indicate" r"|indicated|suggest|suggests|suggested|reveal|revealed|demonstrate|demonstrated|confirm|confirmed)( that)?" r"|(this|these) (results?|findings?) (suggest|suggests|indicate|indicates|impl(y|ies))( that)?" r"|as (a|an) (result|consequence)|on the (other hand|contrary)|in contrast|by contrast" r"|based on (this|these|the)( results?| findings?)?" r"|the (results?|findings?) (of (this|the present) (study )?)?(show|showed|indicate|indicated" r"|suggest|suggested|revealed)( that)?)\s*[,:]?\s*", re.I) CONN = re.compile(r"\s+(?:and|but|whereas|while|although|though|because|since|which|that)\s+") def restyle(s, trim=0): prev = None while prev != s: # rules can cascade: "However, we found that X" -> "X" prev = s; s = DROP.sub("", s).strip() s = re.sub(r"^(and|but|that|which|to|of|in|for|with|as)\s+", "", s, flags=re.I).strip() if trim and len(s.split()) > trim: # optional clause-level truncation parts = CONN.split(s) if len(parts) > 1: acc, keep = 0, [] for p in parts: if keep and acc + len(p.split()) > trim: break keep.append(p); acc += len(p.split()) if keep: s = " and ".join(keep) s = re.sub(r"\s+", " ", s).strip() return (s[0].upper() + s[1:]) if s else s def decode(docs, sc, budget=110, maxs=3, mmr=0.55, trim=0, E=None): out, c = [], 0 for d in docs: n = d["n"]; s = sc[c:c+n]; base = c; c += n order = list(np.argsort(-s)); pick, wc = [], 0 for i in order: if len(pick) >= maxs: break w = len(d["tk"][i]) if wc + w > budget and pick: continue if mmr < 1.0 and pick: if E is not None: sim = max(float(E[base+i] @ E[base+j]) for j in pick) else: a = d["cw"][i] sim = max(len(a & d["cw"][j]) / max(len(a | d["cw"][j]), 1) for j in pick) if sim > mmr: continue pick.append(i); wc += w if not pick: pick = [order[0]] pick.sort() # restore reading order segs = [x for x in (restyle(d["S"][i], trim) for i in pick) if len(x.split()) >= 3] out.append((". ".join(segs) + ".") if segs else d["S"][pick[0]] + ".") return out # ---------------------------------------------------------------- scoring and rank fusion def prob(m, X): return (m.predict_proba(X)[:, 1] if hasattr(m, "predict_proba") else 1 / (1 + np.exp(-m.decision_function(X)))) def fuse(docs, D, names, k=60): z = np.zeros(sum(d["n"] for d in docs)) for n in names: s = D[n]; c = 0 for d in docs: m = d["n"] o = np.argsort(-s[c:c+m]); rr = np.empty(m, int); rr[o] = np.arange(m) z[c:c+m] += 1.0 / (k + rr + 1); c += m return z # ---------------------------------------------------------------- the model class SciHighExtractor: """Loads the saved artifacts and turns raw abstracts into highlights.""" def __init__(self, folder, encoder=None, device="cpu"): self.folder = folder self.config = json.load(open(os.path.join(folder, "config.json"))) self.tv = load_joblib(folder, "tfidf.joblib") self.models = {n: load_joblib(folder, f"clf_{n}.joblib") for n in self.config["ensemble"]} self.use_emb = self.config["embeddings_used"] if self.use_emb: self.svd = load_joblib(folder, "svd.joblib") self.proto = load_npy(folder, "proto.npy") self.Ctr = load_npy(folder, "train_centroids.npy") self.Hemb = load_npy(folder, "train_highlight_emb.npy") self._enc_src, self._enc, self._device = encoder or self.config["encoder"], None, device lex = os.path.join(folder, "highlight_lexicon.json") self.lexicon = json.load(open(lex)) if os.path.exists(lex) else {} self._voc, self._idf = self.tv.vocabulary_, self.tv.idf_ self._idf_oov = float(self._idf.max()) @classmethod def from_pretrained(cls, path_or_repo_id, encoder=None, device="cpu", **hub_kwargs): """path_or_repo_id: a local folder, or a Hugging Face repo id such as "user/scihigh2026-extractive-ranker". encoder: optional local path to all-MiniLM-L6-v2 (default: downloaded from the Hub by name).""" folder = path_or_repo_id if not os.path.isdir(folder): from huggingface_hub import snapshot_download folder = snapshot_download(path_or_repo_id, **hub_kwargs) return cls(folder, encoder=encoder, device=device) def _embed(self, texts): if self._enc is None: self._enc = load_encoder(self._enc_src, self._device) return self._enc.encode(list(texts), batch_size=256, convert_to_numpy=True, normalize_embeddings=True, show_progress_bar=False) def features(self, docs): """Design matrix (one row per sentence) and the sentence embeddings used by the MMR filter.""" F, E = featurize(docs, self.tv), None if self.use_emb: E = self._embed([s for d in docs for s in d["S"]]) proj = pd.DataFrame(self.svd.transform(E), columns=[f"sv{i}" for i in range(32)]) F = pd.concat([F, emb_feats(E, offsets(docs), self.Ctr, self.Hemb, self.proto), proj], axis=1) X = F[self.config["feature_columns"]].astype(np.float32).fillna(0) return X, E def sentence_scores(self, docs, per_model=False): X, E = self.features(docs) S = {n: prob(m, X) for n, m in self.models.items()} z = fuse(docs, S, self.config["ensemble"], k=self.config["rrf_k"]) return (z, E, S) if per_model else (z, E) def _cfg(self, overrides): cfg = dict(self.config["decoder"]) cfg.update({k: v for k, v in overrides.items() if v is not None and k in cfg}) return cfg # ---- word importance (TF-IDF x IDF within the abstract + highlight log-odds lexicon) def idf_of(self, w): parts = [p for p in w.lower().replace("'", "").split("-") if p] v = [self._idf[self._voc[p]] if p in self._voc else self._idf_oov for p in parts] or [self._idf_oov] return float(np.mean(v)) def word_weights(self, doc): from concise import doc_weights return doc_weights(doc["S"], self.idf_of, lambda w: self.lexicon.get(w.lower(), 0.0)) # ---- output modes def predict(self, abstracts, **decoder): """FIRE mode: list of abstract strings -> list of highlight strings (tuned extractive decoder). Optional decoder overrides: budget (words), maxs (sentences), mmr (redundancy threshold), trim.""" if isinstance(abstracts, str): abstracts = [abstracts] docs = [make_doc(a) for a in abstracts] z, E = self.sentence_scores(docs) return decode(docs, z, **self._cfg(decoder), E=E) def concise(self, abstracts, target=18, **decoder): """Concise mode: the same sentence choice, each sentence compressed to about `target` words.""" from concise import compress if isinstance(abstracts, str): abstracts = [abstracts] docs = [make_doc(a) for a in abstracts] z, E = self.sentence_scores(docs) cfg, out, c = self._cfg(decoder), [], 0 for d in docs: n = d["n"]; picks = pick_indices(d, z[c:c + n], None if E is None else E[c:c + n], **cfg); c += n w = self.word_weights(d) hs = [compress(restyle(d["S"][i]), w, target) for i in picks] hs = [h for h in hs if len(h.split()) >= 3] or [d["S"][picks[0]]] out.append(". ".join(hs) + ".") return out def analyse(self, abstract, target=18, n_keywords=4, **decoder): """One abstract -> dict with the FIRE output, the Concise output and a per-sentence table (fused rank, each model's probability, a compressed version of every sentence and its keywords).""" from concise import compress, WORD, FUNC docs = [make_doc(abstract)]; d = docs[0] z, E, S = self.sentence_scores(docs, per_model=True) cfg = self._cfg(decoder) picks = pick_indices(d, z, E, **cfg) w = self.word_weights(d) summaries = [compress(restyle(s), w, target) for s in d["S"]] def keywords(s): ws = [] for t in WORD.findall(s): t = t.lower().strip(".") if t not in FUNC and len(t) > 2 and t not in ws: ws.append(t) return ", ".join(sorted(ws, key=lambda t: -w.get(t, 0.0))[:n_keywords]) order = np.argsort(-z); rank = np.empty(len(z), int); rank[order] = np.arange(1, len(z) + 1) table = pd.DataFrame({"#": np.arange(1, len(z) + 1), "picked": ["yes" if i in picks else "" for i in range(len(z))], "rank": rank, "sentence": d["S"], "summary": summaries, "keywords": [keywords(s) for s in d["S"]], "RRF": np.round(z.astype(float), 4)}) for n in self.config["ensemble"]: table[n] = np.round(np.asarray(S[n], dtype=float), 3) concise_hl = [summaries[i] for i in picks if len(summaries[i].split()) >= 3] or [d["S"][picks[0]]] return {"fire": decode(docs, z, **cfg, E=E)[0], "concise": concise_hl, "picks": picks, "sentences": d["S"], "table": table} def explain(self, abstract, **decoder): """One abstract -> (FIRE highlights, per-sentence table). Kept for backward compatibility.""" r = self.analyse(abstract, **decoder) return r["fire"], r["table"] def pick_indices(d, s, E=None, budget=110, maxs=3, mmr=0.55, trim=0): """The sentence-selection half of decode() for a single document, returning indices (for display).""" order = list(np.argsort(-s)); pick, wc = [], 0 for i in order: if len(pick) >= maxs: break w = len(d["tk"][i]) if wc + w > budget and pick: continue if mmr < 1.0 and pick: if E is not None: sim = max(float(E[i] @ E[j]) for j in pick) else: a = d["cw"][i] sim = max(len(a & d["cw"][j]) / max(len(a | d["cw"][j]), 1) for j in pick) if sim > mmr: continue pick.append(i); wc += w return sorted(pick) if pick else [order[0]] if __name__ == "__main__": import sys m = SciHighExtractor.from_pretrained(os.path.dirname(os.path.abspath(__file__))) text = " ".join(sys.argv[1:]) or sys.stdin.read() print(m.predict([text])[0])