Highlight_Generator / scihigh_model.py
Akash212's picture
Add Concise, LLM rewrite and fine-tuned modes, word importance, per-sentence summaries
4d72966 verified
Raw History Blame Contribute Delete
21.3 kB
"""
SciHigh 2026 - Supervised extractive research-highlight generator (CPU-only).
FIRE 2026 SciHigh, Subtask 1: research highlight generation from MixSub abstracts.
Pipeline for one abstract
1. clean the text and split it into sentences
2. 31 hand-built features per sentence (position, length, lexical, discourse cues, TF-IDF graph)
3. 36 embedding features from all-MiniLM-L6-v2 (4 similarity features + 32-dim SVD projection),
including a retrieval feature built from the 10 nearest training abstracts
4. five classifiers score every sentence; their rankings are merged by Reciprocal Rank Fusion
5. a decoder picks sentences under a word budget with an MMR redundancy filter, restores
reading order and rewrites each sentence into highlight style
Usage
from scihigh_model import SciHighExtractor
model = SciHighExtractor.from_pretrained(".") # local folder or a Hugging Face repo id
print(model.predict(["<abstract text>"])[0])
The text-processing functions below are copied unchanged from SciHigh_2026_CPU_Hybrid.ipynb, so
the saved models see exactly the features they were trained on.
"""
import os, re, io, json, glob, collections
import numpy as np, pandas as pd
from sklearn.preprocessing import normalize
# ---------------------------------------------------------------- file helpers
# Files larger than PART_LIMIT are stored as byte chunks named <stem>.part00<ext>, <stem>.part01<ext>, ...
# so every file stays under 10 MB (browser upload limit). The loader joins them back byte for byte.
PART_LIMIT = 9_500_000
def save_bytes(data, folder, name, limit=PART_LIMIT):
stem, ext = os.path.splitext(name)
for old in glob.glob(os.path.join(folder, f"{stem}.part*{ext}")): os.remove(old)
if len(data) <= limit:
open(os.path.join(folder, name), "wb").write(data); return [name]
names = []
for i in range(0, len(data), limit):
nm = f"{stem}.part{i // limit:02d}{ext}"
open(os.path.join(folder, nm), "wb").write(data[i:i + limit]); names.append(nm)
return names
def read_bytes(folder, name):
p = os.path.join(folder, name)
if os.path.exists(p): return open(p, "rb").read()
stem, ext = os.path.splitext(name)
parts = sorted(glob.glob(os.path.join(folder, f"{stem}.part*{ext}")))
if not parts: raise FileNotFoundError(p)
return b"".join(open(q, "rb").read() for q in parts)
def load_joblib(folder, name):
import joblib
return joblib.load(io.BytesIO(read_bytes(folder, name)))
def load_npy(folder, name):
return np.load(io.BytesIO(read_bytes(folder, name)))
# ---------------------------------------------------------------- cleaning and sentence splitting
def clean(t):
t = str(t)
t = re.sub(r"\.\s*None\.\s*", " ", t) # "value . None . unit" artifact
t = re.sub(r"\bNone\b", " ", t)
t = re.sub(r"\s*\.\s*\.", ".", t) # collapse double periods left behind
return re.sub(r"\s+", " ", t).strip()
def sent_split(t):
# split on " . " (MixSub style) or on ". " followed by a capital
parts = re.split(r"(?<=[A-Za-z0-9\)\]\%])\s+\.\s+|(?<=[a-z0-9\)\]\%])\.\s+(?=[A-Z])", str(t).strip())
out = []
for p in parts:
if not p: continue
p = p.strip().rstrip(".").strip()
if len(p.split()) >= 3:
out.append(p)
elif out:
out[-1] += " " + p # glue fragments onto the previous sentence
return out or [str(t).rstrip(".").strip()]
STOP = set('''a about above after again against all also am among an and any are as at be because been before
being below between both but by can cannot could did do does doing during each few for from further had has
have having he her here hers herself him himself his how i if in into is it its itself me more most my myself
no nor not now of off on once only or other others our ours out over own same she should so some such than
that the their theirs them themselves then there these they this those through to too under until up very
was we were what when where which while who whom why will with would you your yours'''.split())
TOK = re.compile(r"[a-z0-9]+")
def toks(t): return TOK.findall(str(t).lower())
def content(t): return [w for w in toks(t) if w not in STOP and len(w) > 2]
def make_doc(abstract):
"""One cleaned abstract -> the document dict used by featurize() and decode()."""
S = sent_split(clean(abstract)); n = len(S)
return dict(S=S, n=n, tk=[toks(s) for s in S], cw=[set(content(s)) for s in S])
# ---------------------------------------------------------------- hand-built sentence features
CUES = {
"res": r"\b(result|show|showed|shown|found|find|reveal|demonstrat|indicat|observ|obtain|achiev|improv|outperform|increas|decreas|significant)",
"meth": r"\b(propos|present|develop|design|introduc|novel|new |method|approach|model|framework|algorithm|technique|scheme|system|strateg)",
"purp": r"\b(aim|objective|purpose|goal|investigat|examin|explor|evaluat|assess|analys|analyz)",
"bkg": r"\b(previous|literature|recently|recent years|has been|have been|widely|however|although|despite|remains|little is known|few studies)",
"conc": r"\b(conclu|suggest|impl|therefore|thus|hence|overall|consequently|potential|promis|can be used|may be)",
"quant": r"(\d+\.?\d*\s?%|p\s?[<=]|\bn\s?=|\bmean\b|\bmedian\b|\bratio\b|\brate\b|fold)",
}
LEAD_RE = (r"^(in this (study|paper|work|article|research)|in the present|the (aim|purpose|objective|goal) of"
r"|here we|in addition|moreover|furthermore|therefore|thus|however|finally|overall|first|second"
r"|third|also|based on)")
def featurize(docs, tv):
rows = []
for d in docs:
S, n, TKs, CWs = d["S"], d["n"], d["tk"], d["cw"]
tw = max(sum(len(t) for t in TKs), 1)
X = normalize(tv.transform(S))
cen = (X @ normalize(np.asarray(X.mean(0))).T).ravel() # cosine to abstract centroid
M = (X @ X.T).toarray(); np.fill_diagonal(M, 0); deg = M.sum(1)
P = M / np.maximum(deg[:, None], 1e-9); pr = np.ones(n) / n
for _ in range(20): pr = 0.15 / n + 0.85 * (P.T @ pr) # TextRank power iteration
tsum = np.asarray(X.sum(1)).ravel()
mx = M.max(1) if n > 1 else np.zeros(n)
for i, s in enumerate(S):
sw, cw, low = TKs[i], CWs[i], s.lower()
nov = max((len(cw & CWs[j]) / max(len(cw | CWs[j]), 1) for j in range(i)), default=0.0)
f = {"sid": i, "rel_pos": i / max(n-1, 1), "rel_end": (n-1-i) / max(n-1, 1),
"is_first": int(i == 0), "is_last": int(i == n-1), "is_last2": int(i >= n-2),
"is_first2": int(i <= 1), "nsent": n, "nw": len(sw), "nchar": len(s),
"rel_len": len(sw) / tw,
"awlen": float(np.mean([len(w) for w in sw])) if sw else 0.0,
"stop_ratio": sum(1 for w in sw if w in STOP) / max(len(sw), 1),
"n_content": len(cw), "content_dens": len(cw) / max(len(sw), 1),
"n_digit": sum(c.isdigit() for c in s),
"n_acro": len(re.findall(r"\b[A-Z]{2,}\b", s)), "has_pct": int("%" in s),
"lead": int(bool(re.match(LEAD_RE, low))), "redund": nov,
"tf_cent": cen[i], "tf_deg": deg[i], "textrank": pr[i],
"tf_sum": tsum[i], "tf_maxsim": mx[i]}
for k, pat in CUES.items():
f["cue_" + k] = len(re.findall(pat, low))
rows.append(f)
return pd.DataFrame(rows)
# ---------------------------------------------------------------- embedding features
def offsets(docs):
o, c = [], 0
for d in docs: o.append((c, c + d["n"])); c += d["n"]
return o
def cent(E, O):
return np.vstack([normalize(E[a:b].mean(0).reshape(1, -1)) for a, b in O])
def emb_feats(E, O, Ctr, Hemb, proto, is_train=False, k=10):
out = np.zeros((len(E), 4), np.float32)
sim = cent(E, O) @ Ctr.T
if is_train: np.fill_diagonal(sim, -1) # never retrieve yourself
nn = np.argsort(-sim, 1)[:, :k]
for q, (a, b) in enumerate(O):
X = E[a:b]
out[a:b, 0] = (X @ normalize(X.mean(0).reshape(1, -1)).T).ravel()
out[a:b, 1] = (X @ proto.T).ravel()
M = X @ X.T; np.fill_diagonal(M, 0)
out[a:b, 2] = M.max(1) if b - a > 1 else 0
out[a:b, 3] = (X @ normalize(Hemb[nn[q]].mean(0).reshape(1, -1)).T).ravel()
return pd.DataFrame(out, columns=["e_cent", "e_proto", "e_maxsim", "e_rag"])
def load_encoder(name_or_path="sentence-transformers/all-MiniLM-L6-v2", device="cpu"):
"""all-MiniLM-L6-v2 as a SentenceTransformer: Transformer(max_seq_length=256) -> mean pooling -> L2 norm.
Accepts a Hugging Face id, a full sentence-transformers folder, or a folder that holds only
config.json + model.safetensors + tokenizer.json (the same three modules are then assembled here)."""
from sentence_transformers import SentenceTransformer, models
p = name_or_path
if os.path.isdir(p) and not os.path.exists(os.path.join(p, "modules.json")):
tr = models.Transformer(p, max_seq_length=256, do_lower_case=False)
dim = (tr.get_embedding_dimension() if hasattr(tr, "get_embedding_dimension")
else tr.get_word_embedding_dimension())
pool = models.Pooling(dim, pooling_mode="mean")
return SentenceTransformer(modules=[tr, pool, models.Normalize()], device=device)
return SentenceTransformer(p, device=device)
# ---------------------------------------------------------------- decoder and restyling
DROP = re.compile(
r"^(in this (study|paper|work|article|research|comment|letter|note|review|report|contribution)"
r"|in the (present|current) (study|work|paper|article|research)|in the following"
r"|the (aim|aims|purpose|objective|objectives|goal|goals) of (this|the present|the current) "
r"(study|paper|work|research|article) (was|were|is|are) to"
r"|the (present|current) (study|work|paper) (aims?|aimed|sought|seeks?)( to)?"
r"|here (we|the authors)|in addition|additionally|moreover|furthermore|therefore|thus|hence|however"
r"|nevertheless|nonetheless|meanwhile|finally|overall|consequently|accordingly|notably|importantly"
r"|first(ly)?|second(ly)?|third(ly)?|also|in conclusion|in summary|to summari[sz]e|taken together"
r"|it (was|is|has been|had been) (found|shown|observed|concluded|demonstrated|reported|noted|revealed)( that)?"
r"|we (found|find|show|showed|observe|observed|conclude|concluded|report|reported|note|noted"
r"|demonstrate|demonstrated|argue|suggest|propose)( that| with| in)?"
r"|(the|our|these|this) (results?|findings?|data|analysis|study|work) (show|shows|showed|indicate"
r"|indicated|suggest|suggests|suggested|reveal|revealed|demonstrate|demonstrated|confirm|confirmed)( that)?"
r"|(this|these) (results?|findings?) (suggest|suggests|indicate|indicates|impl(y|ies))( that)?"
r"|as (a|an) (result|consequence)|on the (other hand|contrary)|in contrast|by contrast"
r"|based on (this|these|the)( results?| findings?)?"
r"|the (results?|findings?) (of (this|the present) (study )?)?(show|showed|indicate|indicated"
r"|suggest|suggested|revealed)( that)?)\s*[,:]?\s*", re.I)
CONN = re.compile(r"\s+(?:and|but|whereas|while|although|though|because|since|which|that)\s+")
def restyle(s, trim=0):
prev = None
while prev != s: # rules can cascade: "However, we found that X" -> "X"
prev = s; s = DROP.sub("", s).strip()
s = re.sub(r"^(and|but|that|which|to|of|in|for|with|as)\s+", "", s, flags=re.I).strip()
if trim and len(s.split()) > trim: # optional clause-level truncation
parts = CONN.split(s)
if len(parts) > 1:
acc, keep = 0, []
for p in parts:
if keep and acc + len(p.split()) > trim: break
keep.append(p); acc += len(p.split())
if keep: s = " and ".join(keep)
s = re.sub(r"\s+", " ", s).strip()
return (s[0].upper() + s[1:]) if s else s
def decode(docs, sc, budget=110, maxs=3, mmr=0.55, trim=0, E=None):
out, c = [], 0
for d in docs:
n = d["n"]; s = sc[c:c+n]; base = c; c += n
order = list(np.argsort(-s)); pick, wc = [], 0
for i in order:
if len(pick) >= maxs: break
w = len(d["tk"][i])
if wc + w > budget and pick: continue
if mmr < 1.0 and pick:
if E is not None:
sim = max(float(E[base+i] @ E[base+j]) for j in pick)
else:
a = d["cw"][i]
sim = max(len(a & d["cw"][j]) / max(len(a | d["cw"][j]), 1) for j in pick)
if sim > mmr: continue
pick.append(i); wc += w
if not pick: pick = [order[0]]
pick.sort() # restore reading order
segs = [x for x in (restyle(d["S"][i], trim) for i in pick) if len(x.split()) >= 3]
out.append((". ".join(segs) + ".") if segs else d["S"][pick[0]] + ".")
return out
# ---------------------------------------------------------------- scoring and rank fusion
def prob(m, X):
return (m.predict_proba(X)[:, 1] if hasattr(m, "predict_proba")
else 1 / (1 + np.exp(-m.decision_function(X))))
def fuse(docs, D, names, k=60):
z = np.zeros(sum(d["n"] for d in docs))
for n in names:
s = D[n]; c = 0
for d in docs:
m = d["n"]
o = np.argsort(-s[c:c+m]); rr = np.empty(m, int); rr[o] = np.arange(m)
z[c:c+m] += 1.0 / (k + rr + 1); c += m
return z
# ---------------------------------------------------------------- the model
class SciHighExtractor:
"""Loads the saved artifacts and turns raw abstracts into highlights."""
def __init__(self, folder, encoder=None, device="cpu"):
self.folder = folder
self.config = json.load(open(os.path.join(folder, "config.json")))
self.tv = load_joblib(folder, "tfidf.joblib")
self.models = {n: load_joblib(folder, f"clf_{n}.joblib") for n in self.config["ensemble"]}
self.use_emb = self.config["embeddings_used"]
if self.use_emb:
self.svd = load_joblib(folder, "svd.joblib")
self.proto = load_npy(folder, "proto.npy")
self.Ctr = load_npy(folder, "train_centroids.npy")
self.Hemb = load_npy(folder, "train_highlight_emb.npy")
self._enc_src, self._enc, self._device = encoder or self.config["encoder"], None, device
lex = os.path.join(folder, "highlight_lexicon.json")
self.lexicon = json.load(open(lex)) if os.path.exists(lex) else {}
self._voc, self._idf = self.tv.vocabulary_, self.tv.idf_
self._idf_oov = float(self._idf.max())
@classmethod
def from_pretrained(cls, path_or_repo_id, encoder=None, device="cpu", **hub_kwargs):
"""path_or_repo_id: a local folder, or a Hugging Face repo id such as "user/scihigh2026-extractive-ranker".
encoder: optional local path to all-MiniLM-L6-v2 (default: downloaded from the Hub by name)."""
folder = path_or_repo_id
if not os.path.isdir(folder):
from huggingface_hub import snapshot_download
folder = snapshot_download(path_or_repo_id, **hub_kwargs)
return cls(folder, encoder=encoder, device=device)
def _embed(self, texts):
if self._enc is None:
self._enc = load_encoder(self._enc_src, self._device)
return self._enc.encode(list(texts), batch_size=256, convert_to_numpy=True,
normalize_embeddings=True, show_progress_bar=False)
def features(self, docs):
"""Design matrix (one row per sentence) and the sentence embeddings used by the MMR filter."""
F, E = featurize(docs, self.tv), None
if self.use_emb:
E = self._embed([s for d in docs for s in d["S"]])
proj = pd.DataFrame(self.svd.transform(E), columns=[f"sv{i}" for i in range(32)])
F = pd.concat([F, emb_feats(E, offsets(docs), self.Ctr, self.Hemb, self.proto), proj], axis=1)
X = F[self.config["feature_columns"]].astype(np.float32).fillna(0)
return X, E
def sentence_scores(self, docs, per_model=False):
X, E = self.features(docs)
S = {n: prob(m, X) for n, m in self.models.items()}
z = fuse(docs, S, self.config["ensemble"], k=self.config["rrf_k"])
return (z, E, S) if per_model else (z, E)
def _cfg(self, overrides):
cfg = dict(self.config["decoder"])
cfg.update({k: v for k, v in overrides.items() if v is not None and k in cfg})
return cfg
# ---- word importance (TF-IDF x IDF within the abstract + highlight log-odds lexicon)
def idf_of(self, w):
parts = [p for p in w.lower().replace("'", "").split("-") if p]
v = [self._idf[self._voc[p]] if p in self._voc else self._idf_oov for p in parts] or [self._idf_oov]
return float(np.mean(v))
def word_weights(self, doc):
from concise import doc_weights
return doc_weights(doc["S"], self.idf_of, lambda w: self.lexicon.get(w.lower(), 0.0))
# ---- output modes
def predict(self, abstracts, **decoder):
"""FIRE mode: list of abstract strings -> list of highlight strings (tuned extractive decoder).
Optional decoder overrides: budget (words), maxs (sentences), mmr (redundancy threshold), trim."""
if isinstance(abstracts, str): abstracts = [abstracts]
docs = [make_doc(a) for a in abstracts]
z, E = self.sentence_scores(docs)
return decode(docs, z, **self._cfg(decoder), E=E)
def concise(self, abstracts, target=18, **decoder):
"""Concise mode: the same sentence choice, each sentence compressed to about `target` words."""
from concise import compress
if isinstance(abstracts, str): abstracts = [abstracts]
docs = [make_doc(a) for a in abstracts]
z, E = self.sentence_scores(docs)
cfg, out, c = self._cfg(decoder), [], 0
for d in docs:
n = d["n"]; picks = pick_indices(d, z[c:c + n], None if E is None else E[c:c + n], **cfg); c += n
w = self.word_weights(d)
hs = [compress(restyle(d["S"][i]), w, target) for i in picks]
hs = [h for h in hs if len(h.split()) >= 3] or [d["S"][picks[0]]]
out.append(". ".join(hs) + ".")
return out
def analyse(self, abstract, target=18, n_keywords=4, **decoder):
"""One abstract -> dict with the FIRE output, the Concise output and a per-sentence table
(fused rank, each model's probability, a compressed version of every sentence and its keywords)."""
from concise import compress, WORD, FUNC
docs = [make_doc(abstract)]; d = docs[0]
z, E, S = self.sentence_scores(docs, per_model=True)
cfg = self._cfg(decoder)
picks = pick_indices(d, z, E, **cfg)
w = self.word_weights(d)
summaries = [compress(restyle(s), w, target) for s in d["S"]]
def keywords(s):
ws = []
for t in WORD.findall(s):
t = t.lower().strip(".")
if t not in FUNC and len(t) > 2 and t not in ws: ws.append(t)
return ", ".join(sorted(ws, key=lambda t: -w.get(t, 0.0))[:n_keywords])
order = np.argsort(-z); rank = np.empty(len(z), int); rank[order] = np.arange(1, len(z) + 1)
table = pd.DataFrame({"#": np.arange(1, len(z) + 1), "picked": ["yes" if i in picks else "" for i in range(len(z))],
"rank": rank, "sentence": d["S"], "summary": summaries,
"keywords": [keywords(s) for s in d["S"]], "RRF": np.round(z.astype(float), 4)})
for n in self.config["ensemble"]: table[n] = np.round(np.asarray(S[n], dtype=float), 3)
concise_hl = [summaries[i] for i in picks if len(summaries[i].split()) >= 3] or [d["S"][picks[0]]]
return {"fire": decode(docs, z, **cfg, E=E)[0], "concise": concise_hl, "picks": picks,
"sentences": d["S"], "table": table}
def explain(self, abstract, **decoder):
"""One abstract -> (FIRE highlights, per-sentence table). Kept for backward compatibility."""
r = self.analyse(abstract, **decoder)
return r["fire"], r["table"]
def pick_indices(d, s, E=None, budget=110, maxs=3, mmr=0.55, trim=0):
"""The sentence-selection half of decode() for a single document, returning indices (for display)."""
order = list(np.argsort(-s)); pick, wc = [], 0
for i in order:
if len(pick) >= maxs: break
w = len(d["tk"][i])
if wc + w > budget and pick: continue
if mmr < 1.0 and pick:
if E is not None:
sim = max(float(E[i] @ E[j]) for j in pick)
else:
a = d["cw"][i]
sim = max(len(a & d["cw"][j]) / max(len(a | d["cw"][j]), 1) for j in pick)
if sim > mmr: continue
pick.append(i); wc += w
return sorted(pick) if pick else [order[0]]
if __name__ == "__main__":
import sys
m = SciHighExtractor.from_pretrained(os.path.dirname(os.path.abspath(__file__)))
text = " ".join(sys.argv[1:]) or sys.stdin.read()
print(m.predict([text])[0])