Spaces:
Running on Zero
Running on Zero
File size: 21,258 Bytes
d7b7e82 4d72966 d7b7e82 4d72966 d7b7e82 4d72966 d7b7e82 4d72966 d7b7e82 4d72966 d7b7e82 4d72966 d7b7e82 4d72966 d7b7e82 4d72966 d7b7e82 4d72966 d7b7e82 4d72966 d7b7e82 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 | """
SciHigh 2026 - Supervised extractive research-highlight generator (CPU-only).
FIRE 2026 SciHigh, Subtask 1: research highlight generation from MixSub abstracts.
Pipeline for one abstract
1. clean the text and split it into sentences
2. 31 hand-built features per sentence (position, length, lexical, discourse cues, TF-IDF graph)
3. 36 embedding features from all-MiniLM-L6-v2 (4 similarity features + 32-dim SVD projection),
including a retrieval feature built from the 10 nearest training abstracts
4. five classifiers score every sentence; their rankings are merged by Reciprocal Rank Fusion
5. a decoder picks sentences under a word budget with an MMR redundancy filter, restores
reading order and rewrites each sentence into highlight style
Usage
from scihigh_model import SciHighExtractor
model = SciHighExtractor.from_pretrained(".") # local folder or a Hugging Face repo id
print(model.predict(["<abstract text>"])[0])
The text-processing functions below are copied unchanged from SciHigh_2026_CPU_Hybrid.ipynb, so
the saved models see exactly the features they were trained on.
"""
import os, re, io, json, glob, collections
import numpy as np, pandas as pd
from sklearn.preprocessing import normalize
# ---------------------------------------------------------------- file helpers
# Files larger than PART_LIMIT are stored as byte chunks named <stem>.part00<ext>, <stem>.part01<ext>, ...
# so every file stays under 10 MB (browser upload limit). The loader joins them back byte for byte.
PART_LIMIT = 9_500_000
def save_bytes(data, folder, name, limit=PART_LIMIT):
stem, ext = os.path.splitext(name)
for old in glob.glob(os.path.join(folder, f"{stem}.part*{ext}")): os.remove(old)
if len(data) <= limit:
open(os.path.join(folder, name), "wb").write(data); return [name]
names = []
for i in range(0, len(data), limit):
nm = f"{stem}.part{i // limit:02d}{ext}"
open(os.path.join(folder, nm), "wb").write(data[i:i + limit]); names.append(nm)
return names
def read_bytes(folder, name):
p = os.path.join(folder, name)
if os.path.exists(p): return open(p, "rb").read()
stem, ext = os.path.splitext(name)
parts = sorted(glob.glob(os.path.join(folder, f"{stem}.part*{ext}")))
if not parts: raise FileNotFoundError(p)
return b"".join(open(q, "rb").read() for q in parts)
def load_joblib(folder, name):
import joblib
return joblib.load(io.BytesIO(read_bytes(folder, name)))
def load_npy(folder, name):
return np.load(io.BytesIO(read_bytes(folder, name)))
# ---------------------------------------------------------------- cleaning and sentence splitting
def clean(t):
t = str(t)
t = re.sub(r"\.\s*None\.\s*", " ", t) # "value . None . unit" artifact
t = re.sub(r"\bNone\b", " ", t)
t = re.sub(r"\s*\.\s*\.", ".", t) # collapse double periods left behind
return re.sub(r"\s+", " ", t).strip()
def sent_split(t):
# split on " . " (MixSub style) or on ". " followed by a capital
parts = re.split(r"(?<=[A-Za-z0-9\)\]\%])\s+\.\s+|(?<=[a-z0-9\)\]\%])\.\s+(?=[A-Z])", str(t).strip())
out = []
for p in parts:
if not p: continue
p = p.strip().rstrip(".").strip()
if len(p.split()) >= 3:
out.append(p)
elif out:
out[-1] += " " + p # glue fragments onto the previous sentence
return out or [str(t).rstrip(".").strip()]
STOP = set('''a about above after again against all also am among an and any are as at be because been before
being below between both but by can cannot could did do does doing during each few for from further had has
have having he her here hers herself him himself his how i if in into is it its itself me more most my myself
no nor not now of off on once only or other others our ours out over own same she should so some such than
that the their theirs them themselves then there these they this those through to too under until up very
was we were what when where which while who whom why will with would you your yours'''.split())
TOK = re.compile(r"[a-z0-9]+")
def toks(t): return TOK.findall(str(t).lower())
def content(t): return [w for w in toks(t) if w not in STOP and len(w) > 2]
def make_doc(abstract):
"""One cleaned abstract -> the document dict used by featurize() and decode()."""
S = sent_split(clean(abstract)); n = len(S)
return dict(S=S, n=n, tk=[toks(s) for s in S], cw=[set(content(s)) for s in S])
# ---------------------------------------------------------------- hand-built sentence features
CUES = {
"res": r"\b(result|show|showed|shown|found|find|reveal|demonstrat|indicat|observ|obtain|achiev|improv|outperform|increas|decreas|significant)",
"meth": r"\b(propos|present|develop|design|introduc|novel|new |method|approach|model|framework|algorithm|technique|scheme|system|strateg)",
"purp": r"\b(aim|objective|purpose|goal|investigat|examin|explor|evaluat|assess|analys|analyz)",
"bkg": r"\b(previous|literature|recently|recent years|has been|have been|widely|however|although|despite|remains|little is known|few studies)",
"conc": r"\b(conclu|suggest|impl|therefore|thus|hence|overall|consequently|potential|promis|can be used|may be)",
"quant": r"(\d+\.?\d*\s?%|p\s?[<=]|\bn\s?=|\bmean\b|\bmedian\b|\bratio\b|\brate\b|fold)",
}
LEAD_RE = (r"^(in this (study|paper|work|article|research)|in the present|the (aim|purpose|objective|goal) of"
r"|here we|in addition|moreover|furthermore|therefore|thus|however|finally|overall|first|second"
r"|third|also|based on)")
def featurize(docs, tv):
rows = []
for d in docs:
S, n, TKs, CWs = d["S"], d["n"], d["tk"], d["cw"]
tw = max(sum(len(t) for t in TKs), 1)
X = normalize(tv.transform(S))
cen = (X @ normalize(np.asarray(X.mean(0))).T).ravel() # cosine to abstract centroid
M = (X @ X.T).toarray(); np.fill_diagonal(M, 0); deg = M.sum(1)
P = M / np.maximum(deg[:, None], 1e-9); pr = np.ones(n) / n
for _ in range(20): pr = 0.15 / n + 0.85 * (P.T @ pr) # TextRank power iteration
tsum = np.asarray(X.sum(1)).ravel()
mx = M.max(1) if n > 1 else np.zeros(n)
for i, s in enumerate(S):
sw, cw, low = TKs[i], CWs[i], s.lower()
nov = max((len(cw & CWs[j]) / max(len(cw | CWs[j]), 1) for j in range(i)), default=0.0)
f = {"sid": i, "rel_pos": i / max(n-1, 1), "rel_end": (n-1-i) / max(n-1, 1),
"is_first": int(i == 0), "is_last": int(i == n-1), "is_last2": int(i >= n-2),
"is_first2": int(i <= 1), "nsent": n, "nw": len(sw), "nchar": len(s),
"rel_len": len(sw) / tw,
"awlen": float(np.mean([len(w) for w in sw])) if sw else 0.0,
"stop_ratio": sum(1 for w in sw if w in STOP) / max(len(sw), 1),
"n_content": len(cw), "content_dens": len(cw) / max(len(sw), 1),
"n_digit": sum(c.isdigit() for c in s),
"n_acro": len(re.findall(r"\b[A-Z]{2,}\b", s)), "has_pct": int("%" in s),
"lead": int(bool(re.match(LEAD_RE, low))), "redund": nov,
"tf_cent": cen[i], "tf_deg": deg[i], "textrank": pr[i],
"tf_sum": tsum[i], "tf_maxsim": mx[i]}
for k, pat in CUES.items():
f["cue_" + k] = len(re.findall(pat, low))
rows.append(f)
return pd.DataFrame(rows)
# ---------------------------------------------------------------- embedding features
def offsets(docs):
o, c = [], 0
for d in docs: o.append((c, c + d["n"])); c += d["n"]
return o
def cent(E, O):
return np.vstack([normalize(E[a:b].mean(0).reshape(1, -1)) for a, b in O])
def emb_feats(E, O, Ctr, Hemb, proto, is_train=False, k=10):
out = np.zeros((len(E), 4), np.float32)
sim = cent(E, O) @ Ctr.T
if is_train: np.fill_diagonal(sim, -1) # never retrieve yourself
nn = np.argsort(-sim, 1)[:, :k]
for q, (a, b) in enumerate(O):
X = E[a:b]
out[a:b, 0] = (X @ normalize(X.mean(0).reshape(1, -1)).T).ravel()
out[a:b, 1] = (X @ proto.T).ravel()
M = X @ X.T; np.fill_diagonal(M, 0)
out[a:b, 2] = M.max(1) if b - a > 1 else 0
out[a:b, 3] = (X @ normalize(Hemb[nn[q]].mean(0).reshape(1, -1)).T).ravel()
return pd.DataFrame(out, columns=["e_cent", "e_proto", "e_maxsim", "e_rag"])
def load_encoder(name_or_path="sentence-transformers/all-MiniLM-L6-v2", device="cpu"):
"""all-MiniLM-L6-v2 as a SentenceTransformer: Transformer(max_seq_length=256) -> mean pooling -> L2 norm.
Accepts a Hugging Face id, a full sentence-transformers folder, or a folder that holds only
config.json + model.safetensors + tokenizer.json (the same three modules are then assembled here)."""
from sentence_transformers import SentenceTransformer, models
p = name_or_path
if os.path.isdir(p) and not os.path.exists(os.path.join(p, "modules.json")):
tr = models.Transformer(p, max_seq_length=256, do_lower_case=False)
dim = (tr.get_embedding_dimension() if hasattr(tr, "get_embedding_dimension")
else tr.get_word_embedding_dimension())
pool = models.Pooling(dim, pooling_mode="mean")
return SentenceTransformer(modules=[tr, pool, models.Normalize()], device=device)
return SentenceTransformer(p, device=device)
# ---------------------------------------------------------------- decoder and restyling
DROP = re.compile(
r"^(in this (study|paper|work|article|research|comment|letter|note|review|report|contribution)"
r"|in the (present|current) (study|work|paper|article|research)|in the following"
r"|the (aim|aims|purpose|objective|objectives|goal|goals) of (this|the present|the current) "
r"(study|paper|work|research|article) (was|were|is|are) to"
r"|the (present|current) (study|work|paper) (aims?|aimed|sought|seeks?)( to)?"
r"|here (we|the authors)|in addition|additionally|moreover|furthermore|therefore|thus|hence|however"
r"|nevertheless|nonetheless|meanwhile|finally|overall|consequently|accordingly|notably|importantly"
r"|first(ly)?|second(ly)?|third(ly)?|also|in conclusion|in summary|to summari[sz]e|taken together"
r"|it (was|is|has been|had been) (found|shown|observed|concluded|demonstrated|reported|noted|revealed)( that)?"
r"|we (found|find|show|showed|observe|observed|conclude|concluded|report|reported|note|noted"
r"|demonstrate|demonstrated|argue|suggest|propose)( that| with| in)?"
r"|(the|our|these|this) (results?|findings?|data|analysis|study|work) (show|shows|showed|indicate"
r"|indicated|suggest|suggests|suggested|reveal|revealed|demonstrate|demonstrated|confirm|confirmed)( that)?"
r"|(this|these) (results?|findings?) (suggest|suggests|indicate|indicates|impl(y|ies))( that)?"
r"|as (a|an) (result|consequence)|on the (other hand|contrary)|in contrast|by contrast"
r"|based on (this|these|the)( results?| findings?)?"
r"|the (results?|findings?) (of (this|the present) (study )?)?(show|showed|indicate|indicated"
r"|suggest|suggested|revealed)( that)?)\s*[,:]?\s*", re.I)
CONN = re.compile(r"\s+(?:and|but|whereas|while|although|though|because|since|which|that)\s+")
def restyle(s, trim=0):
prev = None
while prev != s: # rules can cascade: "However, we found that X" -> "X"
prev = s; s = DROP.sub("", s).strip()
s = re.sub(r"^(and|but|that|which|to|of|in|for|with|as)\s+", "", s, flags=re.I).strip()
if trim and len(s.split()) > trim: # optional clause-level truncation
parts = CONN.split(s)
if len(parts) > 1:
acc, keep = 0, []
for p in parts:
if keep and acc + len(p.split()) > trim: break
keep.append(p); acc += len(p.split())
if keep: s = " and ".join(keep)
s = re.sub(r"\s+", " ", s).strip()
return (s[0].upper() + s[1:]) if s else s
def decode(docs, sc, budget=110, maxs=3, mmr=0.55, trim=0, E=None):
out, c = [], 0
for d in docs:
n = d["n"]; s = sc[c:c+n]; base = c; c += n
order = list(np.argsort(-s)); pick, wc = [], 0
for i in order:
if len(pick) >= maxs: break
w = len(d["tk"][i])
if wc + w > budget and pick: continue
if mmr < 1.0 and pick:
if E is not None:
sim = max(float(E[base+i] @ E[base+j]) for j in pick)
else:
a = d["cw"][i]
sim = max(len(a & d["cw"][j]) / max(len(a | d["cw"][j]), 1) for j in pick)
if sim > mmr: continue
pick.append(i); wc += w
if not pick: pick = [order[0]]
pick.sort() # restore reading order
segs = [x for x in (restyle(d["S"][i], trim) for i in pick) if len(x.split()) >= 3]
out.append((". ".join(segs) + ".") if segs else d["S"][pick[0]] + ".")
return out
# ---------------------------------------------------------------- scoring and rank fusion
def prob(m, X):
return (m.predict_proba(X)[:, 1] if hasattr(m, "predict_proba")
else 1 / (1 + np.exp(-m.decision_function(X))))
def fuse(docs, D, names, k=60):
z = np.zeros(sum(d["n"] for d in docs))
for n in names:
s = D[n]; c = 0
for d in docs:
m = d["n"]
o = np.argsort(-s[c:c+m]); rr = np.empty(m, int); rr[o] = np.arange(m)
z[c:c+m] += 1.0 / (k + rr + 1); c += m
return z
# ---------------------------------------------------------------- the model
class SciHighExtractor:
"""Loads the saved artifacts and turns raw abstracts into highlights."""
def __init__(self, folder, encoder=None, device="cpu"):
self.folder = folder
self.config = json.load(open(os.path.join(folder, "config.json")))
self.tv = load_joblib(folder, "tfidf.joblib")
self.models = {n: load_joblib(folder, f"clf_{n}.joblib") for n in self.config["ensemble"]}
self.use_emb = self.config["embeddings_used"]
if self.use_emb:
self.svd = load_joblib(folder, "svd.joblib")
self.proto = load_npy(folder, "proto.npy")
self.Ctr = load_npy(folder, "train_centroids.npy")
self.Hemb = load_npy(folder, "train_highlight_emb.npy")
self._enc_src, self._enc, self._device = encoder or self.config["encoder"], None, device
lex = os.path.join(folder, "highlight_lexicon.json")
self.lexicon = json.load(open(lex)) if os.path.exists(lex) else {}
self._voc, self._idf = self.tv.vocabulary_, self.tv.idf_
self._idf_oov = float(self._idf.max())
@classmethod
def from_pretrained(cls, path_or_repo_id, encoder=None, device="cpu", **hub_kwargs):
"""path_or_repo_id: a local folder, or a Hugging Face repo id such as "user/scihigh2026-extractive-ranker".
encoder: optional local path to all-MiniLM-L6-v2 (default: downloaded from the Hub by name)."""
folder = path_or_repo_id
if not os.path.isdir(folder):
from huggingface_hub import snapshot_download
folder = snapshot_download(path_or_repo_id, **hub_kwargs)
return cls(folder, encoder=encoder, device=device)
def _embed(self, texts):
if self._enc is None:
self._enc = load_encoder(self._enc_src, self._device)
return self._enc.encode(list(texts), batch_size=256, convert_to_numpy=True,
normalize_embeddings=True, show_progress_bar=False)
def features(self, docs):
"""Design matrix (one row per sentence) and the sentence embeddings used by the MMR filter."""
F, E = featurize(docs, self.tv), None
if self.use_emb:
E = self._embed([s for d in docs for s in d["S"]])
proj = pd.DataFrame(self.svd.transform(E), columns=[f"sv{i}" for i in range(32)])
F = pd.concat([F, emb_feats(E, offsets(docs), self.Ctr, self.Hemb, self.proto), proj], axis=1)
X = F[self.config["feature_columns"]].astype(np.float32).fillna(0)
return X, E
def sentence_scores(self, docs, per_model=False):
X, E = self.features(docs)
S = {n: prob(m, X) for n, m in self.models.items()}
z = fuse(docs, S, self.config["ensemble"], k=self.config["rrf_k"])
return (z, E, S) if per_model else (z, E)
def _cfg(self, overrides):
cfg = dict(self.config["decoder"])
cfg.update({k: v for k, v in overrides.items() if v is not None and k in cfg})
return cfg
# ---- word importance (TF-IDF x IDF within the abstract + highlight log-odds lexicon)
def idf_of(self, w):
parts = [p for p in w.lower().replace("'", "").split("-") if p]
v = [self._idf[self._voc[p]] if p in self._voc else self._idf_oov for p in parts] or [self._idf_oov]
return float(np.mean(v))
def word_weights(self, doc):
from concise import doc_weights
return doc_weights(doc["S"], self.idf_of, lambda w: self.lexicon.get(w.lower(), 0.0))
# ---- output modes
def predict(self, abstracts, **decoder):
"""FIRE mode: list of abstract strings -> list of highlight strings (tuned extractive decoder).
Optional decoder overrides: budget (words), maxs (sentences), mmr (redundancy threshold), trim."""
if isinstance(abstracts, str): abstracts = [abstracts]
docs = [make_doc(a) for a in abstracts]
z, E = self.sentence_scores(docs)
return decode(docs, z, **self._cfg(decoder), E=E)
def concise(self, abstracts, target=18, **decoder):
"""Concise mode: the same sentence choice, each sentence compressed to about `target` words."""
from concise import compress
if isinstance(abstracts, str): abstracts = [abstracts]
docs = [make_doc(a) for a in abstracts]
z, E = self.sentence_scores(docs)
cfg, out, c = self._cfg(decoder), [], 0
for d in docs:
n = d["n"]; picks = pick_indices(d, z[c:c + n], None if E is None else E[c:c + n], **cfg); c += n
w = self.word_weights(d)
hs = [compress(restyle(d["S"][i]), w, target) for i in picks]
hs = [h for h in hs if len(h.split()) >= 3] or [d["S"][picks[0]]]
out.append(". ".join(hs) + ".")
return out
def analyse(self, abstract, target=18, n_keywords=4, **decoder):
"""One abstract -> dict with the FIRE output, the Concise output and a per-sentence table
(fused rank, each model's probability, a compressed version of every sentence and its keywords)."""
from concise import compress, WORD, FUNC
docs = [make_doc(abstract)]; d = docs[0]
z, E, S = self.sentence_scores(docs, per_model=True)
cfg = self._cfg(decoder)
picks = pick_indices(d, z, E, **cfg)
w = self.word_weights(d)
summaries = [compress(restyle(s), w, target) for s in d["S"]]
def keywords(s):
ws = []
for t in WORD.findall(s):
t = t.lower().strip(".")
if t not in FUNC and len(t) > 2 and t not in ws: ws.append(t)
return ", ".join(sorted(ws, key=lambda t: -w.get(t, 0.0))[:n_keywords])
order = np.argsort(-z); rank = np.empty(len(z), int); rank[order] = np.arange(1, len(z) + 1)
table = pd.DataFrame({"#": np.arange(1, len(z) + 1), "picked": ["yes" if i in picks else "" for i in range(len(z))],
"rank": rank, "sentence": d["S"], "summary": summaries,
"keywords": [keywords(s) for s in d["S"]], "RRF": np.round(z.astype(float), 4)})
for n in self.config["ensemble"]: table[n] = np.round(np.asarray(S[n], dtype=float), 3)
concise_hl = [summaries[i] for i in picks if len(summaries[i].split()) >= 3] or [d["S"][picks[0]]]
return {"fire": decode(docs, z, **cfg, E=E)[0], "concise": concise_hl, "picks": picks,
"sentences": d["S"], "table": table}
def explain(self, abstract, **decoder):
"""One abstract -> (FIRE highlights, per-sentence table). Kept for backward compatibility."""
r = self.analyse(abstract, **decoder)
return r["fire"], r["table"]
def pick_indices(d, s, E=None, budget=110, maxs=3, mmr=0.55, trim=0):
"""The sentence-selection half of decode() for a single document, returning indices (for display)."""
order = list(np.argsort(-s)); pick, wc = [], 0
for i in order:
if len(pick) >= maxs: break
w = len(d["tk"][i])
if wc + w > budget and pick: continue
if mmr < 1.0 and pick:
if E is not None:
sim = max(float(E[i] @ E[j]) for j in pick)
else:
a = d["cw"][i]
sim = max(len(a & d["cw"][j]) / max(len(a | d["cw"][j]), 1) for j in pick)
if sim > mmr: continue
pick.append(i); wc += w
return sorted(pick) if pick else [order[0]]
if __name__ == "__main__":
import sys
m = SciHighExtractor.from_pretrained(os.path.dirname(os.path.abspath(__file__)))
text = " ".join(sys.argv[1:]) or sys.stdin.read()
print(m.predict([text])[0])
|