File size: 21,258 Bytes
d7b7e82
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4d72966
 
 
 
d7b7e82
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4d72966
d7b7e82
 
4d72966
 
d7b7e82
 
 
4d72966
d7b7e82
 
4d72966
 
 
 
 
 
 
 
 
 
 
d7b7e82
4d72966
d7b7e82
 
 
 
 
 
4d72966
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d7b7e82
4d72966
 
 
 
 
 
 
 
 
d7b7e82
4d72966
 
 
d7b7e82
4d72966
 
 
 
 
 
 
 
d7b7e82
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
"""
SciHigh 2026 - Supervised extractive research-highlight generator (CPU-only).

FIRE 2026 SciHigh, Subtask 1: research highlight generation from MixSub abstracts.

Pipeline for one abstract
  1. clean the text and split it into sentences
  2. 31 hand-built features per sentence (position, length, lexical, discourse cues, TF-IDF graph)
  3. 36 embedding features from all-MiniLM-L6-v2 (4 similarity features + 32-dim SVD projection),
     including a retrieval feature built from the 10 nearest training abstracts
  4. five classifiers score every sentence; their rankings are merged by Reciprocal Rank Fusion
  5. a decoder picks sentences under a word budget with an MMR redundancy filter, restores
     reading order and rewrites each sentence into highlight style

Usage
    from scihigh_model import SciHighExtractor
    model = SciHighExtractor.from_pretrained(".")          # local folder or a Hugging Face repo id
    print(model.predict(["<abstract text>"])[0])

The text-processing functions below are copied unchanged from SciHigh_2026_CPU_Hybrid.ipynb, so
the saved models see exactly the features they were trained on.
"""
import os, re, io, json, glob, collections
import numpy as np, pandas as pd
from sklearn.preprocessing import normalize


# ---------------------------------------------------------------- file helpers
# Files larger than PART_LIMIT are stored as byte chunks named <stem>.part00<ext>, <stem>.part01<ext>, ...
# so every file stays under 10 MB (browser upload limit). The loader joins them back byte for byte.
PART_LIMIT = 9_500_000

def save_bytes(data, folder, name, limit=PART_LIMIT):
    stem, ext = os.path.splitext(name)
    for old in glob.glob(os.path.join(folder, f"{stem}.part*{ext}")): os.remove(old)
    if len(data) <= limit:
        open(os.path.join(folder, name), "wb").write(data); return [name]
    names = []
    for i in range(0, len(data), limit):
        nm = f"{stem}.part{i // limit:02d}{ext}"
        open(os.path.join(folder, nm), "wb").write(data[i:i + limit]); names.append(nm)
    return names

def read_bytes(folder, name):
    p = os.path.join(folder, name)
    if os.path.exists(p): return open(p, "rb").read()
    stem, ext = os.path.splitext(name)
    parts = sorted(glob.glob(os.path.join(folder, f"{stem}.part*{ext}")))
    if not parts: raise FileNotFoundError(p)
    return b"".join(open(q, "rb").read() for q in parts)

def load_joblib(folder, name):
    import joblib
    return joblib.load(io.BytesIO(read_bytes(folder, name)))

def load_npy(folder, name):
    return np.load(io.BytesIO(read_bytes(folder, name)))


# ---------------------------------------------------------------- cleaning and sentence splitting
def clean(t):
    t = str(t)
    t = re.sub(r"\.\s*None\.\s*", " ", t)      # "value . None . unit" artifact
    t = re.sub(r"\bNone\b", " ", t)
    t = re.sub(r"\s*\.\s*\.", ".", t)          # collapse double periods left behind
    return re.sub(r"\s+", " ", t).strip()

def sent_split(t):
    # split on " . " (MixSub style) or on ". " followed by a capital
    parts = re.split(r"(?<=[A-Za-z0-9\)\]\%])\s+\.\s+|(?<=[a-z0-9\)\]\%])\.\s+(?=[A-Z])", str(t).strip())
    out = []
    for p in parts:
        if not p: continue
        p = p.strip().rstrip(".").strip()
        if len(p.split()) >= 3:
            out.append(p)
        elif out:
            out[-1] += " " + p          # glue fragments onto the previous sentence
    return out or [str(t).rstrip(".").strip()]

STOP = set('''a about above after again against all also am among an and any are as at be because been before
being below between both but by can cannot could did do does doing during each few for from further had has
have having he her here hers herself him himself his how i if in into is it its itself me more most my myself
no nor not now of off on once only or other others our ours out over own same she should so some such than
that the their theirs them themselves then there these they this those through to too under until up very
was we were what when where which while who whom why will with would you your yours'''.split())

TOK = re.compile(r"[a-z0-9]+")
def toks(t): return TOK.findall(str(t).lower())
def content(t): return [w for w in toks(t) if w not in STOP and len(w) > 2]

def make_doc(abstract):
    """One cleaned abstract -> the document dict used by featurize() and decode()."""
    S = sent_split(clean(abstract)); n = len(S)
    return dict(S=S, n=n, tk=[toks(s) for s in S], cw=[set(content(s)) for s in S])


# ---------------------------------------------------------------- hand-built sentence features
CUES = {
 "res":  r"\b(result|show|showed|shown|found|find|reveal|demonstrat|indicat|observ|obtain|achiev|improv|outperform|increas|decreas|significant)",
 "meth": r"\b(propos|present|develop|design|introduc|novel|new |method|approach|model|framework|algorithm|technique|scheme|system|strateg)",
 "purp": r"\b(aim|objective|purpose|goal|investigat|examin|explor|evaluat|assess|analys|analyz)",
 "bkg":  r"\b(previous|literature|recently|recent years|has been|have been|widely|however|although|despite|remains|little is known|few studies)",
 "conc": r"\b(conclu|suggest|impl|therefore|thus|hence|overall|consequently|potential|promis|can be used|may be)",
 "quant": r"(\d+\.?\d*\s?%|p\s?[<=]|\bn\s?=|\bmean\b|\bmedian\b|\bratio\b|\brate\b|fold)",
}
LEAD_RE = (r"^(in this (study|paper|work|article|research)|in the present|the (aim|purpose|objective|goal) of"
           r"|here we|in addition|moreover|furthermore|therefore|thus|however|finally|overall|first|second"
           r"|third|also|based on)")

def featurize(docs, tv):
    rows = []
    for d in docs:
        S, n, TKs, CWs = d["S"], d["n"], d["tk"], d["cw"]
        tw = max(sum(len(t) for t in TKs), 1)
        X = normalize(tv.transform(S))
        cen = (X @ normalize(np.asarray(X.mean(0))).T).ravel()      # cosine to abstract centroid
        M = (X @ X.T).toarray(); np.fill_diagonal(M, 0); deg = M.sum(1)
        P = M / np.maximum(deg[:, None], 1e-9); pr = np.ones(n) / n
        for _ in range(20): pr = 0.15 / n + 0.85 * (P.T @ pr)       # TextRank power iteration
        tsum = np.asarray(X.sum(1)).ravel()
        mx = M.max(1) if n > 1 else np.zeros(n)
        for i, s in enumerate(S):
            sw, cw, low = TKs[i], CWs[i], s.lower()
            nov = max((len(cw & CWs[j]) / max(len(cw | CWs[j]), 1) for j in range(i)), default=0.0)
            f = {"sid": i, "rel_pos": i / max(n-1, 1), "rel_end": (n-1-i) / max(n-1, 1),
                 "is_first": int(i == 0), "is_last": int(i == n-1), "is_last2": int(i >= n-2),
                 "is_first2": int(i <= 1), "nsent": n, "nw": len(sw), "nchar": len(s),
                 "rel_len": len(sw) / tw,
                 "awlen": float(np.mean([len(w) for w in sw])) if sw else 0.0,
                 "stop_ratio": sum(1 for w in sw if w in STOP) / max(len(sw), 1),
                 "n_content": len(cw), "content_dens": len(cw) / max(len(sw), 1),
                 "n_digit": sum(c.isdigit() for c in s),
                 "n_acro": len(re.findall(r"\b[A-Z]{2,}\b", s)), "has_pct": int("%" in s),
                 "lead": int(bool(re.match(LEAD_RE, low))), "redund": nov,
                 "tf_cent": cen[i], "tf_deg": deg[i], "textrank": pr[i],
                 "tf_sum": tsum[i], "tf_maxsim": mx[i]}
            for k, pat in CUES.items():
                f["cue_" + k] = len(re.findall(pat, low))
            rows.append(f)
    return pd.DataFrame(rows)


# ---------------------------------------------------------------- embedding features
def offsets(docs):
    o, c = [], 0
    for d in docs: o.append((c, c + d["n"])); c += d["n"]
    return o

def cent(E, O):
    return np.vstack([normalize(E[a:b].mean(0).reshape(1, -1)) for a, b in O])

def emb_feats(E, O, Ctr, Hemb, proto, is_train=False, k=10):
    out = np.zeros((len(E), 4), np.float32)
    sim = cent(E, O) @ Ctr.T
    if is_train: np.fill_diagonal(sim, -1)     # never retrieve yourself
    nn = np.argsort(-sim, 1)[:, :k]
    for q, (a, b) in enumerate(O):
        X = E[a:b]
        out[a:b, 0] = (X @ normalize(X.mean(0).reshape(1, -1)).T).ravel()
        out[a:b, 1] = (X @ proto.T).ravel()
        M = X @ X.T; np.fill_diagonal(M, 0)
        out[a:b, 2] = M.max(1) if b - a > 1 else 0
        out[a:b, 3] = (X @ normalize(Hemb[nn[q]].mean(0).reshape(1, -1)).T).ravel()
    return pd.DataFrame(out, columns=["e_cent", "e_proto", "e_maxsim", "e_rag"])

def load_encoder(name_or_path="sentence-transformers/all-MiniLM-L6-v2", device="cpu"):
    """all-MiniLM-L6-v2 as a SentenceTransformer: Transformer(max_seq_length=256) -> mean pooling -> L2 norm.
    Accepts a Hugging Face id, a full sentence-transformers folder, or a folder that holds only
    config.json + model.safetensors + tokenizer.json (the same three modules are then assembled here)."""
    from sentence_transformers import SentenceTransformer, models
    p = name_or_path
    if os.path.isdir(p) and not os.path.exists(os.path.join(p, "modules.json")):
        tr = models.Transformer(p, max_seq_length=256, do_lower_case=False)
        dim = (tr.get_embedding_dimension() if hasattr(tr, "get_embedding_dimension")
               else tr.get_word_embedding_dimension())
        pool = models.Pooling(dim, pooling_mode="mean")
        return SentenceTransformer(modules=[tr, pool, models.Normalize()], device=device)
    return SentenceTransformer(p, device=device)


# ---------------------------------------------------------------- decoder and restyling
DROP = re.compile(
  r"^(in this (study|paper|work|article|research|comment|letter|note|review|report|contribution)"
  r"|in the (present|current) (study|work|paper|article|research)|in the following"
  r"|the (aim|aims|purpose|objective|objectives|goal|goals) of (this|the present|the current) "
  r"(study|paper|work|research|article) (was|were|is|are) to"
  r"|the (present|current) (study|work|paper) (aims?|aimed|sought|seeks?)( to)?"
  r"|here (we|the authors)|in addition|additionally|moreover|furthermore|therefore|thus|hence|however"
  r"|nevertheless|nonetheless|meanwhile|finally|overall|consequently|accordingly|notably|importantly"
  r"|first(ly)?|second(ly)?|third(ly)?|also|in conclusion|in summary|to summari[sz]e|taken together"
  r"|it (was|is|has been|had been) (found|shown|observed|concluded|demonstrated|reported|noted|revealed)( that)?"
  r"|we (found|find|show|showed|observe|observed|conclude|concluded|report|reported|note|noted"
  r"|demonstrate|demonstrated|argue|suggest|propose)( that| with| in)?"
  r"|(the|our|these|this) (results?|findings?|data|analysis|study|work) (show|shows|showed|indicate"
  r"|indicated|suggest|suggests|suggested|reveal|revealed|demonstrate|demonstrated|confirm|confirmed)( that)?"
  r"|(this|these) (results?|findings?) (suggest|suggests|indicate|indicates|impl(y|ies))( that)?"
  r"|as (a|an) (result|consequence)|on the (other hand|contrary)|in contrast|by contrast"
  r"|based on (this|these|the)( results?| findings?)?"
  r"|the (results?|findings?) (of (this|the present) (study )?)?(show|showed|indicate|indicated"
  r"|suggest|suggested|revealed)( that)?)\s*[,:]?\s*", re.I)

CONN = re.compile(r"\s+(?:and|but|whereas|while|although|though|because|since|which|that)\s+")

def restyle(s, trim=0):
    prev = None
    while prev != s:                     # rules can cascade: "However, we found that X" -> "X"
        prev = s; s = DROP.sub("", s).strip()
    s = re.sub(r"^(and|but|that|which|to|of|in|for|with|as)\s+", "", s, flags=re.I).strip()
    if trim and len(s.split()) > trim:   # optional clause-level truncation
        parts = CONN.split(s)
        if len(parts) > 1:
            acc, keep = 0, []
            for p in parts:
                if keep and acc + len(p.split()) > trim: break
                keep.append(p); acc += len(p.split())
            if keep: s = " and ".join(keep)
    s = re.sub(r"\s+", " ", s).strip()
    return (s[0].upper() + s[1:]) if s else s

def decode(docs, sc, budget=110, maxs=3, mmr=0.55, trim=0, E=None):
    out, c = [], 0
    for d in docs:
        n = d["n"]; s = sc[c:c+n]; base = c; c += n
        order = list(np.argsort(-s)); pick, wc = [], 0
        for i in order:
            if len(pick) >= maxs: break
            w = len(d["tk"][i])
            if wc + w > budget and pick: continue
            if mmr < 1.0 and pick:
                if E is not None:
                    sim = max(float(E[base+i] @ E[base+j]) for j in pick)
                else:
                    a = d["cw"][i]
                    sim = max(len(a & d["cw"][j]) / max(len(a | d["cw"][j]), 1) for j in pick)
                if sim > mmr: continue
            pick.append(i); wc += w
        if not pick: pick = [order[0]]
        pick.sort()                                     # restore reading order
        segs = [x for x in (restyle(d["S"][i], trim) for i in pick) if len(x.split()) >= 3]
        out.append((". ".join(segs) + ".") if segs else d["S"][pick[0]] + ".")
    return out


# ---------------------------------------------------------------- scoring and rank fusion
def prob(m, X):
    return (m.predict_proba(X)[:, 1] if hasattr(m, "predict_proba")
            else 1 / (1 + np.exp(-m.decision_function(X))))

def fuse(docs, D, names, k=60):
    z = np.zeros(sum(d["n"] for d in docs))
    for n in names:
        s = D[n]; c = 0
        for d in docs:
            m = d["n"]
            o = np.argsort(-s[c:c+m]); rr = np.empty(m, int); rr[o] = np.arange(m)
            z[c:c+m] += 1.0 / (k + rr + 1); c += m
    return z


# ---------------------------------------------------------------- the model
class SciHighExtractor:
    """Loads the saved artifacts and turns raw abstracts into highlights."""

    def __init__(self, folder, encoder=None, device="cpu"):
        self.folder = folder
        self.config = json.load(open(os.path.join(folder, "config.json")))
        self.tv = load_joblib(folder, "tfidf.joblib")
        self.models = {n: load_joblib(folder, f"clf_{n}.joblib") for n in self.config["ensemble"]}
        self.use_emb = self.config["embeddings_used"]
        if self.use_emb:
            self.svd = load_joblib(folder, "svd.joblib")
            self.proto = load_npy(folder, "proto.npy")
            self.Ctr = load_npy(folder, "train_centroids.npy")
            self.Hemb = load_npy(folder, "train_highlight_emb.npy")
            self._enc_src, self._enc, self._device = encoder or self.config["encoder"], None, device
        lex = os.path.join(folder, "highlight_lexicon.json")
        self.lexicon = json.load(open(lex)) if os.path.exists(lex) else {}
        self._voc, self._idf = self.tv.vocabulary_, self.tv.idf_
        self._idf_oov = float(self._idf.max())

    @classmethod
    def from_pretrained(cls, path_or_repo_id, encoder=None, device="cpu", **hub_kwargs):
        """path_or_repo_id: a local folder, or a Hugging Face repo id such as "user/scihigh2026-extractive-ranker".
        encoder: optional local path to all-MiniLM-L6-v2 (default: downloaded from the Hub by name)."""
        folder = path_or_repo_id
        if not os.path.isdir(folder):
            from huggingface_hub import snapshot_download
            folder = snapshot_download(path_or_repo_id, **hub_kwargs)
        return cls(folder, encoder=encoder, device=device)

    def _embed(self, texts):
        if self._enc is None:
            self._enc = load_encoder(self._enc_src, self._device)
        return self._enc.encode(list(texts), batch_size=256, convert_to_numpy=True,
                                normalize_embeddings=True, show_progress_bar=False)

    def features(self, docs):
        """Design matrix (one row per sentence) and the sentence embeddings used by the MMR filter."""
        F, E = featurize(docs, self.tv), None
        if self.use_emb:
            E = self._embed([s for d in docs for s in d["S"]])
            proj = pd.DataFrame(self.svd.transform(E), columns=[f"sv{i}" for i in range(32)])
            F = pd.concat([F, emb_feats(E, offsets(docs), self.Ctr, self.Hemb, self.proto), proj], axis=1)
        X = F[self.config["feature_columns"]].astype(np.float32).fillna(0)
        return X, E

    def sentence_scores(self, docs, per_model=False):
        X, E = self.features(docs)
        S = {n: prob(m, X) for n, m in self.models.items()}
        z = fuse(docs, S, self.config["ensemble"], k=self.config["rrf_k"])
        return (z, E, S) if per_model else (z, E)

    def _cfg(self, overrides):
        cfg = dict(self.config["decoder"])
        cfg.update({k: v for k, v in overrides.items() if v is not None and k in cfg})
        return cfg

    # ---- word importance (TF-IDF x IDF within the abstract + highlight log-odds lexicon)
    def idf_of(self, w):
        parts = [p for p in w.lower().replace("'", "").split("-") if p]
        v = [self._idf[self._voc[p]] if p in self._voc else self._idf_oov for p in parts] or [self._idf_oov]
        return float(np.mean(v))

    def word_weights(self, doc):
        from concise import doc_weights
        return doc_weights(doc["S"], self.idf_of, lambda w: self.lexicon.get(w.lower(), 0.0))

    # ---- output modes
    def predict(self, abstracts, **decoder):
        """FIRE mode: list of abstract strings -> list of highlight strings (tuned extractive decoder).
        Optional decoder overrides: budget (words), maxs (sentences), mmr (redundancy threshold), trim."""
        if isinstance(abstracts, str): abstracts = [abstracts]
        docs = [make_doc(a) for a in abstracts]
        z, E = self.sentence_scores(docs)
        return decode(docs, z, **self._cfg(decoder), E=E)

    def concise(self, abstracts, target=18, **decoder):
        """Concise mode: the same sentence choice, each sentence compressed to about `target` words."""
        from concise import compress
        if isinstance(abstracts, str): abstracts = [abstracts]
        docs = [make_doc(a) for a in abstracts]
        z, E = self.sentence_scores(docs)
        cfg, out, c = self._cfg(decoder), [], 0
        for d in docs:
            n = d["n"]; picks = pick_indices(d, z[c:c + n], None if E is None else E[c:c + n], **cfg); c += n
            w = self.word_weights(d)
            hs = [compress(restyle(d["S"][i]), w, target) for i in picks]
            hs = [h for h in hs if len(h.split()) >= 3] or [d["S"][picks[0]]]
            out.append(". ".join(hs) + ".")
        return out

    def analyse(self, abstract, target=18, n_keywords=4, **decoder):
        """One abstract -> dict with the FIRE output, the Concise output and a per-sentence table
        (fused rank, each model's probability, a compressed version of every sentence and its keywords)."""
        from concise import compress, WORD, FUNC
        docs = [make_doc(abstract)]; d = docs[0]
        z, E, S = self.sentence_scores(docs, per_model=True)
        cfg = self._cfg(decoder)
        picks = pick_indices(d, z, E, **cfg)
        w = self.word_weights(d)
        summaries = [compress(restyle(s), w, target) for s in d["S"]]
        def keywords(s):
            ws = []
            for t in WORD.findall(s):
                t = t.lower().strip(".")
                if t not in FUNC and len(t) > 2 and t not in ws: ws.append(t)
            return ", ".join(sorted(ws, key=lambda t: -w.get(t, 0.0))[:n_keywords])
        order = np.argsort(-z); rank = np.empty(len(z), int); rank[order] = np.arange(1, len(z) + 1)
        table = pd.DataFrame({"#": np.arange(1, len(z) + 1), "picked": ["yes" if i in picks else "" for i in range(len(z))],
                              "rank": rank, "sentence": d["S"], "summary": summaries,
                              "keywords": [keywords(s) for s in d["S"]], "RRF": np.round(z.astype(float), 4)})
        for n in self.config["ensemble"]: table[n] = np.round(np.asarray(S[n], dtype=float), 3)
        concise_hl = [summaries[i] for i in picks if len(summaries[i].split()) >= 3] or [d["S"][picks[0]]]
        return {"fire": decode(docs, z, **cfg, E=E)[0], "concise": concise_hl, "picks": picks,
                "sentences": d["S"], "table": table}

    def explain(self, abstract, **decoder):
        """One abstract -> (FIRE highlights, per-sentence table). Kept for backward compatibility."""
        r = self.analyse(abstract, **decoder)
        return r["fire"], r["table"]


def pick_indices(d, s, E=None, budget=110, maxs=3, mmr=0.55, trim=0):
    """The sentence-selection half of decode() for a single document, returning indices (for display)."""
    order = list(np.argsort(-s)); pick, wc = [], 0
    for i in order:
        if len(pick) >= maxs: break
        w = len(d["tk"][i])
        if wc + w > budget and pick: continue
        if mmr < 1.0 and pick:
            if E is not None:
                sim = max(float(E[i] @ E[j]) for j in pick)
            else:
                a = d["cw"][i]
                sim = max(len(a & d["cw"][j]) / max(len(a | d["cw"][j]), 1) for j in pick)
            if sim > mmr: continue
        pick.append(i); wc += w
    return sorted(pick) if pick else [order[0]]


if __name__ == "__main__":
    import sys
    m = SciHighExtractor.from_pretrained(os.path.dirname(os.path.abspath(__file__)))
    text = " ".join(sys.argv[1:]) or sys.stdin.read()
    print(m.predict([text])[0])