Spaces:
Running on Zero
Running on Zero
Download concise.py from Akash212/Highlight_Generator: direct link, hf CLI and curl.
- Browser
- Download file 8.27 kB
-
https://huggingface.co/spaces/Akash212/Highlight_Generator/resolve/main/concise.py
- Command line
-
hf download hf://spaces/Akash212/Highlight_Generator/concise.py
-
curl -L -o concise.py https://huggingface.co/spaces/Akash212/Highlight_Generator/resolve/main/concise.py
8.27 kB
| """ | |
| Concise mode: sentence compression guided by word importance. | |
| Every output word still comes from the abstract (plus a few function words such as "is" or "'s"), | |
| so the compressed highlights cannot introduce new facts. | |
| compress(sentence, weights) | |
| 1. remove reporting frames ("This study investigates", "We evaluate", "The research evaluates whether") | |
| 2. "X have shown A, but their B remains limited" -> "X' B remains limited" (keep the contrast clause) | |
| 3. split the rest into clauses/phrases and drop the least important ones until the highlight is short, | |
| scoring each phrase by the importance of its words (abstract keywords x IDF, highlight log-odds, | |
| numbers); the main clause is always kept | |
| """ | |
| import re, math, collections | |
| import numpy as np | |
| TARGET_WORDS = 18 # author highlights: median 12 words, 79 characters | |
| FUNC = set("a an the of in on for to and or with by from as at into is are was were be been this that these those " | |
| "its their our we it they which while whereas".split()) | |
| VERBS = ("investigate examine explore evaluate analyse analyze assess propose present develop introduce describe " | |
| "report study design test compare apply employ identify measure estimate model simulate review discuss " | |
| "provide construct establish derive obtain validate quantify characterize characterise address demonstrate " | |
| "show find use consider focus examine outline summarize summarise highlight determine calculate predict").split() | |
| _V = "|".join(sorted(set(VERBS), key=len, reverse=True)) | |
| def _third_person(v): | |
| v = v.lower() | |
| if re.search(r"(s|x|z|ch|sh)$", v): return v + "es" | |
| if re.search(r"[^aeiou]y$", v): return v[:-1] + "ies" | |
| return v + "s" | |
| FRAME_SUBJ = re.compile( | |
| r"^(?:(?:in )?this|the present|the current|our|the)\s+(?:study|paper|work|research|article|thesis|review|investigation|analysis|project)" | |
| r"\s+(?:also\s+)?(?:aims to |aimed to |seeks to |sought to |attempts to |tries to |is to |was to )?" | |
| r"(" + _V + r")((?:e?s|d|ed)?)\b\s*", re.I) | |
| FRAME_WE = re.compile(r"^(?:here\s*,?\s*)?we\s+(?:also\s+|further\s+|then\s+|first\s+)?(?:have\s+|had\s+)?(" + _V + r")((?:e?s|d|ed)?)\b\s*", re.I) | |
| PURPOSE_LEAD = re.compile(r"^(?:(?:in order )?to\s+)?(?:tackle|address|overcome|solve|fill|bridge|meet|close)\s+" | |
| r"(?:this|these|the|such|this research|the research)\s+(?:\w+\s+){0,2}?" | |
| r"(?:problems?|issues?|challenges?|gaps?|limitations?|needs?|shortcomings?)\s*,?\s+", re.I) | |
| CONCESSIVE = re.compile(r",?\s+(?:but|yet|however,?|although|whereas)\s+", re.I) | |
| PRON = re.compile(r"^(their|its|they|it)\b\s*", re.I) | |
| AUX = re.compile(r"\b(is|are|was|were|has|have|had|can|could|may|might|will|would|remain|remains|show|shows|showed|" | |
| r"demonstrate|demonstrates|demonstrated|exhibit|exhibits|achieve|achieves|achieved|provide|provides)\b", re.I) | |
| SPLIT = re.compile(r"(\s*\([^)]*\)|,\s+(?:particularly|especially|including|such as|which|while|whereas|thereby|" | |
| r"in order to|so as to|due to|leading to|resulting in)\s+|\s+(?:particularly|especially|including|" | |
| r"such as|due to|in order to|so as to|to identify|to determine|to examine|to explore|to investigate|to analy[sz]e|" | |
| r"to assess|to evaluate|to improve|to reduce|to enhance|to support|to predict|to detect|to estimate|" | |
| r"to understand|to characteri[sz]e|to quantify|to provide|" | |
| r"thereby|leading to|resulting in|while|whereas|which)\s+|,\s+)", re.I) | |
| NUM = re.compile(r"\d") | |
| WORD = re.compile(r"[A-Za-z0-9][A-Za-z0-9\-%\.']*") | |
| def doc_weights(sentences, idf_lookup, lo_lookup=None): | |
| """Word importance inside one abstract: normalised (abstract tf x global idf) + positive highlight log-odds.""" | |
| toks = [w.lower() for s in sentences for w in WORD.findall(s)] | |
| tf = collections.Counter(w for w in toks if w not in FUNC) | |
| sc = {w: c * idf_lookup(w) for w, c in tf.items()} | |
| top = max(sc.values()) if sc else 1.0 | |
| out = {} | |
| for w, v in sc.items(): | |
| lo = max(lo_lookup(w), 0.0) if lo_lookup else 0.0 | |
| out[w] = v / top + 0.3 * min(lo, 2.0) | |
| return out | |
| def _importance(text, weights): | |
| ws = [w.lower() for w in WORD.findall(text)] | |
| if not ws: return 0.0 | |
| s = sum(weights.get(w, 0.0) for w in ws if w not in FUNC) | |
| s += 1.5 * sum(1 for w in ws if NUM.search(w)) # numbers carry results | |
| return s / (len(ws) + 2.0) | |
| def _subject(clause): | |
| m = AUX.search(clause) | |
| if not m: return None | |
| subj = clause[:m.start()].strip(" ,") | |
| return subj if 1 <= len(subj.split()) <= 6 else None | |
| def _possessive(np_): | |
| return np_ + ("'" if np_.lower().endswith("s") else "'s") | |
| EVIDENCE = re.compile(r"^(?:our\s+|the\s+)?(?:experiments?|results?|tests?|simulations?|evaluations?|analyses|analysis|" | |
| r"findings|measurements|data)\b[^,;]{0,80}?\b(?:show|shows|showed|shown|demonstrate|demonstrates|" | |
| r"demonstrated|indicate|indicates|indicated|reveal|reveals|revealed|confirm|confirms|confirmed|" | |
| r"suggest|suggests|suggested)\s+that\s+", re.I) | |
| def reframe(s): | |
| """Remove reporting frames and keep the contrast clause of a concessive sentence.""" | |
| s = s.strip().rstrip(".") | |
| s = EVIDENCE.sub("", s) or s | |
| s = PURPOSE_LEAD.sub("", s) or s | |
| m = FRAME_SUBJ.match(s) or FRAME_WE.match(s) | |
| if m and s[m.end():]: | |
| verb = (m.group(1) + m.group(2)) if m.group(2) in ("d", "ed") else _third_person(m.group(1)) | |
| s = verb.capitalize() + " " + s[m.end():] | |
| parts = CONCESSIVE.split(s, maxsplit=1) | |
| if len(parts) == 2 and len(parts[1].split()) >= 5: | |
| first, second = parts | |
| pm = PRON.match(second) | |
| if pm: | |
| subj = _subject(first) | |
| if subj is None: return s | |
| p = pm.group(1).lower() | |
| second = (_possessive(subj) if p in ("their", "its") else subj) + " " + second[pm.end():] | |
| s = second | |
| return s[0].upper() + s[1:] if s else s | |
| PP = re.compile(r"(\s+(?:for|with|using|through|via|by|based on|by means of|under|during|within|across|among|as well as|in terms of)\s+)", re.I) | |
| def _drop(segs, weights, target, protect_first=True): | |
| keep = [True] * len(segs) | |
| length = lambda: sum(len(segs[i].split()) for i in range(len(segs)) if keep[i]) | |
| # keep the segment with the main verb when the first segment has none (otherwise the highlight has no predicate) | |
| verb_seg = None if AUX.search(segs[0]) else next((i for i, g in enumerate(segs) if AUX.search(g)), None) | |
| for i in sorted(range(1 if protect_first else 0, len(segs)), key=lambda i: _importance(segs[i], weights)): | |
| if length() <= target: break | |
| if i == verb_seg: continue | |
| if NUM.search(segs[i]) and _importance(segs[i], weights) > 0.2: continue | |
| keep[i] = False | |
| return "".join(g for g, k in zip(segs, keep) if k) | |
| def _segments(s, rx): | |
| pieces = rx.split(s) | |
| segs = [pieces[0]] + [pieces[i] + pieces[i + 1] for i in range(1, len(pieces) - 1, 2)] | |
| return [g for g in segs if g.strip(" ,")] | |
| FILLER = re.compile(r"\s*\b(?:as described herein|as reported here(?:in)?|described herein|presented here(?:in)?|" | |
| r"in this (?:study|work|paper)|in the present (?:study|work)|herein|here)\b", re.I) | |
| def compress(s, weights, target=TARGET_WORDS): | |
| s = reframe(s) | |
| s = FILLER.sub("", s).strip() | |
| segs = _segments(s, SPLIT) | |
| if len(segs) > 1: s = _drop(segs, weights, target) | |
| if len(s.split()) > target + 4: # still long: drop the least important prepositional phrases | |
| segs = _segments(s, PP) | |
| # never cut inside the first 6 words (subject and main verb) | |
| if len(segs) > 1 and len(segs[0].split()) >= 3: s = _drop(segs, weights, target) | |
| s = re.sub(r"\s+,", ",", re.sub(r"\s+", " ", s)).strip(" ,;:") | |
| s = re.sub(r"\b(and|or|but|with|by|for|to|of|in|the|a|an)$", "", s, flags=re.I).strip(" ,") | |
| return (s[0].upper() + s[1:]) if s else s | |
| def concise_count(n, maxs=3): | |
| """How many highlights to keep in Concise mode: about half the sentences, at least 2, at most maxs.""" | |
| return int(min(maxs, max(2, math.ceil(n / 2)))) if n > 2 else n | |