Spaces:
Running on Zero
Running on Zero
File size: 8,274 Bytes
4d72966 e945472 4d72966 e945472 4d72966 e945472 4d72966 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 | """
Concise mode: sentence compression guided by word importance.
Every output word still comes from the abstract (plus a few function words such as "is" or "'s"),
so the compressed highlights cannot introduce new facts.
compress(sentence, weights)
1. remove reporting frames ("This study investigates", "We evaluate", "The research evaluates whether")
2. "X have shown A, but their B remains limited" -> "X' B remains limited" (keep the contrast clause)
3. split the rest into clauses/phrases and drop the least important ones until the highlight is short,
scoring each phrase by the importance of its words (abstract keywords x IDF, highlight log-odds,
numbers); the main clause is always kept
"""
import re, math, collections
import numpy as np
TARGET_WORDS = 18 # author highlights: median 12 words, 79 characters
FUNC = set("a an the of in on for to and or with by from as at into is are was were be been this that these those "
"its their our we it they which while whereas".split())
VERBS = ("investigate examine explore evaluate analyse analyze assess propose present develop introduce describe "
"report study design test compare apply employ identify measure estimate model simulate review discuss "
"provide construct establish derive obtain validate quantify characterize characterise address demonstrate "
"show find use consider focus examine outline summarize summarise highlight determine calculate predict").split()
_V = "|".join(sorted(set(VERBS), key=len, reverse=True))
def _third_person(v):
v = v.lower()
if re.search(r"(s|x|z|ch|sh)$", v): return v + "es"
if re.search(r"[^aeiou]y$", v): return v[:-1] + "ies"
return v + "s"
FRAME_SUBJ = re.compile(
r"^(?:(?:in )?this|the present|the current|our|the)\s+(?:study|paper|work|research|article|thesis|review|investigation|analysis|project)"
r"\s+(?:also\s+)?(?:aims to |aimed to |seeks to |sought to |attempts to |tries to |is to |was to )?"
r"(" + _V + r")((?:e?s|d|ed)?)\b\s*", re.I)
FRAME_WE = re.compile(r"^(?:here\s*,?\s*)?we\s+(?:also\s+|further\s+|then\s+|first\s+)?(?:have\s+|had\s+)?(" + _V + r")((?:e?s|d|ed)?)\b\s*", re.I)
PURPOSE_LEAD = re.compile(r"^(?:(?:in order )?to\s+)?(?:tackle|address|overcome|solve|fill|bridge|meet|close)\s+"
r"(?:this|these|the|such|this research|the research)\s+(?:\w+\s+){0,2}?"
r"(?:problems?|issues?|challenges?|gaps?|limitations?|needs?|shortcomings?)\s*,?\s+", re.I)
CONCESSIVE = re.compile(r",?\s+(?:but|yet|however,?|although|whereas)\s+", re.I)
PRON = re.compile(r"^(their|its|they|it)\b\s*", re.I)
AUX = re.compile(r"\b(is|are|was|were|has|have|had|can|could|may|might|will|would|remain|remains|show|shows|showed|"
r"demonstrate|demonstrates|demonstrated|exhibit|exhibits|achieve|achieves|achieved|provide|provides)\b", re.I)
SPLIT = re.compile(r"(\s*\([^)]*\)|,\s+(?:particularly|especially|including|such as|which|while|whereas|thereby|"
r"in order to|so as to|due to|leading to|resulting in)\s+|\s+(?:particularly|especially|including|"
r"such as|due to|in order to|so as to|to identify|to determine|to examine|to explore|to investigate|to analy[sz]e|"
r"to assess|to evaluate|to improve|to reduce|to enhance|to support|to predict|to detect|to estimate|"
r"to understand|to characteri[sz]e|to quantify|to provide|"
r"thereby|leading to|resulting in|while|whereas|which)\s+|,\s+)", re.I)
NUM = re.compile(r"\d")
WORD = re.compile(r"[A-Za-z0-9][A-Za-z0-9\-%\.']*")
def doc_weights(sentences, idf_lookup, lo_lookup=None):
"""Word importance inside one abstract: normalised (abstract tf x global idf) + positive highlight log-odds."""
toks = [w.lower() for s in sentences for w in WORD.findall(s)]
tf = collections.Counter(w for w in toks if w not in FUNC)
sc = {w: c * idf_lookup(w) for w, c in tf.items()}
top = max(sc.values()) if sc else 1.0
out = {}
for w, v in sc.items():
lo = max(lo_lookup(w), 0.0) if lo_lookup else 0.0
out[w] = v / top + 0.3 * min(lo, 2.0)
return out
def _importance(text, weights):
ws = [w.lower() for w in WORD.findall(text)]
if not ws: return 0.0
s = sum(weights.get(w, 0.0) for w in ws if w not in FUNC)
s += 1.5 * sum(1 for w in ws if NUM.search(w)) # numbers carry results
return s / (len(ws) + 2.0)
def _subject(clause):
m = AUX.search(clause)
if not m: return None
subj = clause[:m.start()].strip(" ,")
return subj if 1 <= len(subj.split()) <= 6 else None
def _possessive(np_):
return np_ + ("'" if np_.lower().endswith("s") else "'s")
EVIDENCE = re.compile(r"^(?:our\s+|the\s+)?(?:experiments?|results?|tests?|simulations?|evaluations?|analyses|analysis|"
r"findings|measurements|data)\b[^,;]{0,80}?\b(?:show|shows|showed|shown|demonstrate|demonstrates|"
r"demonstrated|indicate|indicates|indicated|reveal|reveals|revealed|confirm|confirms|confirmed|"
r"suggest|suggests|suggested)\s+that\s+", re.I)
def reframe(s):
"""Remove reporting frames and keep the contrast clause of a concessive sentence."""
s = s.strip().rstrip(".")
s = EVIDENCE.sub("", s) or s
s = PURPOSE_LEAD.sub("", s) or s
m = FRAME_SUBJ.match(s) or FRAME_WE.match(s)
if m and s[m.end():]:
verb = (m.group(1) + m.group(2)) if m.group(2) in ("d", "ed") else _third_person(m.group(1))
s = verb.capitalize() + " " + s[m.end():]
parts = CONCESSIVE.split(s, maxsplit=1)
if len(parts) == 2 and len(parts[1].split()) >= 5:
first, second = parts
pm = PRON.match(second)
if pm:
subj = _subject(first)
if subj is None: return s
p = pm.group(1).lower()
second = (_possessive(subj) if p in ("their", "its") else subj) + " " + second[pm.end():]
s = second
return s[0].upper() + s[1:] if s else s
PP = re.compile(r"(\s+(?:for|with|using|through|via|by|based on|by means of|under|during|within|across|among|as well as|in terms of)\s+)", re.I)
def _drop(segs, weights, target, protect_first=True):
keep = [True] * len(segs)
length = lambda: sum(len(segs[i].split()) for i in range(len(segs)) if keep[i])
# keep the segment with the main verb when the first segment has none (otherwise the highlight has no predicate)
verb_seg = None if AUX.search(segs[0]) else next((i for i, g in enumerate(segs) if AUX.search(g)), None)
for i in sorted(range(1 if protect_first else 0, len(segs)), key=lambda i: _importance(segs[i], weights)):
if length() <= target: break
if i == verb_seg: continue
if NUM.search(segs[i]) and _importance(segs[i], weights) > 0.2: continue
keep[i] = False
return "".join(g for g, k in zip(segs, keep) if k)
def _segments(s, rx):
pieces = rx.split(s)
segs = [pieces[0]] + [pieces[i] + pieces[i + 1] for i in range(1, len(pieces) - 1, 2)]
return [g for g in segs if g.strip(" ,")]
FILLER = re.compile(r"\s*\b(?:as described herein|as reported here(?:in)?|described herein|presented here(?:in)?|"
r"in this (?:study|work|paper)|in the present (?:study|work)|herein|here)\b", re.I)
def compress(s, weights, target=TARGET_WORDS):
s = reframe(s)
s = FILLER.sub("", s).strip()
segs = _segments(s, SPLIT)
if len(segs) > 1: s = _drop(segs, weights, target)
if len(s.split()) > target + 4: # still long: drop the least important prepositional phrases
segs = _segments(s, PP)
# never cut inside the first 6 words (subject and main verb)
if len(segs) > 1 and len(segs[0].split()) >= 3: s = _drop(segs, weights, target)
s = re.sub(r"\s+,", ",", re.sub(r"\s+", " ", s)).strip(" ,;:")
s = re.sub(r"\b(and|or|but|with|by|for|to|of|in|the|a|an)$", "", s, flags=re.I).strip(" ,")
return (s[0].upper() + s[1:]) if s else s
def concise_count(n, maxs=3):
"""How many highlights to keep in Concise mode: about half the sentences, at least 2, at most maxs."""
return int(min(maxs, max(2, math.ceil(n / 2)))) if n > 2 else n
|