""" Concise mode: sentence compression guided by word importance. Every output word still comes from the abstract (plus a few function words such as "is" or "'s"), so the compressed highlights cannot introduce new facts. compress(sentence, weights) 1. remove reporting frames ("This study investigates", "We evaluate", "The research evaluates whether") 2. "X have shown A, but their B remains limited" -> "X' B remains limited" (keep the contrast clause) 3. split the rest into clauses/phrases and drop the least important ones until the highlight is short, scoring each phrase by the importance of its words (abstract keywords x IDF, highlight log-odds, numbers); the main clause is always kept """ import re, math, collections import numpy as np TARGET_WORDS = 18 # author highlights: median 12 words, 79 characters FUNC = set("a an the of in on for to and or with by from as at into is are was were be been this that these those " "its their our we it they which while whereas".split()) VERBS = ("investigate examine explore evaluate analyse analyze assess propose present develop introduce describe " "report study design test compare apply employ identify measure estimate model simulate review discuss " "provide construct establish derive obtain validate quantify characterize characterise address demonstrate " "show find use consider focus examine outline summarize summarise highlight determine calculate predict").split() _V = "|".join(sorted(set(VERBS), key=len, reverse=True)) def _third_person(v): v = v.lower() if re.search(r"(s|x|z|ch|sh)$", v): return v + "es" if re.search(r"[^aeiou]y$", v): return v[:-1] + "ies" return v + "s" FRAME_SUBJ = re.compile( r"^(?:(?:in )?this|the present|the current|our|the)\s+(?:study|paper|work|research|article|thesis|review|investigation|analysis|project)" r"\s+(?:also\s+)?(?:aims to |aimed to |seeks to |sought to |attempts to |tries to |is to |was to )?" r"(" + _V + r")((?:e?s|d|ed)?)\b\s*", re.I) FRAME_WE = re.compile(r"^(?:here\s*,?\s*)?we\s+(?:also\s+|further\s+|then\s+|first\s+)?(?:have\s+|had\s+)?(" + _V + r")((?:e?s|d|ed)?)\b\s*", re.I) PURPOSE_LEAD = re.compile(r"^(?:(?:in order )?to\s+)?(?:tackle|address|overcome|solve|fill|bridge|meet|close)\s+" r"(?:this|these|the|such|this research|the research)\s+(?:\w+\s+){0,2}?" r"(?:problems?|issues?|challenges?|gaps?|limitations?|needs?|shortcomings?)\s*,?\s+", re.I) CONCESSIVE = re.compile(r",?\s+(?:but|yet|however,?|although|whereas)\s+", re.I) PRON = re.compile(r"^(their|its|they|it)\b\s*", re.I) AUX = re.compile(r"\b(is|are|was|were|has|have|had|can|could|may|might|will|would|remain|remains|show|shows|showed|" r"demonstrate|demonstrates|demonstrated|exhibit|exhibits|achieve|achieves|achieved|provide|provides)\b", re.I) SPLIT = re.compile(r"(\s*\([^)]*\)|,\s+(?:particularly|especially|including|such as|which|while|whereas|thereby|" r"in order to|so as to|due to|leading to|resulting in)\s+|\s+(?:particularly|especially|including|" r"such as|due to|in order to|so as to|to identify|to determine|to examine|to explore|to investigate|to analy[sz]e|" r"to assess|to evaluate|to improve|to reduce|to enhance|to support|to predict|to detect|to estimate|" r"to understand|to characteri[sz]e|to quantify|to provide|" r"thereby|leading to|resulting in|while|whereas|which)\s+|,\s+)", re.I) NUM = re.compile(r"\d") WORD = re.compile(r"[A-Za-z0-9][A-Za-z0-9\-%\.']*") def doc_weights(sentences, idf_lookup, lo_lookup=None): """Word importance inside one abstract: normalised (abstract tf x global idf) + positive highlight log-odds.""" toks = [w.lower() for s in sentences for w in WORD.findall(s)] tf = collections.Counter(w for w in toks if w not in FUNC) sc = {w: c * idf_lookup(w) for w, c in tf.items()} top = max(sc.values()) if sc else 1.0 out = {} for w, v in sc.items(): lo = max(lo_lookup(w), 0.0) if lo_lookup else 0.0 out[w] = v / top + 0.3 * min(lo, 2.0) return out def _importance(text, weights): ws = [w.lower() for w in WORD.findall(text)] if not ws: return 0.0 s = sum(weights.get(w, 0.0) for w in ws if w not in FUNC) s += 1.5 * sum(1 for w in ws if NUM.search(w)) # numbers carry results return s / (len(ws) + 2.0) def _subject(clause): m = AUX.search(clause) if not m: return None subj = clause[:m.start()].strip(" ,") return subj if 1 <= len(subj.split()) <= 6 else None def _possessive(np_): return np_ + ("'" if np_.lower().endswith("s") else "'s") EVIDENCE = re.compile(r"^(?:our\s+|the\s+)?(?:experiments?|results?|tests?|simulations?|evaluations?|analyses|analysis|" r"findings|measurements|data)\b[^,;]{0,80}?\b(?:show|shows|showed|shown|demonstrate|demonstrates|" r"demonstrated|indicate|indicates|indicated|reveal|reveals|revealed|confirm|confirms|confirmed|" r"suggest|suggests|suggested)\s+that\s+", re.I) def reframe(s): """Remove reporting frames and keep the contrast clause of a concessive sentence.""" s = s.strip().rstrip(".") s = EVIDENCE.sub("", s) or s s = PURPOSE_LEAD.sub("", s) or s m = FRAME_SUBJ.match(s) or FRAME_WE.match(s) if m and s[m.end():]: verb = (m.group(1) + m.group(2)) if m.group(2) in ("d", "ed") else _third_person(m.group(1)) s = verb.capitalize() + " " + s[m.end():] parts = CONCESSIVE.split(s, maxsplit=1) if len(parts) == 2 and len(parts[1].split()) >= 5: first, second = parts pm = PRON.match(second) if pm: subj = _subject(first) if subj is None: return s p = pm.group(1).lower() second = (_possessive(subj) if p in ("their", "its") else subj) + " " + second[pm.end():] s = second return s[0].upper() + s[1:] if s else s PP = re.compile(r"(\s+(?:for|with|using|through|via|by|based on|by means of|under|during|within|across|among|as well as|in terms of)\s+)", re.I) def _drop(segs, weights, target, protect_first=True): keep = [True] * len(segs) length = lambda: sum(len(segs[i].split()) for i in range(len(segs)) if keep[i]) # keep the segment with the main verb when the first segment has none (otherwise the highlight has no predicate) verb_seg = None if AUX.search(segs[0]) else next((i for i, g in enumerate(segs) if AUX.search(g)), None) for i in sorted(range(1 if protect_first else 0, len(segs)), key=lambda i: _importance(segs[i], weights)): if length() <= target: break if i == verb_seg: continue if NUM.search(segs[i]) and _importance(segs[i], weights) > 0.2: continue keep[i] = False return "".join(g for g, k in zip(segs, keep) if k) def _segments(s, rx): pieces = rx.split(s) segs = [pieces[0]] + [pieces[i] + pieces[i + 1] for i in range(1, len(pieces) - 1, 2)] return [g for g in segs if g.strip(" ,")] FILLER = re.compile(r"\s*\b(?:as described herein|as reported here(?:in)?|described herein|presented here(?:in)?|" r"in this (?:study|work|paper)|in the present (?:study|work)|herein|here)\b", re.I) def compress(s, weights, target=TARGET_WORDS): s = reframe(s) s = FILLER.sub("", s).strip() segs = _segments(s, SPLIT) if len(segs) > 1: s = _drop(segs, weights, target) if len(s.split()) > target + 4: # still long: drop the least important prepositional phrases segs = _segments(s, PP) # never cut inside the first 6 words (subject and main verb) if len(segs) > 1 and len(segs[0].split()) >= 3: s = _drop(segs, weights, target) s = re.sub(r"\s+,", ",", re.sub(r"\s+", " ", s)).strip(" ,;:") s = re.sub(r"\b(and|or|but|with|by|for|to|of|in|the|a|an)$", "", s, flags=re.I).strip(" ,") return (s[0].upper() + s[1:]) if s else s def concise_count(n, maxs=3): """How many highlights to keep in Concise mode: about half the sentences, at least 2, at most maxs.""" return int(min(maxs, max(2, math.ceil(n / 2)))) if n > 2 else n