Spaces:
Running
Running
| """ | |
| Rule-based sentence RESTRUCTURING (no AI model): changes the shape of a sentence, not just its words, so that | |
| Parrot's paraphrase of it ends up further from the original. Used for the sentences an uploaded report flagged. | |
| Transforms (each one keeps every word of content; the caller checks terms, numbers and citations with | |
| textparse.guard against the ORIGINAL sentence): | |
| * drop stock openers "Furthermore, X." -> "X." | |
| * fronted clause to the end "By combining A, the method B." -> "The method B by combining A." | |
| * end clause to the front "X improves Y, although Z." -> "Although Z, X improves Y." | |
| * split long sentences "A; B." / "A, whereas B." -> "A. In contrast, B." / "A, and the B" -> "A. The B" | |
| """ | |
| import re | |
| _OPENERS = re.compile( | |
| r"^(Furthermore|Moreover|Additionally|In addition|Also|Overall|Therefore|Thus|Hence|As a result|" | |
| r"Consequently|In conclusion|Finally|Importantly|Notably|In particular|Specifically|Accordingly),\s+", re.I) | |
| _FRONT = re.compile( | |
| r"^(?P<lead>(?:Although|While|Because|Since|When|If|As|By|Through|Using|With|For|To|After|Before|During|" | |
| r"Despite|Unlike|Based on|In order to|In response to|In this|In the|Within|Across|Among)\b[^,;]{6,140}),\s+" | |
| r"(?P<main>(?:the|this|these|those|it|its|they|their|we|our|such|a|an|each|both|all|many|most|[A-Z])[^;]*?)" | |
| r"(?P<end>[.!?])$") | |
| _BACK = re.compile( | |
| r"^(?P<main>[^,;]{12,}?),\s+(?P<conj>although|though|because|since|while|whereas|as|unless)\s+" | |
| r"(?P<clause>[^,;]{8,})(?P<end>[.!?])$", re.I) | |
| _SPLITS = [ | |
| (re.compile(r";\s+(?=[a-z])"), lambda m: ". ", None), | |
| (re.compile(r",\s+whereas\s+"), lambda m: ". In contrast, ", None), | |
| (re.compile(r",\s+while\s+(?=(?:the|this|these|it|they|other|another|[a-z]+s\b))"), lambda m: ". Meanwhile, ", None), | |
| (re.compile(r",\s+and\s+(?=(?:the|this|these|it|they|such|their|its)\b)"), lambda m: ". ", None), | |
| (re.compile(r",\s+which\s+(?=(?:is|are|was|were|has|have|can|could|may|might|will|would|helps|makes|allows|leads|" | |
| r"provides|enables|improves|reduces|results)\b)"), lambda m: ". This ", None), | |
| ] | |
| _OPEN_NOCOMMA = re.compile(r"^(Also|Overall|Therefore|Thus|Hence|Furthermore|Moreover|Additionally)\s+" | |
| r"(?=(?:the|this|these|it|they|we|such|our|its|their)\b)", re.I) | |
| _VERB_AFTER_SUBJ = re.compile(r"^((?:it|they|we)\b|(?:the|this|these|such|our|its|their)\b(?:\s+\S+){1,4}?)\s+" | |
| r"(is|are|was|were|can|could|may|might|will|would|has|have|had|" | |
| r"[a-z]+(?:es|s|ed))\b", re.I) | |
| _TAIL_PP = re.compile(r"\s(in|within|across|for|through|during|among|throughout|under)\s" | |
| r"(?P<np>(?:the\s|a\s|an\s|this\s|these\s|such\s|many\s|both\s|all\s)?[a-z][\w\-]*" | |
| r"(?:\s[\w\-]+){1,6})(?P<end>[.!?])$") | |
| _VERBISH = re.compile(r"\b(is|are|was|were|be|been|being|can|could|may|might|will|would|has|have|had|do|does|" | |
| r"did|which|that|who|whom|whose|when|where|while|and|or)\b") | |
| _KEEP_CAPS = re.compile(r"^(?:[A-Z]{2,}|[A-Z][a-z]*[A-Z]|I\b)") # acronyms / CamelCase / "I" stay capitalised | |
| def _lower_first(s): | |
| return s if _KEEP_CAPS.match(s) else s[:1].lower() + s[1:] | |
| def _upper_first(s): | |
| return s[:1].upper() + s[1:] | |
| def _drop_opener(s): | |
| m = _OPENERS.match(s) | |
| if m and len(s) - m.end() > 25: | |
| return _upper_first(s[m.end():]) | |
| return None | |
| def _drop_opener_nocomma(s): | |
| """'Also the thesis focuses on X.' -> 'The thesis also focuses on X.'""" | |
| m = _OPEN_NOCOMMA.match(s) | |
| if not m: | |
| return None | |
| word, rest = m.group(1).lower(), s[m.end():] | |
| v = _VERB_AFTER_SUBJ.match(rest) | |
| if v and word in ("also", "therefore", "thus", "hence"): | |
| verb = v.group(2) | |
| if verb.lower() in ("is", "are", "was", "were", "can", "could", "may", "might", "will", "would", | |
| "has", "have", "had"): # "It is also ...", "They can also ..." | |
| return _upper_first(f"{v.group(1)} {verb} {word}{rest[v.end(2):]}") | |
| return _upper_first(f"{v.group(1)} {word} {rest[v.start(2):]}") | |
| return _upper_first(rest) if len(rest.split()) >= 6 else None | |
| def _tail_to_front(s): | |
| """'X becomes Y in educational data mining.' -> 'In educational data mining, X becomes Y.'""" | |
| if "," in s[-80:]: | |
| return None | |
| m = _TAIL_PP.search(s) | |
| if not m or _VERBISH.search(m.group("np")): | |
| return None | |
| main = s[:m.start()] | |
| if len(main.split()) < 7 or re.search(r"\b(that|which|who|whom|whose|where|such that|so that)\b", main): | |
| return None # the phrase may belong to a sub-clause: moving it changes meaning | |
| return f"{_upper_first(m.group(1))} {m.group('np')}, {_lower_first(main)}{m.group('end')}" | |
| def _front_to_back(s): | |
| m = _FRONT.match(s) | |
| if not m: | |
| return None | |
| lead, main = m.group("lead"), m.group("main") | |
| if len(main.split()) < 5: | |
| return None | |
| return f"{_upper_first(main)} {_lower_first(lead)}{m.group('end')}" | |
| def _back_to_front(s): | |
| m = _BACK.match(s) | |
| if not m or len(m.group("main").split()) < 4: | |
| return None | |
| conj = m.group("conj").lower() | |
| if conj == "whereas": | |
| return None # handled by a split instead | |
| return f"{_upper_first(conj)} {m.group('clause')}, {_lower_first(m.group('main'))}{m.group('end')}" | |
| def _split(s, min_words=24): | |
| if len(s.split()) < min_words: | |
| return None | |
| for pat, rep, _ in _SPLITS: | |
| m = pat.search(s) | |
| if not m: | |
| continue | |
| left, right = s[:m.start()], s[m.end():] | |
| if len(left.split()) < 7 or len(right.split()) < 6: | |
| continue | |
| joiner = rep(m) | |
| right = _upper_first(right) if joiner.endswith(". ") else right | |
| left = left.rstrip(" ,;") | |
| return f"{left}{joiner}{right}" | |
| return None | |
| def variants(sentence): | |
| """Restructured versions of `sentence` (may be empty). Combinations are tried as well.""" | |
| s = sentence.strip() | |
| out = [] | |
| first = [f(s) for f in (_drop_opener, _drop_opener_nocomma, _front_to_back, _back_to_front, _split, | |
| _tail_to_front)] | |
| for v in first: | |
| if v and v != s and v not in out: | |
| out.append(v) | |
| for v in list(out): # one more step on top (e.g. opener dropped, then split) | |
| for f in (_front_to_back, _back_to_front, _split, _tail_to_front): | |
| w = f(v) | |
| if w and w != v and w != s and w not in out: | |
| out.append(w) | |
| return out | |
| def restructure(sentence, guard=None, overlap=None): | |
| """The restructured version that changes the sentence's shape the most and still passes `guard` | |
| (callable(original, new) -> str|None). Returns the original sentence when nothing applies.""" | |
| best, best_ov = sentence, 101.0 | |
| for v in variants(sentence): | |
| ok = guard(sentence, v) if guard else v | |
| if not ok: | |
| continue | |
| ov = overlap(sentence, ok) if overlap else 0.0 | |
| if ov < best_ov: | |
| best, best_ov = ok, ov | |
| return best | |