Spaces:
Running
Running
| """ | |
| Pasted-text handling shared by the detector, Smart Humanize and the Humanizer. | |
| parse_paste(text) -> [(paragraph, separator_after)] | |
| * blank line -> paragraph break, kept as "\n\n" | |
| * single line break -> paragraph break, kept as "\n" (Word / web copy-paste) | |
| * soft wrap (PDF copy) -> joined into the same paragraph ("in modern\ncommerce" -> "in modern commerce") | |
| join_paste(parts) -> text with the original paragraph breaks restored | |
| chunk_sentence / merge_chunks | |
| Parrot T5 was trained on short sentences and truncates or garbles long academic ones, | |
| so sentences over ~28 words are split at safe clause boundaries, paraphrased piece by | |
| piece and stitched back together. | |
| """ | |
| import contextvars | |
| import difflib | |
| import re | |
| _TERMINAL = re.compile(r"[.!?:;\"”’)\]]\s*$") | |
| _BULLET = re.compile(r"^\s*([•●▪\-\*–]|\(?[0-9ivxIVX]{1,3}[\)\.]|\(?[a-z][\)\.])\s+") | |
| def _soft_wrap(prev, line): | |
| """True if `line` continues `prev` (hard-wrapped text copied from a PDF).""" | |
| p, l = prev.rstrip(), line.lstrip() | |
| if not p or not l or _BULLET.match(l): | |
| return False | |
| if p.endswith("-") and l[:1].islower(): # hyphenated word break | |
| return True | |
| if _TERMINAL.search(p): | |
| return False | |
| if l[:1].islower() or l[:1] in ",;)": # sentence clearly continues | |
| return True | |
| return len(p) >= 55 # long line cut mid-sentence | |
| def parse_paste(text): | |
| lines = (text or "").replace("\r\n", "\n").replace("\r", "\n").split("\n") | |
| parts, cur, blank = [], None, False | |
| for raw in lines: | |
| line = raw.strip() | |
| if not line: | |
| blank = True | |
| continue | |
| if cur is not None and not blank and _soft_wrap(cur, line): | |
| cur = cur[:-1] + line if cur.endswith("-") and line[:1].islower() else cur + " " + line | |
| continue | |
| if cur is not None: | |
| parts.append([cur, "\n\n" if blank else "\n"]) | |
| cur, blank = line, False | |
| if cur is not None: | |
| parts.append([cur, ""]) | |
| return [(re.sub(r"[ \t ]+", " ", p).strip(), s) for p, s in parts] | |
| def join_paste(parts): | |
| return "".join(p + s for p, s in parts).strip() | |
| # ---------------------------------------------------------------- long sentences | |
| LONG_WORDS = 28 | |
| _CONJ = r"(?:and|but|while|whereas|which|although|though|because|so|thus|thereby|yet|whilst|where|when)" | |
| _SPLIT = re.compile(rf"(?<=;)\s+|(?<=,)\s+(?={_CONJ}\b)") | |
| def chunk_sentence(sentence, max_words=LONG_WORDS, min_words=7): | |
| """Split a long sentence at '; ' or ', <conjunction>' so each piece has >= min_words.""" | |
| if len(sentence.split()) <= max_words: | |
| return [sentence] | |
| pieces = [p for p in _SPLIT.split(sentence) if p and p.strip()] | |
| out = [] | |
| for p in pieces: | |
| if out and (len(out[-1].split()) < min_words or len(p.split()) < min_words): | |
| out[-1] = out[-1] + " " + p | |
| else: | |
| out.append(p) | |
| if len(out) > 1 and len(out[-1].split()) < min_words: | |
| last = out.pop() | |
| out[-1] = out[-1] + " " + last | |
| return out | |
| def _fit_chunk(orig, new, first, last): | |
| """Make a paraphrased piece fit back into the sentence it came from.""" | |
| new = (new or "").strip() or orig | |
| tail = orig.rstrip()[-1:] | |
| if not last: | |
| new = re.sub(r"[.!?]+$", "", new).rstrip() | |
| if tail in ",;" and not new.endswith(tail): | |
| new += tail | |
| if not first: | |
| w0 = orig.split()[0] | |
| if re.fullmatch(_CONJ, w0.lower()) and new.split()[0].lower().strip(",") != w0.lower(): | |
| new = w0 + " " + (new[0].lower() + new[1:] if new[1:2].islower() else new) | |
| elif orig[:1].islower() and new[1:2].islower(): | |
| new = new[0].lower() + new[1:] | |
| return new | |
| def merge_chunks(orig_chunks, new_chunks): | |
| n = len(orig_chunks) | |
| return " ".join(_fit_chunk(o, c, i == 0, i == n - 1) for i, (o, c) in enumerate(zip(orig_chunks, new_chunks))) | |
| # ---------------------------------------------------------------- rewrite guard | |
| # citation / reference markers that must survive a rewrite | |
| CITE = re.compile(r"\[\s*\d+(?:\s*[,–\-]\s*\d+)*\s*\]" | |
| r"|\((?:[A-Z][^()]{0,80}?(?:19|20)\d{2}[a-z]?(?:;\s*[^()]{0,80}?(?:19|20)\d{2}[a-z]?)*)\)") | |
| # ---------------------------------------------------------------- technical-term protection | |
| # A rewrite is rejected (original sentence kept) if it loses or alters a technical term: | |
| # * acronyms / symbols / formulas / model names: CNN, LSTM, IoMT, LiFePO4, ResNet-50, 5G, CO2, α, W/m | |
| # * terms defined in the document: "phase change material (PCM)" -> "phase change material" | |
| # * the document's key terms: phrases used repeatedly ("state of charge") and content words used 3+ times | |
| # * words whose everyday synonyms change the technical meaning ("model", "accuracy", "layer", ...) | |
| TECH_TOKEN = re.compile(r"""(?x) | |
| (?<![\w-])(?: | |
| [A-Za-z]*[A-Z][A-Za-z]*\d[\w.\-]* # LiFePO4, ResNet50, CO2, H2O, YOLOv8 | |
| | \d+(?:\.\d+)?[A-Za-z][\w\-]* # 5G, 3D, 2D-CNN, 3C | |
| | [A-Z]{2,}[a-z]?(?:-[A-Za-z0-9]+)* # CNN, LSTM, SVMs, BERT-base | |
| | [a-z]+[A-Z][\w\-]* # mmWave, eNodeB, pH, iOS | |
| | [A-Z][a-z]+[A-Z][\w\-]* # ResNet, PyTorch, IoMT, MobileNet | |
| | [Ͱ-Ͽ][\wͰ-Ͽ]* # Greek letters: α, β-sheet, Δ | |
| | [A-Za-z]+(?:[/^_][A-Za-z0-9]+)+ # W/m, m/s, x_i | |
| )(?!\w)""") | |
| FRAGILE = set(""" | |
| accuracy precision recall specificity sensitivity model training trained learning network layer kernel | |
| feature dataset regression variance gradient algorithm parameter coefficient correlation classification | |
| classifier clustering encoder decoder attention transformer convolutional recurrent neural optimization | |
| optimizer loss epoch inference latency throughput bandwidth voltage resistance capacitance inductance | |
| impedance frequency signal noise power energy thermal temperature heat flux pressure stress strain | |
| viscosity density velocity torque load cathode anode electrolyte catalyst enzyme protein gene cell tissue | |
| hypothesis significance sample respondent mediation moderation reliability validity simulation mesh node | |
| cluster packet protocol routing entropy matrix vector tensor derivative integral equation theorem spectrum | |
| wavelength phase charge discharge battery circuit converter inverter sensor compression sparse sparsity | |
| gender genders male males female females men women participants respondents | |
| reconstruction segmentation detection estimation prediction forecasting pavement asphalt binder aggregate | |
| """.split()) | |
| _STOP = set("""a an the and or but if then than of to in on at by for with from as into onto over under | |
| between among through during before after about above below up down out off again further once here there | |
| when where why how all any both each few more most other some such no nor not only own same so too very | |
| can will just should now is are was were be been being have has had having do does did doing this that | |
| these those it its they them their we our us you your he she his her i me my which who whom whose what | |
| also may might must shall could would however thus therefore hence moreover furthermore additionally | |
| while whereas although though because since whether within without across along toward towards upon via | |
| per et al using used use based show shows shown showed study studies paper research results result | |
| approach proposed propose provides provide provided well new one two three first second third | |
| different various several many much significantly important overall present current""".split()) | |
| GLOSSARY = contextvars.ContextVar("tp_glossary", default=frozenset()) | |
| def _norm(text): | |
| """lower-case, hyphens -> spaces, light plural stripping — "State-of-Charge" == "states of charge".""" | |
| t = re.sub(r"[-‐–]", " ", (text or "").lower()) | |
| t = re.sub(r"\b(\w{3,}?)ies\b", r"\1y", t) | |
| t = re.sub(r"\b(\w{3,}?)(?:es|s)\b", r"\1", t) | |
| return " " + re.sub(r"\s+", " ", re.sub(r"[^\w\s]", " ", t)).strip() + " " | |
| _FRAGILE_N = {_norm(w).strip() for w in FRAGILE} | |
| _STAT_STEMS = ("mediat", "moderat", "correlat", "regress") | |
| _STOP_N = {_norm(w).strip() for w in _STOP} | _STOP | |
| # Narrow term protection (report-flagged sentences): only multi-word technical terms and defined acronyms are | |
| # protected, so everyday words ("educational", "efficient", "associated") are free to change. Acronyms, numbers, | |
| # formulas and citations are always protected (TECH_TOKEN / guard), whatever this setting. | |
| NARROW = contextvars.ContextVar("narrow_terms", default=False) | |
| def narrow(glossary): | |
| """Multi-word entries of a glossary (drop single everyday words).""" | |
| return frozenset(t for t in glossary if " " in t) | |
| def build_glossary(texts): | |
| """Key terms of a document (normalised phrases).""" | |
| joined = " ".join(t for t in texts if t) | |
| terms = set() | |
| for m in re.finditer(r"((?:[A-Za-z][\w\-]*\s+){1,7})\(\s*([A-Z][A-Za-z0-9\-]{1,11}?)s?\s*\)", joined): | |
| words = m.group(1).split() | |
| n = max(1, min(len(words), len(re.findall(r"[A-Z0-9]", m.group(2))) or len(m.group(2)))) | |
| cand = words[-n:] | |
| while cand and cand[0].lower() in _STOP: | |
| cand = cand[1:] | |
| if cand and len(" ".join(cand)) >= 4: | |
| terms.add(_norm(" ".join(cand)).strip()) | |
| defined = set(terms) # terms the author defined with an abbreviation | |
| uni, grams = {}, {} | |
| for s in re.split(r"(?<=[.!?;:])\s+", joined): | |
| toks = _norm(s).split() | |
| for w in toks: | |
| if len(w) >= 5 and w not in _STOP_N and not w.isdigit(): | |
| uni[w] = uni.get(w, 0) + 1 | |
| for n in (2, 3): | |
| for i in range(len(toks) - n + 1): | |
| g = toks[i:i + n] | |
| if g[0] in _STOP_N or g[-1] in _STOP_N or any(len(w) < 3 or w.isdigit() for w in (g[0], g[-1])): | |
| continue | |
| if n == 2 and max(len(g[0]), len(g[1])) < 5: | |
| continue | |
| if n == 3 and g[1] in _STOP_N and g[1] not in ("of", "to", "and", "for", "in", "on", "per"): | |
| continue # "support the preservation" is wording, not a term | |
| k = " ".join(g) | |
| grams[k] = grams.get(k, 0) + 1 | |
| if NARROW.get(): # report-flagged rewriting: real technical terms only | |
| terms |= {k for k, c in grams.items() if c >= 3} | |
| return _no_stock(terms, defined) | |
| terms |= {k for k, c in grams.items() if c >= 2} | |
| # single words: only the document's real key terms (frequent relative to its length), so ordinary | |
| # vocabulary stays free for the paraphraser | |
| min_uni = max(3, len(joined.split()) // 300) | |
| terms |= {w for w, c in uni.items() if c >= min_uni and len(w) >= 6} | |
| return _no_stock(terms, defined) | |
| def _no_stock(terms, defined): | |
| """A phrase that only repeats because the text is AI-written ("crucial role" in three paragraphs) is not a term: | |
| protecting it forced every rewrite to keep it ("acts as a crucial role"). Terms the author defined with an | |
| abbreviation stay protected whatever words they contain.""" | |
| import pubrules as PR | |
| return frozenset(t for t in terms if t and (t in defined or not PR._AI_WORD_RE.search(t))) | |
| # ---------------------------------------------------------------- document key terms (always protected) | |
| # The study's own constructs must never be swapped for synonyms ("travel satisfaction" -> "travel pleasure", | |
| # "intimacy" -> "closeness", "destination loyalty" -> "allegiance"). They are read from the WHOLE document once per | |
| # job and protected in every mode, including report-guided rewriting where the ordinary glossary is narrowed: | |
| # * every term defined with an abbreviation, "travel satisfaction (TS)" -> "travel satisfaction" + "satisfaction" | |
| # * the Keywords line, "Keywords: Travel motivation, Intimacy, ..." -> each keyword + its head word | |
| KEY = contextvars.ContextVar("tp_key_terms", default=frozenset()) | |
| KEY_DEF = contextvars.ContextVar("tp_key_defined", default=frozenset()) # terms defined with an abbreviation | |
| DOC_VOCAB = contextvars.ContextVar("tp_doc_vocab", default=frozenset()) | |
| DOC_PROPER = contextvars.ContextVar("tp_doc_proper", default=frozenset()) # words the document always capitalises | |
| _GENERIC_HEADS = {"study", "analysi", "analysis", "difference", "differenc", "economy", "engagement", "modeling", | |
| "modelling", "council", "organization", "organisation", "approach", "model", "method", "framework", "system", "process", "factor", | |
| "factors", "effect", "effects", "impact", "role", "level", "levels", "rate", "index", "test", | |
| "scale", "theory", "research", "development", "management", "performance", "behaviour", | |
| "behavior", "value", "values", "data", "group", "groups", "practice", "practices"} | |
| def key_terms(texts): | |
| """Construct names of a document (normalised), see KEY.""" | |
| joined = " ".join(t for t in texts if t) | |
| terms = set() | |
| for m in re.finditer(r"((?:[A-Za-z][\w\-]*\s+){1,5})\(\s*([A-Z][A-Za-z0-9\-]{1,11}?)s?\s*\)", joined): | |
| acr, words = m.group(2), m.group(1).split() | |
| core = len(re.findall(r"\b" + re.escape(acr) + r"\b", joined)) >= 4 # a construct the study keeps using | |
| n = len(re.findall(r"[A-Z]", acr)) or len(acr) | |
| cand = words[-n:] | |
| while cand and cand[0].lower() in _STOP: | |
| cand = cand[1:] | |
| if not cand or len(" ".join(cand)) < 4: | |
| continue | |
| terms.add(_norm(" ".join(cand)).strip()) | |
| head = _norm(cand[-1]).strip() | |
| if core and len(head) >= 5 and head not in _GENERIC_HEADS and head not in _STOP_N: | |
| terms.add(head) | |
| KEY_DEF.set(frozenset(terms)) | |
| for line in re.findall(r"(?im)^\s*key\s*-?\s*words?\s*[:\-–]\s*(.+)$", "\n".join(t for t in texts if t)): | |
| for kw in re.split(r"[,;·•]", line): | |
| kw = _norm(kw).strip() | |
| if len(kw) >= 4 and len(kw.split()) <= 5: | |
| terms.add(kw) | |
| head = kw.split()[-1] | |
| if len(head) >= 5 and head not in _GENERIC_HEADS and head not in _STOP_N: | |
| terms.add(head) | |
| return frozenset(t for t in terms if t) | |
| # everyday synonyms a paraphraser uses for common research constructs; when the construct is one of the document's | |
| # key terms, a rewrite may not bring in its synonym ("satisfaction" study: "pleasure", "contentment" are rejected) | |
| _CONSTRUCT_SYN = { | |
| "satisfaction": {"pleasure", "contentment", "happiness", "delight", "gratification", "fulfilment", "fulfillment"}, | |
| "loyalty": {"allegiance", "faithfulness", "devotion", "fidelity"}, | |
| "intimacy": {"closeness", "familiarity", "affinity"}, | |
| "motivation": {"incentive", "incentives", "impetus", "inducement"}, | |
| "well being": {"welfare", "wellness", "health"}, | |
| "wellbeing": {"welfare", "wellness", "health"}, | |
| "trust": {"faith", "confidence", "reliance"}, | |
| "attachment": {"bond", "fondness", "affection"}, | |
| "intention": {"willingness", "inclination"}, | |
| "engagement": {"involvement", "participation", "commitment"}, | |
| "commitment": {"dedication", "devotion", "pledge"}, | |
| "performance": {"output", "achievement", "efficiency"}, | |
| "image": {"picture", "impression", "reputation"}, | |
| "quality": {"standard", "calibre", "caliber", "grade"}, | |
| "anxiety": {"worry", "nervousness", "unease"}, | |
| "stress": {"strain", "pressure", "tension"}, | |
| } | |
| def renamed_construct(original, new): | |
| """A key construct's everyday synonym the rewrite introduces ("allegiance" in a loyalty study), or None.""" | |
| key = KEY.get() | |
| if not key: | |
| return None | |
| nn, no = _norm(new), _norm(original) | |
| for term, syns in _CONSTRUCT_SYN.items(): | |
| if term in key: | |
| for w in syns: | |
| if f" {_norm(w).strip()} " in nn and f" {_norm(w).strip()} " not in no: | |
| return f"{term} -> {w}" | |
| return None | |
| def set_document(texts): | |
| """Call once per job with the whole document: key terms and vocabulary for the checks below.""" | |
| texts = [t for t in texts if t] | |
| KEY.set(key_terms(texts)) | |
| cap, low = {}, {} | |
| for t in texts: | |
| for m in re.finditer(r"(?<![.!?:]\s)(?<=\s)([A-Za-z][a-z]{2,})\b", t): # mid-sentence words only | |
| w = m.group(1) | |
| d = cap if w[:1].isupper() else low | |
| d[w.lower()] = d.get(w.lower(), 0) + 1 | |
| DOC_PROPER.set(frozenset(w.capitalize() for w, c in cap.items() if c >= 2 and not low.get(w))) | |
| vocab = {w.lower() for t in texts for w in re.findall(r"[A-Za-z][A-Za-z\-]+", t)} | |
| DOC_VOCAB.set(frozenset(vocab) if sum(len(t.split()) for t in texts) >= 300 else frozenset()) | |
| def lost_terms(original, new, glossary=None): | |
| """Technical terms of `original` that `new` no longer contains (empty list = safe).""" | |
| glossary = GLOSSARY.get() if glossary is None else glossary | |
| missing = [] | |
| for t in {m.group(0) for m in TECH_TOKEN.finditer(CITE.sub(" ", original))}: | |
| core = t[:-1] if t.endswith("s") and t[:-1].isupper() else t | |
| if not re.search(r"(?<!\w)" + re.escape(core.rstrip(".-")), new): | |
| missing.append(t) | |
| no, nn = _norm(original), _norm(new) | |
| missing += [w for w in set(no.split()) & _FRAGILE_N if f" {w} " not in nn] | |
| missing += [g for g in glossary if f" {g} " in no and f" {g} " not in nn] | |
| # statistical relations in any word form: "does not moderate" must not become "does not modify" | |
| missing += [st + "…" for st in _STAT_STEMS if f" {st}" in no and f" {st}" not in nn] | |
| missing += [g for g in KEY.get() if f" {g} " in no and f" {g} " not in nn] # study constructs | |
| return sorted(set(missing)) | |
| # ---------------------------------------------------------------- sentence rewriting | |
| _TITLES = re.compile(r"\b(?:Mr|Mrs|Ms|Miss|Dr|Prof|Sir|Madam|Mx)\b\.?", re.I) | |
| _PERSONAL = re.compile(r"\b(?:he|she|him|his|her|hers|himself|herself)\b", re.I) | |
| _SENT_START = re.compile(r"(?:^|[.!?]\s+|[\"“(]\s*)$") | |
| def invented(original, new): | |
| """What `new` makes up that `original` does not contain (paraphrase models sometimes add a name, a person or a | |
| number). Returns a short reason, or None if nothing was invented.""" | |
| o_low = original.lower() | |
| o_words = set(re.findall(r"[a-z0-9']+", o_low)) | |
| o_caps = {w.lower() for w in re.findall(r"\b[A-Z][a-z]{3,}\b", original)} | |
| if _TITLES.search(new) and not _TITLES.search(original): | |
| return "title" | |
| if _PERSONAL.search(new) and not _PERSONAL.search(original): | |
| return "person" | |
| nums_o = set(re.findall(r"\d+(?:\.\d+)?", original)) | |
| if any(n not in nums_o for n in re.findall(r"\d+(?:\.\d+)?", CITE.sub(" ", new))): | |
| return "number" | |
| for m in re.finditer(r"\b[A-Z][a-z]{2,}\b", new): # capitalised word mid-sentence = name-like | |
| if _SENT_START.search(new[:m.start()]) or (re.search(r"[\])]\s*$", new[:m.start()]) | |
| and re.fullmatch(r"[A-Z][a-z]+(?:ed|es|s)", m.group(0))): | |
| continue # sentence start, or "[56] Underscored" (citation + verb) | |
| w = m.group(0).lower() | |
| if w in o_words or re.search(r"\b" + re.escape(w) + r"\b", o_low): | |
| continue # ("Men's" in the original counts as "Men") | |
| # another form of a name the author uses ("India's" for "Indian", "Bayes" for "Bayesian") is not invented | |
| if len(w) >= 4 and any(o.startswith(w[:max(4, len(w) - 2)]) or w.startswith(o[:max(4, len(o) - 2)]) | |
| for o in o_caps): | |
| continue | |
| return "name" | |
| return None | |
| # ---------------------------------------------------------------- quality gate (every mode) | |
| _INFORMAL = re.compile( | |
| r"\b(?:absurd|stuff|things like|kinda|sorta|gonna|wanna|gotta|pretty much|a lot of|lots of|tons of|guys|okay|ok|" | |
| r"awesome|cool|huge|super|basically|really|totally|folks|kids|crazy|weird|silly|nope|yeah|yep|dumb|" | |
| r"nice|great deal of fun|anyway|anyhow|like i said|you know|bunch of|figure out|sort of|kind of)\b", re.I) | |
| _FIRST_SING = re.compile(r"\b(?:I|me|my|mine|myself)\b") | |
| _YOU = re.compile(r"\b(?:you|your|yours|yourself)\b", re.I) | |
| _CONTRACTION = re.compile(r"\b\w+(?:n't|'re|'ll|'ve|'m|'d)\b", re.I) | |
| try: | |
| from wordfreq import zipf_frequency as _zipf | |
| except Exception: # optional dependency | |
| _zipf = None | |
| def _known(word, o_words): | |
| """A real English word, or a form of a word the original already uses.""" | |
| w = word.lower() | |
| if w in o_words or len(w) < 4 or _zipf is None: | |
| return True | |
| if _zipf(w, "en") > 0.0: | |
| return True | |
| if "-" in w and all(_zipf(p, "en") > 0.0 or p in o_words for p in w.split("-") if p): | |
| return True | |
| for k in range(3, len(w) - 2): # two words run together ("statisticallyor") | |
| if _zipf(w[:k], "en") >= 3.0 and _zipf(w[k:], "en") >= 3.0: | |
| return False | |
| sfx = ("s", "es", "ed", "d", "ing", "ly", "al", "ally", "ation", "ations", "ity", "ive", "ness", "ment", "er", | |
| "ers", "ic", "ical", "ise", "ize", "ised", "ized") | |
| for o in o_words: # an inflected form of a word the original uses | |
| if len(o) < 5: | |
| continue | |
| for stem in (o, o[:-1], o[:-2]): | |
| if len(stem) >= 4 and w.startswith(stem) and (w[len(stem):] in sfx or w[len(stem):] == ""): | |
| return True | |
| return False | |
| _FOREIGN = re.compile(r"\b[\w\-]*[À-ÖØ-öø-ÿĀ-žƀ-ɏ][\w\-]*") | |
| OFFTOPIC_N = int(__import__("os").getenv("HUM_OFFTOPIC", "2")) # 0 = check off | |
| def _offtopic(original, new): | |
| """Uncommon content words the rewrite brings in that appear nowhere in the document and are not forms of the | |
| sentence's own words ("organizational recruitment" in a tourism paper). None, or the words.""" | |
| vocab = DOC_VOCAB.get() | |
| if not vocab or not OFFTOPIC_N or _zipf is None: | |
| return None | |
| o_low = original.lower() | |
| stems = {w[:5] for w in re.findall(r"[a-z]{5,}", o_low)} | |
| odd = [] | |
| for w in set(re.findall(r"[A-Za-z][a-z\-]{5,}", CITE.sub(" ", new))): | |
| lw = w.lower() | |
| if lw in vocab or lw[:5] in stems or lw in _STOP or w[:1].isupper(): | |
| continue | |
| if _zipf(lw, "en") < 3.3 and not any(v.startswith(lw[:6]) for v in vocab): | |
| odd.append(lw) # rare AND unrelated to any word the document uses | |
| return odd if len(odd) >= OFFTOPIC_N else None | |
| def quality_issue(original, new): | |
| """Why `new` is not acceptable academic English compared with `original` (None = fine): | |
| a non-word the original doesn't contain, a foreign word, words foreign to the document's topic, informal / | |
| conversational wording, first person singular, addressing the reader, contractions, or a question where the | |
| original made a statement.""" | |
| o_words = set(re.findall(r"[a-z][a-z'\-]*", original.lower())) | |
| syn = renamed_construct(original, new) | |
| if syn: | |
| return f"construct renamed ({syn})" | |
| for m in _FOREIGN.finditer(new): # "well-être": accented words the original does not use | |
| if m.group(0).lower() not in original.lower(): | |
| return f"foreign word '{m.group(0)}'" | |
| odd = _offtopic(original, new) | |
| if odd: | |
| return "off-topic words '" + ", ".join(sorted(odd)) + "'" | |
| for w in re.findall(r"[A-Za-z][A-Za-z\-]{3,}", CITE.sub(" ", new)): | |
| if w.isupper() or (w[0].isupper() and w.lower() in o_words): | |
| continue # acronyms / names already in the original | |
| if not _known(w, o_words): | |
| return f"non-word '{w}'" | |
| for pat, why in ((_INFORMAL, "informal wording"), (_FIRST_SING, "first person"), (_YOU, "addresses the reader"), | |
| (_CONTRACTION, "contraction")): | |
| m = pat.search(new) | |
| if m and not pat.search(original): | |
| return f"{why} '{m.group(0)}'" | |
| if "?" in new and "?" not in original: | |
| return "question" | |
| return None | |
| # characters language models like to substitute: the author's own typography is kept | |
| _TYPO = {"‑": "-", "‐": "-", "": "", " ": " ", " ": " ", " ": " ", "": "", | |
| "’": "'", "‘": "'", "“": '"', "”": '"', "−": "-", "²": "2", "³": "3"} | |
| def normalize_typography(original, new): | |
| """Replace no-break hyphens / spaces, curly quotes and the minus sign in `new` by their plain form, unless the | |
| author's sentence uses that character itself (so "lithium-ion" never comes back with a no-break hyphen).""" | |
| if not new: | |
| return new | |
| return "".join(_TYPO[c] if c in _TYPO and c not in original else c for c in new) | |
| _REL = "<>=≤≥≈±×÷~∼≠∝→" | |
| _PAIRS = (("(", ")"), ("[", "]"), ("{", "}")) | |
| def symbol_issue(original, new): | |
| """Relation signs, brackets and citations: None, or why `new` damages them. A paraphraser that turns | |
| "p < 0.01" into "p 0.01" keeps every number but changes the result; an unbalanced bracket or a citation | |
| written twice is always a defect; a numbered citation the author did not write is invented.""" | |
| for c in _REL: | |
| if original.count(c) != new.count(c): | |
| return f"relation sign '{c}' changed" | |
| for a, b in _PAIRS: | |
| if original.count(a) == original.count(b) and new.count(a) != new.count(b): | |
| return "unbalanced bracket" | |
| oc = [re.sub(r"\s+", "", c) for c in CITE.findall(original)] | |
| nc = [re.sub(r"\s+", "", c) for c in CITE.findall(new)] | |
| for c in set(nc): | |
| if c.startswith("[") and c not in oc: | |
| return "citation added" | |
| if c in oc and nc.count(c) > oc.count(c): | |
| return "citation duplicated" | |
| return None | |
| def _ckey(c): | |
| return re.sub(r"\s+", "", c) | |
| def repair_citations(original, new): | |
| """A paraphraser copies citations carelessly: "(Mishra and Mishri, 2012)", "(Sarkar 1973)". A citation in `new` | |
| that the author did not write is replaced by the author's citation it is a damaged copy of; if it is a copy of | |
| none (an invented source), or of one that is already there, the rewrite is rejected (None).""" | |
| orig = CITE.findall(original) | |
| keys = {_ckey(c) for c in orig} | |
| odd = [c for c in CITE.findall(new) if _ckey(c) not in keys] | |
| for c in odd: | |
| have = {_ckey(x) for x in CITE.findall(new)} | |
| free = [o for o in orig if _ckey(o) not in have] | |
| best = max(free, key=lambda o: difflib.SequenceMatcher(None, _ckey(o), _ckey(c)).ratio(), default=None) | |
| if best is None or difflib.SequenceMatcher(None, _ckey(best), _ckey(c)).ratio() < 0.7: | |
| return None | |
| new = new.replace(c, best, 1) | |
| return new | |
| def _symbols(t): | |
| """Comparison / statistics symbols outside citations: < > ≤ ≥ = ± × and their order.""" | |
| return re.findall(r"[<>≤≥=±×]", CITE.sub(" ", t or "")) | |
| def _brackets_ok(t): | |
| return (t or "").count("(") == (t or "").count(")") and (t or "").count("[") == (t or "").count("]") | |
| def _new_acronyms(original, new): | |
| acr = lambda t: set(re.findall(r"\b[A-Z][A-Z0-9]{1,}[a-z]?\b", CITE.sub(" ", t or ""))) | |
| return acr(new) - acr(original) | |
| # ---------------------------------------------------------------- integrity checks shared by EVERY rewrite path | |
| # (Parrot guard, Groq quality check, final polish, proofread, clarity gate). Each one targets an error seen in real | |
| # humanized papers. | |
| _ATTRIB = re.compile(r"\b(?:according to|(?:some |many |several )?(?:experts?|scholars?|researchers?|analysts?|" | |
| r"studies|scientists)\s+(?:say|says|said|argue|argues|contend|contends|believe|believes|" | |
| r"suggest|suggests|claim|claims|found|find|show|shows|note|notes)|a (?:systematic |recent )?" | |
| r"(?:analysis|review|study|survey) (?:found|showed|shows|finds|revealed|suggests))\b", re.I) | |
| _SAID = re.compile(r"\b(?:said|says|told|stated|according to)\b", re.I) | |
| _CAUSAL = re.compile(r"\b(?:because|since (?:they|it|he|she|we|the|this|these|those|their|its)|due to|owing to|" | |
| r"as a result of|thereby|in turn)\b", re.I) | |
| _DANGLING_END = re.compile(r"\b(?:the|a|an|their|its|our|his|her|of|to|and|or|on|in|with|for|by|from|as|than|" | |
| r"that|which|this|these|between|among)\s*[.!?]$", re.I) | |
| _DANGLING_HYPHEN = re.compile(r"[A-Za-z]-(?=[\s,.;:)])") | |
| _EMDASH = re.compile(r"—|\s–\s|\w\s-\s\w") # also a spaced hyphen used as a dash | |
| _SENT_ADV = r"(?:However|Moreover|Furthermore|Therefore|Consequently|Nevertheless|Nonetheless|Additionally|" \ | |
| r"In addition|In general|In contrast|In particular|In summary|In conclusion|For example|For instance|" \ | |
| r"Thus|Hence|Instead|Similarly|Likewise|Meanwhile|Simultaneously|Finally|" \ | |
| r"Accordingly|Specifically|Notably|Importantly|Ultimately|Conversely|Subsequently)" | |
| # ("Further", "Overall", "First" are left out: "Further studies…", "Overall satisfaction…" need no comma) | |
| def _proper_nouns(t): | |
| """Capitalised names inside a sentence (Darjeeling, Singapore, Kumar), not its first word.""" | |
| words = re.findall(r"(?<=[\s(\[,;:])([A-Z][a-z]{2,}(?:-[A-Z][a-z]+)?)\b", " " + CITE.sub(" ", t or "")[1:]) | |
| return {w for w in words if w.lower() not in _STOP and w not in ("Table", "Figure", "Fig", "Section", "Eq", | |
| "Chapter", "Appendix")} | |
| def integrity_issue(original, new): | |
| """Damage a rewrite may never do (None = fine): | |
| * an attribution or a causal explanation the original does not make ("According to some experts … since they | |
| have more muscle"; "a systematic analysis found"); | |
| * a name / place dropped from the sentence ("Digha, Darjeeling, Jhargram" -> "Digha Jhargram"); | |
| * a key construct used more often than in the original ("… destination loyalty (DL) and intimacy"); | |
| * a cut-off ending ("… and to enhance their.") or a cut-off term ("on well-, and"); | |
| * an em dash, or a doubled conjunction ("and, and"); | |
| * two sentences run together ("… or mass According to …").""" | |
| o, n = original or "", new or "" | |
| try: | |
| import adaptive as _AD # words learned to cause bad rewrites | |
| bw = _AD.banned_in(o, n) | |
| if bw: | |
| return f"banned word: {bw}" | |
| except Exception: | |
| pass | |
| if _ATTRIB.search(n) and not _ATTRIB.search(o): | |
| return "attribution added" | |
| if len(_CAUSAL.findall(n)) > len(_CAUSAL.findall(o)): | |
| return "causal claim added" | |
| if len(_SAID.findall(n)) > len(_SAID.findall(o)): | |
| return "attribution added" # "... beyond local capacity the director said" | |
| if set(re.findall(r"\b[A-Z]\.[A-Z]\.?", n)) - set(re.findall(r"\b[A-Z]\.[A-Z]\.?", o)): | |
| return "abbreviation added" # "Darjeeling, and M.P, and purulia" | |
| names = DOC_PROPER.get() | |
| for w in _proper_nouns(o) | ({o.split()[0].strip(",.;:")} if o.split() else set()): | |
| is_name = w in names if names else (_zipf is not None and _zipf(w.lower(), "en") < 2.0) | |
| if len(w) > 3 and is_name and re.search(r"\b" + re.escape(w.lower()) + r"\b", n) and w not in n: | |
| return f"name lower-cased: {w}" # "purulia" (the document always writes "Purulia") | |
| for a, b in re.findall(r"\b([A-Z][a-z]+)\s+([A-Z][a-z]+)\b", o): | |
| if a in names and re.search(r"\b" + a.lower() + r"\s+" + b + r"\b", n): | |
| return f"name lower-cased: {a} {b}" # "west Bengal" | |
| stray = r"[a-z,]\s+(?:The|A|An|Of|For|In|And|To|With|On)\s" | |
| if re.search(stray, n) and not re.search(stray, o): | |
| return "capital inside a sentence" # "variance in The TS" | |
| if re.search(r"(?:\b[A-Z][a-z]{2,}\s+){3,}", n) and not re.search(r"(?:\b[A-Z][a-z]{2,}\s+){3,}", o): | |
| caps = re.findall(r"(?:\b[A-Z][a-z]{2,}\s+){3,}", n)[0].split() | |
| if sum(w.lower() in _STOP or (_zipf is not None and _zipf(w.lower(), "en") >= 4.5) for w in caps) >= 2: | |
| return "title case in a sentence" # "The Strong Effect of Intimacy For Both genders" | |
| if re.match(r"^\[\d[\d,\s–\-]*\]\s+[A-Z][a-z]+(?:ed|es|s)\b", o) and \ | |
| re.match(r"^\[\d[\d,\s–\-]*\]\s+(?:The|A|An|This|These|Its)\b", n): | |
| return "reporting verb dropped" # "[56] Highlighted ..." -> "[56] The importance of ..." | |
| if re.search(r"\b(whereas|while|although|though|but|however|and|or|because)\s+(whereas|while|although|" | |
| r"though|but|however|because)\b", n, re.I): | |
| return "doubled connective" # "whereas while", "but however" | |
| sw = [x.lower() for x in re.findall(r"[A-Za-z]{5,}", n)] | |
| so = [x.lower() for x in re.findall(r"[A-Za-z]{5,}", o)] | |
| rep_n = {a[:5] for a, b in zip(sw, sw[1:]) if a != b and a[:5] == b[:5]} | |
| rep_o = {a[:5] for a, b in zip(so, so[1:]) if a != b and a[:5] == b[:5]} | |
| if rep_n - rep_o: | |
| return "stem repeated" # "intimate intimacy intimation" | |
| dash = lambda t: re.sub(r"[‐‑–—]", "-", t).lower() | |
| lost = {w for w in _proper_nouns(o) if dash(w) not in dash(n) # (case / dash changes are fine) | |
| and not re.search(r"[\])]\s*" + re.escape(w) + r"\b", o) # "[59] Discussed" is a verb | |
| and not (_zipf is not None and _zipf(w.lower(), "en") >= 5.0)} # "for Men" -> "for males" | |
| if lost: | |
| return "name dropped: " + ", ".join(sorted(lost)) | |
| nn, no = _norm(n), _norm(o) | |
| for k in KEY_DEF.get(): | |
| if nn.count(f" {k} ") > max(1, no.count(f" {k} ")): | |
| return f"construct repeated: {k}" | |
| if _DANGLING_END.search(n.strip()) and not _DANGLING_END.search(o.strip()): | |
| return "sentence cut off" | |
| if len(_DANGLING_HYPHEN.findall(n)) > len(_DANGLING_HYPHEN.findall(o)): | |
| return "term cut off" | |
| cut = r"\b(?:of|to|for|with|on|in|by|from|the|a|an|and|or|their|its)\s*," | |
| if len(re.findall(cut, n, re.I)) > len(re.findall(cut, o, re.I)): | |
| return "phrase cut off" # "the port of, and the forest" | |
| op = r"\b(?:Sadly|Unfortunately|Fortunately|Luckily|Interestingly|Surprisingly|Obviously|Clearly|Undoubtedly|" \ | |
| r"Remarkably|Importantly|Notably|Strikingly)\b" | |
| if re.search(op, n) and not re.search(op, o): | |
| return "opinion word added" # "Sadly loyalty does not ..." | |
| letters = lambda t: re.findall(r"(?<![\w.'’-])([b-zB-HJ-Z])(?![\w.'’-])", CITE.sub(" ", t)) | |
| if len(letters(n)) > len(letters(o)): | |
| return "stray letter" # "... is d a vital predictor" | |
| sq = lambda t: t.count("[") - sum(1 for c in CITE.findall(t) if c.startswith("[")) # "[" outside citations | |
| if sq(n) > sq(o): # (author-year citations have no "[": not counted) | |
| return "bracket type changed" # "(explaining ...)" -> "[explaning ...]" | |
| if _EMDASH.search(n) and not _EMDASH.search(o): | |
| return "em dash" | |
| if re.search(r"\b(\w+),?\s+\1\b", n, re.I) and not re.search(r"\b(\w+),?\s+\1\b", o, re.I): | |
| return "doubled word" | |
| if re.search(r"\b[a-z]{3,}\s+(?:According|However|Moreover|Furthermore|Therefore|This|These|The|It|In)\s+[a-z]", n) \ | |
| and not re.search(r"\b[a-z]{3,}\s+(?:According|However|Moreover|Furthermore|Therefore|This|These|The|It|In)\s+[a-z]", o): | |
| return "sentences run together" | |
| return None | |
| def restore_commas(original, new): | |
| """Put back the commas a paraphraser dropped between words that keep their order: list items ("airports, | |
| trains and taxis"), after an opening phrase ("After COVID-19, tourists") and after an abbreviation in brackets | |
| ("(TM), eudaimonic"). Also adds the comma after a sentence-opening linking word ("However the" -> "However, | |
| the").""" | |
| if not new: | |
| return new | |
| out = new | |
| for m in re.finditer(r"(?<![\w\-])([\w)\]%][\w\-)\]%]*),\s+(?=([\w(\[]+))", original or ""): | |
| a, b = m.group(1), m.group(2) | |
| pat = re.compile(r"(?<![\w-])" + re.escape(a) + r"(\s+)" + re.escape(b) + r"(?![\w-])") | |
| if len(pat.findall(out)) == 1 and not re.search(re.escape(a) + r",\s+" + re.escape(b), out): | |
| out = pat.sub(lambda k: a + "," + k.group(1) + b, out, count=1) | |
| out = re.sub(rf"^({_SENT_ADV})\s+(?=[a-z])", r"\1, ", out) | |
| return out | |
| def guard(original, new, glossary=None): | |
| """Reject rewrites that lose meaning-bearing content or invent names / people / numbers; re-attach dropped | |
| citations.""" | |
| new = re.sub(r"\s+([,.;:!?])", r"\1", normalize_typography(original, (new or "").strip())) # " ," -> "," | |
| new = re.sub(r"([.!?])[.!?]+$", r"\1", new) # ".." -> "." | |
| new = re.sub(r"([(\[])\s+", r"\1", re.sub(r"\s+([)\]])", r"\1", new)) # "( Kumar , 2021 )" | |
| if not new: | |
| return None | |
| new = repair_citations(original, new) | |
| if new is None or invented(original, new) or quality_issue(original, new) or symbol_issue(original, new): | |
| return None | |
| if _symbols(new) != _symbols(original): # "p < .001" must not become "p .001" | |
| return None | |
| if _brackets_ok(original) and not _brackets_ok(new): # "(TS amongst" - a bracket lost | |
| return None | |
| if _new_acronyms(original, new): # "EWB" must not become "EWD" | |
| return None | |
| new = restore_commas(original, new) # "airports trains and taxis" -> "airports, trains and taxis" | |
| if integrity_issue(original, new): | |
| return None | |
| ow, nw = len(original.split()), len(new.split()) | |
| if nw < 0.5 * ow or nw > 1.8 * ow + 5: | |
| return None | |
| # numbers (results, years, units) must be kept | |
| nums_o = set(re.findall(r"(?<![\d])\.?\d+(?:\.\d+)?%?", CITE.sub(" ", original))) | |
| nums_n = set(re.findall(r"(?<![\d])\.?\d+(?:\.\d+)?%?", new)) | |
| if not nums_o <= nums_n: | |
| return None | |
| if lost_terms(original, new, glossary): # technical wording must survive unchanged | |
| return None | |
| have = [_ckey(c) for c in CITE.findall(new)] | |
| missing = [c for c in CITE.findall(original) if _ckey(c) not in have] | |
| if missing: | |
| end = re.search(r"[.!?]\s*$", new) | |
| core = new[:end.start()] if end else new | |
| new = core.rstrip() + " " + " ".join(missing) + (end.group(0).strip() if end else "") | |
| if sorted(_ckey(c) for c in CITE.findall(new)) != sorted(_ckey(c) for c in CITE.findall(original)): | |
| return None # every citation once, exactly as the author wrote it | |
| import pubrules as _PR # integrity rules hold for every version, whatever wrote | |
| if _PR.violations(original, new): # it: units, negation, direction, hedging, cause, voice | |
| return None | |
| if original[:1].isupper() and new[:1].islower(): | |
| new = new[0].upper() + new[1:] | |
| elif original[:1].islower() and new[:1].isupper() and not new[1:2].isupper(): | |
| new = new[0].lower() + new[1:] # fragment starting mid-sentence | |
| if not re.search(r"[.!?]$", original.strip()): | |
| new = re.sub(r"\s*[.!?]+$", "", new) # fragment without end punctuation | |
| elif not re.search(r"[.!?][\"”’)\]]?$", new): | |
| # a sentence must end as one: left open ("...; " / "...,") it runs into the next sentence of the paragraph | |
| new = new.rstrip(" ,;:–—-") + "." | |
| return new | |
| _WTOK = re.compile(r"\w+(?:[-'’.]\w+)*|[^\w\s]") | |
| def _term_spans(original, terms): | |
| """Character spans in `original` of the given lost terms (acronyms exact, phrases/words by normalised match).""" | |
| spans = [] | |
| for t in terms: | |
| if not t: | |
| continue | |
| if any(c.isupper() or c.isdigit() for c in t) or not t.replace(" ", "").isalpha(): | |
| pat = r"(?<!\w)" + re.escape(t) + r"(?!\w)" | |
| else: | |
| pat = r"(?<![\w])" + r"[\s\-]+".join(re.escape(w) + r"\w{0,3}" for w in t.split()) + r"(?!\w)" | |
| spans += [(m.start(), m.end()) for m in re.finditer(pat, original, re.I)] | |
| return spans | |
| def _tkey(t): | |
| """Match key: case-insensitive for ordinary words, exact for acronyms/formulas ("PCM" != "pcm").""" | |
| return t if (any(c.isdigit() for c in t) or any(c.isupper() for c in t[1:])) else t.lower() | |
| def restore_terms(original, new, glossary=None): | |
| """Put technical terms that a paraphrase lost or altered back in place, keeping the rest of the paraphrase. | |
| The sentences are aligned word by word; only the words standing in for a lost term are replaced. | |
| If the paraphrase restructured the sentence too much to locate the term, it is returned unchanged | |
| (the guard will then reject it).""" | |
| import difflib | |
| if not new: | |
| return new | |
| lost = lost_terms(original, new, glossary) | |
| if not lost: | |
| return new | |
| spans = _term_spans(original, lost) | |
| ot = [(m.group(0), m.start(), m.end()) for m in _WTOK.finditer(original)] | |
| nt = [(m.group(0), m.start(), m.end()) for m in _WTOK.finditer(new)] | |
| if not spans or not ot or not nt: | |
| return new | |
| ops = difflib.SequenceMatcher(a=[_tkey(t[0]) for t in ot], b=[_tkey(t[0]) for t in nt], | |
| autojunk=False).get_opcodes() | |
| edits, give_up = {}, False | |
| for s0, e0 in sorted(set(spans)): | |
| idx = {k for k, t in enumerate(ot) if t[1] < e0 and t[2] > s0} | |
| for op, i1, i2, j1, j2 in ops: | |
| if op == "equal" or i1 == i2 or not idx & set(range(i1, i2)): | |
| continue | |
| if (i2 - i1) > 5 or (j2 - j1) > 5: # restructured around the term: cannot patch safely | |
| give_up = True | |
| continue | |
| block = original[ot[i1][1]:ot[i2 - 1][2]] | |
| if j1 < j2: | |
| edits[(nt[j1][1], nt[j2 - 1][2])] = block | |
| else: # words deleted -> insert them where they were | |
| pos = nt[j1][1] if j1 < len(nt) else len(new) | |
| nxt = new[pos:pos + 1] | |
| edits[(pos, pos)] = (" " + block) if (not nxt or not nxt.isalnum()) else (block + " ") | |
| if give_up and not edits: | |
| return new | |
| # a paraphrase that moved a term's words ("charge state" for "state of charge") leaves the moved words | |
| # behind as an insertion next to the restored term -> drop them | |
| term_words = {w.lower() for t in lost for w in re.findall(r"\w+", t)} | |
| touched = [j for (a, b) in edits for j, t in enumerate(nt) if a <= t[1] <= b] | |
| for op, i1, i2, j1, j2 in ops: | |
| if op == "insert" and j2 - j1 <= 3 and all(nt[j][0].lower().rstrip("s") in | |
| {w.rstrip("s") for w in term_words} for j in range(j1, j2)): | |
| if touched and min(abs(j - t) for j in (j1, j2 - 1) for t in touched) <= 3: | |
| edits[(nt[j1][1], nt[j2 - 1][2])] = "" | |
| out, last = new, None | |
| for (s0, e0), rep in sorted(edits.items(), reverse=True): | |
| if last is not None and e0 > last: # overlapping edits: keep the first applied | |
| continue | |
| out = out[:s0] + rep + out[e0:] | |
| last = s0 | |
| out = re.sub(r"\s{2,}", " ", out) | |
| out = re.sub(r"\s+([,.;:!?])", r"\1", out).strip() | |
| return out | |
| # --------------------------------------------------------------------------- | |
| # "De-AI" wording pass: typical chatbot vocabulary and set phrases -> plain academic wording. | |
| # Meaning-preserving, rule-based; used as extra candidates next to Parrot's paraphrases (the AI detector | |
| # then picks whichever version reads most human). Technical phrases such as "robust regression" or | |
| # "urban landscape" are left alone. | |
| # --------------------------------------------------------------------------- | |
| _OPENERS = re.compile(r"^(?:Furthermore|Moreover|Additionally|In addition|Notably|Importantly|Indeed|" | |
| r"It is (?:important|worth|crucial) to (?:note|mention|highlight) that|" | |
| r"It is worth noting that|It should be noted that|Ultimately|In essence|Overall|" | |
| r"In today's [\w\s-]{0,40}?(?:world|landscape|era|environment|age|society)),\s+" | |
| r"|^(?:It is (?:important|worth|crucial|essential) to (?:note|mention|highlight|recognize) that|" | |
| r"It is worth noting that|It should be noted that)\s+", re.I) | |
| _WORDS = [ | |
| (r"\bdelve(s|d)? into\b", lambda m: {"": "examine", "s": "examines", "d": "examined"}[m.group(1) or ""]), | |
| (r"\bdelving into\b", "examining"), | |
| (r"\bplays an? (?:crucial|vital|pivotal|key|significant|critical|essential|important) role in\b", "is central to"), | |
| (r"\bplay an? (?:crucial|vital|pivotal|key|significant|critical|essential|important) role in\b", "are central to"), | |
| (r"\bplayed an? (?:crucial|vital|pivotal|key|significant|critical|essential|important) role in\b", "was central to"), | |
| (r"\b(?:crucial|pivotal|paramount)\b", "key"), | |
| (r"\bcomprehensive\b", "detailed"), | |
| (r"\bleverag(e|es|ed|ing)\b", lambda m: {"e": "use", "es": "uses", "ed": "used", "ing": "using"}[m.group(1)]), | |
| (r"\butiliz(e|es|ed|ing|ation)\b", lambda m: {"e": "use", "es": "uses", "ed": "used", "ing": "using", | |
| "ation": "use"}[m.group(1)]), | |
| (r"\bfacilitat(e|es|ed|ing)\b", lambda m: {"e": "help", "es": "helps", "ed": "helped", "ing": "helping"}[m.group(1)]), | |
| (r"\bfoster(s|ed|ing)?\b", lambda m: {"": "encourage", "s": "encourages", "ed": "encouraged", | |
| "ing": "encouraging"}[m.group(1) or ""]), | |
| (r"\benhanc(e|es|ed|ing|ement|ements)\b", lambda m: {"e": "improve", "es": "improves", "ed": "improved", | |
| "ing": "improving", "ement": "improvement", | |
| "ements": "improvements"}[m.group(1)]), | |
| (r"\bunderscor(e|es|ed|ing)\b", lambda m: {"e": "highlight", "es": "highlights", "ed": "highlighted", | |
| "ing": "highlighting"}[m.group(1)]), | |
| (r"\bshowcas(e|es|ed|ing)\b", lambda m: {"e": "show", "es": "shows", "ed": "showed", "ing": "showing"}[m.group(1)]), | |
| (r"\bseamless(ly)?\b", lambda m: "smooth" + (m.group(1) or "")), | |
| (r"\b(?:intricate|multifaceted)\b", "complex"), | |
| (r"\b(?:a )?myriad (?:of )?\b", "many "), | |
| (r"\ba plethora of\b", "many"), | |
| (r"\ba (?:wide|broad) (?:range|array|variety) of\b", "many"), | |
| (r"\ba variety of\b", "various"), | |
| (r"\ba significant number of\b", "many"), | |
| (r"\bin order to\b", "to"), | |
| (r"\bdue to the fact that\b", "because"), | |
| (r"\bwith regard(?:s)? to\b", "on"), | |
| (r"\bin terms of\b", "in"), | |
| (r"\ba testament to\b", "evidence of"), | |
| (r"\bever-evolving\b", "changing"), | |
| (r"\brapidly evolving\b", "fast-changing"), | |
| (r"\bthe (digital|business|educational|competitive|technological|corporate|organi[sz]ational|modern) landscape\b", | |
| lambda m: f"the {m.group(1)} environment"), | |
| (r"\bnavigat(e|es|ing) the complexities of\b", lambda m: {"e": "deal with", "es": "deals with", | |
| "ing": "dealing with"}[m.group(1)]), | |
| (r"\bharness(es|ed|ing)? the power of\b", lambda m: {"": "use", "es": "uses", "ed": "used", | |
| "ing": "using"}[m.group(1) or ""]), | |
| (r"\bembark(s|ed)? on\b", lambda m: {"": "begin", "s": "begins", "ed": "began"}[m.group(1) or ""]), | |
| (r"\bstreamlin(e|es|ed|ing)\b", lambda m: {"e": "simplify", "es": "simplifies", "ed": "simplified", | |
| "ing": "simplifying"}[m.group(1)]), | |
| ] | |
| _WORDS = [(re.compile(p, re.I), r) for p, r in _WORDS] | |
| def _match_case(src, rep): | |
| return rep[:1].upper() + rep[1:] if src[:1].isupper() else rep | |
| def de_ai(sentence): | |
| """Plain-wording version of a sentence (same meaning). Returns the sentence unchanged if nothing applies.""" | |
| s = sentence | |
| m = _OPENERS.match(s) | |
| if m and len(s) - m.end() > 20: | |
| rest = s[m.end():] | |
| s = rest[:1].upper() + rest[1:] | |
| for pat, rep in _WORDS: | |
| s = pat.sub(lambda m: _match_case(m.group(0), rep(m) if callable(rep) else rep), s) | |
| s = re.sub(r"\s+[—–]\s+", ", ", s) # spaced dashes (a chatbot habit) -> comma | |
| s = re.sub(r"(?<=\w)—(?=\w)", ", ", s) | |
| s = re.sub(r"\s{2,}", " ", s).replace(" ,", ",") | |
| return s | |
| # ---------------------------------------------------------------- basic grammar (rules, no model) | |
| _MODALS = r"(?:will|would|can|could|may|might|must|shall|should|to|does|do|did)" | |
| _AN_EXC = re.compile(r"^(?:uni|use|usu|euro|one|once|eu|ewe|u\b|ur|uti)", re.I) # "a university", "a one-off" | |
| _A_EXC = re.compile(r"^(?:hour|honest|honou?r|heir|(?:[FHLMNRSX][A-Z0-9]*)\b)") # "an hour", "an MRI", "an SVM" | |
| def _base_form(w): | |
| """running -> run, making -> make, opening -> open, studying -> study (for 'will opening' -> 'will open').""" | |
| lw = w.lower() | |
| if not lw.endswith("ing") or len(lw) < 6: | |
| return None | |
| stem = lw[:-3] | |
| if len(stem) >= 3 and stem[-1] == stem[-2] and stem[-1] not in "lsz": | |
| stem = stem[:-1] # running -> run | |
| elif re.search(r"(?:[^aeiou][aeiou](?:k|v|z|c|g|s|t|r)|ag|iz|at|ur|ut|ic|uc|ar)$", stem) and not stem.endswith( | |
| ("en", "er", "ow", "ain", "eal", "ess", "ect", "ent", "ist", "est", "ort", "ert", "ast", "ust")): | |
| stem = stem + "e" # making -> make, using -> use, producing -> produce | |
| return stem if w[:1].islower() else stem.capitalize() | |
| def _needs_an(word): | |
| """'an' before vowel SOUNDS: an hour, an MRI, an SVM; but a university, a useful, a one-off.""" | |
| if _A_EXC.match(word): | |
| return True # silent h / letters said with a vowel sound | |
| if _AN_EXC.match(word): | |
| return False # vowel letter said with a consonant sound | |
| return word[0].lower() in "aeiou" | |
| def _article(art, word): | |
| if word.lower() in ("other", "number", "lot", "few", "little") and art.lower() == "a": | |
| return f"{art} {word}" | |
| want_an = _needs_an(word) | |
| cap = art[0].isupper() | |
| new = ("An" if cap else "an") if want_an else ("A" if cap else "a") | |
| return f"{new} {word}" | |
| # leftovers of chatbot output: labels, markdown, assistant phrases | |
| _ARTIFACT_LEAD = re.compile( | |
| r"^\s*(?:(?:sure|certainly|of course|absolutely)[,!.]\s*|here(?:'s| is| are) (?:the |a |your )?" | |
| r"(?:revised|rewritten|corrected|paraphrased|improved|updated)[^:]{0,40}:\s*|" | |
| r"(?:revised|rewritten|corrected|paraphrased|improved|final) (?:sentence|version|text)\s*:\s*|" | |
| r"(?:note|output|answer)\s*:\s*|as an ai(?: language model)?,?\s*)", re.I) | |
| _ARTIFACT_ANY = [ | |
| (re.compile(r"\*\*([^*]+)\*\*"), r"\1"), # **bold** | |
| (re.compile(r"(?<!\w)__([^_]+)__(?!\w)"), r"\1"), # __bold__ | |
| (re.compile(r"`([^`]+)`"), r"\1"), # `code` | |
| (re.compile(r"^\s*#{1,6}\s+"), ""), # ### heading marks | |
| (re.compile(r"^\s*[-*•]\s+(?=[A-Z])"), ""), # stray bullet | |
| (re.compile(r"\s*\((?:revised|rewritten|paraphrased|corrected)\)\s*$", re.I), ""), | |
| (re.compile(r"\s*\[(?:citation needed|insert[^\]]*|source)\]", re.I), ""), | |
| ] | |
| def clean_artifacts(s): | |
| """Remove chatbot leftovers from a sentence (labels, markdown, assistant phrases).""" | |
| if not s: | |
| return s | |
| t, prev = s, None | |
| while t != prev: # "Certainly! Here is the revised sentence: ..." | |
| prev, t = t, _ARTIFACT_LEAD.sub("", t, count=1) | |
| for pat, rep in _ARTIFACT_ANY: | |
| t = pat.sub(rep, t) | |
| t = t.strip() | |
| if len(t) > 2 and t[0] in "\"“" and t[-1] in "\"”" and t.count('"') + t.count("“") + t.count("”") == 2: | |
| t = t[1:-1].strip() # whole sentence wrapped in quotes | |
| if t and t[0].islower() and s[:1].isupper(): | |
| t = t[0].upper() + t[1:] | |
| return t or s | |
| def artifact_only(s): | |
| """A 'sentence' that is nothing but a chatbot leftover ("Certainly!", "Here is the revised text:").""" | |
| t = (s or "").strip() | |
| return bool(t) and len(t.split()) <= 8 and not _ARTIFACT_LEAD.sub("", t).strip(" .!:") | |
| def has_defect(s): | |
| """Doubled word / repeated phrase / chatbot leftover in a sentence.""" | |
| if not s: | |
| return False | |
| return bool(re.search(r"\b(\w{2,})\s+\1\b", s, re.I) | |
| or re.search(r"\b((?:\w+\s+){1,3}\w+)\s+\1\b", s, re.I) | |
| or clean_artifacts(s) != s.strip()) | |
| ABBR = (r"(?:et al|e\.g|i\.e|etc|vs|viz|Fig|Figs|Eq|Eqs|Ref|Refs|No|Nos|Vol|pp|Dr|Mr|Mrs|Ms|Prof|approx|cf|Sec|Ch|" | |
| r"Art|Arts|para|paras|Inc|Ltd|Corp|Jr|Sr|Hon|edn|eds)") | |
| _INITIALS = re.compile(r"(?P<ini>\b(?:[A-Z]\.){2,})(?=\s+(?P<next>[A-Z][A-Za-z]*))") # K.S. Puttaswamy | |
| _INITIAL = re.compile(r"(?P<pre>\b(?:[A-Z][a-z]+|and|by)\s)(?P<ini>[A-Z]\.)(?=\s+(?P<next>[A-Z][a-z]+))") # Justice K. Rao | |
| _VERSUS = re.compile(r"(?P<pre>\b[A-Z][\w.<>]*\s)v\.(?=\s+[A-Z])") # Roe v. Wade | |
| def protect_dots(t): | |
| """Full stops that do not end a sentence are replaced by <DOT>: abbreviations ("et al.", "Fig."), decimals, | |
| the initials of a name ("K.S. Puttaswamy", "Justice K. Rao") and the "v." of a case name. Initials followed | |
| by an ordinary sentence opener ("... in the U.S. The economy") do end the sentence.""" | |
| t = re.sub(rf"\b({ABBR})\.", lambda m: m.group(1).replace(".", "<DOT>") + "<DOT>", t) | |
| t = re.sub(r"(\d)\.(\d)", r"\1<DOT>\2", t) | |
| def name(m): | |
| if m.group("next").lower() in _SENT_OPENERS: | |
| return m.group(0) | |
| return m.groupdict().get("pre", "") + m.group("ini").replace(".", "<DOT>") | |
| t = _INITIALS.sub(lambda m: m.group(0) if m.group("next").lower() in _SENT_OPENERS | |
| else m.group("ini").replace(".", "<DOT>"), t) | |
| t = _INITIAL.sub(name, t) | |
| return _VERSUS.sub(lambda m: m.group("pre") + "v<DOT>", t) | |
| _ABBR_END = re.compile(rf"\b{ABBR}\.\s+$") | |
| def split_sentences(text): | |
| """Sentences of `text`; never split after 'et al.', 'e.g.', 'Fig.' or inside a decimal number.""" | |
| t = protect_dots((text or "").strip()) | |
| parts = re.split(r"(?<=[.!?])\s+(?=[A-Z0-9\"'(\[])", t) | |
| return [p.replace("<DOT>", ".").strip() for p in parts if p.strip()] | |
| def _cap_after(m): | |
| if _ABBR_END.search(m.string[:m.start(2)]): | |
| return m.group(0) | |
| return m.group(1) + m.group(2).upper() | |
| _SENT_OPENERS = {"the", "this", "these", "those", "that", "their", "its", "it", "in", "a", "an", "we", "our", "such", | |
| "however", "moreover", "furthermore", "thus", "hence", "therefore", "for", "on", "at", "as", "to", | |
| "they", "he", "she", "his", "her", "both", "each", "all", "most", "many", "other", "another", "with", | |
| "by", "from", "after", "before", "although", "while", "when", "if", "since", "here", "there"} | |
| def _common_lower(word): | |
| """A capitalised verb after 'et al.' ('Generalised', 'Argue'), not the first word of a new sentence.""" | |
| w = word.lower() | |
| if w in _SENT_OPENERS: | |
| return False | |
| if _zipf is None: | |
| return bool(re.search(r"(?:ed|es|s|ise|ize|ised|ized)$", w)) | |
| return _zipf(w, "en") >= 1.5 | |
| def fix_abbrev_caps(t): | |
| """'Greshake et al. Generalised' -> 'Greshake et al. generalised'.""" | |
| return re.sub(r"(\bet al\.\s+)([A-Z][a-z]+)\b", | |
| lambda m: m.group(1) + (m.group(2).lower() if _common_lower(m.group(2)) else m.group(2)), t) | |
| def _outside_cites(t, fn): | |
| """Apply fn to the text between citations only: "(Mishra and Mishra, 2012)" and "(Duran Duran, 1981)" are two | |
| authors and a name, not doubled words.""" | |
| out, pos = [], 0 | |
| for m in CITE.finditer(t): | |
| out += [fn(t[pos:m.start()]), m.group(0)] | |
| pos = m.end() | |
| return "".join(out) + fn(t[pos:]) | |
| def basic_grammar(s): | |
| """Safe rule-based clean-up of paraphrase output (applied before and after the Groq grammar check).""" | |
| if not s or not s.strip(): | |
| return s | |
| t = clean_artifacts(re.sub(r"\s+", " ", s).strip()) | |
| t = _outside_cites(t, lambda x: re.sub(r"\b((?:[\w\-]+\s+){1,3}[\w\-]+)\s+\1\b", r"\1", x, flags=re.I)) # repeated phrase | |
| t = re.sub(r"\s+([,.;:!?%)\]])", r"\1", t) # space before punctuation | |
| t = re.sub(r"([(\[])\s+", r"\1", t) | |
| t = re.sub(r",\s*,+", ",", t) | |
| t = re.sub(r"\b(and|or|but|the|of|to|in)\s*,\s*\1\b", r"\1", t, flags=re.I) # "and, and" -> "and" | |
| t = re.sub(rf"^({_SENT_ADV})\s+(?=[a-z])", r"\1, ", t) # "However the" -> "However, the" | |
| t = re.sub(r"([.!?])\1+", r"\1", t) | |
| t = _outside_cites(t, lambda x: re.sub(r"\b(\w{2,})\s+\1\b", r"\1", x, flags=re.I)) # "the the", "data data" | |
| t = _outside_cites(t, lambda x: re.sub(r"\b([a-z]{3,})\s+(and|or)\s+\1\b", r"\1", x)) # "informed and informed" | |
| t = re.sub(rf"\b({_MODALS})\s+(\w+ing)\b", | |
| lambda m: f"{m.group(1)} {_base_form(m.group(2)) or m.group(2)}" | |
| if m.group(1).lower() != "to" or _base_form(m.group(2)) else m.group(0), t, flags=re.I) | |
| t = re.sub(r"\b([Aa]n?)\s+([A-Za-z][\w\-]*)", lambda m: _article(m.group(1), m.group(2)), t) | |
| t = re.sub(r"\s*'{2,}\s*", " ", t) # stray quote marks left by the paraphraser | |
| t = re.sub(r"\s+([,.;:!?%)\]])", r"\1", re.sub(r"\s{2,}", " ", t)).strip() | |
| t = re.sub(r"([.!?]\s+)([a-z])", _cap_after, t) # capital after full stop (not after 'et al.', 'e.g.') | |
| t = fix_abbrev_caps(t) | |
| if t[:1].islower() and not re.match(r"^[a-z]+[A-Z0-9]", t): | |
| t = t[0].upper() + t[1:] | |
| if re.search(r"[A-Za-z0-9)\]]$", t) and len(t.split()) >= 4: | |
| t += "." | |
| return t | |
| # ---------------------------------------------------------------- strict acceptance (final polish / proofread) | |
| def _cites(t): | |
| return [re.sub(r"\W+", "", c).lower() for c in CITE.findall(t or "")] | |
| def match_ending(original, new): | |
| """A fragment that does not end its sentence (the words before a superscript citation, a field or a link in a | |
| Word paragraph) must not gain a full stop: "2.1 °C" + superscript 12 would become "2.1 °C.12".""" | |
| if not new or not original: | |
| return new | |
| if original[:1].islower() and new[:1].isupper() and not new[1:2].isupper(): | |
| new = new[0].lower() + new[1:] # ... nor a capital, when it starts in mid-sentence | |
| if not re.search(r"[.!?][\"”’)\]]?\s*$", original): | |
| return re.sub(r"\s*[.!?]+$", "", new) | |
| return new | |
| def strict_accept(cur, cand, glossary, ref=None): | |
| """Strict acceptance of an edited sentence for the proofread. Nothing is auto-repaired (no re-inserted terms | |
| or citations, which is what creates "ethics of care ethics" or duplicated citations): an edit that drops a | |
| term, a number or a citation, adds a citation, invents something, or introduces a defect is rejected.""" | |
| return strict_check(cur, cand, glossary, ref)[0] | |
| def strict_check(cur, cand, glossary, ref=None): | |
| """strict_accept with the reason: (accepted sentence, None) or (None, why it was rejected).""" | |
| if not cand or not cand.strip(): | |
| return None, "empty" | |
| new = match_ending(cur, basic_grammar(normalize_typography(cur, cand.strip()))) | |
| new = re.sub(r'([.!?])\s*["”“]{2,}', r"\1”", new) # .”” -> .” | |
| new = re.sub(r'["“”]{2,}', "”", new) | |
| new = restore_commas(cur, new) | |
| if new.strip() == cur.strip(): | |
| return None, "unchanged" | |
| ow, nw = len(cur.split()), len(new.split()) | |
| if nw < 0.55 * ow or nw > 1.6 * ow + 4: | |
| return None, "length" | |
| why = integrity_issue(cur, new) | |
| if why: | |
| return None, why | |
| if _symbols(new) != _symbols(cur): | |
| return None, "relation sign changed" | |
| if _new_acronyms(cur, new): | |
| return None, "acronym added" | |
| if _brackets_ok(cur) and not _brackets_ok(new): | |
| return None, "unbalanced bracket" | |
| for base in filter(None, (cur, ref)): | |
| why = invented(base, new) | |
| if why: | |
| return None, f"invented {why}" | |
| why = quality_issue(cur, new) or symbol_issue(cur, new) | |
| if why: | |
| return None, why | |
| nums_c = set(re.findall(r"(?<![\d])\.?\d+(?:\.\d+)?%?", CITE.sub(" ", cur))) | |
| if not nums_c <= set(re.findall(r"(?<![\d])\.?\d+(?:\.\d+)?%?", new)): | |
| return None, "number lost" | |
| cc, nc = _cites(cur), _cites(new) | |
| if len(nc) > len(cc) or any(not any(difflib.SequenceMatcher(None, c, n).ratio() >= 0.8 for n in nc) for c in cc): | |
| return None, "citation changed" # citations kept (spelling fixes allowed), none added | |
| lost = lost_terms(cur, new, glossary) | |
| if lost: | |
| return None, "term lost: " + ", ".join(lost[:3]) | |
| if has_defect(new) and not has_defect(cur): | |
| return None, "doubled word / leftover" | |
| if _near_repeats(new) - _near_repeats(cur): | |
| return None, "repeated word" # "ethics of care ethics", "mapping of SDG 16 mapping" | |
| import pubrules as _PR # publication English rulebook | |
| v = _PR.violations(cur, new) | |
| if v: | |
| return None, "; ".join(v) | |
| return new, None | |
| def _near_repeats(t): | |
| """Content words used twice within four words ("ethics of care ethics").""" | |
| w = [x.lower() for x in re.findall(r"[A-Za-z][A-Za-z\-]+", CITE.sub(" ", t or ""))] | |
| out = set() | |
| for i, x in enumerate(w): | |
| if len(x) >= 5 and x in w[i + 1:i + 5]: | |
| j = w.index(x, i + 1) | |
| a, b = (w[i - 1] if i else ""), w[j - 1] | |
| if a != b and len(a) >= 3 and len(b) >= 3 and a not in _STOP and b not in _STOP: | |
| continue # contrast, not a repeat: "female travellers than male travellers" | |
| out.add(x) | |
| return out | |