textparse.py

"""
Pasted-text handling shared by the detector, Smart Humanize and the Humanizer.

parse_paste(text) -> [(paragraph, separator_after)]
  * blank line            -> paragraph break, kept as "\n\n"
  * single line break     -> paragraph break, kept as "\n"  (Word / web copy-paste)
  * soft wrap (PDF copy)  -> joined into the same paragraph ("in modern\ncommerce" -> "in modern commerce")
join_paste(parts)        -> text with the original paragraph breaks restored

chunk_sentence / merge_chunks
  Parrot T5 was trained on short sentences and truncates or garbles long academic ones,
  so sentences over ~28 words are split at safe clause boundaries, paraphrased piece by
  piece and stitched back together.
"""
import contextvars
import difflib
import re

_TERMINAL = re.compile(r"[.!?:;\"”’)\]]\s*$")
_BULLET = re.compile(r"^\s*([•●▪\-\*–]|\(?[0-9ivxIVX]{1,3}[\)\.]|\(?[a-z][\)\.])\s+")


def _soft_wrap(prev, line):
    """True if `line` continues `prev` (hard-wrapped text copied from a PDF)."""
    p, l = prev.rstrip(), line.lstrip()
    if not p or not l or _BULLET.match(l):
        return False
    if p.endswith("-") and l[:1].islower():                        # hyphenated word break
        return True
    if _TERMINAL.search(p):
        return False
    if l[:1].islower() or l[:1] in ",;)":                          # sentence clearly continues
        return True
    return len(p) >= 55                                             # long line cut mid-sentence


def parse_paste(text):
    lines = (text or "").replace("\r\n", "\n").replace("\r", "\n").split("\n")
    parts, cur, blank = [], None, False
    for raw in lines:
        line = raw.strip()
        if not line:
            blank = True
            continue
        if cur is not None and not blank and _soft_wrap(cur, line):
            cur = cur[:-1] + line if cur.endswith("-") and line[:1].islower() else cur + " " + line
            continue
        if cur is not None:
            parts.append([cur, "\n\n" if blank else "\n"])
        cur, blank = line, False
    if cur is not None:
        parts.append([cur, ""])
    return [(re.sub(r"[ \t ]+", " ", p).strip(), s) for p, s in parts]


def join_paste(parts):
    return "".join(p + s for p, s in parts).strip()


# ---------------------------------------------------------------- long sentences
LONG_WORDS = 28
_CONJ = r"(?:and|but|while|whereas|which|although|though|because|so|thus|thereby|yet|whilst|where|when)"
_SPLIT = re.compile(rf"(?<=;)\s+|(?<=,)\s+(?={_CONJ}\b)")


def chunk_sentence(sentence, max_words=LONG_WORDS, min_words=7):
    """Split a long sentence at '; ' or ', <conjunction>' so each piece has >= min_words."""
    if len(sentence.split()) <= max_words:
        return [sentence]
    pieces = [p for p in _SPLIT.split(sentence) if p and p.strip()]
    out = []
    for p in pieces:
        if out and (len(out[-1].split()) < min_words or len(p.split()) < min_words):
            out[-1] = out[-1] + " " + p
        else:
            out.append(p)
    if len(out) > 1 and len(out[-1].split()) < min_words:
        last = out.pop()
        out[-1] = out[-1] + " " + last
    return out


def _fit_chunk(orig, new, first, last):
    """Make a paraphrased piece fit back into the sentence it came from."""
    new = (new or "").strip() or orig
    tail = orig.rstrip()[-1:]
    if not last:
        new = re.sub(r"[.!?]+$", "", new).rstrip()
        if tail in ",;" and not new.endswith(tail):
            new += tail
    if not first:
        w0 = orig.split()[0]
        if re.fullmatch(_CONJ, w0.lower()) and new.split()[0].lower().strip(",") != w0.lower():
            new = w0 + " " + (new[0].lower() + new[1:] if new[1:2].islower() else new)
        elif orig[:1].islower() and new[1:2].islower():
            new = new[0].lower() + new[1:]
    return new


def merge_chunks(orig_chunks, new_chunks):
    n = len(orig_chunks)
    return " ".join(_fit_chunk(o, c, i == 0, i == n - 1) for i, (o, c) in enumerate(zip(orig_chunks, new_chunks)))


# ---------------------------------------------------------------- rewrite guard
# citation / reference markers that must survive a rewrite
CITE = re.compile(r"\[\s*\d+(?:\s*[,–\-]\s*\d+)*\s*\]"
                   r"|\((?:[A-Z][^()]{0,80}?(?:19|20)\d{2}[a-z]?(?:;\s*[^()]{0,80}?(?:19|20)\d{2}[a-z]?)*)\)")


# ---------------------------------------------------------------- technical-term protection
# A rewrite is rejected (original sentence kept) if it loses or alters a technical term:
#   * acronyms / symbols / formulas / model names: CNN, LSTM, IoMT, LiFePO4, ResNet-50, 5G, CO2, α, W/m
#   * terms defined in the document: "phase change material (PCM)" -> "phase change material"
#   * the document's key terms: phrases used repeatedly ("state of charge") and content words used 3+ times
#   * words whose everyday synonyms change the technical meaning ("model", "accuracy", "layer", ...)
TECH_TOKEN = re.compile(r"""(?x)
    (?<![\w-])(?:
        [A-Za-z]*[A-Z][A-Za-z]*\d[\w.\-]*       # LiFePO4, ResNet50, CO2, H2O, YOLOv8
      | \d+(?:\.\d+)?[A-Za-z][\w\-]*            # 5G, 3D, 2D-CNN, 3C
      | [A-Z]{2,}[a-z]?(?:-[A-Za-z0-9]+)*       # CNN, LSTM, SVMs, BERT-base
      | [a-z]+[A-Z][\w\-]*                      # mmWave, eNodeB, pH, iOS
      | [A-Z][a-z]+[A-Z][\w\-]*                 # ResNet, PyTorch, IoMT, MobileNet
      | [Ͱ-Ͽ][\wͰ-Ͽ]*       # Greek letters: α, β-sheet, Δ
      | [A-Za-z]+(?:[/^_][A-Za-z0-9]+)+         # W/m, m/s, x_i
    )(?!\w)""")

FRAGILE = set("""
accuracy precision recall specificity sensitivity model training trained learning network layer kernel
feature dataset regression variance gradient algorithm parameter coefficient correlation classification
classifier clustering encoder decoder attention transformer convolutional recurrent neural optimization
optimizer loss epoch inference latency throughput bandwidth voltage resistance capacitance inductance
impedance frequency signal noise power energy thermal temperature heat flux pressure stress strain
viscosity density velocity torque load cathode anode electrolyte catalyst enzyme protein gene cell tissue
hypothesis significance sample respondent mediation moderation reliability validity simulation mesh node
cluster packet protocol routing entropy matrix vector tensor derivative integral equation theorem spectrum
wavelength phase charge discharge battery circuit converter inverter sensor compression sparse sparsity
gender genders male males female females men women participants respondents
reconstruction segmentation detection estimation prediction forecasting pavement asphalt binder aggregate
""".split())

_STOP = set("""a an the and or but if then than of to in on at by for with from as into onto over under
between among through during before after about above below up down out off again further once here there
when where why how all any both each few more most other some such no nor not only own same so too very
can will just should now is are was were be been being have has had having do does did doing this that
these those it its they them their we our us you your he she his her i me my which who whom whose what
also may might must shall could would however thus therefore hence moreover furthermore additionally
while whereas although though because since whether within without across along toward towards upon via
per et al using used use based show shows shown showed study studies paper research results result
approach proposed propose provides provide provided well new one two three first second third
different various several many much significantly important overall present current""".split())

GLOSSARY = contextvars.ContextVar("tp_glossary", default=frozenset())


def _norm(text):
    """lower-case, hyphens -> spaces, light plural stripping — "State-of-Charge" == "states of charge"."""
    t = re.sub(r"[-‐–]", " ", (text or "").lower())
    t = re.sub(r"\b(\w{3,}?)ies\b", r"\1y", t)
    t = re.sub(r"\b(\w{3,}?)(?:es|s)\b", r"\1", t)
    return " " + re.sub(r"\s+", " ", re.sub(r"[^\w\s]", " ", t)).strip() + " "


_FRAGILE_N = {_norm(w).strip() for w in FRAGILE}
_STAT_STEMS = ("mediat", "moderat", "correlat", "regress")
_STOP_N = {_norm(w).strip() for w in _STOP} | _STOP


# Narrow term protection (report-flagged sentences): only multi-word technical terms and defined acronyms are
# protected, so everyday words ("educational", "efficient", "associated") are free to change. Acronyms, numbers,
# formulas and citations are always protected (TECH_TOKEN / guard), whatever this setting.
NARROW = contextvars.ContextVar("narrow_terms", default=False)


def narrow(glossary):
    """Multi-word entries of a glossary (drop single everyday words)."""
    return frozenset(t for t in glossary if " " in t)


def build_glossary(texts):
    """Key terms of a document (normalised phrases)."""
    joined = " ".join(t for t in texts if t)
    terms = set()
    for m in re.finditer(r"((?:[A-Za-z][\w\-]*\s+){1,7})\(\s*([A-Z][A-Za-z0-9\-]{1,11}?)s?\s*\)", joined):
        words = m.group(1).split()
        n = max(1, min(len(words), len(re.findall(r"[A-Z0-9]", m.group(2))) or len(m.group(2))))
        cand = words[-n:]
        while cand and cand[0].lower() in _STOP:
            cand = cand[1:]
        if cand and len(" ".join(cand)) >= 4:
            terms.add(_norm(" ".join(cand)).strip())
    defined = set(terms)                               # terms the author defined with an abbreviation
    uni, grams = {}, {}
    for s in re.split(r"(?<=[.!?;:])\s+", joined):
        toks = _norm(s).split()
        for w in toks:
            if len(w) >= 5 and w not in _STOP_N and not w.isdigit():
                uni[w] = uni.get(w, 0) + 1
        for n in (2, 3):
            for i in range(len(toks) - n + 1):
                g = toks[i:i + n]
                if g[0] in _STOP_N or g[-1] in _STOP_N or any(len(w) < 3 or w.isdigit() for w in (g[0], g[-1])):
                    continue
                if n == 2 and max(len(g[0]), len(g[1])) < 5:
                    continue
                if n == 3 and g[1] in _STOP_N and g[1] not in ("of", "to", "and", "for", "in", "on", "per"):
                    continue                              # "support the preservation" is wording, not a term
                k = " ".join(g)
                grams[k] = grams.get(k, 0) + 1
    if NARROW.get():                                   # report-flagged rewriting: real technical terms only
        terms |= {k for k, c in grams.items() if c >= 3}
        return _no_stock(terms, defined)
    terms |= {k for k, c in grams.items() if c >= 2}
    # single words: only the document's real key terms (frequent relative to its length), so ordinary
    # vocabulary stays free for the paraphraser
    min_uni = max(3, len(joined.split()) // 300)
    terms |= {w for w, c in uni.items() if c >= min_uni and len(w) >= 6}
    return _no_stock(terms, defined)


def _no_stock(terms, defined):
    """A phrase that only repeats because the text is AI-written ("crucial role" in three paragraphs) is not a term:
    protecting it forced every rewrite to keep it ("acts as a crucial role"). Terms the author defined with an
    abbreviation stay protected whatever words they contain."""
    import pubrules as PR
    return frozenset(t for t in terms if t and (t in defined or not PR._AI_WORD_RE.search(t)))


# ---------------------------------------------------------------- document key terms (always protected)
# The study's own constructs must never be swapped for synonyms ("travel satisfaction" -> "travel pleasure",
# "intimacy" -> "closeness", "destination loyalty" -> "allegiance"). They are read from the WHOLE document once per
# job and protected in every mode, including report-guided rewriting where the ordinary glossary is narrowed:
#   * every term defined with an abbreviation, "travel satisfaction (TS)" -> "travel satisfaction" + "satisfaction"
#   * the Keywords line, "Keywords: Travel motivation, Intimacy, ..." -> each keyword + its head word
KEY = contextvars.ContextVar("tp_key_terms", default=frozenset())
KEY_DEF = contextvars.ContextVar("tp_key_defined", default=frozenset())   # terms defined with an abbreviation
DOC_VOCAB = contextvars.ContextVar("tp_doc_vocab", default=frozenset())
DOC_PROPER = contextvars.ContextVar("tp_doc_proper", default=frozenset())   # words the document always capitalises
_GENERIC_HEADS = {"study", "analysi", "analysis", "difference", "differenc", "economy", "engagement", "modeling",
                  "modelling", "council", "organization", "organisation", "approach", "model", "method", "framework", "system", "process", "factor",
                  "factors", "effect", "effects", "impact", "role", "level", "levels", "rate", "index", "test",
                  "scale", "theory", "research", "development", "management", "performance", "behaviour",
                  "behavior", "value", "values", "data", "group", "groups", "practice", "practices"}


def key_terms(texts):
    """Construct names of a document (normalised), see KEY."""
    joined = " ".join(t for t in texts if t)
    terms = set()
    for m in re.finditer(r"((?:[A-Za-z][\w\-]*\s+){1,5})\(\s*([A-Z][A-Za-z0-9\-]{1,11}?)s?\s*\)", joined):
        acr, words = m.group(2), m.group(1).split()
        core = len(re.findall(r"\b" + re.escape(acr) + r"\b", joined)) >= 4   # a construct the study keeps using
        n = len(re.findall(r"[A-Z]", acr)) or len(acr)
        cand = words[-n:]
        while cand and cand[0].lower() in _STOP:
            cand = cand[1:]
        if not cand or len(" ".join(cand)) < 4:
            continue
        terms.add(_norm(" ".join(cand)).strip())
        head = _norm(cand[-1]).strip()
        if core and len(head) >= 5 and head not in _GENERIC_HEADS and head not in _STOP_N:
            terms.add(head)
    KEY_DEF.set(frozenset(terms))
    for line in re.findall(r"(?im)^\s*key\s*-?\s*words?\s*[:\-–]\s*(.+)$", "\n".join(t for t in texts if t)):
        for kw in re.split(r"[,;·•]", line):
            kw = _norm(kw).strip()
            if len(kw) >= 4 and len(kw.split()) <= 5:
                terms.add(kw)
                head = kw.split()[-1]
                if len(head) >= 5 and head not in _GENERIC_HEADS and head not in _STOP_N:
                    terms.add(head)
    return frozenset(t for t in terms if t)


# everyday synonyms a paraphraser uses for common research constructs; when the construct is one of the document's
# key terms, a rewrite may not bring in its synonym ("satisfaction" study: "pleasure", "contentment" are rejected)
_CONSTRUCT_SYN = {
    "satisfaction": {"pleasure", "contentment", "happiness", "delight", "gratification", "fulfilment", "fulfillment"},
    "loyalty": {"allegiance", "faithfulness", "devotion", "fidelity"},
    "intimacy": {"closeness", "familiarity", "affinity"},
    "motivation": {"incentive", "incentives", "impetus", "inducement"},
    "well being": {"welfare", "wellness", "health"},
    "wellbeing": {"welfare", "wellness", "health"},
    "trust": {"faith", "confidence", "reliance"},
    "attachment": {"bond", "fondness", "affection"},
    "intention": {"willingness", "inclination"},
    "engagement": {"involvement", "participation", "commitment"},
    "commitment": {"dedication", "devotion", "pledge"},
    "performance": {"output", "achievement", "efficiency"},
    "image": {"picture", "impression", "reputation"},
    "quality": {"standard", "calibre", "caliber", "grade"},
    "anxiety": {"worry", "nervousness", "unease"},
    "stress": {"strain", "pressure", "tension"},
}


def renamed_construct(original, new):
    """A key construct's everyday synonym the rewrite introduces ("allegiance" in a loyalty study), or None."""
    key = KEY.get()
    if not key:
        return None
    nn, no = _norm(new), _norm(original)
    for term, syns in _CONSTRUCT_SYN.items():
        if term in key:
            for w in syns:
                if f" {_norm(w).strip()} " in nn and f" {_norm(w).strip()} " not in no:
                    return f"{term} -> {w}"
    return None


def set_document(texts):
    """Call once per job with the whole document: key terms and vocabulary for the checks below."""
    texts = [t for t in texts if t]
    KEY.set(key_terms(texts))
    cap, low = {}, {}
    for t in texts:
        for m in re.finditer(r"(?<![.!?:]\s)(?<=\s)([A-Za-z][a-z]{2,})\b", t):   # mid-sentence words only
            w = m.group(1)
            d = cap if w[:1].isupper() else low
            d[w.lower()] = d.get(w.lower(), 0) + 1
    DOC_PROPER.set(frozenset(w.capitalize() for w, c in cap.items() if c >= 2 and not low.get(w)))
    vocab = {w.lower() for t in texts for w in re.findall(r"[A-Za-z][A-Za-z\-]+", t)}
    DOC_VOCAB.set(frozenset(vocab) if sum(len(t.split()) for t in texts) >= 300 else frozenset())


def lost_terms(original, new, glossary=None):
    """Technical terms of `original` that `new` no longer contains (empty list = safe)."""
    glossary = GLOSSARY.get() if glossary is None else glossary
    missing = []
    for t in {m.group(0) for m in TECH_TOKEN.finditer(CITE.sub(" ", original))}:
        core = t[:-1] if t.endswith("s") and t[:-1].isupper() else t
        if not re.search(r"(?<!\w)" + re.escape(core.rstrip(".-")), new):
            missing.append(t)
    no, nn = _norm(original), _norm(new)
    missing += [w for w in set(no.split()) & _FRAGILE_N if f" {w} " not in nn]
    missing += [g for g in glossary if f" {g} " in no and f" {g} " not in nn]
    # statistical relations in any word form: "does not moderate" must not become "does not modify"
    missing += [st + "…" for st in _STAT_STEMS if f" {st}" in no and f" {st}" not in nn]
    missing += [g for g in KEY.get() if f" {g} " in no and f" {g} " not in nn]       # study constructs
    return sorted(set(missing))


# ---------------------------------------------------------------- sentence rewriting
_TITLES = re.compile(r"\b(?:Mr|Mrs|Ms|Miss|Dr|Prof|Sir|Madam|Mx)\b\.?", re.I)
_PERSONAL = re.compile(r"\b(?:he|she|him|his|her|hers|himself|herself)\b", re.I)
_SENT_START = re.compile(r"(?:^|[.!?]\s+|[\"“(]\s*)$")


def invented(original, new):
    """What `new` makes up that `original` does not contain (paraphrase models sometimes add a name, a person or a
    number). Returns a short reason, or None if nothing was invented."""
    o_low = original.lower()
    o_words = set(re.findall(r"[a-z0-9']+", o_low))
    o_caps = {w.lower() for w in re.findall(r"\b[A-Z][a-z]{3,}\b", original)}
    if _TITLES.search(new) and not _TITLES.search(original):
        return "title"
    if _PERSONAL.search(new) and not _PERSONAL.search(original):
        return "person"
    nums_o = set(re.findall(r"\d+(?:\.\d+)?", original))
    if any(n not in nums_o for n in re.findall(r"\d+(?:\.\d+)?", CITE.sub(" ", new))):
        return "number"
    for m in re.finditer(r"\b[A-Z][a-z]{2,}\b", new):           # capitalised word mid-sentence = name-like
        if _SENT_START.search(new[:m.start()]) or (re.search(r"[\])]\s*$", new[:m.start()])
                                                   and re.fullmatch(r"[A-Z][a-z]+(?:ed|es|s)", m.group(0))):
            continue                                     # sentence start, or "[56] Underscored" (citation + verb)
        w = m.group(0).lower()
        if w in o_words or re.search(r"\b" + re.escape(w) + r"\b", o_low):
            continue                                     # ("Men's" in the original counts as "Men")
        # another form of a name the author uses ("India's" for "Indian", "Bayes" for "Bayesian") is not invented
        if len(w) >= 4 and any(o.startswith(w[:max(4, len(w) - 2)]) or w.startswith(o[:max(4, len(o) - 2)])
                               for o in o_caps):
            continue
        return "name"
    return None


# ---------------------------------------------------------------- quality gate (every mode)
_INFORMAL = re.compile(
    r"\b(?:absurd|stuff|things like|kinda|sorta|gonna|wanna|gotta|pretty much|a lot of|lots of|tons of|guys|okay|ok|"
    r"awesome|cool|huge|super|basically|really|totally|folks|kids|crazy|weird|silly|nope|yeah|yep|dumb|"
    r"nice|great deal of fun|anyway|anyhow|like i said|you know|bunch of|figure out|sort of|kind of)\b", re.I)
_FIRST_SING = re.compile(r"\b(?:I|me|my|mine|myself)\b")
_YOU = re.compile(r"\b(?:you|your|yours|yourself)\b", re.I)
_CONTRACTION = re.compile(r"\b\w+(?:n't|'re|'ll|'ve|'m|'d)\b", re.I)
try:
    from wordfreq import zipf_frequency as _zipf
except Exception:                                         # optional dependency
    _zipf = None


def _known(word, o_words):
    """A real English word, or a form of a word the original already uses."""
    w = word.lower()
    if w in o_words or len(w) < 4 or _zipf is None:
        return True
    if _zipf(w, "en") > 0.0:
        return True
    if "-" in w and all(_zipf(p, "en") > 0.0 or p in o_words for p in w.split("-") if p):
        return True
    for k in range(3, len(w) - 2):                       # two words run together ("statisticallyor")
        if _zipf(w[:k], "en") >= 3.0 and _zipf(w[k:], "en") >= 3.0:
            return False
    sfx = ("s", "es", "ed", "d", "ing", "ly", "al", "ally", "ation", "ations", "ity", "ive", "ness", "ment", "er",
           "ers", "ic", "ical", "ise", "ize", "ised", "ized")
    for o in o_words:                                    # an inflected form of a word the original uses
        if len(o) < 5:
            continue
        for stem in (o, o[:-1], o[:-2]):
            if len(stem) >= 4 and w.startswith(stem) and (w[len(stem):] in sfx or w[len(stem):] == ""):
                return True
    return False


_FOREIGN = re.compile(r"\b[\w\-]*[À-ÖØ-öø-ÿĀ-žƀ-ɏ][\w\-]*")
OFFTOPIC_N = int(__import__("os").getenv("HUM_OFFTOPIC", "2"))   # 0 = check off


def _offtopic(original, new):
    """Uncommon content words the rewrite brings in that appear nowhere in the document and are not forms of the
    sentence's own words ("organizational recruitment" in a tourism paper). None, or the words."""
    vocab = DOC_VOCAB.get()
    if not vocab or not OFFTOPIC_N or _zipf is None:
        return None
    o_low = original.lower()
    stems = {w[:5] for w in re.findall(r"[a-z]{5,}", o_low)}
    odd = []
    for w in set(re.findall(r"[A-Za-z][a-z\-]{5,}", CITE.sub(" ", new))):
        lw = w.lower()
        if lw in vocab or lw[:5] in stems or lw in _STOP or w[:1].isupper():
            continue
        if _zipf(lw, "en") < 3.3 and not any(v.startswith(lw[:6]) for v in vocab):
            odd.append(lw)                                # rare AND unrelated to any word the document uses
    return odd if len(odd) >= OFFTOPIC_N else None


def quality_issue(original, new):
    """Why `new` is not acceptable academic English compared with `original` (None = fine):
    a non-word the original doesn't contain, a foreign word, words foreign to the document's topic, informal /
    conversational wording, first person singular, addressing the reader, contractions, or a question where the
    original made a statement."""
    o_words = set(re.findall(r"[a-z][a-z'\-]*", original.lower()))
    syn = renamed_construct(original, new)
    if syn:
        return f"construct renamed ({syn})"
    for m in _FOREIGN.finditer(new):                      # "well-être": accented words the original does not use
        if m.group(0).lower() not in original.lower():
            return f"foreign word '{m.group(0)}'"
    odd = _offtopic(original, new)
    if odd:
        return "off-topic words '" + ", ".join(sorted(odd)) + "'"
    for w in re.findall(r"[A-Za-z][A-Za-z\-]{3,}", CITE.sub(" ", new)):
        if w.isupper() or (w[0].isupper() and w.lower() in o_words):
            continue                                      # acronyms / names already in the original
        if not _known(w, o_words):
            return f"non-word '{w}'"
    for pat, why in ((_INFORMAL, "informal wording"), (_FIRST_SING, "first person"), (_YOU, "addresses the reader"),
                     (_CONTRACTION, "contraction")):
        m = pat.search(new)
        if m and not pat.search(original):
            return f"{why} '{m.group(0)}'"
    if "?" in new and "?" not in original:
        return "question"
    return None


# characters language models like to substitute: the author's own typography is kept
_TYPO = {"‑": "-", "‐": "-", "­": "", " ": " ", " ": " ", " ": " ", "​": "",
         "’": "'", "‘": "'", "“": '"', "”": '"', "−": "-", "²": "2", "³": "3"}


def normalize_typography(original, new):
    """Replace no-break hyphens / spaces, curly quotes and the minus sign in `new` by their plain form, unless the
    author's sentence uses that character itself (so "lithium-ion" never comes back with a no-break hyphen)."""
    if not new:
        return new
    return "".join(_TYPO[c] if c in _TYPO and c not in original else c for c in new)


_REL = "<>=≤≥≈±×÷~∼≠∝→"
_PAIRS = (("(", ")"), ("[", "]"), ("{", "}"))


def symbol_issue(original, new):
    """Relation signs, brackets and citations: None, or why `new` damages them. A paraphraser that turns
    "p < 0.01" into "p 0.01" keeps every number but changes the result; an unbalanced bracket or a citation
    written twice is always a defect; a numbered citation the author did not write is invented."""
    for c in _REL:
        if original.count(c) != new.count(c):
            return f"relation sign '{c}' changed"
    for a, b in _PAIRS:
        if original.count(a) == original.count(b) and new.count(a) != new.count(b):
            return "unbalanced bracket"
    oc = [re.sub(r"\s+", "", c) for c in CITE.findall(original)]
    nc = [re.sub(r"\s+", "", c) for c in CITE.findall(new)]
    for c in set(nc):
        if c.startswith("[") and c not in oc:
            return "citation added"
        if c in oc and nc.count(c) > oc.count(c):
            return "citation duplicated"
    return None


def _ckey(c):
    return re.sub(r"\s+", "", c)


def repair_citations(original, new):
    """A paraphraser copies citations carelessly: "(Mishra and Mishri, 2012)", "(Sarkar 1973)". A citation in `new`
    that the author did not write is replaced by the author's citation it is a damaged copy of; if it is a copy of
    none (an invented source), or of one that is already there, the rewrite is rejected (None)."""
    orig = CITE.findall(original)
    keys = {_ckey(c) for c in orig}
    odd = [c for c in CITE.findall(new) if _ckey(c) not in keys]
    for c in odd:
        have = {_ckey(x) for x in CITE.findall(new)}
        free = [o for o in orig if _ckey(o) not in have]
        best = max(free, key=lambda o: difflib.SequenceMatcher(None, _ckey(o), _ckey(c)).ratio(), default=None)
        if best is None or difflib.SequenceMatcher(None, _ckey(best), _ckey(c)).ratio() < 0.7:
            return None
        new = new.replace(c, best, 1)
    return new
def _symbols(t):
    """Comparison / statistics symbols outside citations: < > ≤ ≥ = ± × and their order."""
    return re.findall(r"[<>≤≥=±×]", CITE.sub(" ", t or ""))


def _brackets_ok(t):
    return (t or "").count("(") == (t or "").count(")") and (t or "").count("[") == (t or "").count("]")


def _new_acronyms(original, new):
    acr = lambda t: set(re.findall(r"\b[A-Z][A-Z0-9]{1,}[a-z]?\b", CITE.sub(" ", t or "")))
    return acr(new) - acr(original)


# ---------------------------------------------------------------- integrity checks shared by EVERY rewrite path
# (Parrot guard, Groq quality check, final polish, proofread, clarity gate). Each one targets an error seen in real
# humanized papers.
_ATTRIB = re.compile(r"\b(?:according to|(?:some |many |several )?(?:experts?|scholars?|researchers?|analysts?|"
                     r"studies|scientists)\s+(?:say|says|said|argue|argues|contend|contends|believe|believes|"
                     r"suggest|suggests|claim|claims|found|find|show|shows|note|notes)|a (?:systematic |recent )?"
                     r"(?:analysis|review|study|survey) (?:found|showed|shows|finds|revealed|suggests))\b", re.I)
_SAID = re.compile(r"\b(?:said|says|told|stated|according to)\b", re.I)
_CAUSAL = re.compile(r"\b(?:because|since (?:they|it|he|she|we|the|this|these|those|their|its)|due to|owing to|"
                     r"as a result of|thereby|in turn)\b", re.I)
_DANGLING_END = re.compile(r"\b(?:the|a|an|their|its|our|his|her|of|to|and|or|on|in|with|for|by|from|as|than|"
                           r"that|which|this|these|between|among)\s*[.!?]$", re.I)
_DANGLING_HYPHEN = re.compile(r"[A-Za-z]-(?=[\s,.;:)])")
_EMDASH = re.compile(r"—|\s–\s|\w\s-\s\w")       # also a spaced hyphen used as a dash
_SENT_ADV = r"(?:However|Moreover|Furthermore|Therefore|Consequently|Nevertheless|Nonetheless|Additionally|" \
            r"In addition|In general|In contrast|In particular|In summary|In conclusion|For example|For instance|" \
            r"Thus|Hence|Instead|Similarly|Likewise|Meanwhile|Simultaneously|Finally|" \
            r"Accordingly|Specifically|Notably|Importantly|Ultimately|Conversely|Subsequently)"
# ("Further", "Overall", "First" are left out: "Further studies…", "Overall satisfaction…" need no comma)


def _proper_nouns(t):
    """Capitalised names inside a sentence (Darjeeling, Singapore, Kumar), not its first word."""
    words = re.findall(r"(?<=[\s(\[,;:])([A-Z][a-z]{2,}(?:-[A-Z][a-z]+)?)\b", " " + CITE.sub(" ", t or "")[1:])
    return {w for w in words if w.lower() not in _STOP and w not in ("Table", "Figure", "Fig", "Section", "Eq",
                                                                     "Chapter", "Appendix")}


def integrity_issue(original, new):
    """Damage a rewrite may never do (None = fine):
      * an attribution or a causal explanation the original does not make ("According to some experts … since they
        have more muscle"; "a systematic analysis found");
      * a name / place dropped from the sentence ("Digha, Darjeeling, Jhargram" -> "Digha Jhargram");
      * a key construct used more often than in the original ("… destination loyalty (DL) and intimacy");
      * a cut-off ending ("… and to enhance their.") or a cut-off term ("on well-, and");
      * an em dash, or a doubled conjunction ("and, and");
      * two sentences run together ("… or mass According to …")."""
    o, n = original or "", new or ""
    try:
        import adaptive as _AD                            # words learned to cause bad rewrites
        bw = _AD.banned_in(o, n)
        if bw:
            return f"banned word: {bw}"
    except Exception:
        pass
    if _ATTRIB.search(n) and not _ATTRIB.search(o):
        return "attribution added"
    if len(_CAUSAL.findall(n)) > len(_CAUSAL.findall(o)):
        return "causal claim added"
    if len(_SAID.findall(n)) > len(_SAID.findall(o)):
        return "attribution added"                        # "... beyond local capacity the director said"
    if set(re.findall(r"\b[A-Z]\.[A-Z]\.?", n)) - set(re.findall(r"\b[A-Z]\.[A-Z]\.?", o)):
        return "abbreviation added"                       # "Darjeeling, and M.P, and purulia"
    names = DOC_PROPER.get()
    for w in _proper_nouns(o) | ({o.split()[0].strip(",.;:")} if o.split() else set()):
        is_name = w in names if names else (_zipf is not None and _zipf(w.lower(), "en") < 2.0)
        if len(w) > 3 and is_name and re.search(r"\b" + re.escape(w.lower()) + r"\b", n) and w not in n:
            return f"name lower-cased: {w}"               # "purulia" (the document always writes "Purulia")
    for a, b in re.findall(r"\b([A-Z][a-z]+)\s+([A-Z][a-z]+)\b", o):
        if a in names and re.search(r"\b" + a.lower() + r"\s+" + b + r"\b", n):
            return f"name lower-cased: {a} {b}"            # "west Bengal"
    stray = r"[a-z,]\s+(?:The|A|An|Of|For|In|And|To|With|On)\s"
    if re.search(stray, n) and not re.search(stray, o):
        return "capital inside a sentence"                # "variance in The TS"
    if re.search(r"(?:\b[A-Z][a-z]{2,}\s+){3,}", n) and not re.search(r"(?:\b[A-Z][a-z]{2,}\s+){3,}", o):
        caps = re.findall(r"(?:\b[A-Z][a-z]{2,}\s+){3,}", n)[0].split()
        if sum(w.lower() in _STOP or (_zipf is not None and _zipf(w.lower(), "en") >= 4.5) for w in caps) >= 2:
            return "title case in a sentence"             # "The Strong Effect of Intimacy For Both genders"
    if re.match(r"^\[\d[\d,\s–\-]*\]\s+[A-Z][a-z]+(?:ed|es|s)\b", o) and \
            re.match(r"^\[\d[\d,\s–\-]*\]\s+(?:The|A|An|This|These|Its)\b", n):
        return "reporting verb dropped"                   # "[56] Highlighted ..." -> "[56] The importance of ..."
    if re.search(r"\b(whereas|while|although|though|but|however|and|or|because)\s+(whereas|while|although|"
                 r"though|but|however|because)\b", n, re.I):
        return "doubled connective"                       # "whereas while", "but however"
    sw = [x.lower() for x in re.findall(r"[A-Za-z]{5,}", n)]
    so = [x.lower() for x in re.findall(r"[A-Za-z]{5,}", o)]
    rep_n = {a[:5] for a, b in zip(sw, sw[1:]) if a != b and a[:5] == b[:5]}
    rep_o = {a[:5] for a, b in zip(so, so[1:]) if a != b and a[:5] == b[:5]}
    if rep_n - rep_o:
        return "stem repeated"                            # "intimate intimacy intimation"
    dash = lambda t: re.sub(r"[‐‑–—]", "-", t).lower()
    lost = {w for w in _proper_nouns(o) if dash(w) not in dash(n)               # (case / dash changes are fine)
            and not re.search(r"[\])]\s*" + re.escape(w) + r"\b", o)         # "[59] Discussed" is a verb
            and not (_zipf is not None and _zipf(w.lower(), "en") >= 5.0)}    # "for Men" -> "for males"
    if lost:
        return "name dropped: " + ", ".join(sorted(lost))
    nn, no = _norm(n), _norm(o)
    for k in KEY_DEF.get():
        if nn.count(f" {k} ") > max(1, no.count(f" {k} ")):
            return f"construct repeated: {k}"
    if _DANGLING_END.search(n.strip()) and not _DANGLING_END.search(o.strip()):
        return "sentence cut off"
    if len(_DANGLING_HYPHEN.findall(n)) > len(_DANGLING_HYPHEN.findall(o)):
        return "term cut off"
    cut = r"\b(?:of|to|for|with|on|in|by|from|the|a|an|and|or|their|its)\s*,"
    if len(re.findall(cut, n, re.I)) > len(re.findall(cut, o, re.I)):
        return "phrase cut off"                           # "the port of, and the forest"
    op = r"\b(?:Sadly|Unfortunately|Fortunately|Luckily|Interestingly|Surprisingly|Obviously|Clearly|Undoubtedly|" \
         r"Remarkably|Importantly|Notably|Strikingly)\b"
    if re.search(op, n) and not re.search(op, o):
        return "opinion word added"                       # "Sadly loyalty does not ..."
    letters = lambda t: re.findall(r"(?<![\w.'’-])([b-zB-HJ-Z])(?![\w.'’-])", CITE.sub(" ", t))
    if len(letters(n)) > len(letters(o)):
        return "stray letter"                             # "... is d a vital predictor"
    sq = lambda t: t.count("[") - sum(1 for c in CITE.findall(t) if c.startswith("["))   # "[" outside citations
    if sq(n) > sq(o):                                     # (author-year citations have no "[": not counted)
        return "bracket type changed"                     # "(explaining ...)" -> "[explaning ...]"
    if _EMDASH.search(n) and not _EMDASH.search(o):
        return "em dash"
    if re.search(r"\b(\w+),?\s+\1\b", n, re.I) and not re.search(r"\b(\w+),?\s+\1\b", o, re.I):
        return "doubled word"
    if re.search(r"\b[a-z]{3,}\s+(?:According|However|Moreover|Furthermore|Therefore|This|These|The|It|In)\s+[a-z]", n) \
            and not re.search(r"\b[a-z]{3,}\s+(?:According|However|Moreover|Furthermore|Therefore|This|These|The|It|In)\s+[a-z]", o):
        return "sentences run together"
    return None


def restore_commas(original, new):
    """Put back the commas a paraphraser dropped between words that keep their order: list items ("airports,
    trains and taxis"), after an opening phrase ("After COVID-19, tourists") and after an abbreviation in brackets
    ("(TM), eudaimonic"). Also adds the comma after a sentence-opening linking word ("However the" -> "However,
    the")."""
    if not new:
        return new
    out = new
    for m in re.finditer(r"(?<![\w\-])([\w)\]%][\w\-)\]%]*),\s+(?=([\w(\[]+))", original or ""):
        a, b = m.group(1), m.group(2)
        pat = re.compile(r"(?<![\w-])" + re.escape(a) + r"(\s+)" + re.escape(b) + r"(?![\w-])")
        if len(pat.findall(out)) == 1 and not re.search(re.escape(a) + r",\s+" + re.escape(b), out):
            out = pat.sub(lambda k: a + "," + k.group(1) + b, out, count=1)
    out = re.sub(rf"^({_SENT_ADV})\s+(?=[a-z])", r"\1, ", out)
    return out


def guard(original, new, glossary=None):
    """Reject rewrites that lose meaning-bearing content or invent names / people / numbers; re-attach dropped
    citations."""
    new = re.sub(r"\s+([,.;:!?])", r"\1", normalize_typography(original, (new or "").strip()))    # " ," -> ","
    new = re.sub(r"([.!?])[.!?]+$", r"\1", new)                     # ".." -> "."
    new = re.sub(r"([(\[])\s+", r"\1", re.sub(r"\s+([)\]])", r"\1", new))     # "( Kumar , 2021 )"
    if not new:
        return None
    new = repair_citations(original, new)
    if new is None or invented(original, new) or quality_issue(original, new) or symbol_issue(original, new):
        return None
    if _symbols(new) != _symbols(original):              # "p < .001" must not become "p .001"
        return None
    if _brackets_ok(original) and not _brackets_ok(new):  # "(TS amongst" - a bracket lost
        return None
    if _new_acronyms(original, new):                      # "EWB" must not become "EWD"
        return None
    new = restore_commas(original, new)                   # "airports trains and taxis" -> "airports, trains and taxis"
    if integrity_issue(original, new):
        return None
    ow, nw = len(original.split()), len(new.split())
    if nw < 0.5 * ow or nw > 1.8 * ow + 5:
        return None
    # numbers (results, years, units) must be kept
    nums_o = set(re.findall(r"(?<![\d])\.?\d+(?:\.\d+)?%?", CITE.sub(" ", original)))
    nums_n = set(re.findall(r"(?<![\d])\.?\d+(?:\.\d+)?%?", new))
    if not nums_o <= nums_n:
        return None
    if lost_terms(original, new, glossary):                # technical wording must survive unchanged
        return None
    have = [_ckey(c) for c in CITE.findall(new)]
    missing = [c for c in CITE.findall(original) if _ckey(c) not in have]
    if missing:
        end = re.search(r"[.!?]\s*$", new)
        core = new[:end.start()] if end else new
        new = core.rstrip() + " " + " ".join(missing) + (end.group(0).strip() if end else "")
    if sorted(_ckey(c) for c in CITE.findall(new)) != sorted(_ckey(c) for c in CITE.findall(original)):
        return None                                      # every citation once, exactly as the author wrote it
    import pubrules as _PR                                # integrity rules hold for every version, whatever wrote
    if _PR.violations(original, new):                    # it: units, negation, direction, hedging, cause, voice
        return None
    if original[:1].isupper() and new[:1].islower():
        new = new[0].upper() + new[1:]
    elif original[:1].islower() and new[:1].isupper() and not new[1:2].isupper():
        new = new[0].lower() + new[1:]                   # fragment starting mid-sentence
    if not re.search(r"[.!?]$", original.strip()):
        new = re.sub(r"\s*[.!?]+$", "", new)              # fragment without end punctuation
    elif not re.search(r"[.!?][\"”’)\]]?$", new):
        # a sentence must end as one: left open ("...; " / "...,") it runs into the next sentence of the paragraph
        new = new.rstrip(" ,;:–—-") + "."
    return new




_WTOK = re.compile(r"\w+(?:[-'’.]\w+)*|[^\w\s]")


def _term_spans(original, terms):
    """Character spans in `original` of the given lost terms (acronyms exact, phrases/words by normalised match)."""
    spans = []
    for t in terms:
        if not t:
            continue
        if any(c.isupper() or c.isdigit() for c in t) or not t.replace(" ", "").isalpha():
            pat = r"(?<!\w)" + re.escape(t) + r"(?!\w)"
        else:
            pat = r"(?<![\w])" + r"[\s\-]+".join(re.escape(w) + r"\w{0,3}" for w in t.split()) + r"(?!\w)"
        spans += [(m.start(), m.end()) for m in re.finditer(pat, original, re.I)]
    return spans


def _tkey(t):
    """Match key: case-insensitive for ordinary words, exact for acronyms/formulas ("PCM" != "pcm")."""
    return t if (any(c.isdigit() for c in t) or any(c.isupper() for c in t[1:])) else t.lower()


def restore_terms(original, new, glossary=None):
    """Put technical terms that a paraphrase lost or altered back in place, keeping the rest of the paraphrase.
    The sentences are aligned word by word; only the words standing in for a lost term are replaced.
    If the paraphrase restructured the sentence too much to locate the term, it is returned unchanged
    (the guard will then reject it)."""
    import difflib
    if not new:
        return new
    lost = lost_terms(original, new, glossary)
    if not lost:
        return new
    spans = _term_spans(original, lost)
    ot = [(m.group(0), m.start(), m.end()) for m in _WTOK.finditer(original)]
    nt = [(m.group(0), m.start(), m.end()) for m in _WTOK.finditer(new)]
    if not spans or not ot or not nt:
        return new
    ops = difflib.SequenceMatcher(a=[_tkey(t[0]) for t in ot], b=[_tkey(t[0]) for t in nt],
                                  autojunk=False).get_opcodes()

    edits, give_up = {}, False
    for s0, e0 in sorted(set(spans)):
        idx = {k for k, t in enumerate(ot) if t[1] < e0 and t[2] > s0}
        for op, i1, i2, j1, j2 in ops:
            if op == "equal" or i1 == i2 or not idx & set(range(i1, i2)):
                continue
            if (i2 - i1) > 5 or (j2 - j1) > 5:             # restructured around the term: cannot patch safely
                give_up = True
                continue
            block = original[ot[i1][1]:ot[i2 - 1][2]]
            if j1 < j2:
                edits[(nt[j1][1], nt[j2 - 1][2])] = block
            else:                                          # words deleted -> insert them where they were
                pos = nt[j1][1] if j1 < len(nt) else len(new)
                nxt = new[pos:pos + 1]
                edits[(pos, pos)] = (" " + block) if (not nxt or not nxt.isalnum()) else (block + " ")
    if give_up and not edits:
        return new
    # a paraphrase that moved a term's words ("charge state" for "state of charge") leaves the moved words
    # behind as an insertion next to the restored term -> drop them
    term_words = {w.lower() for t in lost for w in re.findall(r"\w+", t)}
    touched = [j for (a, b) in edits for j, t in enumerate(nt) if a <= t[1] <= b]
    for op, i1, i2, j1, j2 in ops:
        if op == "insert" and j2 - j1 <= 3 and all(nt[j][0].lower().rstrip("s") in
                                                  {w.rstrip("s") for w in term_words} for j in range(j1, j2)):
            if touched and min(abs(j - t) for j in (j1, j2 - 1) for t in touched) <= 3:
                edits[(nt[j1][1], nt[j2 - 1][2])] = ""
    out, last = new, None
    for (s0, e0), rep in sorted(edits.items(), reverse=True):
        if last is not None and e0 > last:                 # overlapping edits: keep the first applied
            continue
        out = out[:s0] + rep + out[e0:]
        last = s0
    out = re.sub(r"\s{2,}", " ", out)
    out = re.sub(r"\s+([,.;:!?])", r"\1", out).strip()
    return out


# ---------------------------------------------------------------------------
# "De-AI" wording pass: typical chatbot vocabulary and set phrases -> plain academic wording.
# Meaning-preserving, rule-based; used as extra candidates next to Parrot's paraphrases (the AI detector
# then picks whichever version reads most human). Technical phrases such as "robust regression" or
# "urban landscape" are left alone.
# ---------------------------------------------------------------------------
_OPENERS = re.compile(r"^(?:Furthermore|Moreover|Additionally|In addition|Notably|Importantly|Indeed|"
                      r"It is (?:important|worth|crucial) to (?:note|mention|highlight) that|"
                      r"It is worth noting that|It should be noted that|Ultimately|In essence|Overall|"
                      r"In today's [\w\s-]{0,40}?(?:world|landscape|era|environment|age|society)),\s+"
                      r"|^(?:It is (?:important|worth|crucial|essential) to (?:note|mention|highlight|recognize) that|"
                      r"It is worth noting that|It should be noted that)\s+", re.I)
_WORDS = [
    (r"\bdelve(s|d)? into\b", lambda m: {"": "examine", "s": "examines", "d": "examined"}[m.group(1) or ""]),
    (r"\bdelving into\b", "examining"),
    (r"\bplays an? (?:crucial|vital|pivotal|key|significant|critical|essential|important) role in\b", "is central to"),
    (r"\bplay an? (?:crucial|vital|pivotal|key|significant|critical|essential|important) role in\b", "are central to"),
    (r"\bplayed an? (?:crucial|vital|pivotal|key|significant|critical|essential|important) role in\b", "was central to"),
    (r"\b(?:crucial|pivotal|paramount)\b", "key"),
    (r"\bcomprehensive\b", "detailed"),
    (r"\bleverag(e|es|ed|ing)\b", lambda m: {"e": "use", "es": "uses", "ed": "used", "ing": "using"}[m.group(1)]),
    (r"\butiliz(e|es|ed|ing|ation)\b", lambda m: {"e": "use", "es": "uses", "ed": "used", "ing": "using",
                                                  "ation": "use"}[m.group(1)]),
    (r"\bfacilitat(e|es|ed|ing)\b", lambda m: {"e": "help", "es": "helps", "ed": "helped", "ing": "helping"}[m.group(1)]),
    (r"\bfoster(s|ed|ing)?\b", lambda m: {"": "encourage", "s": "encourages", "ed": "encouraged",
                                          "ing": "encouraging"}[m.group(1) or ""]),
    (r"\benhanc(e|es|ed|ing|ement|ements)\b", lambda m: {"e": "improve", "es": "improves", "ed": "improved",
                                                        "ing": "improving", "ement": "improvement",
                                                        "ements": "improvements"}[m.group(1)]),
    (r"\bunderscor(e|es|ed|ing)\b", lambda m: {"e": "highlight", "es": "highlights", "ed": "highlighted",
                                               "ing": "highlighting"}[m.group(1)]),
    (r"\bshowcas(e|es|ed|ing)\b", lambda m: {"e": "show", "es": "shows", "ed": "showed", "ing": "showing"}[m.group(1)]),
    (r"\bseamless(ly)?\b", lambda m: "smooth" + (m.group(1) or "")),
    (r"\b(?:intricate|multifaceted)\b", "complex"),
    (r"\b(?:a )?myriad (?:of )?\b", "many "),
    (r"\ba plethora of\b", "many"),
    (r"\ba (?:wide|broad) (?:range|array|variety) of\b", "many"),
    (r"\ba variety of\b", "various"),
    (r"\ba significant number of\b", "many"),
    (r"\bin order to\b", "to"),
    (r"\bdue to the fact that\b", "because"),
    (r"\bwith regard(?:s)? to\b", "on"),
    (r"\bin terms of\b", "in"),
    (r"\ba testament to\b", "evidence of"),
    (r"\bever-evolving\b", "changing"),
    (r"\brapidly evolving\b", "fast-changing"),
    (r"\bthe (digital|business|educational|competitive|technological|corporate|organi[sz]ational|modern) landscape\b",
     lambda m: f"the {m.group(1)} environment"),
    (r"\bnavigat(e|es|ing) the complexities of\b", lambda m: {"e": "deal with", "es": "deals with",
                                                               "ing": "dealing with"}[m.group(1)]),
    (r"\bharness(es|ed|ing)? the power of\b", lambda m: {"": "use", "es": "uses", "ed": "used",
                                                        "ing": "using"}[m.group(1) or ""]),
    (r"\bembark(s|ed)? on\b", lambda m: {"": "begin", "s": "begins", "ed": "began"}[m.group(1) or ""]),
    (r"\bstreamlin(e|es|ed|ing)\b", lambda m: {"e": "simplify", "es": "simplifies", "ed": "simplified",
                                               "ing": "simplifying"}[m.group(1)]),
]
_WORDS = [(re.compile(p, re.I), r) for p, r in _WORDS]


def _match_case(src, rep):
    return rep[:1].upper() + rep[1:] if src[:1].isupper() else rep


def de_ai(sentence):
    """Plain-wording version of a sentence (same meaning). Returns the sentence unchanged if nothing applies."""
    s = sentence
    m = _OPENERS.match(s)
    if m and len(s) - m.end() > 20:
        rest = s[m.end():]
        s = rest[:1].upper() + rest[1:]
    for pat, rep in _WORDS:
        s = pat.sub(lambda m: _match_case(m.group(0), rep(m) if callable(rep) else rep), s)
    s = re.sub(r"\s+[—–]\s+", ", ", s)                   # spaced dashes (a chatbot habit) -> comma
    s = re.sub(r"(?<=\w)—(?=\w)", ", ", s)
    s = re.sub(r"\s{2,}", " ", s).replace(" ,", ",")
    return s


# ---------------------------------------------------------------- basic grammar (rules, no model)
_MODALS = r"(?:will|would|can|could|may|might|must|shall|should|to|does|do|did)"
_AN_EXC = re.compile(r"^(?:uni|use|usu|euro|one|once|eu|ewe|u\b|ur|uti)", re.I)       # "a university", "a one-off"
_A_EXC = re.compile(r"^(?:hour|honest|honou?r|heir|(?:[FHLMNRSX][A-Z0-9]*)\b)")   # "an hour", "an MRI", "an SVM"


def _base_form(w):
    """running -> run, making -> make, opening -> open, studying -> study (for 'will opening' -> 'will open')."""
    lw = w.lower()
    if not lw.endswith("ing") or len(lw) < 6:
        return None
    stem = lw[:-3]
    if len(stem) >= 3 and stem[-1] == stem[-2] and stem[-1] not in "lsz":
        stem = stem[:-1]                                  # running -> run
    elif re.search(r"(?:[^aeiou][aeiou](?:k|v|z|c|g|s|t|r)|ag|iz|at|ur|ut|ic|uc|ar)$", stem) and not stem.endswith(
            ("en", "er", "ow", "ain", "eal", "ess", "ect", "ent", "ist", "est", "ort", "ert", "ast", "ust")):
        stem = stem + "e"                                 # making -> make, using -> use, producing -> produce
    return stem if w[:1].islower() else stem.capitalize()


def _needs_an(word):
    """'an' before vowel SOUNDS: an hour, an MRI, an SVM; but a university, a useful, a one-off."""
    if _A_EXC.match(word):
        return True                                       # silent h / letters said with a vowel sound
    if _AN_EXC.match(word):
        return False                                      # vowel letter said with a consonant sound
    return word[0].lower() in "aeiou"


def _article(art, word):
    if word.lower() in ("other", "number", "lot", "few", "little") and art.lower() == "a":
        return f"{art} {word}"
    want_an = _needs_an(word)
    cap = art[0].isupper()
    new = ("An" if cap else "an") if want_an else ("A" if cap else "a")
    return f"{new} {word}"


# leftovers of chatbot output: labels, markdown, assistant phrases
_ARTIFACT_LEAD = re.compile(
    r"^\s*(?:(?:sure|certainly|of course|absolutely)[,!.]\s*|here(?:'s| is| are) (?:the |a |your )?"
    r"(?:revised|rewritten|corrected|paraphrased|improved|updated)[^:]{0,40}:\s*|"
    r"(?:revised|rewritten|corrected|paraphrased|improved|final) (?:sentence|version|text)\s*:\s*|"
    r"(?:note|output|answer)\s*:\s*|as an ai(?: language model)?,?\s*)", re.I)
_ARTIFACT_ANY = [
    (re.compile(r"\*\*([^*]+)\*\*"), r"\1"),                       # **bold**
    (re.compile(r"(?<!\w)__([^_]+)__(?!\w)"), r"\1"),                  # __bold__
    (re.compile(r"`([^`]+)`"), r"\1"),                                # `code`
    (re.compile(r"^\s*#{1,6}\s+"), ""),                               # ### heading marks
    (re.compile(r"^\s*[-*•]\s+(?=[A-Z])"), ""),                        # stray bullet
    (re.compile(r"\s*\((?:revised|rewritten|paraphrased|corrected)\)\s*$", re.I), ""),
    (re.compile(r"\s*\[(?:citation needed|insert[^\]]*|source)\]", re.I), ""),
]


def clean_artifacts(s):
    """Remove chatbot leftovers from a sentence (labels, markdown, assistant phrases)."""
    if not s:
        return s
    t, prev = s, None
    while t != prev:                                     # "Certainly! Here is the revised sentence: ..."
        prev, t = t, _ARTIFACT_LEAD.sub("", t, count=1)
    for pat, rep in _ARTIFACT_ANY:
        t = pat.sub(rep, t)
    t = t.strip()
    if len(t) > 2 and t[0] in "\"“" and t[-1] in "\"”" and t.count('"') + t.count("“") + t.count("”") == 2:
        t = t[1:-1].strip()                                 # whole sentence wrapped in quotes
    if t and t[0].islower() and s[:1].isupper():
        t = t[0].upper() + t[1:]
    return t or s


def artifact_only(s):
    """A 'sentence' that is nothing but a chatbot leftover ("Certainly!", "Here is the revised text:")."""
    t = (s or "").strip()
    return bool(t) and len(t.split()) <= 8 and not _ARTIFACT_LEAD.sub("", t).strip(" .!:")


def has_defect(s):
    """Doubled word / repeated phrase / chatbot leftover in a sentence."""
    if not s:
        return False
    return bool(re.search(r"\b(\w{2,})\s+\1\b", s, re.I)
                or re.search(r"\b((?:\w+\s+){1,3}\w+)\s+\1\b", s, re.I)
                or clean_artifacts(s) != s.strip())


ABBR = (r"(?:et al|e\.g|i\.e|etc|vs|viz|Fig|Figs|Eq|Eqs|Ref|Refs|No|Nos|Vol|pp|Dr|Mr|Mrs|Ms|Prof|approx|cf|Sec|Ch|"
        r"Art|Arts|para|paras|Inc|Ltd|Corp|Jr|Sr|Hon|edn|eds)")
_INITIALS = re.compile(r"(?P<ini>\b(?:[A-Z]\.){2,})(?=\s+(?P<next>[A-Z][A-Za-z]*))")            # K.S. Puttaswamy
_INITIAL = re.compile(r"(?P<pre>\b(?:[A-Z][a-z]+|and|by)\s)(?P<ini>[A-Z]\.)(?=\s+(?P<next>[A-Z][a-z]+))")  # Justice K. Rao
_VERSUS = re.compile(r"(?P<pre>\b[A-Z][\w.<>]*\s)v\.(?=\s+[A-Z])")                                # Roe v. Wade


def protect_dots(t):
    """Full stops that do not end a sentence are replaced by <DOT>: abbreviations ("et al.", "Fig."), decimals,
    the initials of a name ("K.S. Puttaswamy", "Justice K. Rao") and the "v." of a case name. Initials followed
    by an ordinary sentence opener ("... in the U.S. The economy") do end the sentence."""
    t = re.sub(rf"\b({ABBR})\.", lambda m: m.group(1).replace(".", "<DOT>") + "<DOT>", t)
    t = re.sub(r"(\d)\.(\d)", r"\1<DOT>\2", t)

    def name(m):
        if m.group("next").lower() in _SENT_OPENERS:
            return m.group(0)
        return m.groupdict().get("pre", "") + m.group("ini").replace(".", "<DOT>")
    t = _INITIALS.sub(lambda m: m.group(0) if m.group("next").lower() in _SENT_OPENERS
                      else m.group("ini").replace(".", "<DOT>"), t)
    t = _INITIAL.sub(name, t)
    return _VERSUS.sub(lambda m: m.group("pre") + "v<DOT>", t)
_ABBR_END = re.compile(rf"\b{ABBR}\.\s+$")


def split_sentences(text):
    """Sentences of `text`; never split after 'et al.', 'e.g.', 'Fig.' or inside a decimal number."""
    t = protect_dots((text or "").strip())
    parts = re.split(r"(?<=[.!?])\s+(?=[A-Z0-9\"'(\[])", t)
    return [p.replace("<DOT>", ".").strip() for p in parts if p.strip()]


def _cap_after(m):
    if _ABBR_END.search(m.string[:m.start(2)]):
        return m.group(0)
    return m.group(1) + m.group(2).upper()


_SENT_OPENERS = {"the", "this", "these", "those", "that", "their", "its", "it", "in", "a", "an", "we", "our", "such",
                 "however", "moreover", "furthermore", "thus", "hence", "therefore", "for", "on", "at", "as", "to",
                 "they", "he", "she", "his", "her", "both", "each", "all", "most", "many", "other", "another", "with",
                 "by", "from", "after", "before", "although", "while", "when", "if", "since", "here", "there"}


def _common_lower(word):
    """A capitalised verb after 'et al.' ('Generalised', 'Argue'), not the first word of a new sentence."""
    w = word.lower()
    if w in _SENT_OPENERS:
        return False
    if _zipf is None:
        return bool(re.search(r"(?:ed|es|s|ise|ize|ised|ized)$", w))
    return _zipf(w, "en") >= 1.5


def fix_abbrev_caps(t):
    """'Greshake et al. Generalised' -> 'Greshake et al. generalised'."""
    return re.sub(r"(\bet al\.\s+)([A-Z][a-z]+)\b",
                  lambda m: m.group(1) + (m.group(2).lower() if _common_lower(m.group(2)) else m.group(2)), t)


def _outside_cites(t, fn):
    """Apply fn to the text between citations only: "(Mishra and Mishra, 2012)" and "(Duran Duran, 1981)" are two
    authors and a name, not doubled words."""
    out, pos = [], 0
    for m in CITE.finditer(t):
        out += [fn(t[pos:m.start()]), m.group(0)]
        pos = m.end()
    return "".join(out) + fn(t[pos:])


def basic_grammar(s):
    """Safe rule-based clean-up of paraphrase output (applied before and after the Groq grammar check)."""
    if not s or not s.strip():
        return s
    t = clean_artifacts(re.sub(r"\s+", " ", s).strip())
    t = _outside_cites(t, lambda x: re.sub(r"\b((?:[\w\-]+\s+){1,3}[\w\-]+)\s+\1\b", r"\1", x, flags=re.I))   # repeated phrase
    t = re.sub(r"\s+([,.;:!?%)\]])", r"\1", t)            # space before punctuation
    t = re.sub(r"([(\[])\s+", r"\1", t)
    t = re.sub(r",\s*,+", ",", t)
    t = re.sub(r"\b(and|or|but|the|of|to|in)\s*,\s*\1\b", r"\1", t, flags=re.I)    # "and, and" -> "and"
    t = re.sub(rf"^({_SENT_ADV})\s+(?=[a-z])", r"\1, ", t)                     # "However the" -> "However, the"
    t = re.sub(r"([.!?])\1+", r"\1", t)
    t = _outside_cites(t, lambda x: re.sub(r"\b(\w{2,})\s+\1\b", r"\1", x, flags=re.I))   # "the the", "data data"
    t = _outside_cites(t, lambda x: re.sub(r"\b([a-z]{3,})\s+(and|or)\s+\1\b", r"\1", x))  # "informed and informed"
    t = re.sub(rf"\b({_MODALS})\s+(\w+ing)\b",
               lambda m: f"{m.group(1)} {_base_form(m.group(2)) or m.group(2)}"
               if m.group(1).lower() != "to" or _base_form(m.group(2)) else m.group(0), t, flags=re.I)
    t = re.sub(r"\b([Aa]n?)\s+([A-Za-z][\w\-]*)", lambda m: _article(m.group(1), m.group(2)), t)
    t = re.sub(r"\s*'{2,}\s*", " ", t)                    # stray quote marks left by the paraphraser
    t = re.sub(r"\s+([,.;:!?%)\]])", r"\1", re.sub(r"\s{2,}", " ", t)).strip()
    t = re.sub(r"([.!?]\s+)([a-z])", _cap_after, t)       # capital after full stop (not after 'et al.', 'e.g.')
    t = fix_abbrev_caps(t)
    if t[:1].islower() and not re.match(r"^[a-z]+[A-Z0-9]", t):
        t = t[0].upper() + t[1:]
    if re.search(r"[A-Za-z0-9)\]]$", t) and len(t.split()) >= 4:
        t += "."
    return t


# ---------------------------------------------------------------- strict acceptance (final polish / proofread)
def _cites(t):
    return [re.sub(r"\W+", "", c).lower() for c in CITE.findall(t or "")]


def match_ending(original, new):
    """A fragment that does not end its sentence (the words before a superscript citation, a field or a link in a
    Word paragraph) must not gain a full stop: "2.1 °C" + superscript 12 would become "2.1 °C.12"."""
    if not new or not original:
        return new
    if original[:1].islower() and new[:1].isupper() and not new[1:2].isupper():
        new = new[0].lower() + new[1:]                   # ... nor a capital, when it starts in mid-sentence
    if not re.search(r"[.!?][\"”’)\]]?\s*$", original):
        return re.sub(r"\s*[.!?]+$", "", new)
    return new


def strict_accept(cur, cand, glossary, ref=None):
    """Strict acceptance of an edited sentence for the proofread. Nothing is auto-repaired (no re-inserted terms
    or citations, which is what creates "ethics of care ethics" or duplicated citations): an edit that drops a
    term, a number or a citation, adds a citation, invents something, or introduces a defect is rejected."""
    return strict_check(cur, cand, glossary, ref)[0]


def strict_check(cur, cand, glossary, ref=None):
    """strict_accept with the reason: (accepted sentence, None) or (None, why it was rejected)."""
    if not cand or not cand.strip():
        return None, "empty"
    new = match_ending(cur, basic_grammar(normalize_typography(cur, cand.strip())))
    new = re.sub(r'([.!?])\s*["”“]{2,}', r"\1”", new)                 # .”” -> .”
    new = re.sub(r'["“”]{2,}', "”", new)
    new = restore_commas(cur, new)
    if new.strip() == cur.strip():
        return None, "unchanged"
    ow, nw = len(cur.split()), len(new.split())
    if nw < 0.55 * ow or nw > 1.6 * ow + 4:
        return None, "length"
    why = integrity_issue(cur, new)
    if why:
        return None, why
    if _symbols(new) != _symbols(cur):
        return None, "relation sign changed"
    if _new_acronyms(cur, new):
        return None, "acronym added"
    if _brackets_ok(cur) and not _brackets_ok(new):
        return None, "unbalanced bracket"
    for base in filter(None, (cur, ref)):
        why = invented(base, new)
        if why:
            return None, f"invented {why}"
    why = quality_issue(cur, new) or symbol_issue(cur, new)
    if why:
        return None, why
    nums_c = set(re.findall(r"(?<![\d])\.?\d+(?:\.\d+)?%?", CITE.sub(" ", cur)))
    if not nums_c <= set(re.findall(r"(?<![\d])\.?\d+(?:\.\d+)?%?", new)):
        return None, "number lost"
    cc, nc = _cites(cur), _cites(new)
    if len(nc) > len(cc) or any(not any(difflib.SequenceMatcher(None, c, n).ratio() >= 0.8 for n in nc) for c in cc):
        return None, "citation changed"                   # citations kept (spelling fixes allowed), none added
    lost = lost_terms(cur, new, glossary)
    if lost:
        return None, "term lost: " + ", ".join(lost[:3])
    if has_defect(new) and not has_defect(cur):
        return None, "doubled word / leftover"
    if _near_repeats(new) - _near_repeats(cur):
        return None, "repeated word"                      # "ethics of care ethics", "mapping of SDG 16 mapping"
    import pubrules as _PR                                # publication English rulebook
    v = _PR.violations(cur, new)
    if v:
        return None, "; ".join(v)
    return new, None


def _near_repeats(t):
    """Content words used twice within four words ("ethics of care ethics")."""
    w = [x.lower() for x in re.findall(r"[A-Za-z][A-Za-z\-]+", CITE.sub(" ", t or ""))]
    out = set()
    for i, x in enumerate(w):
        if len(x) >= 5 and x in w[i + 1:i + 5]:
            j = w.index(x, i + 1)
            a, b = (w[i - 1] if i else ""), w[j - 1]
            if a != b and len(a) >= 3 and len(b) >= 3 and a not in _STOP and b not in _STOP:
                continue                                  # contrast, not a repeat: "female travellers than male travellers"
            out.add(x)
    return out