smart.py

"""
Smart Humanize = detector + humanizer working together.

Loop (up to `passes` times):
  1. score every sentence with the RoBERTa AI detector
  2. stop if the AI % is at/below the target (checked from pass 2 on, so every run rewrites at least once)
  3. paraphrase ONLY the sentences the detector flags (P(AI) >= moderate threshold),
     with slightly bolder variation on each later pass
  4. write the new sentences back in place (Word runs outside the sentence keep their formatting)
Then score once more and report before/after.
"""
import io
import os
import re
import time

import gradio as gr
from docx.oxml.ns import qn

import detector as D
import textparse as TP
import docx_report as DX
import humanizer as H
import groqai as G
import report_flags as R
import restructure as RS
import adaptive as AD
import meaning as MEANING
import profile_settings as PS
import fluency as FL

W_T = qn("w:t")
_XML_SPACE = "{http://www.w3.org/XML/1998/namespace}space"
_UNSAFE_ANCESTORS = {qn("w:fldSimple"), qn("w:hyperlink"), qn("w:ins"), qn("w:smartTag"), qn("w:sdtContent")}

from textparse import guard as _guard  # noqa: E402  (shared with the Humanizer)


def _rewrite(sentences, temperature, mode, rep, lo, hi):
    if not sentences:
        return [], ""
    out, note = H.paraphrase_paragraphs(sentences, temperature, rep, lo=lo, hi=hi, mode=mode)
    return [_guard(s, n) for s, n in zip(sentences, out)], note


# ---------------------------------------------------------------- docx: in-place span replacement
_MATH_TAGS = {"{http://schemas.openxmlformats.org/officeDocument/2006/math}oMath",
              "{http://schemas.openxmlformats.org/officeDocument/2006/math}oMathPara"}


def math_inside(p_el, a, b):
    """True if an equation sits inside the sentence at [a, b) (such sentences are never rewritten)."""
    return any(r is None and a <= s < b for r, s, _, _ in _run_map(p_el))


def _run_map(p_el):
    """[(run, start, end, safe)] over DX.para_text(p_el)."""
    items, pos, depth = [], 0, 0
    runs = DX.para_runs(p_el)
    keep = set(runs)
    seq = [el for el in p_el.iter() if el in keep or el.tag in _MATH_TAGS]
    for r in seq:
        if r.tag in _MATH_TAGS:                           # an equation: a barrier no rewrite may cross
            if not items or items[-1][0] is not None or items[-1][1] != pos:
                items.append((None, pos, pos, False))
            continue
        for fc in r.iter(qn("w:fldChar")):
            t = fc.get(qn("w:fldCharType"))
            depth += 1 if t == "begin" else -1 if t == "end" else 0
        t = DX.run_text(r)
        s, e = pos, pos + len(t)
        pos = e
        safe = DX._simple(r) and depth == 0
        if safe:
            a = r.getparent()
            while a is not None and a.tag != qn("w:p"):
                if a.tag in _UNSAFE_ANCESTORS:
                    safe = False
                    break
                a = a.getparent()
        rpr = r.find(qn("w:rPr"))
        if safe and rpr is not None and rpr.find(qn("w:vertAlign")) is not None:
            safe = False                                  # superscript citations, formulas
        items.append((r, s, e, safe))
    return items


_FMT_TAGS = tuple(qn("w:" + t) for t in ("b", "i", "u", "strike", "dstrike", "caps", "smallCaps", "color", "highlight",
                                          "shd", "vertAlign", "rStyle", "sz", "rFonts", "vanish"))


def _fmt(r):
    """Visible formatting of a run (bold, italic, underline, colour, font, size, style ...)."""
    rpr = r.find(qn("w:rPr"))
    if rpr is None:
        return ()
    return tuple(sorted((c.tag, tuple(sorted(c.attrib.items()))) for c in rpr if c.tag in _FMT_TAGS))


def _replace_keeping_format(hit, a, b, new):
    """Replace [a, b) when the sentence mixes formats (a bold term, an italic model name): a formatted run whose
    text is still in the new sentence word for word keeps its run and its formatting; the text between such runs
    goes into the plain runs between them. Returns False (nothing changed) when the new sentence cannot be laid
    out that way, and the caller then writes it in the sentence's first run."""
    segs = []                                            # [run, w:t, text before the span, text in it, text after]
    for r, s0, e0, _ in hit:
        te = r.find(W_T)
        t = te.text or ""
        lo, up = max(0, a - s0), min(len(t), b - s0)
        segs.append((r, te, t[:lo], t[lo:up], t[up:]))
    weight = {}
    for r, _, _, seg, _ in segs:
        weight[_fmt(r)] = weight.get(_fmt(r), 0) + len(seg)
    base = max(weight, key=weight.get)                   # the sentence's own format: the one most of it is in
    anchors, pos = {}, 0
    for i, (r, _, _, seg, _) in enumerate(segs):
        core = seg.strip()
        if _fmt(r) == base or len(core) < 2:
            continue
        j = new.find(core, pos)
        while j >= 0 and ((j > 0 and new[j - 1].isalnum() and core[0].isalnum())
                          or (j + len(core) < len(new) and new[j + len(core)].isalnum() and core[-1].isalnum())):
            j = new.find(core, j + 1)                    # inside another word: not the same term
        if j >= 0:
            anchors[i] = (j, j + len(core))
            pos = j + len(core)
    if not anchors:
        return False
    out, prev_i, prev_end = [""] * len(segs), -1, 0
    for i in sorted(anchors) + [len(segs)]:
        gap = new[prev_end:anchors[i][0]] if i < len(segs) else new[prev_end:]
        zone = [k for k in range(prev_i + 1, i) if _fmt(segs[k][0]) == base]
        lead = ""
        if gap:
            if zone:
                out[zone[0]] = gap
            elif gap.strip():
                return False                             # words with no plain run to hold them
            elif i < len(segs):
                lead = gap                               # only a space: it stays next to the formatted term
            else:
                out[prev_i] += gap
        if i < len(segs):
            out[i] = lead + new[anchors[i][0]:anchors[i][1]]
            prev_i, prev_end = i, anchors[i][1]
    for i, (r, te, before, _, after) in enumerate(segs):
        te.text = before + out[i] + after
        te.set(_XML_SPACE, "preserve")
        if i > 0 and not te.text and r.getparent() is not None:
            r.getparent().remove(r)
    return True


def replace_span(p_el, a, b, new):
    """Replace characters [a, b) of the paragraph text. Returns False (and leaves the
    paragraph untouched) if the span crosses fields, hyperlinks, sub/superscripts or
    non-text runs."""
    items = _run_map(p_el)
    if any(r is None and a <= s < b for r, s, _, _ in items):
        return False                                     # an equation inside the span
    hit = [(r, s, e, safe) for r, s, e, safe in items if r is not None and e > a and s < b and e > s]
    if not hit or not all(x[3] for x in hit):
        return False
    if len({_fmt(x[0]) for x in hit}) > 1 and _replace_keeping_format(hit, a, b, new):
        return True
    for i, (r, s, e, _) in enumerate(hit):
        te = r.find(W_T)
        t = te.text or ""
        before = t[:max(0, a - s)]
        after = t[b - s:] if b < e else ""
        te.text = (before + new + after) if i == 0 else after
        te.set(_XML_SPACE, "preserve")
        if i > 0 and not te.text and r.getparent() is not None:
            r.getparent().remove(r)                      # drop runs emptied by the merge
    return True


_REASONS = [(qn("w:hyperlink"), "link"), (qn("w:fldSimple"), "citation field"), (qn("w:ins"), "tracked change"),
            (qn("w:sdtContent"), "content control"), (qn("w:smartTag"), "smart tag")]


def lock_reason(p_el, a, b):
    """Why the sentence at [a, b) cannot be edited safely (shown to the customer)."""
    reasons = []
    items = _run_map(p_el)
    if any(r is None and a <= s < b for r, s, _, _ in items):
        reasons.append("equation")
    for r, s, e, safe in items:
        if r is None or safe or not (e > a and s < b and e > s):
            continue
        anc, why = r.getparent(), None
        while anc is not None and anc.tag != qn("w:p") and why is None:
            why = dict(_REASONS).get(anc.tag)
            anc = anc.getparent()
        if why is None:
            rpr = r.find(qn("w:rPr"))
            if rpr is not None and rpr.find(qn("w:vertAlign")) is not None:
                why = "superscript / subscript"
            elif r.find(qn("w:fldChar")) is not None or r.find(qn("w:instrText")) is not None:
                why = "citation field"
            elif any(c.tag in (qn("w:tab"), qn("w:br"), qn("w:cr")) for c in r):
                why = "tab or line break inside the sentence"
            elif r.find(qn("w:t")) is None:
                why = "special object (symbol, footnote mark or image)"
            else:
                why = "citation field"                 # text inside a field (e.g. Zotero / EndNote / Mendeley)
        if why not in reasons:
            reasons.append(why)
    return ", ".join(reasons) or "special formatting"


def safe_subspan(p_el, a, b, min_words=5):
    """Largest part of [a, b) that lies in plain text runs (so a sentence ending in a
    superscript citation or a field can still have its prose rewritten)."""
    items = _run_map(p_el)
    if any(r is None and a <= s < b for r, s, _, _ in items):
        return None                                      # sentences with an equation are never rewritten
    runs = [(s, e, safe) for r, s, e, safe in items if r is not None and e > a and s < b and e > s]
    if runs and all(x[2] for x in runs):
        return a, b
    best, cur = None, None
    for s, e, safe in runs + [(b, b, False)]:
        if safe:
            cur = (max(a, s) if cur is None else cur[0], min(b, e))
        elif cur is not None:
            if best is None or cur[1] - cur[0] > best[1] - best[0]:
                best = cur
            cur = None
    if best is None:
        return None
    raw = DX.para_text(p_el)
    x, y = best
    while x < y and raw[x].isspace():
        x += 1
    while y > x and raw[y - 1].isspace():
        y -= 1
    frag = raw[x:y]
    if len(frag.split()) < min_words or (y - x) < 0.5 * (b - a):
        return None
    return x, y


# ---------------------------------------------------------------- scoring helpers
def _score_doc(doc):
    blocks = D.docx_blocks(doc)
    if not any(b.kind == "para" for b in blocks):
        raise gr.Error("No readable sentences were found in this document. If the text is inside images, "
                       "text boxes or a scanned page, paste it into the text tab instead.")
    D.analyse(blocks)
    return blocks


def _score_paras(paras):
    blocks = D.classify([D.Block(D._norm(p)) for p in paras], D.PASTE_MIN_WORDS)
    if not any(b.kind == "para" for b in blocks):
        raise gr.Error("No qualifying prose found — paste full sentences (headings and short "
                       "fragments are not analysed).")
    D.analyse(blocks)
    return blocks


def detector_pct_text(text, hi=0.80, mod=0.50):
    try:
        paras = [p for p, _ in TP.parse_paste(text)]
        return D.summarise(_score_paras(paras), hi, mod)["pct"]
    except Exception:
        return None


def detector_pct_docx(path, hi=0.80, mod=0.50):
    try:
        return D.summarise(_score_doc(DX.open_clean(path)), hi, mod)["pct"]
    except Exception:
        return None


# ---------------------------------------------------------------- pipelines
def _log_line(k, pct, n_flag=None, n_done=None, n_locked=0):
    s = f"Pass {k}: AI score {pct}%"
    if n_flag is not None:
        s += f" → rewrote {n_done} of {n_flag} " + ("sentences (100% AI mode)" if _full_ai() else
                                                     f"sentences (vocabulary {_vocab():.0f}%)" if _vocab() else "flagged sentences")
        if n_locked:
            s += f" ({n_locked} skipped: citation fields / links / sub-superscripts)"
    return s


def _rewrite_retry(sentences, temperature, mode, rep, lo, hi):
    """Paraphrase; sentences that come back unchanged or rejected get one bolder retry.
    100% AI mode: the humanizer already re-runs weak sentences itself (1 extra round per pass here, since
    every Smart pass rewrites again), so the separate retry is skipped — it only repeated the same work."""
    if _full_ai():
        tok = H.EXTRA_ROUNDS.set(int(__import__("os").getenv("SMART_AI_ROUNDS", "1")))
        try:
            return _rewrite(sentences, temperature, mode, rep, lo, hi)
        finally:
            H.EXTRA_ROUNDS.reset(tok)
    new, note = _rewrite(sentences, temperature, mode, rep, lo, hi)
    redo = [i for i, (s, n) in enumerate(zip(sentences, new)) if not n or n.strip() == s.strip()]
    if redo:
        again, _ = _rewrite([sentences[i] for i in redo], min(1.5, temperature + 0.3), mode, rep, hi, hi)
        for i, n in zip(redo, again):
            if n and n.strip() != sentences[i].strip():
                new[i] = n
    return new, note


# setting groups of one pass rewritten at the same time (SMART_PARALLEL_GROUPS=1 = one after another)
# OFF by default: the checking models' fast tokenizers are not thread-safe ("Already borrowed" when two groups
# tokenize with the same model at once, 2026-10-19 on the live Space). Only with per-model locks could this be on.
PARALLEL_GROUPS = max(1, int(os.getenv("SMART_PARALLEL_GROUPS", "1")))


def _adaptive(fn, sentences, temperature, mode, rep, lo, hi):
    """Self-adaptive pass: every sentence gets the setting its own history in this document calls for
    (adaptive.settings): bolder (higher temperature + detector-guided decoding) when its AI score did not fall last
    pass, calmer when its rewrites keep failing the checks, left as written after ADAPT_MAX_FAILS failures.
    Sentences with the same setting are rewritten together."""
    anchors = [H.anchor_of(t) or t for t in sentences]
    sets = [AD.settings(a) for a in anchors]
    groups = {}
    for i, (dt, bold, skip) in enumerate(sets):
        if not skip:
            groups.setdefault((round(dt, 2), bold), []).append(i)
    new, note = [None] * len(sentences), ""
    total = max(1, sum(len(v) for v in groups.values()))
    specs, pos = [], 0
    for (dt, bold), idx in sorted(groups.items()):
        specs.append((dt, bold, idx, lo + (hi - lo) * pos / total, lo + (hi - lo) * (pos + len(idx)) / total))
        pos += len(idx)

    def run(spec, rep_):
        dt, bold, idx, a_, b_ = spec
        tok = H.FORCE_ADV.set(bool(bold))
        try:
            return fn([sentences[i] for i in idx], min(1.5, max(0.3, temperature + dt)), mode, rep_, a_, b_)
        finally:
            H.FORCE_ADV.reset(tok)
    if PARALLEL_GROUPS > 1 and len(specs) > 1:
        # The groups are independent (different sentences, own settings). One after another, each paid the fixed
        # cost of a rewrite call (model, checks, several Groq round trips) in turn; together they overlap. Each runs
        # in a copy of this job's context (its account, glossary, settings); the bar only moves forward.
        import concurrent.futures as _cf
        import contextvars as _cv
        bar = {"at": 0.0}

        def forward(frac, desc=""):
            if frac >= bar["at"]:
                bar["at"] = frac
                rep(frac, desc)
        with _cf.ThreadPoolExecutor(max_workers=min(PARALLEL_GROUPS, len(specs))) as pool:
            outs = list(pool.map(lambda sp: _cv.copy_context().run(run, sp, forward), specs))
    else:
        outs = [run(sp, rep) for sp in specs]
    for (_, _, idx, _, _), (out, n_) in zip(specs, outs):
        note = note or n_
        for i, o in zip(idx, out):
            new[i] = o
    for a, t, n in zip(anchors, sentences, new):
        AD.outcome(a, bool(n and n.strip() != t.strip()))
    skipped = sum(1 for x in sets if x[2])
    if skipped:
        note += f" 🧠 {skipped} sentence(s) kept as written: their rewrites failed the checks in {AD.MAX_FAILS} passes."
    return new, note


def _full_ai():
    return bool(H.FULL_AI.get())


def _vocab():
    """Vocabulary change (%) slider value (0 = off)."""
    return float(H.VOCAB_TARGET.get() or 0)


def _all_sentences():
    """Rewrite every sentence, not only flagged ones: 100% AI box ticked, or a vocabulary level is set."""
    return _full_ai() or _vocab() > 0


def _target_change():
    return H.change_target() if _full_ai() else _vocab()


REPORT_VOCAB = PS.get("report_vocab")          # AI report: at least this much new wording is aimed for in flagged sentences
REPORT_VOCAB_SIM = PS.get("report_vocab_sim")      # similarity report: copied wording is exactly what is matched, so aim higher


REPORT_EXTRA = PS.get("report_extra")
# Automatic passes: Smart Humanize keeps running passes (each one rewriting what is still flagged, starting from the
# previous pass's output) until the AI score reaches the target (0% by default), until STALL_PASSES passes in a row
# bring no improvement, or until TIME_BUDGET seconds have passed. The plan sets the most passes allowed.
STALL_PASSES = int(os.getenv("SMART_STALL_PASSES", "2"))
TIME_BUDGET = float(os.getenv("SMART_TIME_BUDGET", "1500"))   # follow-up rounds for weak sentences
# When a pass brings no improvement, the still-flagged sentences get the deeper rewrite (humanizer.BOOST) for the
# remaining passes, so the customer never has to choose a mode. SMART_AUTO_DEEP=0 = off.
AUTO_DEEP = os.getenv("SMART_AUTO_DEEP", "1") != "0"


def _report_rewrite(sentences, temperature, mode, rep, lo, hi):
    """Report-flagged sentences get the strongest rewrite:
      1. restructured first (clause order, splits, openers; rule-based),
      2. Parrot with the 100% AI settings (more versions, plain-wording options, more rounds, >= 60% new-wording
         target),
      3. follow-up rounds for every sentence that is still close to the original (less than half new wording) or
         had no safe rewrite: Parrot rewrites its own latest version with more variation each round, so the
         changes stack up. Per sentence, the version with the MOST new wording that still passes the check against
         the ORIGINAL sentence (terms, numbers, citations, length) is kept."""
    gl = TP.narrow(TP.GLOSSARY.get())                  # protect real technical terms, not everyday words
    guard = lambda o, n: TP.guard(o, TP.restore_terms(o, n, gl), gl) if n else None
    t_gl, t_nw = TP.GLOSSARY.set(gl), TP.NARROW.set(True)
    ov = lambda o, n: H.similarity_to_source(o, n) / 100.0
    shaped = [RS.restructure(s, guard, H.similarity_to_source) for s in sentences]
    t1 = H.FULL_AI.set(True)
    rpt = R.ACTIVE.get()
    level = REPORT_VOCAB_SIM if (rpt is not None and rpt.kind == "Similarity") else REPORT_VOCAB
    t2 = H.VOCAB_TARGET.set(max(float(H.VOCAB_TARGET.get() or 0), level))
    try:
        new, note = _rewrite(shaped, temperature, mode, rep, lo, hi)
        new = [guard(o, n) for o, n in zip(sentences, new)]
        new = [n if n and m and f else None for n, m, f in
               zip(new, MEANING.keeps_meaning([(o, n or o) for o, n in zip(sentences, new)]),
                   FL.acceptable([(o, n or o) for o, n in zip(sentences, new)]))]
        best = [n if n else sh for n, sh in zip(new, shaped)]   # failed a check: restructured (or original)
        cur = list(best)                                  # chain: next round rewrites the latest version
        for r in range(1, REPORT_EXTRA + 1):
            weak = [i for i, (o, b) in enumerate(zip(sentences, best))
                    if b.strip() == o.strip() or ov(o, b) >= R.KEEP_OVERLAP]
            if not weak:
                break
            if rep is not None:
                rep(hi, desc=f"Extra rewrite round {r}: {len(weak)} sentence(s) still close to the original")
            # always from the ORIGINAL sentence: rewriting a rewrite is what breaks grammar and technical meaning
            again, _ = _rewrite([sentences[i] for i in weak], min(1.5, float(temperature) + 0.15 * r), mode, rep, hi, hi)
            again = [guard(sentences[i], n) for i, n in zip(weak, again)]
            same = [a_ and b_ for a_, b_ in zip(
                MEANING.keeps_meaning([(sentences[i], n or sentences[i]) for i, n in zip(weak, again)]),
                FL.acceptable([(sentences[i], n or sentences[i]) for i, n in zip(weak, again)]))]
            for i, ok, m in zip(weak, again, same):
                if not ok or not m:                       # chained rounds drift: compare with the ORIGINAL
                    continue
                cur[i] = ok
                if best[i].strip() == sentences[i].strip() or ov(sentences[i], ok) < ov(sentences[i], best[i]):
                    best[i] = ok
        # final meaning check against the ORIGINAL (the chained rounds can drift): Groq rejects a changed meaning
        idx = [i for i, (o, b) in enumerate(zip(sentences, best)) if b.strip() != o.strip()]
        if idx and G.available():
            try:
                res = G.review([(sentences[i], best[i]) for i in idx], gl, rep, hi, hi)
                dropped = 0
                for i, r in zip(idx, res):
                    o = sentences[i]
                    if r is None:                         # meaning changed: fall back to the safe restructured form
                        best[i] = shaped[i] if shaped[i] != o else o
                        dropped += 1
                    elif r:
                        ok = guard(o, r)                  # Groq's minimal repair, if it keeps the new wording
                        if ok and ov(o, ok) <= ov(o, best[i]) + 0.15 and MEANING.ok(o, ok):
                            best[i] = ok
                if dropped:
                    print(f"[report] {dropped} rewrite(s) rejected by the meaning check", flush=True)
            except G.Unavailable as err:
                print(f"[report] meaning check skipped: {err}", flush=True)
        # grammar now, so the results table shows exactly the wording that goes into the document
        idx = [i for i, (o, b) in enumerate(zip(sentences, best)) if b.strip() != o.strip()]
        if idx and H.GRAMMAR:
            fixed, _, _ = H.grammar_fix_sentences([best[i] for i in idx], gl, rep, hi, hi,
                                                  orig_sent=[sentences[i] for i in idx])
            for i, f in zip(idx, fixed):
                ok = guard(sentences[i], f)
                if ok and MEANING.ok(sentences[i], ok):
                    best[i] = ok
    finally:
        H.FULL_AI.reset(t1)
        H.VOCAB_TARGET.reset(t2)
        TP.GLOSSARY.reset(t_gl)
        TP.NARROW.reset(t_nw)
    return best, note


def _report_summary(rpt, rows):
    """Report-guided run -> dict for the result panel: report score, estimate, and one row per flagged sentence
    (report sentence, highlight colour, final sentence, status)."""
    if rpt is None or rows is None:
        return None
    for r in rows:                                     # still likely flagged: kept, or less than half the wording changed
        lim = 40 if rpt.kind == "Similarity" else R.KEEP_OVERLAP * 100
        r["likely"] = r["status"] != "rephrased" or H.similarity_to_source(r["orig"], r["new"]) >= lim
    pairs = [(r["orig"], r["new"]) for r in rows]
    return dict(score=rpt.score, kind=rpt.kind, tool=rpt.tool, ocr=rpt.used_ocr,
                est=R.estimate_after(rpt.score, pairs, rpt.kind), rows=rows,
                flagged=len(rows), rephrased=sum(1 for r in rows if r["status"] == "rephrased"),
                likely=sum(1 for r in rows if r["likely"]))


def _not_lower(pct, prev):
    """A pass is undone only when it made the AI score worse. (Undoing passes that kept the same score threw away
    sentence-level progress, so later passes started from the original again and flagged text stayed unchanged.)"""
    return pct > prev


def _done_early(k, pct, target, chg):
    if R.ACTIVE.get() is not None:                     # report-guided: always rewrite what the report flagged
        return False
    return _done_early_inner(k, pct, target, chg)


def _done_early_inner(k, pct, target, chg):
    """Normal mode: stop when the AI score is 0 or (from pass 2) at/below the target.
    100% AI mode: at least one rewrite, and the new-wording target must also be reached."""
    if not _all_sentences():
        return pct == 0 or (k > 1 and pct <= target)
    return k > 1 and pct <= target and chg is not None and chg >= _target_change()


def _better(cand, best, k):
    """cand/best = (pct, change, payload). Normal mode: lowest AI score (ties -> the rewritten version).
    100% AI mode: the original is never returned; versions reaching the new-wording target come first,
    then the lowest AI score; below the target, the version with the most new wording wins."""
    if best is None:
        return True
    if R.ACTIVE.get() is not None:                     # report-guided: the rewritten version is the result
        return k > 1
    if not _all_sentences():
        return cand[0] < best[0] or (k > 1 and cand[0] == best[0])
    if best[1] is None:                                # best is still the untouched original
        return k > 1
    t = _target_change()
    ca, ba = cand[1] >= t, best[1] >= t
    if ca != ba:
        return ca
    if ca:                                             # both reach the level: lowest AI score, then more change
        return (cand[0], -cand[1]) <= (best[0], -best[1])
    # neither reaches the level: never trade AI score for wording; within 2 points, more new wording wins
    if cand[0] < best[0] - 2:
        return True
    if cand[0] > best[0] + 2:
        return False
    return cand[1] >= best[1]


def _grammar_docx(doc, rep, orig_path=None):
    """Final Groq grammar check of every sentence in the document's prose, written back in place (only plain
    text runs are touched, so citation fields, links and sub/superscripts keep their formatting)."""
    jobs, op, os_ = [], [], []
    blocks = list(D.docx_blocks(doc))
    orig = None
    if orig_path:                                      # the author's original paragraphs, block by block
        try:
            ob = list(D.docx_blocks(DX.open_clean(orig_path)))
            orig = ob if len(ob) == len(blocks) else None
        except Exception as err:
            print(f"[smart] original paragraphs not available for the final polish: {err}", flush=True)
    for bi, b in enumerate(blocks):
        if b.kind != "para" or b.el is None or not b.raw:
            continue
        raws = D.split_sentences(b.raw)
        o_raw = orig[bi].raw if orig is not None and orig[bi].kind == "para" else None
        ctx = H.originals_for(raws, o_raw)
        for sp, (cp, cs) in zip(DX.sentence_char_spans(b.raw, raws), ctx):
            if not sp:
                continue
            sub = safe_subspan(b.el, *sp)
            if sub:
                jobs.append((b, sub, b.raw[sub[0]:sub[1]]))
                op.append(cp)
                os_.append(cs if sub == tuple(sp) or list(sub) == list(sp) else None)
    if not jobs:
        return 0, ""
    fixed, n, note = H.grammar_fix_sentences([j[2] for j in jobs], None, rep, 0.9, 0.95,
                                             groups=[id(j[0]) for j in jobs], orig_par=op, orig_sent=os_)
    done = 0
    for (b, (a, e), t), f in sorted(zip(jobs, fixed), key=lambda x: (id(x[0][0]), -x[0][1][0])):
        if f != t and replace_span(b.el, a, e, f):
            done += 1
    if done:
        m = re.search(r"improved \((.*?)\)\.", note or "")
        return done, (f" 📝 Final polish by Groq (clarity, grammar, academic wording, flow): {done} sentence(s) "
                      f"improved" + (f" ({m.group(1)})." if m else "."))
    return 0, note if "unavailable" in note else ""


def _overall_note(orig, new):
    """Whole-text new wording vs the original (the same measure as the 'Vocabulary changed' gauge)."""
    if not _all_sentences():
        return ""
    chg, t = H.text_change(orig, new), _target_change()
    return (f" 🔤 Whole text: {chg:.0f}% new wording vs your original"
            + (f" (level {t:.0f}%: {'reached' if chg >= t else 'not reached — try more passes'})." if t else "."))


def _snapshot(doc):
    buf = io.BytesIO()
    doc.save(buf)
    return buf.getvalue()


def _clean_stem(path):
    stem = re.sub(r"\.docx$", "", re.split(r"[\\/]", path)[-1], flags=re.I)
    return re.sub(r"(_humanized|_paraphrased)+$", "", stem)


def smart_docx(path, target, passes, temperature, mode, hi, mod, rep):
    H.GRAMMAR_DEFER.set(True)
    G.start_account()                                # tokens and cost of this job (logged at the end)
    H.BOOST.set(False)                          # grammar runs once on the final version (below)
    rep(0.03, "Opening Word document")
    doc = DX.open_clean(path)
    H.ANCHOR.set({})                                   # rewritten sentence -> author's original sentence
    log, first, note = [], None, ""
    orig_text = "\n".join(b.text for b in D.docx_blocks(doc) if b.kind == "para")
    TP.GLOSSARY.set(TP.build_glossary([orig_text]))     # the document's key terms stay untouched
    TP.set_document([orig_text])                        # its constructs (defined terms, keywords) never change
    best = None                                        # (pct, docx bytes, scored blocks)
    rpt, rows = R.ACTIVE.get(), None                   # uploaded report: rewrite only the sentences it flags
    if rpt is not None:
        passes = 1
    acc, from_orig = None, True                        # last version whose AI score went DOWN (passes build on it)
    stall, low, t0 = 0, None, time.time()           # automatic passes: stop when no pass improves any more
    AD.start_job()                                     # self-adaptive settings per sentence for this document
    pend = None                                        # (temperature offset, AI % before, jobs, rewritten) of last pass
    step = 0.8 / max(1, passes)
    for k in range(1, int(passes) + 2):                # last iteration = final check only
        base = 0.05 + step * (k - 1)
        rep(min(base, 0.88), f"Pass {k}: running AI detector" if k <= passes else "Final AI check")
        blocks = _score_doc(doc)
        stats = D.summarise(blocks, hi, mod)
        first = first or stats
        if pend is not None:                           # learn how well last pass's setting worked
            AD.record_pass(pend[0], pend[1], stats["pct"], pend[2], pend[3])
            pend = None
        cur = "\n".join(b.text for b in blocks if b.kind == "para")
        chg = H.text_change(orig_text, cur) if k > 1 else None
        cand_ = (stats["pct"], chg, (_snapshot(doc), blocks))
        if _better(cand_, best, k):
            best = cand_
        if k > 1 and acc is not None and rpt is None and _not_lower(stats["pct"], acc[0]):
            # this pass did not lower the AI score: undo it and retry from the better version
            log.append(f"Pass {k - 1} did not lower the AI score ({acc[0]}% → {stats['pct']}%), so it was undone")
            from docx import Document
            doc = Document(io.BytesIO(acc[1]))
            blocks = _score_doc(doc)
            stats = D.summarise(blocks, hi, mod)
            chg, from_orig = acc[2], acc[3]
        else:
            acc, from_orig = (stats["pct"], _snapshot(doc), chg, k == 1), k == 1
        for b_ in blocks:                              # per-sentence learning: did its AI score fall?
            if b_.kind == "para":
                for s_, p_ in zip(b_.sents, b_.scores):
                    AD.observe(H.anchor_of(s_) or s_, p_)
        if k > passes:
            break
        if _done_early(k, stats["pct"], target, chg):
            log.append(_log_line(k, stats["pct"]) + " — target reached")
            break
        if low is not None and stats["pct"] >= low:
            stall += 1
        else:
            stall, low = 0, stats["pct"]
        if stall >= 1 and AUTO_DEEP and not H.BOOST.get() and not _all_sentences() and rpt is None and k < passes:
            H.BOOST.set(True)                          # the normal rewrite is not enough for this text
            stall = 0
            log.append(f"Pass {k - 1} did not lower the AI score: switching to the deeper rewrite automatically")
        if stall >= STALL_PASSES:
            log.append(_log_line(k, stats["pct"]) + f" — stopped: {STALL_PASSES} pass(es) without further improvement")
            break
        if time.time() - t0 > TIME_BUDGET or H.past_deadline():
            log.append(_log_line(k, stats["pct"]) + " — stopped: time limit for one document reached")
            break
        full = _all_sentences()                        # 100% AI box / vocabulary level: every sentence
        jobs, locked = [], 0                           # (block, span, sentence)
        jinfo, linfo = [], []                          # report rows for rewritten / locked sentences
        for b in blocks:
            if b.kind != "para" or b.el is None:
                continue
            for sp, s, p in zip(b.cspans, b.sents, b.scores):
                if rpt is not None:
                    hit, rsent, colour = rpt.match(s)
                    pick = hit
                else:
                    pick = full or p >= mod
                if sp and pick and len(s.split()) >= 5:
                    sub = safe_subspan(b.el, *sp)
                    if sub:
                        jobs.append((b, sub, b.raw[sub[0]:sub[1]]))
                        if rpt is not None:
                            jinfo.append((rsent, colour))
                    else:
                        locked += 1
                        if rpt is not None:
                            linfo.append(dict(report=rsent, colour=colour, orig=s, new=s, status="locked",
                                              reason=lock_reason(b.el, *sp)))
        ctx = {}                                       # sentence -> neighbours, for in-context scoring
        for b in blocks:
            if b.kind == "para":
                for i, t in enumerate(b.sents):
                    ctx[t.strip()] = (b.sents[i - 1] if i else "", b.sents[i + 1] if i + 1 < len(b.sents) else "")
        for b_, sub_, t_ in jobs:
            full_ = b_.raw[sub_[0]:sub_[1]].strip()
            if full_ not in ctx:
                hit_ = next((v for k, v in ctx.items() if full_ in k or k in full_), ("", ""))
                ctx[full_] = hit_
        H.SENT_CONTEXT.set(ctx)
        if rpt is not None and not jobs and not locked:
            raise gr.Error("None of the sentences in your document match the sentences flagged in the report. "
                           "Check that the report belongs to this document.")
        if not jobs:
            if rpt is not None:
                rows = linfo
            log.append(_log_line(k, stats["pct"]) + f" — no rewritable sentences left ({locked} locked)")
            break
        t_off = AD.pick_temp_offset()                  # learned across documents (Thompson sampling)
        temp = min(1.5, max(0.3, float(temperature) + 0.15 * (k - 1) + t_off))
        # pass 1 pushes each sentence to the new-wording level (measured against the ORIGINAL sentence);
        # later passes only lower the AI score: their input is already rewritten, so a per-sentence level there
        # would compare against the previous pass and keep churning. The whole-text level is checked below.
        vtok = H.VOCAB_TARGET.set(0.0) if not from_orig else None
        try:
            if rpt is not None:
                new, note = _adaptive(_report_rewrite, [j[2] for j in jobs], temp, mode, rep, base + 0.1 * step,
                                      base + 0.9 * step)
            else:
                new, note = _adaptive(_rewrite_retry, [j[2] for j in jobs], temp, mode, rep, base + 0.1 * step,
                                      base + 0.9 * step)
        finally:
            if vtok is not None:
                H.VOCAB_TARGET.reset(vtok)
        done = 0
        # right-to-left inside each paragraph so earlier offsets stay valid
        for (b, (a, e), t), n in sorted(zip(jobs, new), key=lambda x: (id(x[0][0]), -x[0][1][0])):
            if n and n != b.raw[a:e] and replace_span(b.el, a, e, n):
                H.remember_anchor(t, n)
                done += 1
        log.append(_log_line(k, stats["pct"], len(jobs), done, locked))
        pend = (t_off, stats["pct"], len(jobs), done)
        if rpt is not None:
            rows = [dict(report=ri[0], colour=ri[1], orig=j[2], new=(n if (n and n != j[2]) else j[2]),
                         status="rephrased" if (n and n != j[2]) else "kept")
                    for j, n, ri in zip(jobs, new, jinfo)] + linfo
        if not done:
            break
    AD.save(force=True)
    # keep the lowest-scoring version seen (a pass can occasionally make things worse)
    rep(0.9, "Preparing results")
    from docx import Document
    doc = Document(io.BytesIO(best[2][0]))
    blocks = best[2][1]
    after = D.summarise(blocks, hi, mod)
    if H.GRAMMAR:                                      # final step: Groq grammar check of the finished text
        n, gnote = _grammar_docx(doc, rep, path)
        if n:                                          # always kept: the final text must read correctly
            blocks = _score_doc(doc)
            after = D.summarise(blocks, hi, mod)
        note += gnote
    new_text = "\n".join(b.text for b in blocks if b.kind == "para")
    out = H._save_output(doc, _clean_stem(path), "_humanized")
    note += _overall_note(orig_text, new_text)
    return dict(before=first, after=after, log=log, note=note, blocks=blocks, path=out,
                report=_report_summary(rpt, rows),
                orig=orig_text, new=new_text)


def smart_text(text, target, passes, temperature, mode, hi, mod, rep):
    H.GRAMMAR_DEFER.set(True)
    G.start_account()                                # tokens and cost of this job (logged at the end)
    H.BOOST.set(False)                          # grammar runs once on the final version (below)
    rep(0.03, "Reading pasted text")
    parts = TP.parse_paste(text)
    seps = [sep for _, sep in parts]
    TP.GLOSSARY.set(TP.build_glossary([p for p, _ in parts]))   # the text's key terms stay untouched
    TP.set_document([p for p, _ in parts])
    paras = [p for p, _ in parts]
    orig = TP.join_paste(parts)
    H.ANCHOR.set({})                                   # rewritten sentence -> author's original sentence
    log, first, note, best = [], None, "", None       # best = (pct, paras, blocks)
    rpt, rows = R.ACTIVE.get(), None                   # uploaded report: rewrite only the sentences it flags
    if rpt is not None:
        passes = 1
    acc, from_orig = None, True                        # last version whose AI score went DOWN (passes build on it)
    stall, low, t0 = 0, None, time.time()           # automatic passes: stop when no pass improves any more
    AD.start_job()                                     # self-adaptive settings per sentence for this document
    pend = None                                        # (temperature offset, AI % before, jobs, rewritten) of last pass
    step = 0.8 / max(1, passes)
    for k in range(1, int(passes) + 2):
        base = 0.05 + step * (k - 1)
        rep(min(base, 0.88), f"Pass {k}: running AI detector" if k <= passes else "Final AI check")
        blocks = _score_paras(paras)
        stats = D.summarise(blocks, hi, mod)
        first = first or stats
        if pend is not None:                           # learn how well last pass's setting worked
            AD.record_pass(pend[0], pend[1], stats["pct"], pend[2], pend[3])
            pend = None
        chg = H.text_change(orig, TP.join_paste(list(zip(paras, seps)))) if k > 1 else None
        cand_ = (stats["pct"], chg, (list(paras), blocks))
        if _better(cand_, best, k):
            best = cand_
        if k > 1 and acc is not None and rpt is None and _not_lower(stats["pct"], acc[0]):
            # this pass did not lower the AI score: undo it and retry from the better version
            log.append(f"Pass {k - 1} did not lower the AI score ({acc[0]}% → {stats['pct']}%), so it was undone")
            paras, blocks = list(acc[1]), acc[4]
            stats = D.summarise(blocks, hi, mod)
            chg, from_orig = acc[2], acc[3]
        else:
            acc, from_orig = (stats["pct"], list(paras), chg, k == 1, blocks), k == 1
        for b_ in blocks:                              # per-sentence learning: did its AI score fall?
            if b_.kind == "para":
                for s_, p_ in zip(b_.sents, b_.scores):
                    AD.observe(H.anchor_of(s_) or s_, p_)
        if k > passes:
            break
        if _done_early(k, stats["pct"], target, chg):
            log.append(_log_line(k, stats["pct"]) + " — target reached")
            break
        if low is not None and stats["pct"] >= low:
            stall += 1
        else:
            stall, low = 0, stats["pct"]
        if stall >= 1 and AUTO_DEEP and not H.BOOST.get() and not _all_sentences() and rpt is None and k < passes:
            H.BOOST.set(True)                          # the normal rewrite is not enough for this text
            stall = 0
            log.append(f"Pass {k - 1} did not lower the AI score: switching to the deeper rewrite automatically")
        if stall >= STALL_PASSES:
            log.append(_log_line(k, stats["pct"]) + f" — stopped: {STALL_PASSES} pass(es) without further improvement")
            break
        if time.time() - t0 > TIME_BUDGET or H.past_deadline():
            log.append(_log_line(k, stats["pct"]) + " — stopped: time limit for one document reached")
            break
        full = _all_sentences()                        # 100% AI box / vocabulary level: every sentence
        jinfo = []
        if rpt is not None:
            jobs = []
            for bi, b in enumerate(blocks):
                if b.kind != "para":
                    continue
                for si, s in enumerate(b.sents):
                    hit, rsent, colour = rpt.match(s)
                    if hit and len(s.split()) >= 5:
                        jobs.append((bi, si, s))
                        jinfo.append((rsent, colour))
            if not jobs:
                raise gr.Error("None of your sentences match the sentences flagged in the report. "
                               "Check that the report belongs to this text.")
        else:
            jobs = [(bi, si, s) for bi, b in enumerate(blocks) if b.kind == "para"
                    for si, (s, p) in enumerate(zip(b.sents, b.scores))
                    if (full or p >= mod) and len(s.split()) >= 5]
        H.SENT_CONTEXT.set({b.sents[si].strip(): (b.sents[si - 1] if si else "",
                                                b.sents[si + 1] if si + 1 < len(b.sents) else "")
                             for bi, b in enumerate(blocks) if b.kind == "para" for si in range(len(b.sents))})
        if not jobs:
            log.append(_log_line(k, stats["pct"]) + " — no rewritable sentences left")
            break
        t_off = AD.pick_temp_offset()                  # learned across documents (Thompson sampling)
        temp = min(1.5, max(0.3, float(temperature) + 0.15 * (k - 1) + t_off))
        # pass 1 pushes each sentence to the new-wording level (measured against the ORIGINAL sentence);
        # later passes only lower the AI score: their input is already rewritten, so a per-sentence level there
        # would compare against the previous pass and keep churning. The whole-text level is checked below.
        vtok = H.VOCAB_TARGET.set(0.0) if not from_orig else None
        try:
            if rpt is not None:
                new, note = _adaptive(_report_rewrite, [j[2] for j in jobs], temp, mode, rep, base + 0.1 * step,
                                      base + 0.9 * step)
            else:
                new, note = _adaptive(_rewrite_retry, [j[2] for j in jobs], temp, mode, rep, base + 0.1 * step,
                                      base + 0.9 * step)
        finally:
            if vtok is not None:
                H.VOCAB_TARGET.reset(vtok)
        sents = [list(b.sents) for b in blocks]
        done = 0
        for (bi, si, s), n in zip(jobs, new):
            if n and n != s:
                sents[bi][si] = n
                H.remember_anchor(s, n)
                done += 1
        paras = [" ".join(sents[i]) if b.kind == "para" else b.text for i, b in enumerate(blocks)]
        log.append(_log_line(k, stats["pct"], len(jobs), done))
        pend = (t_off, stats["pct"], len(jobs), done)
        if rpt is not None:
            rows = [dict(report=ri[0], colour=ri[1], orig=j[2], new=(n if (n and n != j[2]) else j[2]),
                         status="rephrased" if (n and n != j[2]) else "kept")
                    for j, n, ri in zip(jobs, new, jinfo)]
        if not done:
            break
    AD.save(force=True)
    rep(0.9, "Preparing results")
    paras, blocks = best[2]
    after = D.summarise(blocks, hi, mod)
    if H.GRAMMAR:                                      # final step: Groq grammar check of the finished text
        idx = [i for i, b in enumerate(blocks) if b.kind == "para" and i < len(paras)]
        o_paras = [p for p, _ in parts]
        newp, n, gnote = H.grammar_pass([paras[i] for i in idx], rep, 0.9, 0.95,
                                        originals=[o_paras[i] if i < len(o_paras) else None for i in idx])
        if n:                                          # always kept: the final text must read correctly
            for i, p in zip(idx, newp):
                paras[i] = p
            blocks = _score_paras(paras)
            after = D.summarise(blocks, hi, mod)
        note += gnote
    out_text = TP.join_paste(list(zip(paras, seps)))
    from docx import Document
    d = Document()
    for p in paras:
        d.add_paragraph(p)
    path = H._save_output(d, "pasted_text", "_humanized")
    note += _overall_note(orig, out_text)
    return dict(before=first, after=after, log=log, note=note, blocks=blocks, path=path,
                report=_report_summary(rpt, rows),
                orig=orig, new=out_text, text=out_text)