humanizer.py

"""Humanizer engine (Parrot T5 paraphraser, optional detector-guided "adversarial" decoding) + stylometric indicators + DOCX helpers."""
import base64
import contextvars
import hashlib
import hmac
import secrets
import sqlite3
import os
import re
import statistics
import tempfile
import threading
import time
from pathlib import Path

import gradio as gr
import torch

# HF installs the `spaces` package on EVERY Space; only SPACES_ZERO_GPU=true means ZeroGPU hardware.
ZERO = os.getenv("SPACES_ZERO_GPU", "").strip().lower() in ("1", "true", "yes")
try:
    import spaces
except Exception:  # pragma: no cover
    spaces = None
if ZERO and spaces is not None:
    _GPU = spaces.GPU
else:
    def _GPU(duration=None):
        return lambda fn: fn
from docx import Document
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer

import runtime as RT
import adversarial as ADV
import adaptive as AD
import textparse as TP
import groqai as G
import meaning as MEANING
import privacy as PV
import fluency as FL
import lt_grammar as LT
import clarity as CL
import lexsub as LX
import profile_settings as PS

# ---------------------------------------------------------------------------
# Brand (colours sampled from the CS Software Solutions logo)
# ---------------------------------------------------------------------------
BRAND = {
    "navy": "#07228B",
    "purple": "#47148B",
    "magenta": "#E92E7F",
    "red": "#E12B41",
    "orange": "#FE5828",
    "yellow": "#F5C518",
    "green": "#50BF49",
    "sky": "#50B4EF",
    "grey": "#4A4A4A",
}
RAINBOW = (
    f"linear-gradient(90deg, {BRAND['purple']}, {BRAND['navy']}, {BRAND['sky']}, "
    f"{BRAND['green']}, {BRAND['yellow']}, {BRAND['orange']}, {BRAND['red']}, {BRAND['magenta']})"
)
APP_NAME = "CS Academic Paraphraser"
SITE_URL = "https://cssoftwaresolutions.in"

LOGO_PATH = Path(__file__).parent / "logo.png"
LOGO_SRC = ""
FAVICON_PATH = Path(__file__).parent / "favicon.png"
FAVICON_SRC = ""
if FAVICON_PATH.exists():
    FAVICON_SRC = "data:image/png;base64," + base64.b64encode(FAVICON_PATH.read_bytes()).decode()
if LOGO_PATH.exists():
    LOGO_SRC = "data:image/png;base64," + base64.b64encode(LOGO_PATH.read_bytes()).decode()

# ---------------------------------------------------------------------------
# Model (kept in CPU RAM while idle; moved to GPU only inside @spaces.GPU)
# ---------------------------------------------------------------------------
# Paraphrase model: Parrot (prithivida/parrot_paraphraser_on_T5) by default. PARA_MODEL (or the older PARROT_MODEL)
# can name another T5 paraphraser with the same "paraphrase: " input, e.g. humarin/chatgpt_paraphraser_on_T5_base.
# If the chosen model cannot be loaded, Parrot is used so the Space still starts.
FALLBACK_MODEL = "prithivida/parrot_paraphraser_on_T5"
MODEL_ID = (os.getenv("PARA_MODEL") or os.getenv("PARROT_MODEL") or FALLBACK_MODEL).strip()
tokenizer = model = None
# mild penalty on repeating input tokens: pushes the model away from copying the sentence back (1.0 = off);
# set again in load_model() if the chosen model could not be loaded and Parrot is used instead
REP_PENALTY = float(os.getenv("PARA_REP_PENALTY", "1.0" if "parrot" in MODEL_ID.lower() else "1.15"))
# ZeroGPU / dedicated GPU: keep the model on "cuda" permanently (ZeroGPU restores it for each
# @spaces.GPU call), so several users can paraphrase at the same time — no global lock.
ON_GPU = ZERO or torch.cuda.is_available()
_cpu_model = None                                     # CPU copy (int8 on CPU hardware)
_cpu_lock = threading.Lock()
_load_lock = threading.Lock()


def load_model():
    """Load the paraphraser once (at startup; SKIP_MODEL_LOAD=1 delays it to the first use, for tests and tools)."""
    global tokenizer, model, _cpu_model, MODEL_ID, REP_PENALTY
    with _load_lock:
        if model is None:
            try:
                tok = AutoTokenizer.from_pretrained(MODEL_ID)
                m = AutoModelForSeq2SeqLM.from_pretrained(MODEL_ID).eval()
            except Exception as _err:
                if MODEL_ID == FALLBACK_MODEL:
                    raise
                print(f"[humanizer] {MODEL_ID} could not be loaded ({_err}); using {FALLBACK_MODEL}", flush=True)
                MODEL_ID = FALLBACK_MODEL
                REP_PENALTY = float(os.getenv("PARA_REP_PENALTY", "1.0"))
                tok = AutoTokenizer.from_pretrained(MODEL_ID)
                m = AutoModelForSeq2SeqLM.from_pretrained(MODEL_ID).eval()
            print(f"[humanizer] paraphrase model: {MODEL_ID}", flush=True)
            if ON_GPU:
                m.to("cuda")
            else:
                # CPU hardware only ever runs the int8 copy: keep that one and let the full-precision weights go
                # (about 0.9 GB of RAM that was held for nothing)
                m = _cpu_model = RT.quantize(m)
            tokenizer, model = tok, m
    return model


if os.getenv("SKIP_MODEL_LOAD") != "1":
    load_model()


def _get_cpu_model():
    global _cpu_model
    load_model()
    with _cpu_lock:
        if _cpu_model is None:
            _cpu_model = RT.quantize(AutoModelForSeq2SeqLM.from_pretrained(MODEL_ID).eval())
    return _cpu_model

GPU_CHUNK = 48          # sentences per ZeroGPU allocation (short reservations)
GPU_BATCH = int(os.getenv("HUM_GPU_BATCH", "32"))   # sentences per generate() call on GPU (x versions each)
CPU_CHUNK = 16          # sentences per CPU step (keeps progress updating)
OUT_DIR = Path(tempfile.gettempdir()) / "cs_outputs"
OUT_DIR.mkdir(parents=True, exist_ok=True)
MODES = ["Fast (recommended)", "Standard"]
MAX_PASTE_WORDS = int(os.getenv("MAX_PASTE_WORDS", "300"))  # max words per paste (all tools)


# ---------------------------------------------------------------------------
# Text helpers
# ---------------------------------------------------------------------------
def split_sentences(text):
    """Sentences of `text`; never split after 'et al.', 'e.g.', 'Fig.' or inside a decimal number."""
    return TP.split_sentences(text)


def should_process(text, min_words):
    text = (text or "").strip()
    if len(text.split()) < int(min_words):
        return False
    if re.fullmatch(r"https?://\S+|www\.\S+|\S+@\S+\.\S+", text):
        return False
    alpha = sum(ch.isalpha() for ch in text)
    if alpha < max(5, int(len(text) * 0.35)):
        return False
    return True


def words(text):
    return re.findall(r"[A-Za-z0-9']+", (text or "").lower())


# ---------------------------------------------------------------------------
# Tentative indicators (local heuristics — NOT Turnitin / iThenticate / GPTZero)
# ---------------------------------------------------------------------------
def _ngrams(tokens, n):
    return {tuple(tokens[i:i + n]) for i in range(len(tokens) - n + 1)}


def similarity_to_source(original, revised):
    """Share of the revised text's word n-grams that also occur in the original."""
    a, b = words(original), words(revised)
    if len(b) < 4 or len(a) < 4:
        return 0.0
    scores, weights = [], []
    for n, w in ((2, 0.2), (3, 0.35), (4, 0.45)):
        gb = _ngrams(b, n)
        if not gb:
            continue
        scores.append(len(gb & _ngrams(a, n)) / len(gb))
        weights.append(w)
    if not weights:
        return 0.0
    return 100.0 * sum(s * w for s, w in zip(scores, weights)) / sum(weights)


AI_MARKERS = [
    "delve", "delves", "delving", "furthermore", "moreover", "additionally",
    "in conclusion", "it is important to note", "it is worth noting", "plays a crucial role",
    "plays a pivotal role", "pivotal", "crucial", "comprehensive", "leverage", "leveraging",
    "seamless", "seamlessly", "robust", "landscape", "underscore", "underscores",
    "notably", "in today's", "fostering", "foster", "multifaceted", "intricate",
    "realm", "tapestry", "navigate", "navigating", "holistic", "paramount",
    "a testament to", "shed light", "sheds light", "ever-evolving", "cutting-edge",
    "in the realm of", "overall,", "ultimately", "enhance", "enhancing", "streamline",
    "valuable insights", "key role", "vital role", "significantly",
]


def ai_likelihood(text):
    """Stylometric estimate (0–100) of machine-like writing."""
    toks = words(text)
    if len(toks) < 30:
        return None
    sents = split_sentences(text) or [text]
    lengths = [len(words(s)) for s in sents if words(s)]

    # 1. Burstiness: humans vary sentence length more than LLMs.
    if len(lengths) >= 3:
        cv = statistics.pstdev(lengths) / max(1.0, statistics.mean(lengths))
        burst = min(1.0, max(0.0, (0.60 - cv) / 0.45))
    else:
        burst = 0.5

    # 2. Density of stock "LLM" vocabulary per 100 words.
    low = " " + " ".join(toks) + " "
    raw = (text or "").lower()
    hits = sum(raw.count(m) if " " in m or "," in m else low.count(f" {m} ") for m in AI_MARKERS)
    marker = min(1.0, (hits / len(toks) * 100) / 2.5)

    # 3. Uniformity: mid-length sentences, repeated openers, low lexical variety.
    mean_len = statistics.mean(lengths)
    band = 1.0 if 16 <= mean_len <= 28 else 0.4
    openers = [words(s)[0] for s in sents if words(s)]
    repeat = 1 - len(set(openers)) / len(openers) if openers else 0
    window = toks[:400]
    ttr = len(set(window)) / len(window)
    variety = min(1.0, max(0.0, (0.62 - ttr) / 0.25))
    uniform = 0.45 * band + 0.3 * min(1.0, repeat * 2) + 0.25 * variety

    score = 100 * (0.40 * burst + 0.35 * marker + 0.25 * uniform)
    return float(max(3.0, min(97.0, score)))


def _content(ws):
    """Vocabulary words: no function words (the, of, and ...), numbers, or one- to three-letter tokens."""
    return [w for w in ws if len(w) >= 4 and not w.isdigit() and w not in TP._STOP]


def words_changed(original, revised):
    """Vocabulary change (%): share of the revised text's vocabulary words (content words; function words, numbers
    and citations cannot change and are not counted) that are not in the original."""
    a, b = words(original), words(revised)
    if not b:
        return 0.0
    ca, cb = _content(a), _content(b)
    if len(set(cb)) >= 3:
        a, b = ca, cb
    return 100.0 * (1 - len(set(b) & set(a)) / len(set(b)))


def text_change(original, revised):
    """Vocabulary change of a whole text, measured the way it is set: sentence by sentence against the sentence it
    came from, weighted by length. (Comparing the word sets of two whole documents understates the change, because a
    synonym used in one sentence often appears somewhere else in the original.)"""
    import difflib
    op = [p for p in (original or "").split("\n") if p.strip()]
    np_ = [p for p in (revised or "").split("\n") if p.strip()]
    if not np_:
        return 0.0
    if len(op) != len(np_):
        sm = difflib.SequenceMatcher(None, [words(p)[:6] and " ".join(words(p)[:6]) for p in op],
                                     [words(p)[:6] and " ".join(words(p)[:6]) for p in np_], autojunk=False)
        pairs = []
        for tag, i1, i2, j1, j2 in sm.get_opcodes():
            if tag == "equal" or (i2 - i1) == (j2 - j1):
                pairs += list(zip(op[i1:i2], np_[j1:j2]))
            else:
                pairs.append((" ".join(op[i1:i2]), " ".join(np_[j1:j2])))
    else:
        pairs = list(zip(op, np_))
    tot = wsum = 0.0
    for a, b in pairs:
        sa, sb = split_sentences(a) or [a], split_sentences(b) or [b]
        if len(sa) == len(sb):
            units = list(zip(sa, sb))
        else:                                          # split / merged sentences: compare with the closest one
            ws = [set(words(x)) for x in sa]
            units = []
            for y in sb:
                wy = set(words(y))
                k = max(range(len(sa)), key=lambda i: len(ws[i] & wy) / max(1, len(ws[i] | wy)))
                units.append((sa[k], y))
        for x, y in units:
            k = len(words(y))
            if k:
                tot += k * words_changed(x, y)
                wsum += k
    return tot / wsum if wsum else 0.0


def _level(value, invert=False):
    if value is None:
        return "#9aa0a6", "n/a"
    v = 100 - value if invert else value
    if v < 30:
        return BRAND["green"], "Low"
    if v < 55:
        return BRAND["orange"], "Moderate"
    return BRAND["red"], "High"


def _gauge(title, value, caption, invert=False, fixed_color=None, chip=None):
    color, label = _level(value, invert)
    if fixed_color:
        color = fixed_color
        label = chip or "Rewrite depth"
    pct = 0 if value is None else round(value)
    shown = "—" if value is None else f"{pct}%"
    return f"""
    <div class="cs-gauge">
      <div class="cs-ring" style="--p:{pct};--c:{color}">
        <div class="cs-ring-inner"><span>{shown}</span></div>
      </div>
      <div class="cs-gauge-title">{title}</div>
      <div class="cs-chip" style="background:{color}1f;color:{color}">{label}</div>
      <div class="cs-gauge-cap">{caption}</div>
    </div>"""


def score_panel(original, revised):
    sim = similarity_to_source(original, revised)
    ai_in = ai_likelihood(original)
    ai_out = ai_likelihood(revised)
    changed = text_change(original, revised)
    return f"""
    <div class="cs-scores">
      {_gauge("Similarity to source", sim, "Word n-gram overlap with your original")}
      {_gauge("AI indicator · input", ai_in, "Stylometric estimate of the original")}
      {_gauge("AI indicator · output", ai_out, "Stylometric estimate after paraphrasing")}
      {_gauge("Vocabulary changed", changed, "Share of new wording", fixed_color=BRAND["navy"])}
    </div>
    <div class="cs-disclaimer">
      <b>Tentative indicators only.</b> Similarity is measured against your own original text,
      not against internet or journal databases, and the AI indicator is a stylometric heuristic.
      Results will differ from Turnitin, iThenticate or other detectors. Always review the
      output for meaning, technical terms and citations before use.
    </div>"""


EMPTY_SCORES = """
<div class="cs-scores cs-empty">
  <div>Scores appear here after paraphrasing — similarity to source, AI indicators and rewrite depth.</div>
</div>"""


# ---------------------------------------------------------------------------
# DOCX structure helpers
# ---------------------------------------------------------------------------
def iter_table_paragraphs(table):
    for row in table.rows:
        for cell in row.cells:
            for paragraph in cell.paragraphs:
                yield paragraph
            for nested_table in cell.tables:
                yield from iter_table_paragraphs(nested_table)


def collect_paragraphs(doc):
    seen = set()

    def unique(paragraphs):
        for paragraph in paragraphs:
            key = id(paragraph._p)
            if key not in seen:
                seen.add(key)
                yield paragraph

    yield from unique(doc.paragraphs)
    for table in doc.tables:
        yield from unique(iter_table_paragraphs(table))
    for section in doc.sections:
        for part in (
            section.header, section.footer,
            section.first_page_header, section.first_page_footer,
            section.even_page_header, section.even_page_footer,
        ):
            yield from unique(part.paragraphs)
            for table in part.tables:
                yield from unique(iter_table_paragraphs(table))


def _is_protected(run):
    return bool(
        run._r.xpath(".//w:drawing")
        or run._r.xpath(".//w:object")
        or run._r.xpath(".//w:fldChar")
    )


def replace_text_preserving_paragraph(paragraph, new_text):
    runs = list(paragraph.runs)
    if not runs:
        paragraph.add_run(new_text)
        return
    target = next((r for r in runs if not _is_protected(r)), None)
    if target is None:
        target = paragraph.add_run()
    target.text = new_text
    for run in runs:
        if run is not target and not _is_protected(run):
            run.text = ""


# ---------------------------------------------------------------------------
# Paraphrasing
# ---------------------------------------------------------------------------
def _generate(texts, temperature, device, batch_size, m=None, k=1):
    """Paraphrase `texts`; returns len(texts) * k strings (k sampled candidates per text, in order).
    Output length is capped just above the input length, so the decoder never runs on idle."""
    m = m or load_model()
    temperature = max(0.3, min(float(temperature), PS.get("max_temp")))
    k = max(1, int(k))
    outputs = []
    with torch.inference_mode():
        for start in range(0, len(texts), batch_size):
            chunk = texts[start:start + batch_size]
            encoded = tokenizer(
                ["paraphrase: " + t for t in chunk],
                return_tensors="pt", padding=True, truncation=True, max_length=256,
            )
            longest = int(encoded["input_ids"].shape[1])
            encoded = {kk: v.to(device) for kk, v in encoded.items()}
            _sup = AD.suppress_ids(tokenizer)
            _t0 = time.time()
            generated = m.generate(
                **encoded,
                do_sample=True,
                temperature=temperature,
                top_p=0.92,
                num_return_sequences=k,
                max_new_tokens=min(180, int(longest * 1.35) + 12),
                no_repeat_ngram_size=3,
                repetition_penalty=REP_PENALTY,
                suppress_tokens=_sup,                           # words learned to cause rejected rewrites
            )
            print(f"[timing] parrot generate on {device}: {len(chunk)} text(s) x {k}, {len(_sup or [])} banned "
                  f"tokens, {time.time() - _t0:.1f}s", flush=True)
            decoded = tokenizer.batch_decode(generated.detach().cpu(), skip_special_tokens=True)
            for n_, src in enumerate(chunk):
                for c in decoded[n_ * k:(n_ + 1) * k]:
                    outputs.append(c.strip() if c.strip() else src)
            del encoded, generated
    return outputs


def _gpu_duration(texts, temperature, k=1):
    # Reserve only what this chunk needs, so visitors' daily ZeroGPU quota lasts longer.
    return int(min(90, 12 + 0.3 * len(texts or []) * max(1, int(k))))


@_GPU(duration=_gpu_duration)
def paraphrase_batch_gpu(texts, temperature, k=1):
    if not texts:
        return []
    if not ON_GPU or not torch.cuda.is_available():
        raise RuntimeError("no gpu available")
    return _generate(texts, temperature, "cuda", batch_size=GPU_BATCH, k=k)


def paraphrase_batch_cpu(texts, temperature, k=1):
    if not texts:
        return []
    m = _get_cpu_model()
    with RT.cpu_job():                       # share the cores fairly between simultaneous users
        return _generate(texts, temperature, "cpu", batch_size=8, m=m, k=k)


def _is_gpu_unavailable(err):
    msg = str(err).lower()
    return any(k in msg for k in ("quota", "zerogpu", "gpu task aborted", "cuda", "no gpu", "queue is full"))


# Candidates per sentence: Parrot samples several paraphrases in one pass and the one the AI detector finds
# most human is kept (HUM_CANDIDATES=1 turns this off; HUM_SELECT=0 keeps the first valid candidate).
CANDIDATES = max(1, int(os.getenv("HUM_CANDIDATES", "3")))
# Set per job when the customer ticks "My text is 100% AI-written": Parrot writes CANDIDATES_AI versions of
# every sentence (default 5), the plain-wording (de-AI) method is added, and Groq checks the GROQ_SHORTLIST
# most human versions of each sentence (default 2) so a rejected best version falls back to a checked one.
CANDIDATES_AI = max(1, int(os.getenv("HUM_CANDIDATES_AI", "5")))
GROQ_SHORTLIST = max(1, int(os.getenv("HUM_GROQ_SHORTLIST", "2")))
# 100% AI mode: target share of new wording per sentence (same measure as the "Vocabulary changed" gauge)
# and the number of extra Parrot rounds for sentences below it.
AI_CHANGE_TARGET = float(os.getenv("HUM_AI_CHANGE_TARGET", "45"))
# safety limits: above these the paraphraser starts breaking grammar and technical meaning
MAX_TEMP = float(os.getenv("HUM_MAX_TEMP", "1.0"))        # highest sampling variation ever used
MAX_VOCAB = float(os.getenv("HUM_MAX_VOCAB", "45"))       # highest new-wording target per sentence (%)
AI_ROUNDS = max(0, int(os.getenv("HUM_AI_ROUNDS", "2")))
# Smart Humanize (adaptive): sentences whose AI score did not fall last pass get detector-guided decoding too.
FORCE_ADV = contextvars.ContextVar("force_adv", default=False)
# a job paid from the wallet starts no new pass / round after this moment (epoch seconds), so its cost stays inside
# what was reserved for it (wallet.plan_job); None = the engines' own time budgets only
JOB_DEADLINE = contextvars.ContextVar("job_deadline", default=None)


def past_deadline():
    d = JOB_DEADLINE.get()
    return d is not None and time.time() > d
# Smart Humanize sets this per pass (its passes already rewrite again, so fewer inner rounds are needed).
EXTRA_ROUNDS = contextvars.ContextVar("extra_rounds", default=None)
# Vocabulary change (%) slider, per job. Box ticked: the new-wording target (None -> AI_CHANGE_TARGET).
# Box unticked: 0 = off (the original behaviour); above 0 = same target logic as the 100% AI mode.
VOCAB_TARGET = contextvars.ContextVar("vocab_target", default=None)


def change_target():
    """New-wording target per sentence: the customer's Vocabulary change setting when they chose one (up to 90%),
    otherwise the automatic level (capped at HUM_MAX_VOCAB)."""
    v = VOCAB_TARGET.get()
    if v:
        return min(90.0, float(v))
    return min(PS.get("max_vocab"), float(os.getenv("HUM_AI_CHANGE_TARGET", "90")))
FULL_AI = contextvars.ContextVar("full_ai", default=False)
# Set by Smart Humanize itself when the normal rewrite stops lowering the score: the sentences that are still flagged
# get the deeper rewrite (more versions per sentence, plain-wording versions, two versions checked) without the
# customer having to tick anything. Unlike FULL_AI, sentences that read as human stay untouched.
BOOST = contextvars.ContextVar("boost", default=False)
# Kept for compatibility with smart.py / app.py versions that set it per pass. The 100% AI mode is now
# "5 Parrot versions + Groq check" (Parrot does all rewriting), so this switch has no effect.
DEEP = contextvars.ContextVar("deep_rewrite", default=False)
SELECT = os.getenv("HUM_SELECT", "1") != "0"
# Smart Humanize: current sentence text -> the AUTHOR'S original sentence, so every pass is checked against the
# original (rewrites of rewrites cannot drift or pile up damage)
ANCHOR = contextvars.ContextVar("anchor", default=None)
# a sentence with no clear version keeps its original wording (strict, default) or keeps the best version for the
# final polish to repair (lenient)
CLARITY_SELECT = os.getenv("CLARITY_SELECT", "strict").lower()
# rescue rewrites (language model, written to the publication rules) for sentences that have no clear rewrite or
# still read as AI-written; every rescue rewrite passes the same checks
LEX_N = int(os.getenv("LEXSUB_N", "3"))                  # vocabulary-replaced versions per sentence
DEBUG = os.getenv("HUM_DEBUG", "0") == "1"                # print every candidate with its AI score (server log)
RESCUE = os.getenv("HUM_RESCUE", "1") != "0"
RESCUE_AI = float(os.getenv("HUM_RESCUE_AI", "0.35"))      # sentences scoring at least this get rescue versions
RESCUE_N = int(os.getenv("HUM_RESCUE_N", "4"))            # versions per sentence
RESCUE_MAX = int(os.getenv("HUM_RESCUE_MAX", "300"))      # at most this many sentences per run
RESCUE_SWEEPS = max(1, int(os.getenv("HUM_RESCUE_SWEEPS", "3")))   # rounds of the joint choice between versions
# Speed: steps that do not depend on each other run at the same time inside one rewrite call: the vocabulary
# versions (lexsub) while Parrot writes, and the rescue rewrites (Groq) from the start of the call instead of after
# the review. HUM_OVERLAP=0 = one after another, as before.
OVERLAP = os.getenv("HUM_OVERLAP", "0") == "1"           # off by default (shared tokenizers: see smart.PARALLEL_GROUPS)


def _rescue_versions(sents, glossary, refs, n, progress, lo, hi, min_change):
    """Rescue rewrites from the language model (the job's provider, then OpenAI when that one is set and the first is
    out of quota). None = no provider could take the request."""
    for prov in [G.PROVIDER.get()] + (["openai"] if G.PROVIDER.get() != "openai" and G.openai_available() else []):
        try:
            return G._in_provider(prov, G.variants, sents, glossary, refs, n, progress, lo, hi, min_change)
        except G.Unavailable as err:
            print(f"[humanizer] rescue rewrites unavailable ({prov}): {err}", flush=True)
    return None
# independent reading of every rewritten sentence against the author's sentence (HUM_VERIFY=0 = off)
VERIFY = os.getenv("HUM_VERIFY", "1") != "0"
VERIFY_ALTS = max(0, int(os.getenv("HUM_VERIFY_ALTS", "3")))      # other versions tried for a sentence that fails


def anchor_of(s):
    d = ANCHOR.get()
    return d.get((s or "").strip()) if d else None


def remember_anchor(old, new):
    """Smart Humanize: `new` replaced `old`; both lead back to the same original sentence."""
    d = ANCHOR.get()
    if d is not None and new and old and new.strip() != old.strip():
        d[new.strip()] = d.get(old.strip(), old.strip())


# Smart Humanize: sentence -> (previous sentence, next sentence) in the document. The detector judges text in
# context (~90-word windows); a sentence scored on its own is scored unreliably, so candidates are scored together
# with their neighbours, the way the final report sees them.
SENT_CONTEXT = contextvars.ContextVar("sent_context", default=None)


def _ctx_scores(pairs, ctx=None, current=None):
    """[(current sentence, candidate)] -> AI probability of each candidate read in its document context.
    ctx: sentence -> (previous, next); default = the document context set by Smart Humanize.
    current: sentence -> the wording it has now, so a candidate is read next to its neighbours as they WILL be in
    the result (a rewritten neighbour changes how the detector reads the sentence)."""
    if not pairs:
        return []
    ctx = ctx if ctx is not None else (SENT_CONTEXT.get() or {})
    current = current or {}
    texts = []
    for s, c in pairs:
        p, n = ctx.get((s or "").strip(), ("", ""))
        p, n = current.get((p or "").strip(), p), current.get((n or "").strip(), n)
        texts.append(" ".join(x for x in (p, c, n) if x))
    return _detector_scores(texts)


def _detector_scores(texts):
    """AI probability per text (lower = more human), or None if the detector is unavailable."""
    if not texts:
        return []
    try:
        import detector as D
        return list(D.score_texts(texts))
    except Exception as err:
        print(f"[humanizer] candidate scoring skipped: {err}", flush=True)
        return None


def _adversarial(chunks, flat, cand, temp, progress, at):
    """Adversarial Paraphrasing: one detector-guided Parrot version per piece, added to cand[piece] (every piece of
    a sentence gets one, so the pieces stay aligned). Returns True if it ran."""
    if ZERO and not ADV.ON_ZERO:
        return False                                   # ZeroGPU: no nested GPU allocation for the detector
    todo = sorted(set(chunks))
    if not todo or len(todo) > ADV.MAX_PIECES:         # over the per-job cap (ADV_MAX): Parrot's versions only
        if todo:
            print(f"[humanizer] adversarial paraphrasing skipped: {len(todo)} pieces > ADV_MAX={ADV.MAX_PIECES}",
                  flush=True)
        return False
    if progress is not None:
        progress(at, desc=f"Detector-guided rewriting of {len(todo)} sentence piece(s)")
    try:
        if ON_GPU and not ZERO and torch.cuda.is_available():
            outs = ADV.generate([flat[x] for x in todo], load_model(), tokenizer, "cuda", _detector_scores, temp)
        else:
            m = _get_cpu_model()
            with RT.cpu_job():
                outs = ADV.generate([flat[x] for x in todo], m, tokenizer, "cpu", _detector_scores, temp)
    except Exception as err:
        print(f"[humanizer] adversarial paraphrasing skipped: {err}", flush=True)
        return False
    for x, o in zip(todo, outs):
        cand[x].append(o if o and o.strip() else flat[x])
    print(f"[humanizer] adversarial paraphrasing: {len(todo)} piece(s)", flush=True)
    return True


def _paraphrase_local(texts, temperature, progress=None, lo=0.15, hi=0.85, mode="Auto"):
    """Returns (paragraphs, engine_note).
    1. Parrot writes CANDIDATES paraphrases per sentence (long sentences clause by clause), batched
       shortest-first so batches carry little padding.
    2. Technical terms a candidate lost are copied back in place (textparse.restore_terms); candidates that
       still lose terms, numbers or citations are dropped (textparse.guard).
    3. The AI detector scores the surviving candidates; the most human one is kept.
    4. Groq (optional) checks the chosen sentence's academic wording with minimal edits, and rescues
       sentences where no candidate survived."""
    import time as _time
    _tm = [("start", _time.time())]             # stage timing, one log line per call (where the waiting goes)
    groups, flat, sent_map = [], [], []        # sent_map: (sentence, first_chunk, n_chunks)
    for text in texts:
        sents = split_sentences(text) or [text]
        g0 = len(sent_map)
        for s in sents:
            pieces = TP.chunk_sentence(s)
            sent_map.append((s, len(flat), len(pieces)))
            flat.extend(pieces)
        groups.append((g0, len(sent_map)))

    doc_ctx = SENT_CONTEXT.get() or {}
    ctx = {}
    for a, b in groups:                                  # plain Humanizer: neighbours inside the paragraph
        for si in range(a, b):
            key = sent_map[si][0].strip()
            ctx[key] = doc_ctx.get(key) or (sent_map[si - 1][0] if si > a else "",
                                            sent_map[si + 1][0] if si + 1 < b else "")
    full_ai = FULL_AI.get()
    deep = full_ai or BOOST.get()                      # deeper rewrite: asked for, or switched on by Smart Humanize
    k = AD.candidates(CANDIDATES_AI if deep else CANDIDATES)   # more samples when few survive the checks
    target = change_target() if full_ai else min(90.0, float(VOCAB_TARGET.get() or 0))
    er = EXTRA_ROUNDS.get()
    rounds = 1 + ((AI_ROUNDS if er is None else er) if (full_ai or target) else 0)
    temp0 = min(1.5, float(temperature) + (0.2 if deep else 0.0))
    p_hi = lo + (hi - lo) * (0.7 if G.available() else 0.85)   # room on the bar for scoring / Groq
    p_gen = lo + (p_hi - lo) * (0.75 if full_ai else 1.0)      # 100% AI: room for the extra rounds
    use_gpu = ON_GPU and not (str(mode) == MODES[1] or str(mode).lower().startswith("cpu"))   # CPU-only hardware: straight to CPU
    fell_back = False
    cand = [[] for _ in flat]                  # cand[i] = all paraphrases of flat[i] (k per round)

    def generate(indices, temp, a_, b_, label):
        nonlocal use_gpu, fell_back
        order = sorted(indices, key=lambda x: len(flat[x]))
        total = max(1, len(order))
        step = GPU_CHUNK if use_gpu else CPU_CHUNK
        i = 0
        while i < len(order):
            idx = order[i:i + step]
            chunk = [flat[x] for x in idx]
            engine = "GPU" if use_gpu else "CPU"
            if progress is not None:
                progress(a_ + (b_ - a_) * i / total,
                         desc=f"{label} {i + 1}–{i + len(chunk)} of {total} ({engine}, {k} versions each)")
            if use_gpu:
                try:
                    outs = paraphrase_batch_gpu(chunk, temp, k)
                except Exception as err:
                    if not _is_gpu_unavailable(err):
                        raise
                    use_gpu, fell_back, step = False, True, CPU_CHUNK
                    continue  # retry the same chunk on CPU
            else:
                outs = paraphrase_batch_cpu(chunk, temp, k)
            for n_, x in enumerate(idx):
                got = outs[n_ * k:(n_ + 1) * k] or [flat[x]]
                cand[x].extend(got + [got[-1]] * (k - len(got)))   # keep k per round so chunks line up
            i += len(chunk)

    glossary = TP.build_glossary(texts) | TP.GLOSSARY.get()

    def checked(orig, new, source, guarded):
        """Guard result of one version + what the adaptive learner records about it."""
        ok = guarded
        if ok and AD.banned_in(orig, ok):                # a word learned to cause bad rewrites
            ok = None
        if new and new.strip() != orig.strip():
            why = None if ok else (AD.banned_in(orig, new) and "banned word") or TP.integrity_issue(orig, new) \
                or TP.quality_issue(orig, new) or ("lost term" if TP.lost_terms(orig, new, glossary) else "other")
            AD.record_candidate(orig, new, source, bool(ok), why)
        return ok

    def build(si):
        """Valid candidates of sentence si (terms restored, guard passed) + first raw paraphrase."""
        s, c0, n = sent_map[si]
        opts, raw0 = [], None
        m = min(len(cand[c]) for c in range(c0, c0 + n)) if n else 0
        for j in range(m):
            pieces = [cand[c][j] for c in range(c0, c0 + n)]
            new = TP.merge_chunks(flat[c0:c0 + n], pieces) if n > 1 else pieces[0]
            raw0 = raw0 or new
            source = "adversarial" if (adv_used and j == k) else "parrot"
            ok = checked(s, new, source, TP.guard(s, TP.restore_terms(s, new, glossary), glossary))
            if ok and TP._near_repeats(ok) - TP._near_repeats(s):
                ok = None                                # restored term doubled up ("ethics of care ethics")
            if ok and ok.strip() != s.strip() and ok not in opts:
                opts.append(ok)
                src_of[(si, ok)] = source
        # "100% AI-written" mode only: plain-wording versions (chatbot vocabulary / set phrases removed) of each
        # paraphrase and of the original itself, so fully AI-written sentences always get a safe, more human option
        for v in lex[si]:                                # vocabulary-replaced versions of the ORIGINAL sentence
            ok = checked(ref[si], v, "lexsub", TP.guard(ref[si], v, glossary))
            if ok and ok.strip() != s.strip() and ok not in opts:
                opts.append(ok)
                src_of[(si, ok)] = "lexsub"
        for base in ((list(opts) + [s]) if deep and AD.use_source("de_ai") else []):
            v = TP.de_ai(base)
            if v != base:
                ok = checked(s, v, "de_ai", TP.guard(s, TP.restore_terms(s, v, glossary), glossary))
                if ok and ok.strip() != s.strip() and ok not in opts:
                    opts.append(ok)
                    src_of[(si, ok)] = "de_ai"
        return opts, raw0 or s

    def change(si, o):
        return words_changed(sent_map[si][0], o)

    _tm.append(("split", _time.time()))
    # ---- 1–2. Parrot versions; in 100% AI mode, sentences below the new-wording target (or with no safe
    #           version) go back to Parrot for extra rounds with more variation
    todo = list(range(len(sent_map)))
    options, first_raw = [[] for _ in sent_map], [s for s, _, _ in sent_map]
    extra_rounds = meaning_dropped = 0
    adv_used = False
    src_of = {}                                          # (sentence, version) -> source, for the adaptive learner
    qstat = dict(n=0, meaning=0, fluency=0, second=0, sims=[])
    ref = [anchor_of(s) or s for s, _, _ in sent_map]          # the author's original sentence
    # stricter for technical sentences; a high vocabulary-change setting allows a little more distance (the
    # faithfulness check still rejects changed claims)
    relax = 0.05 if target >= 50 else 0.03 if target >= 35 else 0.0
    thr = [MEANING.min_for(r) - relax for r in ref]
    soft = [MEANING.min_for(r, MEANING.soft_sim()) - relax for r in ref]
    _tm.append(("setup", _time.time()))
    import concurrent.futures as _cf
    _bg = _cf.ThreadPoolExecutor(max_workers=2) if OVERLAP else None
    # the rescue rewrites (step 5) need only each sentence and its original, not the choices made before them: they
    # are requested now and arrive while Parrot, the checks and the review run. Step 5 still decides which
    # sentences use them (in practice nearly all that are sent).
    rescue_pre = None
    want0 = [si for si, (s, _, _) in enumerate(sent_map) if len(s.split()) >= 6]
    if _bg is not None and RESCUE and want0 and len(want0) <= RESCUE_MAX and (G.available() or G.openai_available()):
        rescue_pre = (want0, _bg.submit(
            contextvars.copy_context().run, _rescue_versions, [sent_map[si][0] for si in want0], glossary,
            [ref[si] if ref[si] != sent_map[si][0] else None for si in want0], RESCUE_N + (2 if BOOST.get() else 0),
            None, hi, hi, min(80, target) if target else 0))
    # correct sentences with replaced vocabulary: suitable in-context synonyms, structure unchanged (always available
    # as a clear option, so a rejected paraphrase is replaced by new wording instead of the original sentence)
    def vocabulary():
        lex_ = [[] for _ in sent_map]
        if LX.ENABLED and LX.available():
            try:
                # chained passes: new vocabulary is built on the CURRENT text (the previous pass's output), so every
                # pass adds to the last one; terms, numbers and meaning are still checked against the original
                lex_ = LX.variants([s for s, _, _ in sent_map], glossary, n=LEX_N,
                                   ratios=LX.ratios_for(target, LEX_N + 1) if target else None)
            except Exception as err:
                print(f"[humanizer] vocabulary replacement skipped: {err}", flush=True)
            print(f"[humanizer] vocabulary replacement: {sum(1 for x in lex_ if x)} of {len(lex_)} sentences got "
                  f"versions", flush=True)
        return lex_

    if _bg is not None:                                  # written while Parrot writes (both needed only below)
        lex_job, lex = _bg.submit(contextvars.copy_context().run, vocabulary), None
    else:
        if progress is not None and LX.ENABLED:
            progress(lo, desc="Finding suitable replacement vocabulary")
        lex_job, lex = None, vocabulary()
    _tm.append(("vocab", _time.time()))
    for r in range(rounds):
        if not todo:
            break
        a_ = lo if r == 0 else p_gen + (p_hi - p_gen) * (r - 1) / max(1, rounds - 1)
        b_ = p_gen if r == 0 else p_gen + (p_hi - p_gen) * r / max(1, rounds - 1)
        chunks = [c for si in todo for c in range(sent_map[si][1], sent_map[si][1] + sent_map[si][2])]
        generate(chunks, min(1.5, temp0 + 0.25 * r), a_, b_,
                 "Paraphrasing" if r == 0 else f"Extra rewrite round {r} for")
        _tm.append(("parrot", _time.time()))
        if r == 0 and (ADV.wanted(full_ai) or FORCE_ADV.get()) and AD.use_source("adversarial"):
            if _adversarial(chunks, flat, cand, temp0, progress, b_):
                adv_used = True
        extra_rounds += r > 0
        if lex is None:
            lex = lex_job.result()
            _tm.append(("vocab_wait", _time.time()))
        for si in todo:
            options[si], first_raw[si] = build(si)
        _tm.append(("guards", _time.time()))
        # quality checks of every version: meaning (vs the original sentence) and fluency (well-formed English)
        pairs = [(si, o) for si in todo for o in options[si]]
        if pairs:
            sims = MEANING.similarities([(ref[si], o) for si, o in pairs])
            flu = FL.acceptable([(ref[si], o) for si, o in pairs])
            keep_ = {}
            for (si, o), sm, fl in zip(pairs, sims, flu):
                keep_[(si, o)] = (sm >= thr[si] and fl, sm, fl)
                qstat["n"] += 1
                qstat["meaning"] += sm < thr[si]
                qstat["fluency"] += not fl
                qstat["sims"].append(sm)
            for si in todo:
                good = [o for o in options[si] if keep_[(si, o)][0]]
                if not good and options[si]:
                    # second chance: no version passed both checks -> keep the closest-meaning version that is
                    # fluent English and only slightly below the meaning threshold
                    soft_ = [(keep_[(si, o)][1], o) for o in options[si]
                            if keep_[(si, o)][2] and keep_[(si, o)][1] >= soft[si]]
                    if soft_:
                        good = [max(soft_)[1]]
                        qstat["second"] += 1
                meaning_dropped += len(options[si]) - len(good)
                options[si] = good
        if not target:
            break
        todo = [si for si in todo
                if not options[si] or max(change(si, o) for o in options[si]) < target]

    if fell_back:
        note = "⚠️ GPU quota was unavailable, so the remaining text was processed on CPU (slower, same quality)."
    elif use_gpu:
        note = "⚡ Processed on GPU."
    else:
        note = "🖥️ Processed on CPU."
    if adv_used:
        note += " 🧭 Detector-guided (adversarial) rewriting added one extra version per sentence."

    _tm.append(("meaning+fluency", _time.time()))
    # ---- 3. most human candidate (AI detector); 100% AI mode: versions meeting the new-wording target first
    score_of = {}
    multi = [(si, o) for si, opts in enumerate(options) if len(opts) > 1 for o in opts]
    if SELECT and multi:
        if progress is not None:
            progress(p_hi, desc="Choosing the most human-sounding version of each sentence")
        sc = _ctx_scores([(sent_map[si][0], o) for si, o in multi], ctx)
        if sc:
            score_of = {key: v for key, v in zip(multi, sc)}
            note += (f" 🎯 Best of {k} rewritten versions per sentence chosen by the AI detector." if k > 1 else
                     " 🎯 The AI detector chose between the rewritten version and the plain-wording version.")

    def rank_key(si, o):
        det = score_of.get((si, o), 1.0 if score_of else 0.0)
        if target:
            ch = change(si, o)
            return (0, det) if ch >= target else (1, -ch, det)
        return (det,)

    ranked = [sorted(opts, key=lambda o, si=si: rank_key(si, o)) if (score_of or target) else list(opts)
              for si, opts in enumerate(options)]
    if DEBUG:
        for si, opts in enumerate(ranked):
            print(f"[debug] s{si} | {sent_map[si][0]}", flush=True)
            for o in opts:
                print(f"[debug]    {score_of.get((si, o), float('nan')):.2f} chg {change(si, o):3.0f}% | {o}", flush=True)
    chosen = [r_[0] if r_ else None for r_ in ranked]
    for si, opts in enumerate(options):                  # adaptive learner: which source's version won
        if len(opts) > 1 and chosen[si]:
            for o in opts:
                AD.record_win(src_of.get((si, o), "parrot"), o == chosen[si])
    AD.save()

    _tm.append(("detector_pick", _time.time()))
    # ---- 4. Groq: academic-wording check of the chosen sentences + rescue of sentences with no valid version.
    #      100% AI mode: Groq checks the GROQ_SHORTLIST most human versions of each sentence, and the most human
    #      version Groq accepts is kept (so one rejection does not fall back to an unchecked version).
    if G.available():
        shortlist = GROQ_SHORTLIST if deep else 1
        pairs, where = [], []
        for si, (s, _, _) in enumerate(sent_map):
            if chosen[si]:
                todo = [chosen[si]] + [o for o in ranked[si] if o != chosen[si]][:shortlist - 1]
                for o in todo:
                    pairs.append((s, o)); where.append((si, "check", o))
            elif first_raw[si] and first_raw[si].strip() != s.strip():
                pairs.append((s, first_raw[si])); where.append((si, "rescue", first_raw[si]))
        if pairs:
            try:
                res = G.review(pairs, glossary, progress, p_hi + (hi - p_hi) * 0.3, hi)
                verdict = {}                           # si -> list of (candidate, result) in rank order
                for (si, kind, cand_), r in zip(where, res):
                    verdict.setdefault(si, []).append((kind, cand_, r))
                fixes = []                             # (si, kind, groq_text, candidate it came from)
                for si, items in verdict.items():
                    s = sent_map[si][0]
                    kind = items[0][0]
                    accepted, unchecked, rejected = [], [], 0
                    for _, cand_, r in items:
                        if r is None:
                            rejected += 1
                            continue
                        if r is False:
                            unchecked.append(cand_)
                            continue
                        ok = TP.guard(s, TP.restore_terms(s, r, glossary), glossary)
                        if ok and ok.strip() != s.strip():
                            accepted.append((ok, cand_))
                        elif ok and kind == "check":
                            accepted.append((None, cand_))  # Groq's repair = the original
                    if kind == "rescue":
                        if accepted and accepted[0][0]:
                            chosen[si] = accepted[0][0]
                        continue
                    if accepted:
                        real = [x for x in accepted if x[0]]
                        ok, cand_ = (real or accepted)[0]   # most human version Groq accepted
                        if ok is None:
                            chosen[si] = None          # keep the original sentence
                        else:
                            chosen[si] = cand_
                            if ok != cand_:
                                fixes.append((si, ok, cand_))
                    elif unchecked:
                        chosen[si] = unchecked[0]      # Groq could not check it: local rules decide
                    elif rejected:
                        # every checked version rejected: try an unchecked lower-ranked one, else the original
                        tried = {c for _, c, _ in items}
                        chosen[si] = next((o for o in ranked[si] if o not in tried), None)
                # keep Groq's wording fix unless it makes the sentence clearly more AI-like
                sc = _ctx_scores([(sent_map[si][0], ok) for si, ok, _ in fixes]
                                 + [(sent_map[si][0], c) for si, _, c in fixes], ctx) if fixes else []
                m = len(fixes)
                for n_, (si, ok, cand_) in enumerate(fixes):
                    if sc and sc[n_] > sc[m + n_] + 0.02:       # never trade AI score for polish
                        continue
                    if target and change(si, ok) < 0.5 * change(si, cand_):
                        continue                                 # 100% AI: Groq must not undo the rewrite
                    chosen[si] = ok
                note += (f" ✅ Groq checked the top {shortlist} versions of each sentence." if shortlist > 1
                         else " ✅ Academic wording checked by Groq.")
            except G.Unavailable as err:
                note += " (Groq check unavailable right now — local term checks used.)"
                print(f"[humanizer] Groq review skipped: {err}", flush=True)

    _tm.append(("groq_review", _time.time()))
    # final meaning check of what will actually be used (Groq rescues / repairs included)
    final = [si for si in range(len(sent_map)) if chosen[si] and chosen[si].strip() != sent_map[si][0].strip()]
    fsims = MEANING.similarities([(ref[si], chosen[si]) for si in final])
    for si, k_ in zip(final, [sm >= soft[si] for si, sm in zip(final, fsims)]):
        if not k_:
            meaning_dropped += 1
            chosen[si] = next((o for o in ranked[si] if o != chosen[si]), None)   # already meaning-checked
    _tm.append(("meaning", _time.time()))
    # clarity gate: the chosen version must read as clear, correct English (compared with the ORIGINAL sentence);
    # otherwise the clearest acceptable lower-ranked version is used, else the original wording is kept
    unclear = switched = 0
    if CL.ENABLED and PS.get("select") != "off":
        if progress is not None:
            progress(hi, desc="Checking every rewritten sentence for clear, correct English")
        final = [si for si in range(len(sent_map)) if chosen[si] and chosen[si].strip() != sent_map[si][0].strip()]
        verdict = CL.assess([(ref[si], chosen[si]) for si in final], use_meaning=False)
        bad = [si for si, v in zip(final, verdict) if v]
        if bad:
            alts = [(si, o) for si in bad for o in ranked[si] if o != chosen[si]]
            good = {}
            for (si, o), v in zip(alts, CL.assess([(ref[si], o) for si, o in alts], use_meaning=False)):
                if not v and si not in good:
                    good[si] = o
            for si in bad:
                if si in good:
                    chosen[si] = good[si]
                    switched += 1
                else:
                    unclear += 1
                    if PS.get("select") == "strict":
                        chosen[si] = None
        CL.report()
    _tm.append(("clarity", _time.time()))
    # rescue: sentences still kept in the author's wording, or still reading as AI-written, get rule-compliant,
    # human-style rewrites from the language model (written from the ORIGINAL sentence); a rewrite is used only if
    # it passes every check and reads as more human than what the sentence has now
    rescued = 0
    pool, before_rescue = {}, list(chosen)             # rescue versions per sentence; the choice made without them
    if RESCUE and (G.available() or G.openai_available()):
        want = [si for si, (s, _, _) in enumerate(sent_map) if len(s.split()) >= 6]
        cur = [chosen[si] or sent_map[si][0] for si in want]
        def wording():
            """sentence -> its wording right now (rewritten neighbours are read as rewritten)."""
            return {sent_map[sj][0].strip(): chosen[sj] for sj in range(len(sent_map)) if chosen[sj]}
        sc = _ctx_scores([(sent_map[si][0], c) for si, c in zip(want, cur)], ctx, wording()) or [1.0] * len(cur)
        now = dict(zip(want, sc))
        short = {si for si in want if target and (chosen[si] is None or change(si, chosen[si]) < target)}
        todo = sorted([si for si in want if chosen[si] is None or now[si] >= PS.get("rescue_ai") or si in short],
                      key=lambda si: (si not in short or chosen[si] is not None, -now[si]))[:RESCUE_MAX]
        if todo:
            if rescue_pre is not None:                 # requested at the start of the call (see above)
                got = rescue_pre[1].result()
                got = None if got is None else dict(zip(rescue_pre[0], got))
                vs = None if got is None else [got.get(si) for si in todo]
            else:
                vs = _rescue_versions([sent_map[si][0] for si in todo], glossary,
                                      [ref[si] if ref[si] != sent_map[si][0] else None for si in todo],
                                      RESCUE_N + (2 if BOOST.get() else 0), progress, hi, hi,
                                      min(80, target) if target else 0)
            if vs is None:
                vs = [None] * len(todo)
                note += " (Advanced rewriting is busy right now: fewer sentences could be rewritten.)"
            print(f"[humanizer] rescue: {len(todo)} sentence(s) sent, "
                  f"{sum(1 for v in vs if v)} got versions", flush=True)
            cands = []
            for si, lst in zip(todo, vs):
                for v in lst or []:
                    ok, why = TP.strict_check(ref[si], v, glossary)
                    if ok and (si, ok) not in cands:
                        cands.append((si, ok))
                    elif DEBUG and why:
                        print(f"[debug] rescue s{si} rejected ({why}) | {v}", flush=True)
            n_all = len(cands)
            if cands:
                verdict = CL.assess([(ref[si], v) for si, v in cands])
                if DEBUG:
                    for (si, v), why in zip(cands, verdict):
                        if why:
                            print(f"[debug] rescue s{si} unclear ({why}) | {v}", flush=True)
                cands = [c for c, why in zip(cands, verdict) if not why]
            print(f"[humanizer] rescue: {n_all} version(s) passed the term/number checks, {len(cands)} passed "
                  f"the clarity checks", flush=True)
            if cands:
                # Joint choice: a sentence is read by the detector together with its neighbours, so the versions
                # are compared next to the neighbours' CURRENT wording, alternating between even and odd sentences
                # (neighbours never change in the same step) until nothing improves.
                for si, v in cands:
                    pool.setdefault(si, []).append(v)
                switched = set()
                for sweep in range(RESCUE_SWEEPS):
                    moved = 0
                    for parity in (0, 1):
                        part = [si for si in sorted(pool) if si % 2 == parity]
                        if not part:
                            continue
                        items = [(si, t) for si in part
                                 for t in dict.fromkeys([chosen[si] or sent_map[si][0]] + pool[si])]
                        sc = _ctx_scores([(sent_map[si][0], t) for si, t in items], ctx, wording())
                        if not sc:
                            break
                        score = dict(zip(items, sc))
                        for si in part:
                            cur_t = chosen[si] or sent_map[si][0]
                            now_ = score[(si, cur_t)]
                            if DEBUG and sweep == 0:
                                for v in pool[si]:
                                    print(f"[debug] rescue s{si} now {now_:.2f} -> {score[(si, v)]:.2f} | {v}",
                                          flush=True)
                            # with a vocabulary-change setting, versions that reach it come first, then the most human
                            opts = sorted(((0 if not target or change(si, v) >= target else 1), score[(si, v)], v)
                                          for v in pool[si] if v != cur_t)
                            if not opts:
                                continue
                            miss, d, v = opts[0]
                            reaches = target and not miss and (chosen[si] is None or change(si, chosen[si]) < target)
                            if d < now_ - 0.05 or (chosen[si] is None and d < now_) or (reaches and d <= now_ + 0.05):
                                chosen[si] = v
                                switched.add(si)
                                moved += 1
                    if not moved:
                        break
                rescued = len(switched)
            CL.report()
        if rescued:
            note += f" ✨ {rescued} sentence(s) rewritten in a natural academic style."
    _tm.append(("rescue", _time.time()))
    # Faithfulness check of what will be used. The checks so far are rules (numbers, terms, citations, negation ...)
    # and a similarity score; neither sees that "increased by 38% compared with the control" has become "the
    # control was 38% lower", that a method was left out, or that "but" turned into "so". An independent reading
    # of every rewritten sentence against the AUTHOR'S sentence does. A sentence that fails gets the next version
    # that passes, else the author's wording.
    if VERIFY and G.available():
        final = [si for si in range(len(sent_map)) if chosen[si] and chosen[si].strip() != ref[si].strip()]
        try:
            verdict = G.verify([(ref[si], chosen[si]) for si in final], progress, hi, hi) if final else []
            bad = [si for si, v in zip(final, verdict) if isinstance(v, str)]
            if DEBUG:
                for si, v in zip(final, verdict):
                    if isinstance(v, str):
                        print(f"[debug] unfaithful s{si} ({v}) | {chosen[si]}", flush=True)
            alts = []
            for si in bad:
                seen = {chosen[si].strip(), ref[si].strip()}
                for o in pool.get(si, []) + [before_rescue[si]] + ranked[si]:
                    if o and o.strip() not in seen and sum(1 for x, _ in alts if x == si) < VERIFY_ALTS:
                        seen.add(o.strip())
                        alts.append((si, o))
            second = G.verify([(ref[si], o) for si, o in alts], progress, hi, hi) if alts else []
            good = {}
            for (si, o), v in zip(alts, second):
                if v is True and si not in good:
                    good[si] = o
            for si in bad:
                chosen[si] = good.get(si)               # None = the author's sentence stays
            if bad:
                print(f"[humanizer] faithfulness check: {len(final)} rewritten sentence(s) read against the "
                      f"original, {len(bad)} changed the meaning ({len(good)} replaced by another version, "
                      f"{len(bad) - len(good)} kept in the author's wording)", flush=True)
                note += (f" 🔍 Meaning check: {len(bad)} rewrite(s) that changed what the sentence says were "
                         f"replaced ({len(bad) - len(good)} kept in your wording).")
        except G.Unavailable as err:
            print(f"[humanizer] faithfulness check skipped: {err}", flush=True)
    if CL.ENABLED:
        kept = sum(1 for si, (s, _, _) in enumerate(sent_map) if not chosen[si] and len(s.split()) >= 6)
        if kept and PS.get("select") == "strict":
            note += f" 🧹 {kept} sentence(s) kept in your wording (no clear rewrite found)."
    if qstat["n"]:
        import statistics as _st
        med = _st.median(qstat["sims"]) if qstat["sims"] else 0
        print(f"[humanizer] quality checks: {qstat['n']} versions | below meaning {MEANING.min_sim():.2f}: "
              f"{qstat['meaning']} (median similarity {med:.2f}) | not fluent: {qstat['fluency']} | "
              f"rejected in total: {meaning_dropped} | sentences saved by second chance: {qstat['second']}",
              flush=True)

    sentences = [chosen[si] or s for si, (s, _, _) in enumerate(sent_map)]
    changed = sum(1 for si, (s, _, _) in enumerate(sent_map) if sentences[si].strip() != s.strip())
    if target and not full_ai:
        hit = sum(1 for si, (s0, _, _) in enumerate(sent_map)
                  if sentences[si].strip() != s0.strip() and change(si, sentences[si]) >= target)
        note += (f" 🔤 Vocabulary change {target:.0f}%"
                 + (f", {extra_rounds} extra round(s)" if extra_rounds else "")
                 + f": {hit} of {len(sent_map)} sentences reach it.")
    if full_ai:
        hit = sum(1 for si, (s0, _, _) in enumerate(sent_map)
                  if sentences[si].strip() != s0.strip() and change(si, sentences[si]) >= target)
        note += (f" 🤖 100% AI mode: {k} rewritten version{'s' if k != 1 else ''} per sentence"
                 + (f", {extra_rounds} extra round(s)" if extra_rounds else "")
                 + (f"; {hit} of {len(sent_map)} sentences reach {target:.0f}% new wording." if target else ".")) \
            + f" ✍️ {changed} of {len(sent_map)} sentences rewritten."
    if changed < len(sent_map):
        print(f"[humanizer] kept {len(sent_map) - changed} sentences unchanged "
              f"(no safe version: {sum(1 for o in options if not o)})", flush=True)
    _tm.append(("faithfulness", _time.time()))
    print(f"[timing] {len(sent_map)} sentence(s): " + " | ".join(
        f"{n} {t - t0:.1f}s" for (_, t0), (n, t) in zip(_tm, _tm[1:])) + f" | total {_tm[-1][1] - _tm[0][1]:.1f}s",
        flush=True)
    if _bg is not None:
        _bg.shutdown(wait=False)
    return [" ".join(sentences[a:b]) for a, b in groups], note


# Final grammar pass: after ALL rewriting, Groq corrects grammar / spelling / punctuation only (HUM_GRAMMAR=0 = off).
GRAMMAR = os.getenv("HUM_GRAMMAR", "1") != "0"
GRAMMAR_MAX_EDIT = float(os.getenv("HUM_GRAMMAR_MAX_EDIT", "0.45"))   # max share of words Groq may change
# Smart Humanize sets this so its passes skip the grammar step; it runs grammar once on the final text instead.
GRAMMAR_DEFER = contextvars.ContextVar("grammar_defer", default=False)
# Set by the app when the automatic proofread follows the humanizing: that proofread IS the final check (grammar,
# clarity, wording, flow, consistency), so the language-model polish here is not run a second time and paid twice.
# The rule-based clean-up still runs.
PROOFREAD_FOLLOWS = contextvars.ContextVar("proofread_follows", default=False)


def _small_edit(a, b):
    """True if b differs from a by at most GRAMMAR_MAX_EDIT of the words (a correction, not a rewrite)."""
    import difflib
    wa, wb = a.split(), b.split()
    if not wa:
        return False
    sm = difflib.SequenceMatcher(None, wa, wb, autojunk=False)
    changed = sum(max(i2 - i1, j2 - j1) for op, i1, i2, j1, j2 in sm.get_opcodes() if op != "equal")
    return changed <= max(2, GRAMMAR_MAX_EDIT * len(wa))


# Final polish (default on): Groq edits whole paragraphs for grammar, technical academic wording and flow, instead of
# a grammar-only check. POLISH=0 -> grammar-only. A polished sentence must keep every term / number / citation,
# keep its meaning, change at most POLISH_MAX_EDIT of its words, and (POLISH_KEEP_HUMAN=1) must not turn a sentence
# the detector reads as human into one it reads as AI-written; otherwise the grammar-only correction is used.
POLISH = os.getenv("POLISH", "1") != "0"
POLISH_MAX_EDIT = float(os.getenv("POLISH_MAX_EDIT", "0.6"))
POLISH_KEEP_HUMAN = os.getenv("POLISH_KEEP_HUMAN", "1") != "0"
POLISH_CHANGED_ONLY = os.getenv("POLISH_CHANGED_ONLY", "1") != "0"   # 0 = polish every paragraph of the document


def _edit_share(a, b):
    import difflib
    wa, wb = a.split(), b.split()
    if not wa:
        return 1.0
    sm = difflib.SequenceMatcher(None, wa, wb, autojunk=False)
    return sum(max(i2 - i1, j2 - j1) for op, i1, i2, j1, j2 in sm.get_opcodes() if op != "equal") / len(wa)


def _polish(sentences, out, idx, groups, glossary, progress, lo, hi, orig_par, orig_sent, stats):
    """Groq repair + polish of out[idx]. Garbled sentences are REPAIRED to the meaning of the author's original
    (clarity wins over the AI score); clear sentences get the light polish (grammar, academic wording, flow) with
    the usual limits. A garbled sentence that cannot be repaired safely gets the original sentence back when it is
    known. Returns the indices that still need the grammar-only check."""
    ctx = [orig_par[i] or orig_sent[i] for i in idx]
    ctx = [c if c and c.strip() != out[i].strip() else None for c, i in zip(ctx, idx)]
    res = G.polish([out[i] for i in idx], [groups[i] for i in idx], glossary, progress, lo, hi, originals=ctx)
    edited, repaired, missing, broken = {}, {}, [], []

    def ctx_known(i):
        return bool(orig_sent[i] or orig_par[i])
    for i, (st, r) in zip(idx, res):
        if not r:
            missing.append(i)
            continue
        if r.strip() == out[i].strip():
            continue
        ref = orig_sent[i] or sentences[i]
        if st == "repaired":
            ok = TP.strict_accept(out[i], r, glossary, orig_sent[i]) or \
                (TP.strict_accept(orig_sent[i], r, glossary) if orig_sent[i] else None)
            if ok:
                repaired[i] = ok
            else:
                broken.append(i)
            continue
        ok = TP.strict_accept(out[i], r, glossary, orig_sent[i])
        if ok and _edit_share(out[i], ok) <= max(POLISH_MAX_EDIT, 3 / max(1, len(out[i].split()))):
            edited[i] = TP.basic_grammar(ok)
        elif ctx_known(i):                             # a deep correction: judge it as a repair
            ok = ok or (TP.strict_accept(orig_sent[i], r, glossary) if orig_sent[i] else None)
            if ok:
                repaired[i] = TP.basic_grammar(ok)
            else:
                missing.append(i)
        else:
            missing.append(i)
    # meaning: a repair must be at least as close to the original as the garbled sentence was
    if repaired:
        keys = list(repaired)
        refs = [orig_sent[i] or orig_par[i] or sentences[i] for i in keys]
        sim = MEANING.similarities([(r_, repaired[i]) for r_, i in zip(refs, keys)]
                                   + [(r_, out[i]) for r_, i in zip(refs, keys)])
        m = len(keys)
        for n_, i in enumerate(keys):
            exact = orig_sent[i] is not None
            if (exact and sim[n_] < MEANING.min_sim()) or (not exact and sim[n_] < sim[m + n_] - 0.02):
                repaired.pop(i)
                broken.append(i)
    if edited:                                         # light edits must keep the meaning ...
        keys = list(edited)
        for i, k in zip(keys, MEANING.keeps_meaning([(sentences[i], edited[i]) for i in keys])):
            if not k:
                edited.pop(i)
                missing.append(i)

    def in_context(i, text):
        """The sentence as the detector reads it: together with its neighbours in the paragraph."""
        prev = out[i - 1] if i > 0 and groups[i - 1] == groups[i] else ""
        nxt = out[i + 1] if i + 1 < len(out) and groups[i + 1] == groups[i] else ""
        return " ".join(x for x in (prev, text, nxt) if x)

    def more_ai(keys, new_of, margin=0.35):
        """Keys whose new wording turns a human-reading sentence into an AI-reading one."""
        sc = _detector_scores([in_context(i, new_of[i]) for i in keys] + [in_context(i, out[i]) for i in keys])
        m = len(keys)
        if DEBUG and sc:
            for n_, i in enumerate(keys):
                print(f"[debug] polish {sc[m + n_]:.2f} -> {sc[n_]:.2f} | {out[i]} => {new_of[i]}", flush=True)
        return [i for n_, i in enumerate(keys) if sc and (sc[n_] >= 0.5 > sc[m + n_] or sc[n_] > sc[m + n_] + margin)]
    if edited and POLISH_KEEP_HUMAN:                   # ... and must not undo the humanization
        for i in more_ai(list(edited), edited):
            edited.pop(i)
            missing.append(i)
    # Which of the sentences the editor called unclear really are? Measured against the author's sentence with the
    # clarity checks (defects, grammar, structure, meaning); a sentence that passes them is clear.
    clear = [i for i in list(repaired) + broken if orig_sent[i] and not TP.has_defect(out[i])]
    if clear and CL.ENABLED:
        clear = [i for i, why in zip(clear, CL.assess([(orig_sent[i], out[i]) for i in clear])) if not why]
    clear = set(clear)
    optional = set()
    if repaired and POLISH_KEEP_HUMAN:
        # A "repair" rewrites the sentence towards the author's original. That is right for a sentence that IS
        # unclear (clarity wins over the AI score), but the editor also calls clear, correctly humanized sentences
        # "repaired" and so puts the machine-like wording back. A sentence that passes the clarity checks against
        # the original is not broken: its repair counts as an edit and may not raise the AI score.
        optional = clear & set(repaired)
        for i in more_ai(sorted(optional), repaired, 0.05) if optional else []:
            repaired.pop(i)                            # the clear humanized sentence stays
    before = list(out)
    for i, t in list(edited.items()) + list(repaired.items()):
        out[i] = t
    if POLISH_KEEP_HUMAN:
        # the paragraph as a whole: several small edits together can read as machine-written again even when each
        # one passed on its own; then the optional ones (edits, and repairs of sentences that were clear) are undone
        undo = {}
        for i in list(edited) + [i for i in repaired if i in optional]:
            undo.setdefault(groups[i], []).append(i)
        if undo:
            members = {g: [k for k in range(len(out)) if groups[k] == g] for g in undo}
            sc = _detector_scores([" ".join(before[k] for k in members[g]) for g in undo]
                                  + [" ".join(out[k] for k in members[g]) for g in undo]) or []
            m = len(undo)
            for n_, g in enumerate(undo):
                if sc and (sc[m + n_] >= 0.5 > sc[n_] or sc[m + n_] > sc[n_] + 0.15):
                    for i in undo[g]:
                        out[i] = before[i]
                        edited.pop(i, None)
                        repaired.pop(i, None)
                        missing.append(i)
    restored = 0
    for i in broken:                                   # unclear and not safely repairable: the clear original
        if i in clear and POLISH_KEEP_HUMAN:           # (a clear humanized sentence is not thrown away)
            missing.append(i)
        elif orig_sent[i] and orig_sent[i].strip() != out[i].strip():
            out[i] = orig_sent[i]
            restored += 1
        else:
            missing.append(i)
    stats.update(edited=len(edited), repaired=len(repaired), restored=restored)
    return sorted(set(missing))


def grammar_fix_sentences(sentences, glossary=None, progress=None, lo=0.9, hi=0.95, groups=None,
                          orig_par=None, orig_sent=None):
    """Final correction of `sentences`. Returns (sentences, n_fixed, note).
    1. rule-based clean-up (doubled words, a/an, 'will opening', spacing, capitals, final full stop) - always runs;
    2. Groq final polish (POLISH=1, default): whole paragraphs (`groups` = paragraph key per sentence) are edited
       for grammar, technical academic wording and flow between sentences, with the checks described at POLISH;
       sentences whose polish was not accepted get the Groq grammar-only check (smallest correction);
    3. rules once more;
    4. LanguageTool (free, self-hosted) for the remaining mechanical errors. Every correction must keep every term,
       number and citation and invent nothing."""
    if not GRAMMAR or not sentences:
        return list(sentences), 0, ""
    out = [TP.basic_grammar(t) if len(t.split()) >= 3 else t for t in sentences]
    groups = list(groups) if groups is not None else list(range(len(sentences)))
    orig_par = list(orig_par) if orig_par is not None else [None] * len(sentences)
    orig_sent = list(orig_sent) if orig_sent is not None else [None] * len(sentences)
    orig_sent = [o or anchor_of(t) for o, t in zip(orig_sent, sentences)]     # Smart Humanize: original sentence
    if glossary is None and (any(orig_par) or any(orig_sent)):
        # protected terms come from the AUTHOR'S text, never from garbled rewritten phrases
        src = list(dict.fromkeys([p_ or s_ for p_, s_ in zip(orig_par, orig_sent) if p_ or s_]))
        glossary = TP.build_glossary(src) | TP.GLOSSARY.get()
    note, stats = "", dict(edited=0, repaired=0, restored=0)
    if G.available() and not PROOFREAD_FOLLOWS.get():
        glossary = glossary if glossary is not None else (TP.build_glossary(out) | TP.GLOSSARY.get())
        idx = [i for i, t in enumerate(out) if len(t.split()) >= 3]
        if POLISH_CHANGED_ONLY:
            # A paragraph the tool did not change is the author's own text: sending it to the language model costs
            # tokens for nothing and invites edits nobody asked for. Only paragraphs with at least one rewritten
            # sentence are polished (as a whole, for flow); the rule-based clean-up and LanguageTool still see all.
            def unchanged(i):
                t = sentences[i].strip()
                return bool((orig_sent[i] and orig_sent[i].strip() == t) or (orig_par[i] and t in orig_par[i]))
            touched = {groups[i] for i in range(len(sentences)) if not unchanged(i)}
            skipped = sum(1 for i in idx if groups[i] not in touched)
            idx = [i for i in idx if groups[i] in touched]
            if skipped:
                print(f"[humanizer] final polish: {skipped} sentence(s) in unchanged paragraphs not sent "
                      f"({len(idx)} sent)", flush=True)
        todo = idx
        if POLISH and idx:
            try:
                todo = _polish(sentences, out, idx, groups, glossary, progress, lo, lo + (hi - lo) * 0.7,
                               orig_par, orig_sent, stats)
            except G.Unavailable as err:               # fall back to the grammar-only check
                print(f"[humanizer] Groq final polish skipped, grammar-only check instead: {err}", flush=True)
        try:
            if todo:
                res = G.grammar([out[i] for i in todo], glossary, progress, lo + (hi - lo) * 0.7, hi)
                for i, r in zip(todo, res):
                    s0 = out[i]
                    if not r or r.strip() == s0.strip():
                        continue
                    ok = TP.strict_accept(s0, r, glossary, orig_sent[i])
                    if ok and _small_edit(s0, ok):
                        out[i] = ok
        except G.Unavailable as err:
            print(f"[humanizer] Groq grammar check skipped: {err}", flush=True)
            note = " (Groq final polish unavailable right now: rule-based corrections applied.)"
    # 4. LanguageTool (free, open source): last mechanical check - agreement, confused words, a/an, punctuation
    if LT.MODE != "off":
        glossary = glossary if glossary is not None else (TP.build_glossary(out) | TP.GLOSSARY.get())
        idx = [i for i, t in enumerate(out) if len(t.split()) >= 3]
        n_lt = 0
        for i, r in zip(idx, LT.correct([out[i] for i in idx])):
            if r.strip() == out[i].strip():
                continue
            ok = TP.strict_accept(out[i], r, glossary)
            if ok and _small_edit(out[i], ok):
                out[i], n_lt = ok, n_lt + 1
        if n_lt:
            note += f" 🔎 LanguageTool fixed {n_lt} sentence(s)."
    # last safety net (clarity gate): a sentence that still reads as broken, ungrammatical or awkward English
    # compared with the author's original sentence gets the original sentence back
    refs = [orig_sent[i] or anchor_of(sentences[i]) for i in range(len(out))]
    # the author's spelling (British / American) is kept
    vocab = {w.lower() for t in (list(filter(None, refs)) + list(filter(None, orig_par)) + list(sentences))
             for w in re.findall(r"[A-Za-z]{4,}", t)}
    for i in range(len(out)):
        if out[i] != sentences[i] or (refs[i] and out[i] != refs[i]):
            out[i] = CL.match_spelling(out[i], vocab)
    known = [i for i in range(len(out)) if refs[i] and refs[i].strip() != out[i].strip() and len(out[i].split()) >= 4]
    if known and CL.ENABLED and PS.get("final") == "restore":
        failed = [(i, why) for i, why in zip(known, CL.assess([(refs[i], out[i]) for i in known])) if why]
        lexv = LX.variants([refs[i] for i, _ in failed], glossary or set(), n=LEX_N) if failed and LX.available() \
            else [[] for _ in failed]
        cands = [(i, v) for (i, _), vs in zip(failed, lexv) for v in vs]
        good = {}
        for (i, v), why in zip(cands, CL.assess([(refs[i], v) for i, v in cands]) if cands else []):
            if not why and i not in good:
                good[i] = v
        for i, why in failed:
            print(f"[clarity] {'new vocabulary' if i in good else 'restored original'} ({why}): "
                  f"{out[i][:90]!r}", flush=True)
            out[i] = good.get(i, refs[i])
            stats["revocab" if i in good else "restored"] = stats.get("revocab" if i in good else "restored", 0) + 1
        CL.report()
    elif known:
        sc = FL.scores([out[i] for i in known] + [refs[i] for i in known])
        m = len(known)
        for k, i in enumerate(known):
            broken = sc[k] < FL.MIN or (FL.doubled_word(out[i]) and not FL.doubled_word(refs[i]))
            if broken and (sc[m + k] >= sc[k] + 0.15 or FL.doubled_word(out[i])):
                out[i] = refs[i]
                stats["restored"] = stats.get("restored", 0) + 1
    out = [TP.match_ending(a, b) for a, b in zip(sentences, out)]   # fragments stay fragments (no added full stop)
    n = sum(1 for a, b in zip(sentences, out) if a.strip() != b.strip())
    parts = [f"{stats[k]} {lbl}" for k, lbl in (("revocab", "unclear sentence(s) rewritten with suitable vocabulary"),
                                                 ("repaired", "unclear sentence(s) repaired to the original meaning"),
                                                 ("restored", "restored to your original wording"),
                                                 ("edited", "edited for wording and flow")) if stats.get(k)]
    head = (f" 📝 Final polish (clarity, grammar, academic wording, flow): {n} sentence(s) improved"
            + (f" ({'; '.join(parts)})." if parts else ".")) if n else " 📝 Final polish: no changes needed."
    return out, n, head + note


def originals_for(new_sents, orig_paragraph):
    """Per sentence of a rewritten paragraph: (original paragraph, original sentence or None). The sentence is
    known only when the original paragraph has the same number of sentences (one-to-one rewrite)."""
    if not orig_paragraph:
        return [(None, None)] * len(new_sents)
    os_ = split_sentences(orig_paragraph) or [orig_paragraph]
    same = len(os_) == len(new_sents)
    return [(orig_paragraph, os_[k] if same else None) for k in range(len(new_sents))]


def grammar_pass(paragraphs, progress=None, lo=0.9, hi=0.95, originals=None):
    """Final polish of whole paragraphs, sentence by sentence. `originals` (same order as `paragraphs`) are the
    author's original paragraphs, used to repair garbled sentences. Returns (paragraphs, n_fixed, note)."""
    groups, sents, op, os_ = [], [], [], []
    if originals is not None and len(originals) != len(paragraphs):
        originals = None
    for k, p in enumerate(paragraphs):
        ss = split_sentences(p) or [p]
        groups.append((len(sents), len(sents) + len(ss), len(ss) > 1 or ss[0] != p))
        sents.extend(ss)
        for a_, b_ in originals_for(ss, originals[k] if originals is not None else None):
            op.append(a_)
            os_.append(b_)
    para_of = [k for k, (a, b, _) in enumerate(groups) for _ in range(a, b)]
    fixed, n, note = grammar_fix_sentences(sents, None, progress, lo, hi, groups=para_of, orig_par=op, orig_sent=os_)
    if not n:
        return list(paragraphs), 0, note
    out = []
    for (a, b, split), p in zip(groups, paragraphs):
        same = all(fixed[k] == sents[k] for k in range(a, b))
        out.append(p if same else " ".join(fixed[a:b]))
    return out, n, note


def paraphrase_paragraphs(texts, temperature, progress=None, lo=0.15, hi=0.85, mode="Auto"):
    """Returns (paragraphs, engine_note). Parrot rewrites; Groq (if GROQ_API_KEY is set) checks and repairs
    Parrot's academic/technical wording; the local guard has the final say. Finally (unless Smart Humanize
    defers it to its own end) Groq corrects the grammar of the finished paragraphs."""
    out, note = _paraphrase_local(texts, temperature, progress, lo, hi, mode)
    if GRAMMAR and not GRAMMAR_DEFER.get():
        out, _, gnote = grammar_pass(out, progress, hi, min(0.97, hi + 0.08), originals=list(texts))
        note += gnote
    return out, note

def _preview(pairs, limit=10):
    blocks = []
    for i, (before, after) in enumerate(pairs[:limit], 1):
        blocks.append(
            f"#### Change {i}\n**Original:** {before}\n\n**Paraphrased:** {after}"
        )
    return "\n\n---\n\n".join(blocks) if blocks else "No textual changes were generated."


# ---------------------------------------------------------------------------
# Handlers
# ---------------------------------------------------------------------------
def _docx_core(file_path, temperature, min_words, mode, progress):
    if not file_path:
        raise gr.Error("Please upload a .docx file.")
    source_path = Path(file_path)
    if source_path.suffix.lower() != ".docx":
        raise gr.Error("Only .docx Word documents are supported.")

    progress(0.05, desc="Opening Word document")
    doc = Document(str(source_path))
    TP.set_document([p.text for p in collect_paragraphs(doc)])   # constructs / vocabulary of the whole document

    import detector as _D
    targets, originals = [], []
    in_refs = False
    for paragraph in collect_paragraphs(doc):
        text = paragraph.text
        if _D.REF_HEAD.match(text):                    # reference list: never rewritten
            in_refs = True
            continue
        if in_refs or (_D.REF_ENTRY.match(text) and _D.REF_HINT.search(text)):
            continue
        if paragraph._p.xpath(".//*[local-name()='oMath' or local-name()='oMathPara']"):
            continue                                   # inline equations: rewriting would move them
        if should_process(text, min_words):
            targets.append(paragraph)
            originals.append(text)

    if not originals:
        raise gr.Error("No eligible prose paragraphs were found in the document.")

    revised, engine_note = paraphrase_paragraphs(originals, temperature, progress, mode=mode)

    progress(0.9, desc="Injecting revised text into the original layout")
    changed, pairs, final_after = 0, [], []
    for paragraph, before, after in zip(targets, originals, revised):
        after = (after or "").strip()
        if after and after != before.strip():
            replace_text_preserving_paragraph(paragraph, after)
            changed += 1
            pairs.append((before, after))
            final_after.append(after)
        else:
            final_after.append(before)

    output_path = _save_output(doc, source_path.stem)
    progress(1.0, desc="Completed")

    summary = (
        f"✅ **Completed** — {changed} of {len(originals)} eligible paragraphs rewritten. "
        f"Layout, styles, tables, images and headers/footers were kept in place.\n\n{engine_note}"
    )
    scores = score_panel("\n".join(originals), "\n".join(final_after))
    return output_path, download_button(output_path), summary, scores, _preview(pairs)


def _text_core(text, temperature, min_words, mode, progress, max_words=None):
    text = (text or "").strip()
    if not text:
        raise gr.Error("Please paste some text to paraphrase.")
    n_words = len(text.split())
    if n_words > (max_words or MAX_PASTE_WORDS):
        raise gr.Error(
            f"Pasted text has {n_words} words. Please keep it under {max_words or MAX_PASTE_WORDS} words, "
            "or upload it as a .docx instead."
        )

    progress(0.05, desc="Reading pasted text")
    parts = TP.parse_paste(text)
    paragraphs = [p for p, _ in parts]
    TP.set_document(paragraphs)
    idx = [i for i, p in enumerate(paragraphs) if should_process(p, min_words)]
    if not idx:
        raise gr.Error(f"No paragraph has at least {int(min_words)} words of prose. Lower the minimum and try again.")

    revised, engine_note = paraphrase_paragraphs([paragraphs[i] for i in idx], temperature, progress, mode=mode)
    result = list(paragraphs)
    pairs = []
    for i, after in zip(idx, revised):
        after = (after or "").strip()
        if after and after != paragraphs[i]:
            result[i] = after
            pairs.append((paragraphs[i], after))

    output_text = TP.join_paste([(p, sep) for p, (_, sep) in zip(result, parts)])

    progress(0.95, desc="Preparing download")
    doc = Document()
    for p in result:
        if p:
            doc.add_paragraph(p)
    output_path = _save_output(doc, "pasted_text")
    progress(1.0, desc="Completed")

    summary = f"✅ **Completed** — {len(pairs)} of {len(idx)} eligible paragraphs rewritten ({n_words} words in).\n\n{engine_note}"
    scores = score_panel(
        "\n".join(paragraphs[i] for i in idx),
        "\n".join(result[i] for i in idx),
    )
    return output_text, output_path, download_button(output_path), summary, scores


# Single branded progress panel (replaces Gradio's per-component loaders)
# ---------------------------------------------------------------------------
class Cancelled(Exception):
    """The customer pressed Stop & reset: the job ends at its next progress step."""


class _Reporter:
    def __init__(self):
        self.frac, self.desc, self.start = 0.0, "Starting…", time.time()
        self.cancel = threading.Event()

    def __call__(self, frac, desc=None):
        if self.cancel.is_set():
            raise Cancelled("Stopped by you.")
        self.frac = max(self.frac, min(1.0, float(frac)))
        if desc:
            self.desc = desc


def _md2html(md):
    html = re.sub(r"\*\*(.+?)\*\*", r"<b>\1</b>", md)
    return html.replace("\n\n", "<br><br>").replace("\n", "<br>")


def status_idle(msg):
    return f'<div class="cs-status cs-idle">{_md2html(PV.clean(msg))}</div>'


def status_done(msg):
    return f'<div class="cs-status cs-done">{_md2html(PV.clean(msg))}</div>'


def status_error(msg):
    return f'<div class="cs-status cs-err">⚠️ {PV.clean(str(msg))}</div>'


def status_running(rep):
    pct = round(rep.frac * 100)
    secs = int(time.time() - rep.start)
    return f"""
    <div class="cs-status cs-run">
      <div class="cs-spin-logo"></div>
      <div class="cs-st-text">{PV.clean(rep.desc)}</div>
      <div class="cs-bar"><div style="width:{pct}%"></div></div>
      <div class="cs-st-meta">{pct}% · {secs}s</div>
    </div>"""


# ---------------------------------------------------------------------------
# Background jobs that survive a dropped connection
#   A job runs in its own thread, independent of the browser connection. If the connection drops (mobile data,
#   switching apps, Wi-Fi hiccup) the job keeps going; when the page reconnects and the same request is sent again
#   (the page does this automatically), the running job is picked up again instead of starting over, and a
#   finished job's result is shown straight away. Finished jobs are kept for JOB_KEEP seconds.
# ---------------------------------------------------------------------------
JOB_KEEP = int(os.getenv("JOB_KEEP_SECONDS", str(3 * 3600)))
_JOBS = {}
_JOBS_LOCK = threading.Lock()


class Job:
    def __init__(self, key, owner=None):
        self.key, self.rep, self.box = key, _Reporter(), {}
        self.owner = owner
        self.id = secrets.token_hex(4)
        self.meta, self.extra = {}, {}
        self.done = threading.Event()
        self.delivered = False
        self.created, self.finished_at = time.time(), None


def _purge_jobs():
    now = time.time()
    for k in [k for k, j in _JOBS.items() if j.finished_at and now - j.finished_at > JOB_KEEP]:
        _JOBS.pop(k, None)


def find_job(key):
    """The running or recently finished (successful) job for `key`, or None."""
    if not key:
        return None
    with _JOBS_LOCK:
        _purge_jobs()
        return _JOBS.get(key)


def cancel_jobs(owner):
    """Stop every running job of this visitor (Stop & reset). Returns how many were stopped."""
    if not owner:
        return 0
    n = 0
    with _JOBS_LOCK:
        for k, j in list(_JOBS.items()):
            if j.owner == owner and not j.done.is_set():
                j.rep.cancel.set()
                _JOBS.pop(k, None)                       # a new run starts afresh instead of re-attaching
                n += 1
    return n


def start_job(key, core, args, on_finish=None, owner=None):
    """Run core(*args, rep) in the background. on_finish(job) runs once when it ends (even if nobody is
    watching any more) — used for logging, usage counting and the "ready" push notification."""
    job = Job(key, owner)
    with _JOBS_LOCK:
        _purge_jobs()
        if key:
            _JOBS[key] = job
    job.rep.meta = job.meta

    def work():
        try:
            job.box["result"] = core(*args, job.rep)
        except Cancelled:
            job.box["error"] = gr.Error("Stopped. Nothing was charged; start again whenever you like.")
        except Exception as err:
            job.box["error"] = err
        job.finished_at = time.time()
        if "error" in job.box and key:
            with _JOBS_LOCK:                              # a failed job is not reused: a retry starts afresh
                if _JOBS.get(key) is job:
                    _JOBS.pop(key, None)
        if on_finish is not None:
            try:
                on_finish(job)
            except Exception as err:
                print(f"[job] finish step failed: {err}", flush=True)
        job.done.set()

    # copy_context: the worker thread keeps the Gradio request context, so ZeroGPU knows WHO the
    # visitor is (their own quota and queue priority) instead of treating every run as anonymous
    ctx = contextvars.copy_context()
    threading.Thread(target=ctx.run, args=(work,), daemon=True).start()
    return job


def follow(job, n_outputs, status_index):
    """Stream the status panel of `job` until it ends, then yield its result (or raise its error)."""
    keep = [gr.update()] * n_outputs
    while not job.done.is_set():
        frame = list(keep)
        frame[status_index] = status_running(job.rep)
        yield tuple(frame)
        job.done.wait(timeout=0.6)
    if "error" in job.box:
        err = job.box["error"]
        frame = list(keep)
        frame[status_index] = status_error(str(err) or "Something went wrong.")
        yield tuple(frame)
        if isinstance(err, gr.Error):
            raise err
        raise gr.Error(f"Processing failed: {err}")
    yield job.box["result"]


def _run_live(core, args, n_outputs, status_index):
    """Run `core` in a thread and stream only the status panel until it finishes."""
    yield from follow(start_job(None, core, args), n_outputs, status_index)


def _save_output(doc, stem, suffix="_paraphrased"):
    """Save to a unique folder with a clean, readable file name."""
    safe = re.sub(r"[^A-Za-z0-9_-]+", "_", stem).strip("_") or "document"
    folder = Path(tempfile.mkdtemp(dir=OUT_DIR))
    path = folder / f"{safe}{suffix}.docx"
    doc.save(str(path))
    return str(path)


_MIME = {".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
         ".pdf": "application/pdf"}


def download_button(path):
    """Direct download link embedded in the page (works even when the file widget's link is
    blocked, e.g. inside iframes / custom domains)."""
    if not path or not Path(path).exists():
        return ""
    data = base64.b64encode(Path(path).read_bytes()).decode()
    mime = _MIME.get(Path(path).suffix.lower(), "application/octet-stream")
    name = Path(path).name
    return (f'<a class="cs-dl" download="{name}" href="data:{mime};base64,{data}">'
            f'⬇️ Download {name}</a>')


def count_words(text, limit=None):
    limit = limit or MAX_PASTE_WORDS
    n = len((text or "").split())
    color = BRAND["red"] if n > limit else BRAND["grey"]
    return f"<div class='cs-count' style='color:{color}'>{n:,} / {limit:,} words</div>"