"""
Fast engine: a Word document of any length humanized in about a minute.

The Smart engine rewrites sentence by sentence: Parrot versions, about ten checks per version, several passes one
after another; its time grows with the length of the document. This engine works on the whole document at once:

  1. The AI detector reads the whole document once (GPU, batched).
  2. Every flagged sentence goes to SEVERAL language models in parallel batches (FAST_GENERATORS; FAST_PARALLEL
     requests at once per model), with the author's sentence as the meaning reference; each model writes several
     human-style versions. Measured on 30 AI-written sentences (devtools/bench_generators.py), versions the detector
     reads as human (< 0.5): gpt-oss-120b 2/30, Qwen 3.8 9/30, Gemini 3.5 Flash-Lite 14/30, the two pooled 16/30.
  3. Each version must keep every term, number, citation and symbol (textparse.guard) and the meaning (sentence
     embeddings, GPU). The detector reads the surviving versions IN CONTEXT (with their neighbours) and the most
     human one is used, but only if it reads as more human than what the sentence has now.
  4. The document is scored again. Sentences that are still flagged get another round with a stronger rewrite
     (more new wording), until the score is 0 or the time budget (FAST_BUDGET seconds) is used up.
  The version with the lowest score is kept. Only the text of each sentence is replaced, so formatting, citation
  fields, equations, tables and figures stay as they were.

Space variables: FAST_BUDGET (50 s), FAST_PARALLEL (16 requests at once per model), FAST_ROUNDS (4),
FAST_VERSIONS (2 per model), FAST_GENERATORS ("provider:model,..."; Groq only by default; other providers only if
named here).
"""
import io
import os
import time
from concurrent.futures import FIRST_EXCEPTION, ThreadPoolExecutor, wait

import requests

import detector as D
import docx_report as DX
import fluency as FL
import groqai as G
import humanizer as H
import meaning as MEANING
import pubrules as PR
import smart as S
import textparse as TP

# 0% first: rounds continue until the document is at 0% on our detector (each round rewriting more strongly), until
# STALL rounds in a row bring no improvement, or until FAST_BUDGET seconds (a safety cap; about a minute for 2,000
# words, a few minutes for a thesis on a 250k-tokens-a-minute Groq key). Requests still running at the cap are dropped.
BUDGET = float(os.getenv("FAST_BUDGET", "600"))
ROUND_LM_MAX = float(os.getenv("FAST_ROUND_LM_MAX", "90"))   # seconds one round may wait for the language models
STALL = int(os.getenv("FAST_STALL", "3"))
CAP_PER_1K = float(os.getenv("FAST_CAP_PER_1K_WORDS", "0.60"))   # US$ per 1,000 words a job may spend at most
PARALLEL = int(os.getenv("FAST_PARALLEL", "16"))
# sentences per request: the instructions (~1,500 tokens) go with every request, so larger batches cost far fewer
# tokens per sentence (they were most of the tokens sent)
BATCH = int(os.getenv("FAST_BATCH", "24"))
ALWAYS_ROUNDS = int(os.getenv("FAST_ALWAYS_ROUNDS", "1"))   # rounds in which every flagged sentence is replaced
ROUNDS = int(os.getenv("FAST_ROUNDS", "10"))
VERSIONS = int(os.getenv("FAST_VERSIONS", "2"))           # per model (4 models: 8 versions per sentence)
# round -> (versions per sentence, share of new wording asked for (%), sentences above this score are rewritten)
# FAST_CHANGES: the new-wording share asked for in each round (measured on 4 AI paragraphs with the 3-model detector:
# "0,40,60,70" needed 4 rounds for 0%; the first, gentle round only took the score from 100% to 79%)
_CHG = [int(x) for x in os.getenv("FAST_CHANGES", "50,60,70,75").split(",") if x.strip()]
PLAN = [(VERSIONS + min(k, 2), c, thr) for k, (c, thr) in enumerate(zip(_CHG, (0.50, 0.40, 0.30, 0.25)))]
# rounds after the plan: the sentences left are the stubborn ones, so more versions and the strongest rewrite
PLAN += [(VERSIONS + 3, max(_CHG[-1], 80), 0.25)] * max(0, ROUNDS - len(PLAN))

# Long documents (a 40,000+ word thesis) get limits that grow with their length, so they too reach 0%: a flat
# 10-minute cap and 90-second rounds dropped most of a thesis's flagged sentences in each round and stopped it early.
BUDGET_PER_1K = float(os.getenv("FAST_BUDGET_PER_1K_WORDS", "30"))   # seconds per 1,000 words (22,000 words: ~4 min)
ROUND_LM_PER_SENT = float(os.getenv("FAST_ROUND_LM_PER_SENTENCE", "0.08"))   # a round's wait per flagged sentence
ROUNDS_PER_10K = int(os.getenv("FAST_ROUNDS_PER_10K_WORDS", "2"))     # extra rounds per 10,000 words


def budget_for(words):
    """The job's time cap: FAST_BUDGET, or more for a long document."""
    return max(BUDGET, BUDGET_PER_1K * max(0, int(words or 0)) / 1000)


def rounds_for(words):
    return ROUNDS + ROUNDS_PER_10K * (max(0, int(words or 0)) // 10000)


def _plan(rounds):
    return PLAN[:rounds] + [PLAN[-1]] * max(0, rounds - len(PLAN))


def _stock(s):
    """The sentence uses a stock AI word or phrase ("plays a crucial role", "Furthermore", "It is important to note").
    Such a sentence is rewritten even when the detector no longer flags it: once its neighbours were rewritten the
    detector passed it, but a reader (or another checker) still sees it."""
    return bool(PR._AI_WORD_RE.search(s or "") or any(r.search(s or "") for r in PR._AI_PHRASE_RE))


def _jobs(blocks, thr, min_words=5, also=None, around=None):
    """(block, sub-span, current sentence text) for every sentence at or above `thr` (or with a stock AI phrase, or
    picked by `also`, e.g. the sentences an uploaded checker report highlights) that can be edited safely."""
    jobs, locked = [], 0
    for b in blocks:
        if b.kind != "para" or b.el is None:
            continue
        for sp, s, p in zip(b.cspans, b.sents, b.scores):
            picked = bool(also and also(s))            # asked for every sentence (the caller records the matches)
            if not sp or (p < thr and not _stock(s) and not picked) or len(s.split()) < min_words:
                continue
            sub = S.safe_subspan(b.el, *sp)
            # only the whole sentence, or its beginning (a sentence ending in a citation field or superscript):
            # a rewrite spliced into the middle of a sentence read "It is worth noting that Performance drops"
            if sub and b.raw[sp[0]:sub[0]].strip() == "" and sub == tuple(sp):
                jobs.append((b, sub, b.raw[sub[0]:sub[1]]))
                continue
            segs = _segments(b, *sp) if (AROUND and around is not None) else None
            if segs:                                   # fields inside: rewrite around them (markers)
                text, refs = _tokenized(b, segs)
                around[(id(b), sp[0])] = (segs, refs)
                jobs.append((b, tuple(sp), text))
            elif sub and b.raw[sp[0]:sub[0]].strip() == "":
                jobs.append((b, sub, b.raw[sub[0]:sub[1]]))
            else:
                locked += 1
    return jobs, locked


# ---------------------------------------------------------------------------------------------------- around fields
# Sentences with a citation field (Zotero / Mendeley / EndNote), a link or a superscript citation in the middle used to
# be left as written ("locked"): in a thesis that was a third of the flagged sentences, and a checker kept flagging
# them. Now the locked parts become markers ([REF1], [REF2] ...) for the language model; a version is used only if it
# keeps every marker once, in the same order, and puts words only where the sentence has plain text to hold them. Its
# words go into the plain text around the untouched fields, so the citations stay live (and are never retyped).
AROUND = os.getenv("FAST_AROUND_FIELDS", "1") != "0"
_TOKEN = "[REF{}]"
_TOKEN_RE = __import__("re").compile(r"\[REF(\d+)\]")


def _segments(b, a, e):
    """The sentence at [a, e) of block b as [("text", s, t) | ("lock", s, t)] (paragraph offsets), or None when it holds
    an equation or has too little plain text to rewrite."""
    items = S._run_map(b.el)
    if any(r is None and a <= s < e for r, s, _, _ in items):
        return None
    segs = []
    for r, s, t, safe in items:
        if r is None or t <= a or s >= e or t <= s:
            continue
        kind = "text" if safe else "lock"
        s, t = max(a, s), min(e, t)
        if segs and segs[-1][0] == kind and segs[-1][2] == s:
            segs[-1] = (kind, segs[-1][1], t)
        else:
            segs.append((kind, s, t))
    if not any(k == "lock" for k, _, _ in segs):
        return None
    words = sum(len(b.raw[s:t].split()) for k, s, t in segs if k == "text")
    return segs if words >= 5 else None


def _tokenized(b, segs):
    """The sentence with each locked part replaced by its marker, and the markers' real text."""
    out, refs = [], []
    for k, s, t in segs:
        if k == "lock":
            refs.append(b.raw[s:t])
            out.append(_TOKEN.format(len(refs)))
        else:
            out.append(b.raw[s:t])
    return "".join(out), refs


def _fits(segs, version):
    """Words of a version for each plain-text part of the sentence (None: the version cannot be laid out around the
    fields: a marker missing, repeated or moved, or words where the sentence has no plain text)."""
    nums = [int(m) for m in _TOKEN_RE.findall(version)]
    locks = sum(1 for k, _, _ in segs if k == "lock")
    if nums != list(range(1, locks + 1)):
        return None
    pieces = _TOKEN_RE.split(version)[::2]              # text before the first marker, between markers, after the last
    texts, i = [], 0
    for k, s, t in segs:                                # each lock sits between two plain-text slots (maybe empty)
        if k == "lock":
            i += 1
            continue
        texts.append((i, s, t))
    out = []
    for slot, piece in enumerate(pieces):
        span = next(((s, t) for i_, s, t in texts if i_ == slot), None)
        if span is None:
            if piece.strip():
                return None                            # words where the sentence has no plain text to hold them
            continue
        out.append((span, piece))
    return out


def _replace_around(b, segs, version):
    """Write a version into the plain text around the fields (right to left, so offsets stay valid)."""
    parts = _fits(segs, version)
    if parts is None:
        return False
    for (s, t), piece in sorted(parts, key=lambda x: -x[0][0]):
        if not S.replace_span(b.el, s, t, piece):
            return False
    return True


def _tok(text, refs):
    """A sentence with its citations' real text -> with markers (the author's sentence for the language model)."""
    for i, r in enumerate(refs, 1):
        text = text.replace(r, _TOKEN.format(i), 1) if r else text
    return text


def _detok(text, refs):
    """A sentence with markers -> with the citations' real text."""
    return _TOKEN_RE.sub(lambda m: refs[int(m.group(1)) - 1] if int(m.group(1)) <= len(refs) else m.group(0), text)


def _context(blocks):
    ctx = {}
    for b in blocks:
        if b.kind == "para":
            for i, t in enumerate(b.sents):
                ctx[t.strip()] = (b.sents[i - 1] if i else "", b.sents[i + 1] if i + 1 < len(b.sents) else "")
    return ctx


def _recase(anchor, v, lower):
    """The word that opened the author's sentence keeps its capital only at the start: "relies heavily on Data
    protection legislation" -> "data protection legislation". Only for words the document also writes in lower case
    (`lower`), so names ("India", "Kumar") keep theirs."""
    import re
    m = re.match(r"\s*([A-Z][a-z]+)\b", anchor or "")
    if not v or not m or m.group(1).lower() not in lower:
        return v
    w = m.group(1)
    return v[:1] + re.sub(rf"(?<=[a-z,;:] ){w}\b", w.lower(), v[1:])


def _fit(cur, v):
    """A rewrite of a sentence's beginning (the rest is a citation field etc.) must not end the sentence early."""
    if v and cur and cur.rstrip()[-1:] not in ".!?" and v.rstrip()[-1:] in ".!?":
        return v.rstrip()[:-1]
    return v


_URLS = {"groq": ("https://api.groq.com/openai/v1/chat/completions", "GROQ_API_KEY"),
         "gemini": ("https://generativelanguage.googleapis.com/v1beta/openai/chat/completions", "GEMINI_API_KEY"),
         "anthropic": ("https://api.anthropic.com/v1/chat/completions", "ANTHROPIC_API_KEY")}
GENERATORS = [g.strip() for g in os.getenv(
    "FAST_GENERATORS",
    # Groq only: four models, each with its own limit (250,000 tokens a minute per model on the paid key), so about
    # a million tokens a minute together. (With two models a 14,600-word document had only 11 of 688 sentences
    # rewritten within the minute.)
    "groq:qwen/qwen3.8-27b,groq:qwen/qwen3.6-27b,groq:openai/gpt-oss-20b,groq:openai/gpt-oss-120b").split(",")
    if g.strip()]
FALLBACK = os.getenv("FAST_FALLBACK", "groq:openai/gpt-oss-20b")
# sentences per request by provider: Gemini keys have a low requests-per-minute limit (about 20 on this key), so it
# gets fewer, larger requests; a batch it refuses goes to the fallback model
BATCH_BY = {"gemini": int(os.getenv("FAST_BATCH_GEMINI", "40"))}


def _size(gen):
    return BATCH_BY.get(gen.split(":", 1)[0], BATCH)


def _key(gen):
    prov = gen.split(":", 1)[0]
    if prov == "groq":
        return G.KEY
    return os.getenv(_URLS.get(prov, ("", ""))[1], "").strip()


def _live_generators():
    gens = [g for g in GENERATORS if g.split(":", 1)[0] in _URLS and _key(g)]
    return gens or ([FALLBACK] if _key(FALLBACK) else [])


def _system(n):
    tok = G.FAITHFUL.set(True)
    try:
        return G._variants_system(n)
    finally:
        G.FAITHFUL.reset(tok)


def _request(gen, system, user, max_tokens, deadline=None):
    """One chat request to an OpenAI-compatible endpoint; two retries on rate limits / server errors."""
    prov, model = gen.split(":", 1)
    url = _URLS[prov][0]
    body = {"model": model, "max_tokens": max_tokens,
            "messages": [{"role": "system", "content": system}, {"role": "user", "content": user}]}
    if prov != "anthropic":
        body["temperature"] = 0.7
    if "gpt-oss" in model:
        body["reasoning_effort"] = "low"
    elif "qwen" in model:
        body["reasoning_effort"] = "none"
    for attempt in range(3):
        try:
            left = 60 if deadline is None else deadline - time.time()
            if left < 2:
                raise RuntimeError("deadline reached")
            r = requests.post(url, headers={"Authorization": f"Bearer {_key(gen)}", "Content-Type": "application/json"},
                              json=body, timeout=min(60, left))
        except requests.RequestException as err:
            last = str(err)
        else:
            if r.ok:
                d = r.json()
                u = d.get("usage") or {}
                G._account("variants", model, {"prompt_tokens": u.get("prompt_tokens", 0),
                                               "completion_tokens": u.get("completion_tokens", 0)})
                return d["choices"][0]["message"]["content"]
            last = f"{r.status_code} {r.text[:160]}"
            if r.status_code not in (429, 500, 502, 503, 504):
                break
        if deadline is not None and time.time() + 1.5 * (attempt + 1) + 3 > deadline:
            break
        time.sleep(1.5 * (attempt + 1))
    raise RuntimeError(f"{gen}: {last}")


def _batch(gen, items, glossary, n, min_change, lower=frozenset(), deadline=None):
    """Versions of a batch of (current sentence, author's sentence) from one model, each already checked for terms,
    numbers, citations and symbols (the check runs here, in the worker thread, while other requests are waiting)."""
    lines = [f"{i}. ORIGINAL: {a}\n   SENTENCE: {t}" if a and a != t else f"{i}. SENTENCE: {t}"
             for i, (t, a) in enumerate(items, 1)]
    terms = G._surface_terms(" ".join(t for t, _ in items), glossary)
    user = "PROTECTED TERMS: " + ("; ".join(terms) if terms else "(none)") + "\n\n" + "\n".join(lines)
    if min_change:
        user += (f"\n\nVOCABULARY CHANGE: in every version, at least {int(min_change)}% of the words must differ "
                 "from the sentence: use different (correct, common) vocabulary and change the clause order or "
                 "voice where natural. Protected terms, numbers and citations stay exactly as written, and every "
                 "rule above still holds.")
    max_tokens = min(8000, G._out_budget(sum(len(t) for t, _ in items) * n, len(items) * n, 1.1) + 800)
    data = G._load_json(_request(gen, _system(n), user, max_tokens, deadline))
    out = [[] for _ in items]
    for k, it in enumerate(data.get("items") or []):
        if not isinstance(it, dict):
            continue
        try:
            i = int(it.get("id", k + 1)) - 1
        except (TypeError, ValueError):
            continue
        if not 0 <= i < len(items):
            continue
        cur, anchor = items[i][0], items[i][1] or items[i][0]
        for v in (it.get("versions") or [])[:n]:
            if isinstance(v, str) and v.strip():
                ok = _fit(cur, _recase(anchor, TP.guard(anchor, TP.restore_terms(anchor, v.strip(), glossary),
                                                        glossary), lower))
                if ok and ok.strip() != cur.strip():
                    out[i].append(ok)
    return out


def _versions(texts, anchors, glossary, n, min_change, lower=frozenset(), deadline=None):
    """Checked versions of every sentence from every model, all batches of all models in parallel. A batch a model
    could not do goes to the fallback model."""
    gens = _live_generators()
    if not gens:
        raise G.Unavailable("no language model key is set")
    items = [(t, a) for t, a in zip(texts, anchors)]
    jobs = [(g, i, min(len(items), i + _size(g))) for g in gens for i in range(0, len(items), _size(g))]
    out = [[] for _ in items]
    failed = []

    acct = G.ACCT.get()                                # the job's cost book (not visible in worker threads)

    def run(gen, a, b):
        tok = G.ACCT.set(acct)
        try:
            return gen, a, _batch(gen, items[a:b], glossary, n, min_change, lower, deadline)
        except Exception as err:                       # this model, this batch: the others still count
            return gen, a, err
        finally:
            G.ACCT.reset(tok)
    pool = ThreadPoolExecutor(max_workers=PARALLEL * len(gens))
    res, late = [], 0

    def collect(futs):
        nonlocal late
        left = None if deadline is None else max(0.5, deadline - time.time())
        done, rest = wait(list(futs), timeout=left)
        late += len(rest)                              # still running at the deadline: dropped
        return [f.result() for f in done]
    try:
        futs = {pool.submit(run, *j): j for j in jobs}
        got = collect(futs)
        res += got
        bad = [(g, a) for g, a, r in got if isinstance(r, Exception)]
        failed += [f"{g}: {str(r)[:120]}" for g, _, r in got if isinstance(r, Exception)]
        enough = deadline is None or deadline - time.time() > 6
        if bad and _key(FALLBACK) and enough:         # refused batches (rate limit): the fallback model does them
            size = {(g, a): b for g, a, b in jobs}
            res += collect({pool.submit(run, FALLBACK, a, size[(g, a)]): (g, a) for g, a in bad})
    finally:
        pool.shutdown(wait=False, cancel_futures=True)
    if late:
        print(f"[fast] {late} request(s) still running at the deadline were dropped", flush=True)
    for gen, a, r in res:
        if isinstance(r, Exception):
            continue
        for k, vs in enumerate(r):
            out[a + k] += [v for v in vs if v not in out[a + k]]
    if failed:
        print(f"[fast] {len(failed)} request(s) failed: " + " | ".join(sorted(set(failed))[:3]), flush=True)
    if not any(out) and failed:
        raise G.Unavailable(failed[0])
    return out


def _verify(pairs):
    """Faithfulness verdicts (True = says the same) for [(author's sentence, version)], all batches in parallel."""
    if not pairs:
        return []
    old = G.PARALLEL
    G.PARALLEL = max(old, PARALLEL)
    try:
        return G.verify(pairs)
    except G.Unavailable as err:
        print(f"[fast] faithfulness check unavailable: {err}", flush=True)
        return [None] * len(pairs)
    finally:
        G.PARALLEL = old


TOP = 2                                                # versions per sentence sent to the faithfulness check
# CPU only (no GPU): every version is first ranked by the two small detector models, and only the best PREFILTER per
# sentence go to the full detector (its large model was about 2/3 of the time). FAST_PREFILTER=0 = off.
PREFILTER = int(os.getenv("FAST_PREFILTER", "2"))
PREFILTER_GPU_MIN = int(os.getenv("FAST_PREFILTER_GPU_MIN", "400"))
CPU_VERSIONS = int(os.getenv("FAST_CPU_VERSIONS", "0"))   # CPU only: cap on versions per model (0 = no cap; 2 stalled at 23%)
TIMES = {}                                             # time of each step of the last choice (log)


# A run with an uploaded checker report: the sentences it highlights keep being rewritten (each round asking for more
# new wording) until little of their original wording is left, because the checker matches reworded text too and the
# report estimate counts a sentence as cleared only then. FAST_REPORT_OVERLAP: share of the original word sequences a
# highlighted sentence may keep; FAST_REPORT_CHANGE: the least new wording asked for in those rounds (%).
REPORT_OVERLAP = float(os.getenv("FAST_REPORT_OVERLAP", "0.15"))
REPORT_CHANGE = int(os.getenv("FAST_REPORT_CHANGE", "75"))
REPORT_MIN_CLEARED = int(os.getenv("FAST_REPORT_MIN_CLEARED", "3"))   # per round at 0%, else the run stops


def _overlap(a, b):
    return H.similarity_to_source(a, b) / 100.0


def _choose(jobs, anchors, outs, glossary, ctx, always=False, fresh=()):
    """Per job: the most human version (read in context) that keeps terms, numbers, citations, the meaning
    (embeddings + an independent reading against the author's sentence) and reads as fluent English, if it reads as
    more human than the sentence does now; else None. -> (choices, versions rejected as unfaithful)"""
    # (versions arrive already checked for terms, numbers, citations and symbols; see _batch)
    cands = list(dict.fromkeys((j, v) for j, vs in enumerate(outs) for v in vs or ()))
    if not cands:
        return [None] * len(jobs), 0
    t_ = time.time()
    sims = MEANING.similarities([(anchors[j], v) for j, v in cands])
    lim = MEANING.min_sim()
    cands = [c for c, s_ in zip(cands, sims) if s_ >= lim]
    TIMES["meaning"] = time.time() - t_
    # CPU: always; GPU: when the candidates are many (a long document), so choosing stays within the deadline
    cpu = PREFILTER and (D.on_cpu() or len(cands) > PREFILTER_GPU_MIN)
    if cpu and cands:
        # CPU: rank every version with the two small models, on its own (3x less text than in context), and keep
        # one more than PREFILTER per sentence for the fluency check below
        t_ = time.time()
        light = D.score_light([v for _, v in cands])
        keep, per = [], {}
        for c, s_ in sorted(zip(cands, light), key=lambda x: x[1]):
            if per.get(c[0], 0) < PREFILTER + 1:
                per[c[0]] = per.get(c[0], 0) + 1
                keep.append(c)
        cands = keep
        TIMES["prefilter"] = time.time() - t_
    t_ = time.time()
    flu = FL.acceptable([(anchors[j], v) for j, v in cands])
    cands = [c for c, f_ in zip(cands, flu) if f_]
    TIMES["fluency"] = time.time() - t_
    if cpu and cands:                                  # the PREFILTER best fluent versions go to the full detector
        per, keep = {}, []
        for c in cands:
            if per.get(c[0], 0) < PREFILTER:
                per[c[0]] = per.get(c[0], 0) + 1
                keep.append(c)
        cands = keep
    # Each version is scored on its own. (Scored inside its window, a version could not win while its neighbours were
    # still the AI-written ones: on long documents only a quarter of the flagged sentences were replaced. Now every
    # flagged sentence is replaced at once, so whole windows change together.)
    t_ = time.time()
    if always:
        sc = H._detector_scores([v for _, v in cands]) or [0.0] * len(cands)
        now = H._detector_scores([j[2].strip() for j in jobs]) or [1.0] * len(jobs)
    else:
        # later rounds: most neighbours are rewritten already, so a version read in its context (next to the
        # neighbours as they are NOW) says how the detector will read it in the document
        cur = {t.strip(): t.strip() for t in ctx}
        sc = H._ctx_scores([(jobs[j][2].strip(), v) for j, v in cands], ctx, cur) or [0.0] * len(cands)
        now = H._ctx_scores([(j[2].strip(), j[2].strip()) for j in jobs], ctx, cur) or [1.0] * len(jobs)
    TIMES["detector"] = time.time() - t_
    ranked = {}
    for (j, v), s_ in sorted(zip(cands, sc), key=lambda x: x[1]):
        if j in fresh:
            continue
        if (always or s_ < now[j] - 0.02) and len(ranked.setdefault(j, [])) < TOP:
            ranked[j].append(v)
    # report sentences ("fresh"): fewest of the author's word sequences first, provided the version does not read
    # clearly more AI-like than the sentence does now; a later round takes it when it is clearly more reworded
    for j in fresh:
        mine = [(v, s_) for (jj, v), s_ in zip(cands, sc) if jj == j]
        if not mine:
            continue
        cur = _overlap(anchors[j], jobs[j][2].strip())
        for v, s_ in sorted(mine, key=lambda x: (_overlap(anchors[j], x[0]), x[1])):
            if s_ > now[j] + 0.10 and not always:
                continue
            if (always or _overlap(anchors[j], v) < cur - 0.05) and len(ranked.setdefault(j, [])) < TOP:
                ranked[j].append(v)
    pairs = [(j, v) for j, vs in ranked.items() for v in vs]
    t_ = time.time()
    verdict = _verify([(anchors[j], v) for j, v in pairs])
    TIMES["verify"] = time.time() - t_
    # used only if the faithfulness check SAID it keeps the meaning: when the check could not run (service busy,
    # spending cap), the sentence keeps its wording rather than taking an unchecked rewrite
    ok = {p for p, r in zip(pairs, verdict) if r is True}
    best, bad = [None] * len(jobs), 0
    for j, vs in ranked.items():
        good = [v for v in vs if (j, v) in ok]
        bad += len(vs) - len(good)
        best[j] = good[0] if good else None
    return best, bad


def humanize_docx(path, hi=0.80, mod=0.50, rep=None, budget=None, also=None, also_overlap=None):
    """-> dict like smart.smart_docx (before, after, log, note, blocks, path, report, orig, new, final_of).
    also(sentence) -> True: rewrite that sentence in round 1 even if our detector does not flag it (the sentences an
    uploaded checker report highlights); later rounds take what our detector still flags, until 0%."""
    rep = rep or (lambda *a, **k: None)
    t0 = time.time()
    G.start_account()
    rep(0.03, "Reading your document")
    doc = DX.open_clean(path)
    orig_text = "\n".join(b.text for b in D.docx_blocks(doc) if b.kind == "para")
    n_words = len(orig_text.split())
    budget = budget_for(n_words) if budget is None else budget
    glossary = TP.build_glossary([orig_text]) | TP.GLOSSARY.get()
    import re as _re
    lower = frozenset(_re.findall(r"\b[a-z][a-z]+\b", orig_text))     # words the document writes in lower case
    TP.GLOSSARY.set(glossary)
    TP.set_document([orig_text])
    # spending cap scaled to the length (the flat $1 stopped the faithfulness check mid-way on a 5,500-word paper)
    G.set_cap(max(G.JOB_BUDGET, CAP_PER_1K * len(orig_text.split()) / 1000))
    rep(0.08, "Checking the whole document for AI-written passages")
    blocks = S._score_doc(doc)
    first = stats = D.summarise(blocks, hi, mod)
    best = (stats["pct"], S._snapshot(doc), blocks)
    anchor = {}                                        # current sentence -> the author's sentence
    # the author's sentence -> its latest wording, recorded when it is replaced (re-splitting the finished text
    # cannot find it when a rewrite changed the sentence boundaries); kept with the best version of the document
    replaced, best_replaced = {}, {}
    log, note = [], ""
    end = min(t0 + budget, H.JOB_DEADLINE.get() or float("inf"))   # the job's safety cap (wallet: its deadline)

    def anchor_of(t):
        return anchor.get(t.strip(), t.strip())

    def needs_rewording(t):
        """A highlighted sentence (asked about by its ORIGINAL wording) that still keeps too much of that wording."""
        a = anchor_of(t)
        return bool(also and also(a)) and _overlap(a, t.strip()) > (REPORT_OVERLAP if also_overlap is None
                                                                    else also_overlap)

    def report_left(blocks_):
        return sum(1 for b_ in blocks_ if b_.kind == "para" for t in b_.sents if needs_rewording(t)) if also else 0
    rleft = report_left(blocks)
    best_key = (stats["pct"], rleft)
    last_lm = last_after = None
    stalled = zero_rounds = 0
    for rnd, (n, chg, thr) in enumerate(_plan(rounds_for(n_words)), 1):
        if CPU_VERSIONS and D.on_cpu():
            n = min(n, CPU_VERSIONS)
        if stats["pct"] == 0 and not rleft and (zero_rounds >= 1 or
                                               not any(_stock(t) for b in blocks if b.kind == "para" for t in b.sents)):
            break                                      # at 0%: one more round at most, for leftover stock phrases
        zero_rounds += stats["pct"] == 0
        around = {}
        jobs, locked = _jobs(blocks, thr if rnd == 1 else min(thr, mod), also=needs_rewording if also else None,
                             around=around)
        if not jobs:
            break
        # time for choosing + re-scoring after the language model (measured on the T4: about 0.03 s per sentence)
        after_lm = 4 + 0.035 * len(jobs) if last_after is None else last_after * 1.15
        left = min(max(ROUND_LM_MAX, ROUND_LM_PER_SENT * len(jobs)), end - time.time() - after_lm)
        if rnd > 1 and left < 8.0:                     # the safety cap is reached
            log.append(f"Round {rnd}: stopped, time limit reached")
            break
        # sentences with fields: the language model sees markers ([REF1] ...); everything else (anchors, choosing,
        # the faithfulness check, the report) reads the real text with the citations in place
        refs_of = [around[(id(b_), sp_[0])][1] if (id(b_), sp_[0]) in around else [] for b_, sp_, _ in jobs]
        llm_texts = [j[2].strip() for j in jobs]
        texts = [_detok(t, r) for t, r in zip(llm_texts, refs_of)]
        anchors = [anchor.get(t, t) for t in texts]
        llm_anchors = [_tok(a, r) for a, r in zip(anchors, refs_of)]
        fresh = {i for i, t in enumerate(texts) if needs_rewording(t)} if also else set()
        if fresh:                                      # report sentences: ask for more new wording
            chg = max(chg, REPORT_CHANGE + 5 * (rnd - 1))
        frac = 0.12 + 0.83 * (1 - 0.6 ** (rnd - 1))     # moves on every round without promising an end
        rep(frac, f"Round {rnd}: AI score {stats['pct']}%, rewriting {len(jobs)} sentence(s) at once")
        tg = time.time()
        try:
            outs = _versions(llm_texts, llm_anchors, glossary, n, chg, lower, deadline=time.time() + max(8.0, left))
        except G.Unavailable as err:
            note += " (The rewriting service is busy right now: fewer sentences could be rewritten.)"
            print(f"[fast] round {rnd}: rewriting unavailable: {err}", flush=True)
            break
        tg = time.time() - tg
        marked = {}                                     # (job, real version) -> its version with markers
        for i_, (b_, sp_, _) in enumerate(jobs):         # around fields: only versions that fit around them
            key_ = (id(b_), sp_[0])
            if key_ in around and outs[i_]:
                vs_ = [v for v in outs[i_] if _fits(around[key_][0], v) is not None]
                outs[i_] = [_detok(v, refs_of[i_]) for v in vs_]
                marked.update({(i_, r_): v for r_, v in zip(outs[i_], vs_)})
        real_jobs = [(b_, sp_, t) for (b_, sp_, _), t in zip(jobs, texts)]
        rep(frac + 0.83 * 0.6 ** (rnd - 1) * 0.2, f"Round {rnd}: choosing the most natural version of each sentence")
        tc = time.time()
        # round 1: every flagged sentence takes its best valid version, even one that scores no lower on its own: a
        # window is only read as human when (nearly) all its sentences changed; on a 5,500-word document the "must
        # score lower" rule left 4 in 10 sentences as they were and the score stayed at 95%
        chosen, unfaithful = _choose(real_jobs, anchors, outs, glossary, _context(blocks), always=rnd <= ALWAYS_ROUNDS,
                                     fresh=fresh)
        tc = time.time() - tc
        done = 0
        for i_, (b, (a, e), t), new, anc in sorted(zip(range(len(jobs)), jobs, chosen, anchors),
                                                   key=lambda x: (id(x[1][0]), -x[1][1][0])):
            info = around.get((id(b), a))
            if new and info:
                ok = _replace_around(b, info[0], marked.get((i_, new), new))
            else:
                ok = bool(new) and S.replace_span(b.el, a, e, new)
            if ok:
                anchor[new.strip()] = anc
                replaced[anc.strip()] = new.strip()
                done += 1
        ts = time.time()
        blocks = S._score_doc(doc)
        prev, stats = stats["pct"], D.summarise(blocks, hi, mod)
        rprev, rleft = rleft, report_left(blocks)
        ts = time.time() - ts
        last_lm, last_after = tg, tc + ts
        log.append(f"Round {rnd}: AI score {prev}% → {stats['pct']}% (rewrote {done} of {len(jobs)} sentences"
                   + (f", {locked} locked" if locked else "")
                   + (f"; {rleft} highlighted sentence(s) still close to the original" if also else "") + ")")
        print(f"[fast] round {rnd}: {len(jobs)} sentences, {done} rewritten, {unfaithful} version(s) rejected as "
              f"unfaithful, AI {prev}% -> {stats['pct']}% | "
              f"language model {tg:.1f}s, choosing {tc:.1f}s ("
              + ", ".join(f"{k} {v:.1f}s" for k, v in TIMES.items()) + f"), scoring {ts:.1f}s", flush=True)
        key = (stats["pct"], rleft)                    # lower AI score first, then fewer report sentences left
        if key < best_key:
            best, best_key = (stats["pct"], S._snapshot(doc), blocks), key
            best_replaced = dict(replaced)
            stalled = 0
        else:
            stalled += 1
            # at 0% only the report's sentences are being reworded: one round without progress is enough
            if stalled >= STALL or (stats["pct"] == 0 and stalled >= 1):
                log.append(f"Round {rnd}: stopped, {STALL} rounds without further improvement")
                break
        # at 0% with only the report's sentences left: a round that clears very few of them is not worth another
        # minute (a thesis part spent 5 of its 12 minutes taking 80 such sentences to 72)
        if also and prev == 0 and stats["pct"] == 0 and rprev - rleft < max(REPORT_MIN_CLEARED, 0.05 * rprev):
            log.append(f"Round {rnd}: stopped, few highlighted sentences left to reword further")
            break
        if stats["pct"] > best[0]:                   # this round made it worse: the next one starts from the best
            from docx import Document as _Doc
            doc = _Doc(io.BytesIO(best[1]))
            replaced = dict(best_replaced)
            blocks = S._score_doc(doc)
            stats = D.summarise(blocks, hi, mod)
            rleft = report_left(blocks)
        if not done:
            break
    from docx import Document
    doc = Document(io.BytesIO(best[1]))
    blocks = best[2]
    after = D.summarise(blocks, hi, mod)
    new_text = "\n".join(b.text for b in blocks if b.kind == "para")
    out = H._save_output(doc, S._clean_stem(path), "_humanized")
    secs = time.time() - t0
    print(f"[fast] {first['pct']}% -> {after['pct']}% in {secs:.1f}s ({first['analysed']} words) | "
          f"{G.usage_line()}", flush=True)
    note = "⚡ Fast engine: every flagged sentence rewritten at once by the language model." + note
    left_ = [s_ for b_ in blocks if b_.kind == "para" for s_, p_ in zip(b_.sents, b_.scores) if p_ >= mod]
    final_of = dict(best_replaced)                    # the author's sentence -> its wording in the delivered file
    return dict(before=first, after=after, log=log, note=note, blocks=blocks, path=out, report=None,
                orig=orig_text, new=new_text, seconds=secs, zero_goal=True, left=len(left_), final_of=final_of)


def humanize_text(text, hi=0.80, mod=0.50, rep=None, budget=None):
    """Pasted text: written to a Word file (one paragraph per paragraph), humanized, read back."""
    import tempfile
    from docx import Document
    d = Document()
    for p in [p.strip() for p in (text or "").split("\n") if p.strip()]:
        d.add_paragraph(p)
    path = os.path.join(tempfile.mkdtemp(prefix="cs_fast_"), "Pasted_text.docx")
    d.save(path)
    res = humanize_docx(path, hi, mod, rep, budget)
    res["text"] = "\n\n".join(p.text for p in Document(res["path"]).paragraphs if p.text.strip())
    return res


def selftest(sizes=(1, 3, 8)):
    """FAST_SELFTEST=1 (Space variable): after start-up, time the engine on test documents of about 1,800 x size
    words (the subject samples; every copy gets different numbers, so no cache helps). Results go to the log."""
    import random
    import re
    import sys
    import tempfile
    from pathlib import Path
    from docx import Document
    sys.path.insert(0, str(Path(__file__).resolve().parent / "devtools"))
    from samples_domains import DOMAINS
    time.sleep(float(os.getenv("FAST_SELFTEST_DELAY", "60")))          # let start-up finish
    for size in sizes:
        rnd = random.Random(size)
        d = Document()
        d.add_heading(f"Self-test document x{size}", 1)
        for c in range(size):
            for k, v in DOMAINS.items():
                d.add_heading(f"{k.title()} ({c + 1})", 2)
                d.add_paragraph(v if c == 0 else re.sub(r"\d+", lambda m: str(int(m.group()) + rnd.randint(1, 9)), v))
        path = os.path.join(tempfile.mkdtemp(prefix="cs_selftest_"), f"selftest_x{size}.docx")
        d.save(path)
        try:
            r = humanize_docx(path, budget=float(os.getenv("FAST_SELFTEST_BUDGET", str(BUDGET))))
            print(f"[selftest] x{size}: {r['before']['analysed']} words, AI {r['before']['pct']}% -> "
                  f"{r['after']['pct']}% in {r['seconds']:.1f}s | " + " | ".join(r["log"]), flush=True)
        except Exception as err:
            print(f"[selftest] x{size}: failed: {type(err).__name__}: {err}", flush=True)