Spaces:
Running
Running
unknown
Claude Opus 5.5
Theses over 40,000 words reach 0%: the fast engine's limits grow with the length (time cap 30 s per 1,000 words, at least 10 min; a round waits 0.08 s per flagged sentence, at least 90 s; 2 extra rounds per 10,000 words); a wallet run's deadline follows the same cap. A flat 10-minute cap and 90-second rounds dropped most flagged sentences of a long thesis each round
c7ef1c5 | """ | |
| Fast engine: a Word document of any length humanized in about a minute. | |
| The Smart engine rewrites sentence by sentence: Parrot versions, about ten checks per version, several passes one | |
| after another; its time grows with the length of the document. This engine works on the whole document at once: | |
| 1. The AI detector reads the whole document once (GPU, batched). | |
| 2. Every flagged sentence goes to SEVERAL language models in parallel batches (FAST_GENERATORS; FAST_PARALLEL | |
| requests at once per model), with the author's sentence as the meaning reference; each model writes several | |
| human-style versions. Measured on 30 AI-written sentences (devtools/bench_generators.py), versions the detector | |
| reads as human (< 0.5): gpt-oss-120b 2/30, Qwen 3.8 9/30, Gemini 3.5 Flash-Lite 14/30, the two pooled 16/30. | |
| 3. Each version must keep every term, number, citation and symbol (textparse.guard) and the meaning (sentence | |
| embeddings, GPU). The detector reads the surviving versions IN CONTEXT (with their neighbours) and the most | |
| human one is used, but only if it reads as more human than what the sentence has now. | |
| 4. The document is scored again. Sentences that are still flagged get another round with a stronger rewrite | |
| (more new wording), until the score is 0 or the time budget (FAST_BUDGET seconds) is used up. | |
| The version with the lowest score is kept. Only the text of each sentence is replaced, so formatting, citation | |
| fields, equations, tables and figures stay as they were. | |
| Space variables: FAST_BUDGET (50 s), FAST_PARALLEL (16 requests at once per model), FAST_ROUNDS (4), | |
| FAST_VERSIONS (2 per model), FAST_GENERATORS ("provider:model,..."; Groq only by default; other providers only if | |
| named here). | |
| """ | |
| import io | |
| import os | |
| import time | |
| from concurrent.futures import FIRST_EXCEPTION, ThreadPoolExecutor, wait | |
| import requests | |
| import detector as D | |
| import docx_report as DX | |
| import fluency as FL | |
| import groqai as G | |
| import humanizer as H | |
| import meaning as MEANING | |
| import pubrules as PR | |
| import smart as S | |
| import textparse as TP | |
| # 0% first: rounds continue until the document is at 0% on our detector (each round rewriting more strongly), until | |
| # STALL rounds in a row bring no improvement, or until FAST_BUDGET seconds (a safety cap; about a minute for 2,000 | |
| # words, a few minutes for a thesis on a 250k-tokens-a-minute Groq key). Requests still running at the cap are dropped. | |
| BUDGET = float(os.getenv("FAST_BUDGET", "600")) | |
| ROUND_LM_MAX = float(os.getenv("FAST_ROUND_LM_MAX", "90")) # seconds one round may wait for the language models | |
| STALL = int(os.getenv("FAST_STALL", "3")) | |
| CAP_PER_1K = float(os.getenv("FAST_CAP_PER_1K_WORDS", "0.60")) # US$ per 1,000 words a job may spend at most | |
| PARALLEL = int(os.getenv("FAST_PARALLEL", "16")) | |
| # sentences per request: the instructions (~1,500 tokens) go with every request, so larger batches cost far fewer | |
| # tokens per sentence (they were most of the tokens sent) | |
| BATCH = int(os.getenv("FAST_BATCH", "24")) | |
| ALWAYS_ROUNDS = int(os.getenv("FAST_ALWAYS_ROUNDS", "1")) # rounds in which every flagged sentence is replaced | |
| ROUNDS = int(os.getenv("FAST_ROUNDS", "10")) | |
| VERSIONS = int(os.getenv("FAST_VERSIONS", "2")) # per model (4 models: 8 versions per sentence) | |
| # round -> (versions per sentence, share of new wording asked for (%), sentences above this score are rewritten) | |
| # FAST_CHANGES: the new-wording share asked for in each round (measured on 4 AI paragraphs with the 3-model detector: | |
| # "0,40,60,70" needed 4 rounds for 0%; the first, gentle round only took the score from 100% to 79%) | |
| _CHG = [int(x) for x in os.getenv("FAST_CHANGES", "50,60,70,75").split(",") if x.strip()] | |
| PLAN = [(VERSIONS + min(k, 2), c, thr) for k, (c, thr) in enumerate(zip(_CHG, (0.50, 0.40, 0.30, 0.25)))] | |
| # rounds after the plan: the sentences left are the stubborn ones, so more versions and the strongest rewrite | |
| PLAN += [(VERSIONS + 3, max(_CHG[-1], 80), 0.25)] * max(0, ROUNDS - len(PLAN)) | |
| # Long documents (a 40,000+ word thesis) get limits that grow with their length, so they too reach 0%: a flat | |
| # 10-minute cap and 90-second rounds dropped most of a thesis's flagged sentences in each round and stopped it early. | |
| BUDGET_PER_1K = float(os.getenv("FAST_BUDGET_PER_1K_WORDS", "30")) # seconds per 1,000 words (22,000 words: ~4 min) | |
| ROUND_LM_PER_SENT = float(os.getenv("FAST_ROUND_LM_PER_SENTENCE", "0.08")) # a round's wait per flagged sentence | |
| ROUNDS_PER_10K = int(os.getenv("FAST_ROUNDS_PER_10K_WORDS", "2")) # extra rounds per 10,000 words | |
| def budget_for(words): | |
| """The job's time cap: FAST_BUDGET, or more for a long document.""" | |
| return max(BUDGET, BUDGET_PER_1K * max(0, int(words or 0)) / 1000) | |
| def rounds_for(words): | |
| return ROUNDS + ROUNDS_PER_10K * (max(0, int(words or 0)) // 10000) | |
| def _plan(rounds): | |
| return PLAN[:rounds] + [PLAN[-1]] * max(0, rounds - len(PLAN)) | |
| def _stock(s): | |
| """The sentence uses a stock AI word or phrase ("plays a crucial role", "Furthermore", "It is important to note"). | |
| Such a sentence is rewritten even when the detector no longer flags it: once its neighbours were rewritten the | |
| detector passed it, but a reader (or another checker) still sees it.""" | |
| return bool(PR._AI_WORD_RE.search(s or "") or any(r.search(s or "") for r in PR._AI_PHRASE_RE)) | |
| def _jobs(blocks, thr, min_words=5, also=None, around=None): | |
| """(block, sub-span, current sentence text) for every sentence at or above `thr` (or with a stock AI phrase, or | |
| picked by `also`, e.g. the sentences an uploaded checker report highlights) that can be edited safely.""" | |
| jobs, locked = [], 0 | |
| for b in blocks: | |
| if b.kind != "para" or b.el is None: | |
| continue | |
| for sp, s, p in zip(b.cspans, b.sents, b.scores): | |
| picked = bool(also and also(s)) # asked for every sentence (the caller records the matches) | |
| if not sp or (p < thr and not _stock(s) and not picked) or len(s.split()) < min_words: | |
| continue | |
| sub = S.safe_subspan(b.el, *sp) | |
| # only the whole sentence, or its beginning (a sentence ending in a citation field or superscript): | |
| # a rewrite spliced into the middle of a sentence read "It is worth noting that Performance drops" | |
| if sub and b.raw[sp[0]:sub[0]].strip() == "" and sub == tuple(sp): | |
| jobs.append((b, sub, b.raw[sub[0]:sub[1]])) | |
| continue | |
| segs = _segments(b, *sp) if (AROUND and around is not None) else None | |
| if segs: # fields inside: rewrite around them (markers) | |
| text, refs = _tokenized(b, segs) | |
| around[(id(b), sp[0])] = (segs, refs) | |
| jobs.append((b, tuple(sp), text)) | |
| elif sub and b.raw[sp[0]:sub[0]].strip() == "": | |
| jobs.append((b, sub, b.raw[sub[0]:sub[1]])) | |
| else: | |
| locked += 1 | |
| return jobs, locked | |
| # ---------------------------------------------------------------------------------------------------- around fields | |
| # Sentences with a citation field (Zotero / Mendeley / EndNote), a link or a superscript citation in the middle used to | |
| # be left as written ("locked"): in a thesis that was a third of the flagged sentences, and a checker kept flagging | |
| # them. Now the locked parts become markers ([REF1], [REF2] ...) for the language model; a version is used only if it | |
| # keeps every marker once, in the same order, and puts words only where the sentence has plain text to hold them. Its | |
| # words go into the plain text around the untouched fields, so the citations stay live (and are never retyped). | |
| AROUND = os.getenv("FAST_AROUND_FIELDS", "1") != "0" | |
| _TOKEN = "[REF{}]" | |
| _TOKEN_RE = __import__("re").compile(r"\[REF(\d+)\]") | |
| def _segments(b, a, e): | |
| """The sentence at [a, e) of block b as [("text", s, t) | ("lock", s, t)] (paragraph offsets), or None when it holds | |
| an equation or has too little plain text to rewrite.""" | |
| items = S._run_map(b.el) | |
| if any(r is None and a <= s < e for r, s, _, _ in items): | |
| return None | |
| segs = [] | |
| for r, s, t, safe in items: | |
| if r is None or t <= a or s >= e or t <= s: | |
| continue | |
| kind = "text" if safe else "lock" | |
| s, t = max(a, s), min(e, t) | |
| if segs and segs[-1][0] == kind and segs[-1][2] == s: | |
| segs[-1] = (kind, segs[-1][1], t) | |
| else: | |
| segs.append((kind, s, t)) | |
| if not any(k == "lock" for k, _, _ in segs): | |
| return None | |
| words = sum(len(b.raw[s:t].split()) for k, s, t in segs if k == "text") | |
| return segs if words >= 5 else None | |
| def _tokenized(b, segs): | |
| """The sentence with each locked part replaced by its marker, and the markers' real text.""" | |
| out, refs = [], [] | |
| for k, s, t in segs: | |
| if k == "lock": | |
| refs.append(b.raw[s:t]) | |
| out.append(_TOKEN.format(len(refs))) | |
| else: | |
| out.append(b.raw[s:t]) | |
| return "".join(out), refs | |
| def _fits(segs, version): | |
| """Words of a version for each plain-text part of the sentence (None: the version cannot be laid out around the | |
| fields: a marker missing, repeated or moved, or words where the sentence has no plain text).""" | |
| nums = [int(m) for m in _TOKEN_RE.findall(version)] | |
| locks = sum(1 for k, _, _ in segs if k == "lock") | |
| if nums != list(range(1, locks + 1)): | |
| return None | |
| pieces = _TOKEN_RE.split(version)[::2] # text before the first marker, between markers, after the last | |
| texts, i = [], 0 | |
| for k, s, t in segs: # each lock sits between two plain-text slots (maybe empty) | |
| if k == "lock": | |
| i += 1 | |
| continue | |
| texts.append((i, s, t)) | |
| out = [] | |
| for slot, piece in enumerate(pieces): | |
| span = next(((s, t) for i_, s, t in texts if i_ == slot), None) | |
| if span is None: | |
| if piece.strip(): | |
| return None # words where the sentence has no plain text to hold them | |
| continue | |
| out.append((span, piece)) | |
| return out | |
| def _replace_around(b, segs, version): | |
| """Write a version into the plain text around the fields (right to left, so offsets stay valid).""" | |
| parts = _fits(segs, version) | |
| if parts is None: | |
| return False | |
| for (s, t), piece in sorted(parts, key=lambda x: -x[0][0]): | |
| if not S.replace_span(b.el, s, t, piece): | |
| return False | |
| return True | |
| def _tok(text, refs): | |
| """A sentence with its citations' real text -> with markers (the author's sentence for the language model).""" | |
| for i, r in enumerate(refs, 1): | |
| text = text.replace(r, _TOKEN.format(i), 1) if r else text | |
| return text | |
| def _detok(text, refs): | |
| """A sentence with markers -> with the citations' real text.""" | |
| return _TOKEN_RE.sub(lambda m: refs[int(m.group(1)) - 1] if int(m.group(1)) <= len(refs) else m.group(0), text) | |
| def _context(blocks): | |
| ctx = {} | |
| for b in blocks: | |
| if b.kind == "para": | |
| for i, t in enumerate(b.sents): | |
| ctx[t.strip()] = (b.sents[i - 1] if i else "", b.sents[i + 1] if i + 1 < len(b.sents) else "") | |
| return ctx | |
| def _recase(anchor, v, lower): | |
| """The word that opened the author's sentence keeps its capital only at the start: "relies heavily on Data | |
| protection legislation" -> "data protection legislation". Only for words the document also writes in lower case | |
| (`lower`), so names ("India", "Kumar") keep theirs.""" | |
| import re | |
| m = re.match(r"\s*([A-Z][a-z]+)\b", anchor or "") | |
| if not v or not m or m.group(1).lower() not in lower: | |
| return v | |
| w = m.group(1) | |
| return v[:1] + re.sub(rf"(?<=[a-z,;:] ){w}\b", w.lower(), v[1:]) | |
| def _fit(cur, v): | |
| """A rewrite of a sentence's beginning (the rest is a citation field etc.) must not end the sentence early.""" | |
| if v and cur and cur.rstrip()[-1:] not in ".!?" and v.rstrip()[-1:] in ".!?": | |
| return v.rstrip()[:-1] | |
| return v | |
| _URLS = {"groq": ("https://api.groq.com/openai/v1/chat/completions", "GROQ_API_KEY"), | |
| "gemini": ("https://generativelanguage.googleapis.com/v1beta/openai/chat/completions", "GEMINI_API_KEY"), | |
| "anthropic": ("https://api.anthropic.com/v1/chat/completions", "ANTHROPIC_API_KEY")} | |
| GENERATORS = [g.strip() for g in os.getenv( | |
| "FAST_GENERATORS", | |
| # Groq only: four models, each with its own limit (250,000 tokens a minute per model on the paid key), so about | |
| # a million tokens a minute together. (With two models a 14,600-word document had only 11 of 688 sentences | |
| # rewritten within the minute.) | |
| "groq:qwen/qwen3.8-27b,groq:qwen/qwen3.6-27b,groq:openai/gpt-oss-20b,groq:openai/gpt-oss-120b").split(",") | |
| if g.strip()] | |
| FALLBACK = os.getenv("FAST_FALLBACK", "groq:openai/gpt-oss-20b") | |
| # sentences per request by provider: Gemini keys have a low requests-per-minute limit (about 20 on this key), so it | |
| # gets fewer, larger requests; a batch it refuses goes to the fallback model | |
| BATCH_BY = {"gemini": int(os.getenv("FAST_BATCH_GEMINI", "40"))} | |
| def _size(gen): | |
| return BATCH_BY.get(gen.split(":", 1)[0], BATCH) | |
| def _key(gen): | |
| prov = gen.split(":", 1)[0] | |
| if prov == "groq": | |
| return G.KEY | |
| return os.getenv(_URLS.get(prov, ("", ""))[1], "").strip() | |
| def _live_generators(): | |
| gens = [g for g in GENERATORS if g.split(":", 1)[0] in _URLS and _key(g)] | |
| return gens or ([FALLBACK] if _key(FALLBACK) else []) | |
| def _system(n): | |
| tok = G.FAITHFUL.set(True) | |
| try: | |
| return G._variants_system(n) | |
| finally: | |
| G.FAITHFUL.reset(tok) | |
| def _request(gen, system, user, max_tokens, deadline=None): | |
| """One chat request to an OpenAI-compatible endpoint; two retries on rate limits / server errors.""" | |
| prov, model = gen.split(":", 1) | |
| url = _URLS[prov][0] | |
| body = {"model": model, "max_tokens": max_tokens, | |
| "messages": [{"role": "system", "content": system}, {"role": "user", "content": user}]} | |
| if prov != "anthropic": | |
| body["temperature"] = 0.7 | |
| if "gpt-oss" in model: | |
| body["reasoning_effort"] = "low" | |
| elif "qwen" in model: | |
| body["reasoning_effort"] = "none" | |
| for attempt in range(3): | |
| try: | |
| left = 60 if deadline is None else deadline - time.time() | |
| if left < 2: | |
| raise RuntimeError("deadline reached") | |
| r = requests.post(url, headers={"Authorization": f"Bearer {_key(gen)}", "Content-Type": "application/json"}, | |
| json=body, timeout=min(60, left)) | |
| except requests.RequestException as err: | |
| last = str(err) | |
| else: | |
| if r.ok: | |
| d = r.json() | |
| u = d.get("usage") or {} | |
| G._account("variants", model, {"prompt_tokens": u.get("prompt_tokens", 0), | |
| "completion_tokens": u.get("completion_tokens", 0)}) | |
| return d["choices"][0]["message"]["content"] | |
| last = f"{r.status_code} {r.text[:160]}" | |
| if r.status_code not in (429, 500, 502, 503, 504): | |
| break | |
| if deadline is not None and time.time() + 1.5 * (attempt + 1) + 3 > deadline: | |
| break | |
| time.sleep(1.5 * (attempt + 1)) | |
| raise RuntimeError(f"{gen}: {last}") | |
| def _batch(gen, items, glossary, n, min_change, lower=frozenset(), deadline=None): | |
| """Versions of a batch of (current sentence, author's sentence) from one model, each already checked for terms, | |
| numbers, citations and symbols (the check runs here, in the worker thread, while other requests are waiting).""" | |
| lines = [f"{i}. ORIGINAL: {a}\n SENTENCE: {t}" if a and a != t else f"{i}. SENTENCE: {t}" | |
| for i, (t, a) in enumerate(items, 1)] | |
| terms = G._surface_terms(" ".join(t for t, _ in items), glossary) | |
| user = "PROTECTED TERMS: " + ("; ".join(terms) if terms else "(none)") + "\n\n" + "\n".join(lines) | |
| if min_change: | |
| user += (f"\n\nVOCABULARY CHANGE: in every version, at least {int(min_change)}% of the words must differ " | |
| "from the sentence: use different (correct, common) vocabulary and change the clause order or " | |
| "voice where natural. Protected terms, numbers and citations stay exactly as written, and every " | |
| "rule above still holds.") | |
| max_tokens = min(8000, G._out_budget(sum(len(t) for t, _ in items) * n, len(items) * n, 1.1) + 800) | |
| data = G._load_json(_request(gen, _system(n), user, max_tokens, deadline)) | |
| out = [[] for _ in items] | |
| for k, it in enumerate(data.get("items") or []): | |
| if not isinstance(it, dict): | |
| continue | |
| try: | |
| i = int(it.get("id", k + 1)) - 1 | |
| except (TypeError, ValueError): | |
| continue | |
| if not 0 <= i < len(items): | |
| continue | |
| cur, anchor = items[i][0], items[i][1] or items[i][0] | |
| for v in (it.get("versions") or [])[:n]: | |
| if isinstance(v, str) and v.strip(): | |
| ok = _fit(cur, _recase(anchor, TP.guard(anchor, TP.restore_terms(anchor, v.strip(), glossary), | |
| glossary), lower)) | |
| if ok and ok.strip() != cur.strip(): | |
| out[i].append(ok) | |
| return out | |
| def _versions(texts, anchors, glossary, n, min_change, lower=frozenset(), deadline=None): | |
| """Checked versions of every sentence from every model, all batches of all models in parallel. A batch a model | |
| could not do goes to the fallback model.""" | |
| gens = _live_generators() | |
| if not gens: | |
| raise G.Unavailable("no language model key is set") | |
| items = [(t, a) for t, a in zip(texts, anchors)] | |
| jobs = [(g, i, min(len(items), i + _size(g))) for g in gens for i in range(0, len(items), _size(g))] | |
| out = [[] for _ in items] | |
| failed = [] | |
| acct = G.ACCT.get() # the job's cost book (not visible in worker threads) | |
| def run(gen, a, b): | |
| tok = G.ACCT.set(acct) | |
| try: | |
| return gen, a, _batch(gen, items[a:b], glossary, n, min_change, lower, deadline) | |
| except Exception as err: # this model, this batch: the others still count | |
| return gen, a, err | |
| finally: | |
| G.ACCT.reset(tok) | |
| pool = ThreadPoolExecutor(max_workers=PARALLEL * len(gens)) | |
| res, late = [], 0 | |
| def collect(futs): | |
| nonlocal late | |
| left = None if deadline is None else max(0.5, deadline - time.time()) | |
| done, rest = wait(list(futs), timeout=left) | |
| late += len(rest) # still running at the deadline: dropped | |
| return [f.result() for f in done] | |
| try: | |
| futs = {pool.submit(run, *j): j for j in jobs} | |
| got = collect(futs) | |
| res += got | |
| bad = [(g, a) for g, a, r in got if isinstance(r, Exception)] | |
| failed += [f"{g}: {str(r)[:120]}" for g, _, r in got if isinstance(r, Exception)] | |
| enough = deadline is None or deadline - time.time() > 6 | |
| if bad and _key(FALLBACK) and enough: # refused batches (rate limit): the fallback model does them | |
| size = {(g, a): b for g, a, b in jobs} | |
| res += collect({pool.submit(run, FALLBACK, a, size[(g, a)]): (g, a) for g, a in bad}) | |
| finally: | |
| pool.shutdown(wait=False, cancel_futures=True) | |
| if late: | |
| print(f"[fast] {late} request(s) still running at the deadline were dropped", flush=True) | |
| for gen, a, r in res: | |
| if isinstance(r, Exception): | |
| continue | |
| for k, vs in enumerate(r): | |
| out[a + k] += [v for v in vs if v not in out[a + k]] | |
| if failed: | |
| print(f"[fast] {len(failed)} request(s) failed: " + " | ".join(sorted(set(failed))[:3]), flush=True) | |
| if not any(out) and failed: | |
| raise G.Unavailable(failed[0]) | |
| return out | |
| def _verify(pairs): | |
| """Faithfulness verdicts (True = says the same) for [(author's sentence, version)], all batches in parallel.""" | |
| if not pairs: | |
| return [] | |
| old = G.PARALLEL | |
| G.PARALLEL = max(old, PARALLEL) | |
| try: | |
| return G.verify(pairs) | |
| except G.Unavailable as err: | |
| print(f"[fast] faithfulness check unavailable: {err}", flush=True) | |
| return [None] * len(pairs) | |
| finally: | |
| G.PARALLEL = old | |
| TOP = 2 # versions per sentence sent to the faithfulness check | |
| # CPU only (no GPU): every version is first ranked by the two small detector models, and only the best PREFILTER per | |
| # sentence go to the full detector (its large model was about 2/3 of the time). FAST_PREFILTER=0 = off. | |
| PREFILTER = int(os.getenv("FAST_PREFILTER", "2")) | |
| PREFILTER_GPU_MIN = int(os.getenv("FAST_PREFILTER_GPU_MIN", "400")) | |
| CPU_VERSIONS = int(os.getenv("FAST_CPU_VERSIONS", "0")) # CPU only: cap on versions per model (0 = no cap; 2 stalled at 23%) | |
| TIMES = {} # time of each step of the last choice (log) | |
| # A run with an uploaded checker report: the sentences it highlights keep being rewritten (each round asking for more | |
| # new wording) until little of their original wording is left, because the checker matches reworded text too and the | |
| # report estimate counts a sentence as cleared only then. FAST_REPORT_OVERLAP: share of the original word sequences a | |
| # highlighted sentence may keep; FAST_REPORT_CHANGE: the least new wording asked for in those rounds (%). | |
| REPORT_OVERLAP = float(os.getenv("FAST_REPORT_OVERLAP", "0.15")) | |
| REPORT_CHANGE = int(os.getenv("FAST_REPORT_CHANGE", "75")) | |
| REPORT_MIN_CLEARED = int(os.getenv("FAST_REPORT_MIN_CLEARED", "3")) # per round at 0%, else the run stops | |
| def _overlap(a, b): | |
| return H.similarity_to_source(a, b) / 100.0 | |
| def _choose(jobs, anchors, outs, glossary, ctx, always=False, fresh=()): | |
| """Per job: the most human version (read in context) that keeps terms, numbers, citations, the meaning | |
| (embeddings + an independent reading against the author's sentence) and reads as fluent English, if it reads as | |
| more human than the sentence does now; else None. -> (choices, versions rejected as unfaithful)""" | |
| # (versions arrive already checked for terms, numbers, citations and symbols; see _batch) | |
| cands = list(dict.fromkeys((j, v) for j, vs in enumerate(outs) for v in vs or ())) | |
| if not cands: | |
| return [None] * len(jobs), 0 | |
| t_ = time.time() | |
| sims = MEANING.similarities([(anchors[j], v) for j, v in cands]) | |
| lim = MEANING.min_sim() | |
| cands = [c for c, s_ in zip(cands, sims) if s_ >= lim] | |
| TIMES["meaning"] = time.time() - t_ | |
| # CPU: always; GPU: when the candidates are many (a long document), so choosing stays within the deadline | |
| cpu = PREFILTER and (D.on_cpu() or len(cands) > PREFILTER_GPU_MIN) | |
| if cpu and cands: | |
| # CPU: rank every version with the two small models, on its own (3x less text than in context), and keep | |
| # one more than PREFILTER per sentence for the fluency check below | |
| t_ = time.time() | |
| light = D.score_light([v for _, v in cands]) | |
| keep, per = [], {} | |
| for c, s_ in sorted(zip(cands, light), key=lambda x: x[1]): | |
| if per.get(c[0], 0) < PREFILTER + 1: | |
| per[c[0]] = per.get(c[0], 0) + 1 | |
| keep.append(c) | |
| cands = keep | |
| TIMES["prefilter"] = time.time() - t_ | |
| t_ = time.time() | |
| flu = FL.acceptable([(anchors[j], v) for j, v in cands]) | |
| cands = [c for c, f_ in zip(cands, flu) if f_] | |
| TIMES["fluency"] = time.time() - t_ | |
| if cpu and cands: # the PREFILTER best fluent versions go to the full detector | |
| per, keep = {}, [] | |
| for c in cands: | |
| if per.get(c[0], 0) < PREFILTER: | |
| per[c[0]] = per.get(c[0], 0) + 1 | |
| keep.append(c) | |
| cands = keep | |
| # Each version is scored on its own. (Scored inside its window, a version could not win while its neighbours were | |
| # still the AI-written ones: on long documents only a quarter of the flagged sentences were replaced. Now every | |
| # flagged sentence is replaced at once, so whole windows change together.) | |
| t_ = time.time() | |
| if always: | |
| sc = H._detector_scores([v for _, v in cands]) or [0.0] * len(cands) | |
| now = H._detector_scores([j[2].strip() for j in jobs]) or [1.0] * len(jobs) | |
| else: | |
| # later rounds: most neighbours are rewritten already, so a version read in its context (next to the | |
| # neighbours as they are NOW) says how the detector will read it in the document | |
| cur = {t.strip(): t.strip() for t in ctx} | |
| sc = H._ctx_scores([(jobs[j][2].strip(), v) for j, v in cands], ctx, cur) or [0.0] * len(cands) | |
| now = H._ctx_scores([(j[2].strip(), j[2].strip()) for j in jobs], ctx, cur) or [1.0] * len(jobs) | |
| TIMES["detector"] = time.time() - t_ | |
| ranked = {} | |
| for (j, v), s_ in sorted(zip(cands, sc), key=lambda x: x[1]): | |
| if j in fresh: | |
| continue | |
| if (always or s_ < now[j] - 0.02) and len(ranked.setdefault(j, [])) < TOP: | |
| ranked[j].append(v) | |
| # report sentences ("fresh"): fewest of the author's word sequences first, provided the version does not read | |
| # clearly more AI-like than the sentence does now; a later round takes it when it is clearly more reworded | |
| for j in fresh: | |
| mine = [(v, s_) for (jj, v), s_ in zip(cands, sc) if jj == j] | |
| if not mine: | |
| continue | |
| cur = _overlap(anchors[j], jobs[j][2].strip()) | |
| for v, s_ in sorted(mine, key=lambda x: (_overlap(anchors[j], x[0]), x[1])): | |
| if s_ > now[j] + 0.10 and not always: | |
| continue | |
| if (always or _overlap(anchors[j], v) < cur - 0.05) and len(ranked.setdefault(j, [])) < TOP: | |
| ranked[j].append(v) | |
| pairs = [(j, v) for j, vs in ranked.items() for v in vs] | |
| t_ = time.time() | |
| verdict = _verify([(anchors[j], v) for j, v in pairs]) | |
| TIMES["verify"] = time.time() - t_ | |
| # used only if the faithfulness check SAID it keeps the meaning: when the check could not run (service busy, | |
| # spending cap), the sentence keeps its wording rather than taking an unchecked rewrite | |
| ok = {p for p, r in zip(pairs, verdict) if r is True} | |
| best, bad = [None] * len(jobs), 0 | |
| for j, vs in ranked.items(): | |
| good = [v for v in vs if (j, v) in ok] | |
| bad += len(vs) - len(good) | |
| best[j] = good[0] if good else None | |
| return best, bad | |
| def humanize_docx(path, hi=0.80, mod=0.50, rep=None, budget=None, also=None, also_overlap=None): | |
| """-> dict like smart.smart_docx (before, after, log, note, blocks, path, report, orig, new, final_of). | |
| also(sentence) -> True: rewrite that sentence in round 1 even if our detector does not flag it (the sentences an | |
| uploaded checker report highlights); later rounds take what our detector still flags, until 0%.""" | |
| rep = rep or (lambda *a, **k: None) | |
| t0 = time.time() | |
| G.start_account() | |
| rep(0.03, "Reading your document") | |
| doc = DX.open_clean(path) | |
| orig_text = "\n".join(b.text for b in D.docx_blocks(doc) if b.kind == "para") | |
| n_words = len(orig_text.split()) | |
| budget = budget_for(n_words) if budget is None else budget | |
| glossary = TP.build_glossary([orig_text]) | TP.GLOSSARY.get() | |
| import re as _re | |
| lower = frozenset(_re.findall(r"\b[a-z][a-z]+\b", orig_text)) # words the document writes in lower case | |
| TP.GLOSSARY.set(glossary) | |
| TP.set_document([orig_text]) | |
| # spending cap scaled to the length (the flat $1 stopped the faithfulness check mid-way on a 5,500-word paper) | |
| G.set_cap(max(G.JOB_BUDGET, CAP_PER_1K * len(orig_text.split()) / 1000)) | |
| rep(0.08, "Checking the whole document for AI-written passages") | |
| blocks = S._score_doc(doc) | |
| first = stats = D.summarise(blocks, hi, mod) | |
| best = (stats["pct"], S._snapshot(doc), blocks) | |
| anchor = {} # current sentence -> the author's sentence | |
| # the author's sentence -> its latest wording, recorded when it is replaced (re-splitting the finished text | |
| # cannot find it when a rewrite changed the sentence boundaries); kept with the best version of the document | |
| replaced, best_replaced = {}, {} | |
| log, note = [], "" | |
| end = min(t0 + budget, H.JOB_DEADLINE.get() or float("inf")) # the job's safety cap (wallet: its deadline) | |
| def anchor_of(t): | |
| return anchor.get(t.strip(), t.strip()) | |
| def needs_rewording(t): | |
| """A highlighted sentence (asked about by its ORIGINAL wording) that still keeps too much of that wording.""" | |
| a = anchor_of(t) | |
| return bool(also and also(a)) and _overlap(a, t.strip()) > (REPORT_OVERLAP if also_overlap is None | |
| else also_overlap) | |
| def report_left(blocks_): | |
| return sum(1 for b_ in blocks_ if b_.kind == "para" for t in b_.sents if needs_rewording(t)) if also else 0 | |
| rleft = report_left(blocks) | |
| best_key = (stats["pct"], rleft) | |
| last_lm = last_after = None | |
| stalled = zero_rounds = 0 | |
| for rnd, (n, chg, thr) in enumerate(_plan(rounds_for(n_words)), 1): | |
| if CPU_VERSIONS and D.on_cpu(): | |
| n = min(n, CPU_VERSIONS) | |
| if stats["pct"] == 0 and not rleft and (zero_rounds >= 1 or | |
| not any(_stock(t) for b in blocks if b.kind == "para" for t in b.sents)): | |
| break # at 0%: one more round at most, for leftover stock phrases | |
| zero_rounds += stats["pct"] == 0 | |
| around = {} | |
| jobs, locked = _jobs(blocks, thr if rnd == 1 else min(thr, mod), also=needs_rewording if also else None, | |
| around=around) | |
| if not jobs: | |
| break | |
| # time for choosing + re-scoring after the language model (measured on the T4: about 0.03 s per sentence) | |
| after_lm = 4 + 0.035 * len(jobs) if last_after is None else last_after * 1.15 | |
| left = min(max(ROUND_LM_MAX, ROUND_LM_PER_SENT * len(jobs)), end - time.time() - after_lm) | |
| if rnd > 1 and left < 8.0: # the safety cap is reached | |
| log.append(f"Round {rnd}: stopped, time limit reached") | |
| break | |
| # sentences with fields: the language model sees markers ([REF1] ...); everything else (anchors, choosing, | |
| # the faithfulness check, the report) reads the real text with the citations in place | |
| refs_of = [around[(id(b_), sp_[0])][1] if (id(b_), sp_[0]) in around else [] for b_, sp_, _ in jobs] | |
| llm_texts = [j[2].strip() for j in jobs] | |
| texts = [_detok(t, r) for t, r in zip(llm_texts, refs_of)] | |
| anchors = [anchor.get(t, t) for t in texts] | |
| llm_anchors = [_tok(a, r) for a, r in zip(anchors, refs_of)] | |
| fresh = {i for i, t in enumerate(texts) if needs_rewording(t)} if also else set() | |
| if fresh: # report sentences: ask for more new wording | |
| chg = max(chg, REPORT_CHANGE + 5 * (rnd - 1)) | |
| frac = 0.12 + 0.83 * (1 - 0.6 ** (rnd - 1)) # moves on every round without promising an end | |
| rep(frac, f"Round {rnd}: AI score {stats['pct']}%, rewriting {len(jobs)} sentence(s) at once") | |
| tg = time.time() | |
| try: | |
| outs = _versions(llm_texts, llm_anchors, glossary, n, chg, lower, deadline=time.time() + max(8.0, left)) | |
| except G.Unavailable as err: | |
| note += " (The rewriting service is busy right now: fewer sentences could be rewritten.)" | |
| print(f"[fast] round {rnd}: rewriting unavailable: {err}", flush=True) | |
| break | |
| tg = time.time() - tg | |
| marked = {} # (job, real version) -> its version with markers | |
| for i_, (b_, sp_, _) in enumerate(jobs): # around fields: only versions that fit around them | |
| key_ = (id(b_), sp_[0]) | |
| if key_ in around and outs[i_]: | |
| vs_ = [v for v in outs[i_] if _fits(around[key_][0], v) is not None] | |
| outs[i_] = [_detok(v, refs_of[i_]) for v in vs_] | |
| marked.update({(i_, r_): v for r_, v in zip(outs[i_], vs_)}) | |
| real_jobs = [(b_, sp_, t) for (b_, sp_, _), t in zip(jobs, texts)] | |
| rep(frac + 0.83 * 0.6 ** (rnd - 1) * 0.2, f"Round {rnd}: choosing the most natural version of each sentence") | |
| tc = time.time() | |
| # round 1: every flagged sentence takes its best valid version, even one that scores no lower on its own: a | |
| # window is only read as human when (nearly) all its sentences changed; on a 5,500-word document the "must | |
| # score lower" rule left 4 in 10 sentences as they were and the score stayed at 95% | |
| chosen, unfaithful = _choose(real_jobs, anchors, outs, glossary, _context(blocks), always=rnd <= ALWAYS_ROUNDS, | |
| fresh=fresh) | |
| tc = time.time() - tc | |
| done = 0 | |
| for i_, (b, (a, e), t), new, anc in sorted(zip(range(len(jobs)), jobs, chosen, anchors), | |
| key=lambda x: (id(x[1][0]), -x[1][1][0])): | |
| info = around.get((id(b), a)) | |
| if new and info: | |
| ok = _replace_around(b, info[0], marked.get((i_, new), new)) | |
| else: | |
| ok = bool(new) and S.replace_span(b.el, a, e, new) | |
| if ok: | |
| anchor[new.strip()] = anc | |
| replaced[anc.strip()] = new.strip() | |
| done += 1 | |
| ts = time.time() | |
| blocks = S._score_doc(doc) | |
| prev, stats = stats["pct"], D.summarise(blocks, hi, mod) | |
| rprev, rleft = rleft, report_left(blocks) | |
| ts = time.time() - ts | |
| last_lm, last_after = tg, tc + ts | |
| log.append(f"Round {rnd}: AI score {prev}% → {stats['pct']}% (rewrote {done} of {len(jobs)} sentences" | |
| + (f", {locked} locked" if locked else "") | |
| + (f"; {rleft} highlighted sentence(s) still close to the original" if also else "") + ")") | |
| print(f"[fast] round {rnd}: {len(jobs)} sentences, {done} rewritten, {unfaithful} version(s) rejected as " | |
| f"unfaithful, AI {prev}% -> {stats['pct']}% | " | |
| f"language model {tg:.1f}s, choosing {tc:.1f}s (" | |
| + ", ".join(f"{k} {v:.1f}s" for k, v in TIMES.items()) + f"), scoring {ts:.1f}s", flush=True) | |
| key = (stats["pct"], rleft) # lower AI score first, then fewer report sentences left | |
| if key < best_key: | |
| best, best_key = (stats["pct"], S._snapshot(doc), blocks), key | |
| best_replaced = dict(replaced) | |
| stalled = 0 | |
| else: | |
| stalled += 1 | |
| # at 0% only the report's sentences are being reworded: one round without progress is enough | |
| if stalled >= STALL or (stats["pct"] == 0 and stalled >= 1): | |
| log.append(f"Round {rnd}: stopped, {STALL} rounds without further improvement") | |
| break | |
| # at 0% with only the report's sentences left: a round that clears very few of them is not worth another | |
| # minute (a thesis part spent 5 of its 12 minutes taking 80 such sentences to 72) | |
| if also and prev == 0 and stats["pct"] == 0 and rprev - rleft < max(REPORT_MIN_CLEARED, 0.05 * rprev): | |
| log.append(f"Round {rnd}: stopped, few highlighted sentences left to reword further") | |
| break | |
| if stats["pct"] > best[0]: # this round made it worse: the next one starts from the best | |
| from docx import Document as _Doc | |
| doc = _Doc(io.BytesIO(best[1])) | |
| replaced = dict(best_replaced) | |
| blocks = S._score_doc(doc) | |
| stats = D.summarise(blocks, hi, mod) | |
| rleft = report_left(blocks) | |
| if not done: | |
| break | |
| from docx import Document | |
| doc = Document(io.BytesIO(best[1])) | |
| blocks = best[2] | |
| after = D.summarise(blocks, hi, mod) | |
| new_text = "\n".join(b.text for b in blocks if b.kind == "para") | |
| out = H._save_output(doc, S._clean_stem(path), "_humanized") | |
| secs = time.time() - t0 | |
| print(f"[fast] {first['pct']}% -> {after['pct']}% in {secs:.1f}s ({first['analysed']} words) | " | |
| f"{G.usage_line()}", flush=True) | |
| note = "⚡ Fast engine: every flagged sentence rewritten at once by the language model." + note | |
| left_ = [s_ for b_ in blocks if b_.kind == "para" for s_, p_ in zip(b_.sents, b_.scores) if p_ >= mod] | |
| final_of = dict(best_replaced) # the author's sentence -> its wording in the delivered file | |
| return dict(before=first, after=after, log=log, note=note, blocks=blocks, path=out, report=None, | |
| orig=orig_text, new=new_text, seconds=secs, zero_goal=True, left=len(left_), final_of=final_of) | |
| def humanize_text(text, hi=0.80, mod=0.50, rep=None, budget=None): | |
| """Pasted text: written to a Word file (one paragraph per paragraph), humanized, read back.""" | |
| import tempfile | |
| from docx import Document | |
| d = Document() | |
| for p in [p.strip() for p in (text or "").split("\n") if p.strip()]: | |
| d.add_paragraph(p) | |
| path = os.path.join(tempfile.mkdtemp(prefix="cs_fast_"), "Pasted_text.docx") | |
| d.save(path) | |
| res = humanize_docx(path, hi, mod, rep, budget) | |
| res["text"] = "\n\n".join(p.text for p in Document(res["path"]).paragraphs if p.text.strip()) | |
| return res | |
| def selftest(sizes=(1, 3, 8)): | |
| """FAST_SELFTEST=1 (Space variable): after start-up, time the engine on test documents of about 1,800 x size | |
| words (the subject samples; every copy gets different numbers, so no cache helps). Results go to the log.""" | |
| import random | |
| import re | |
| import sys | |
| import tempfile | |
| from pathlib import Path | |
| from docx import Document | |
| sys.path.insert(0, str(Path(__file__).resolve().parent / "devtools")) | |
| from samples_domains import DOMAINS | |
| time.sleep(float(os.getenv("FAST_SELFTEST_DELAY", "60"))) # let start-up finish | |
| for size in sizes: | |
| rnd = random.Random(size) | |
| d = Document() | |
| d.add_heading(f"Self-test document x{size}", 1) | |
| for c in range(size): | |
| for k, v in DOMAINS.items(): | |
| d.add_heading(f"{k.title()} ({c + 1})", 2) | |
| d.add_paragraph(v if c == 0 else re.sub(r"\d+", lambda m: str(int(m.group()) + rnd.randint(1, 9)), v)) | |
| path = os.path.join(tempfile.mkdtemp(prefix="cs_selftest_"), f"selftest_x{size}.docx") | |
| d.save(path) | |
| try: | |
| r = humanize_docx(path, budget=float(os.getenv("FAST_SELFTEST_BUDGET", str(BUDGET)))) | |
| print(f"[selftest] x{size}: {r['before']['analysed']} words, AI {r['before']['pct']}% -> " | |
| f"{r['after']['pct']}% in {r['seconds']:.1f}s | " + " | ".join(r["log"]), flush=True) | |
| except Exception as err: | |
| print(f"[selftest] x{size}: failed: {type(err).__name__}: {err}", flush=True) | |