Upload 3 files

app.py

@@ -1586,11 +1586,12 @@ with gr.Blocks(title=f"{APP_NAME} · CS Software Solutions", theme=theme, css=CS
                                 sd_status = gr.HTML(H.status_idle("Upload a .docx and press **Detect & humanize document**."))
                                 sd_file = gr.File(label="Download humanized .docx", interactive=False)
                                 sd_dl = gr.HTML()
-                                sd_pr_btn = gr.Button("🔍 Rigorous proofread (keeps the AI score)", variant="secondary")
-                                gr.Markdown("<small>After humanizing: every sentence is proofread (grammar first, then "
-                                            "clarity, academic wording and flow) and removes doubled terms and AI "
-                                            "leftovers. Grammar fixes are always made; style edits only if the AI score "
-                                            "holds. Free, does not use a document from your plan.</small>")
+                                sd_pr_btn = gr.Button("🔍 Final check & proofread (keeps the AI score)", variant="secondary")
+                                gr.Markdown("<small>After humanizing: one last full check of the whole document in a "
+                                            "single pass: grammar, unclear sentences, academic wording, flow and "
+                                            "consistent terms, compared with your original. Grammar fixes are always "
+                                            "made; style edits only if the AI score holds. Problems only the author can "
+                                            "decide are listed. Free, does not use a document from your plan.</small>")
                                 sd_pr_status = gr.HTML()
                                 sd_pr_file = gr.File(label="Download proofread .docx", interactive=False)
                                 gr.HTML(next_steps_html())
@@ -1733,7 +1734,7 @@ with gr.Blocks(title=f"{APP_NAME} · CS Software Solutions", theme=theme, css=CS
                                 hd_status = gr.HTML(H.status_idle("Upload a document and press **Paraphrase document**."))
                                 hd_file = gr.File(label="Download paraphrased .docx", interactive=False)
                                 hd_dl = gr.HTML()
-                                hd_pr_btn = gr.Button("🔍 Rigorous proofread (keeps the AI score)", variant="secondary")
+                                hd_pr_btn = gr.Button("🔍 Final check & proofread (keeps the AI score)", variant="secondary")
                                 hd_pr_status = gr.HTML()
                                 hd_pr_file = gr.File(label="Download proofread .docx", interactive=False)
                                 hd_pr_dl = gr.HTML()

groqai.py

@@ -217,6 +217,8 @@ OA_STATS = {"calls": 0, "tokens": 0, "last_error": ""}
 
 PAR_LIMIT = _cv.ContextVar("llm_parallel", default=0)    # >0: fewer requests at once (proofread on Groq)
 REASONING = _cv.ContextVar("llm_reasoning", default="")  # "" = GROQ_REASONING / OPENAI_REASONING
+OUT_CAP = _cv.ContextVar("llm_out_cap", default=0)        # >0: larger max_tokens cap for this call
+TIMEOUT = _cv.ContextVar("llm_timeout", default=0)        # >0: longer HTTP timeout (s) for this call
 
 
 def _par():
@@ -277,7 +279,8 @@ def _chat_openai(messages, temperature, max_tokens):
     waited = 0.0
     for attempt in range(10):
         try:
-            r = httpx.post(OPENAI_URL + "/chat/completions", json=body, headers=_oa_headers(), timeout=180)
+            r = httpx.post(OPENAI_URL + "/chat/completions", json=body, headers=_oa_headers(),
+                           timeout=TIMEOUT.get() or 180)
         except Exception as err:
             OA_STATS["last_error"] = f"network: {err}"
             if attempt >= 2:
@@ -421,8 +424,10 @@ def _chat(messages, temperature, max_tokens):
     model = models[0]
     effort = REASONING.get() or REASONING_DEFAULT
 
+    cap = OUT_CAP.get() or MAX_OUT
+
     def budget(m, base):
-        return min(MAX_OUT, base + (_REASON_ALLOW.get(effort, 700) if _is_reasoning(m) else 0))
+        return min(cap, base + (_REASON_ALLOW.get(effort, 700) if _is_reasoning(m) else 0))
     body = {"model": model, "messages": messages, "temperature": temperature,
             "max_tokens": budget(model, max_tokens), "response_format": {"type": "json_object"}}
     if "gpt-oss" in model:
@@ -442,7 +447,7 @@ def _chat(messages, temperature, max_tokens):
             continue
         try:
             try:
-                r = httpx.post(URL + "/chat/completions", json=body, headers=_headers(k), timeout=120)
+                r = httpx.post(URL + "/chat/completions", json=body, headers=_headers(k), timeout=TIMEOUT.get() or 120)
             except Exception as err:
                 STATS["errors"] += 1
                 STATS["last_error"] = f"network: {err}"
@@ -465,10 +470,14 @@ def _chat(messages, temperature, max_tokens):
             _pause_key(k, min(max(ra, 60), 6 * 3600) if daily else ra, "daily quota" if daily else "rate limit")
             STATS["last_error"] = f"rate limit (key {k + 1}): {txt[:160]}"
             continue                                   # next key at once
-        if r.status_code == 413 or (r.status_code == 400 and "max_tokens" in txt and "must be" in txt.lower()):
+        if (r.status_code in (400, 413) and "max_tokens" in txt and ("must be" in txt.lower() or
+                                                                      "too large" in txt.lower())):
             if body["max_tokens"] > 1024:              # budget above what this model accepts: shrink and retry
                 body["max_tokens"] = max(1024, body["max_tokens"] // 2)
                 continue
+        if r.status_code == 413:                       # the request itself is too large for this key / model
+            STATS["last_error"] = f"request too large: {txt[:160]}"
+            raise Unavailable(STATS["last_error"])
         if r.status_code == 401:
             _pause_key(k, 3600, "key rejected (401)")
             STATS["last_error"] = f"Groq key {k + 1} rejected (401) — check that secret."
@@ -511,9 +520,9 @@ def _chat(messages, temperature, max_tokens):
             _KEY_STATE[k]["tokens"] += used
         choice = (data.get("choices") or [{}])[0]
         content = (choice.get("message") or {}).get("content") or ""
-        if choice.get("finish_reason") == "length" and grown < 1 and body["max_tokens"] < MAX_OUT:
+        if choice.get("finish_reason") == "length" and grown < 1 and body["max_tokens"] < cap:
             grown += 1                                 # reply cut off: one retry with a larger budget
-            body["max_tokens"] = min(MAX_OUT, int(body["max_tokens"] * 1.8) + 256)
+            body["max_tokens"] = min(cap, int(body["max_tokens"] * 1.8) + 256)
             STATS["truncated"] = STATS.get("truncated", 0) + 1
             continue
         return content
@@ -995,3 +1004,66 @@ def variants(sentences, glossary=(), originals=None, n=3, progress=None, lo=0.8,
     sizes = [len(s) * (n + 1) + len(o or "") for s, o in zip(sentences, originals)]
     return _run_batches("variants", list(zip(sentences, originals)), sizes, 8, BATCH_CHARS * 2,
                         lambda its: _variants_batch(its, glossary, n, min_change), None, lo, hi, "Alternatives")
+
+
+# ---------------------------------------------------------------- one-call final check of a whole document
+DOC_SYSTEM = POLISH_SYSTEM.replace(
+    "It is given paragraph by paragraph; each numbered item is\none sentence of the CURRENT text.",
+    "The WHOLE document is given at once, paragraph by paragraph; each numbered item is one sentence of the CURRENT "
+    "text.").replace(
+    "Answer only with JSON:", "DOCUMENT-LEVEL CHECKS (you see the whole text, so also):\n"
+    "- every construct listed under CONSTRUCTS keeps exactly that name everywhere (never pleasure/contentment/"
+    "happiness/enjoyment for satisfaction, closeness for intimacy, allegiance/devotion for loyalty);\n"
+    "- the same group words as the original (men/women, male/female, genders), the same numbers and the same "
+    "statistics, including operators: never add, remove or change <, >, = (if one is missing, leave it and list it "
+    "in notes);\n"
+    "- every sentence complete (no cut-off endings, no fragments, no two sentences run together), sentences flow "
+    "into each other within each paragraph, commas in lists and after introductory phrases.\n"
+    "In \"notes\", list (short, at most 12) problems you must NOT fix yourself because they need the author: "
+    "inconsistent abbreviations (e.g. EWB vs EWD), a construct named two ways, missing statistics operators, "
+    "results described in the wrong direction, an unexplained place or name.\n"
+    "List in \"items\" ONLY the sentences you changed; leave every correct sentence out.\n"
+    "Answer only with JSON:").replace(
+    '"unchanged": [1, 4]}. ' + _DIFF_RULE, '"notes": ["..."]}').replace(
+    'List in \"items\" ONLY the sentences you changed;',
+    'status: "repaired" = unclear/garbled/meaning fixed, "fixed" = a grammar, spelling, punctuation or wrong-term '
+    'error corrected, "edited" = style or flow only. List in \"items\" ONLY the sentences you changed;')
+
+
+def proofread_document(sentences, groups, originals, glossary, constructs=()):
+    """ONE request for the whole document: clarity repair + grammar + academic wording + flow + document-level
+    consistency. Returns ([(status, text) per sentence], notes). Unchanged sentences come back as ("ok", text).
+    Raises Unavailable when the provider cannot take the request (too large, quota): the caller falls back to the
+    batched proofread."""
+    n = len(sentences)
+    terms = _surface_terms(" ".join(sentences), glossary)
+    lines, last, para = [], object(), 0
+    for i, (g, t, orig) in enumerate(zip(groups, sentences, originals), 1):
+        if g != last:
+            para += 1
+            lines.append(f"\n[Paragraph {para}]")
+            if orig:
+                lines.append(f"ORIGINAL: {orig}")
+                lines.append("CURRENT:")
+            last = g
+        lines.append(f"{i}. {t}")
+    user = ("CONSTRUCTS: " + ("; ".join(constructs) if constructs else "(none)") + "\n"
+            + "PROTECTED TERMS: " + ("; ".join(terms) if terms else "(none)") + "\n" + "\n".join(lines))
+    chars = sum(len(t) for t in sentences)
+    max_tokens = int(chars / 3.3 * 0.6) + 20 * n + 800       # a reply that changes up to ~60% of the text
+    tok_c, tok_t = OUT_CAP.set(int(os.getenv("PROOF_SINGLE_MAX_OUT", "32000"))), \
+        TIMEOUT.set(float(os.getenv("PROOF_SINGLE_TIMEOUT", "600")))
+    try:
+        content = _chat([{"role": "system", "content": DOC_SYSTEM}, {"role": "user", "content": user}], 0.2,
+                        max_tokens)
+    finally:
+        OUT_CAP.reset(tok_c)
+        TIMEOUT.reset(tok_t)
+    data = _load_json(content)
+    if not data or ("items" not in data and "notes" not in data):
+        raise Unavailable("one-call proofread: no usable answer")
+    res = _parse(content, n, sentences)
+    if data.get("_salvaged"):                              # cut-off reply: the missing ids were NOT checked
+        res = [r if r != (None, None) else (None, None) for r in res]
+    notes = [str(x).strip() for x in (data.get("notes") or []) if str(x).strip()][:12]
+    return res, notes

proofread.py

@@ -59,6 +59,11 @@ REPAIR_SIM = float(os.getenv("PROOF_REPAIR_SIM", "0.60"))
 REPAIR_SIM_NOREF = float(os.getenv("PROOF_REPAIR_SIM_NOREF", "0.70"))
 # reasoning effort of the full proofread on reasoning models (the grammar audit stays on GROQ_REASONING)
 REASONING = os.getenv("PROOF_REASONING", "medium").lower()
+# single (default): the whole document is checked and corrected in ONE request (grammar, clarity, academic wording,
+# flow and document-wide consistency), and the reply lists problems that need the author. Every correction still
+# goes through the local checks. If the provider cannot take one request of that size, or it fails, the batched
+# proofread (several smaller requests) runs instead. batched = always the batched proofread.
+MODE = os.getenv("PROOF_MODE", "single").lower()
 PAR_TOL = int(os.getenv("PROOF_PAR_TOL", "0"))       # flagged words a paragraph may gain after balancing
 PARROT = os.getenv("PROOF_PARROT", "0") == "1"
 # which AI does the proofreading: Groq by default (PROOF_PROVIDER=openai for ChatGPT). If the chosen one is out of
@@ -309,35 +314,66 @@ def _proofread(path, orig_path=None, rep=None, hi=0.80, mod=0.50):
     for i in FL.bad(base):
         defect[i] = True
 
-    # 1. grammar audit: Groq's grammar-only correction tells which sentences contain an error (and gives the
-    #    minimal fix for them). Sentences with an error or a defect are REQUIRED fixes: correctness comes first.
-    rep(0.15, "Grammar audit of every sentence")
     work = [i for i in range(len(jobs)) if i not in drops]
-    minimal = {}
-    g = llm.run(G.grammar, [base[i] for i in work], glossary, rep, 0.15, 0.35, size=BATCH)
-    unchecked = sum(1 for t in g if t is None)
-    if work and unchecked == len(work):
-        raise G.Unavailable("The proofreading engine has reached its usage limit for now. Please try again in a "
-                            "while; nothing in your document was changed.")
-    cands = [None] * len(jobs)
-    for i, t in zip(work, g):
-        cands[i] = t
-    minimal = _checked(jobs, cands, glossary, minimal=True)
-    required = {i for i in work if defect[i] or i in minimal}
-
-    # 2. full proofread (clarity + grammar + academic wording + flow)
     ctx = [j["par"] or j["sent"] for j in jobs]
     ctx = [c if c and c.strip() != t.strip() else None for c, t in zip(ctx, texts)]
     groups = [id(j["b"]) for j in jobs]
     idx_all = list(range(len(jobs)))
-    tok_r = G.REASONING.set(REASONING)                 # deeper reading for the full proofread
-    try:
-        res = llm.run(lambda ids, *a: G.polish([base[i] for i in ids], [groups[i] for i in ids], glossary, rep,
-                                                0.35, 0.65, originals=[ctx[i] for i in ids], size=BATCH),
-                      idx_all, missing=lambda r: r is None or r[1] is None)
-    finally:
+    minimal, notes, single, unchecked = {}, [], False, 0
+
+    # ONE request for the whole document (PROOF_MODE=single): grammar, clarity, wording, flow and consistency
+    if MODE == "single":
+        rep(0.2, "Final check: the whole document is read and corrected in one pass")
+        constructs = sorted({re.sub(r"\s+", " ", m.group(0)).strip() for m in re.finditer(
+            r"(?:[A-Za-z][\w\-]*\s+){1,4}\([A-Z][A-Za-z0-9\-]{1,10}\)",
+            " ".join(b.raw for b in D.docx_blocks(doc) if b.kind == "para" and b.raw))})[:40]
+        tok_r = G.REASONING.set(REASONING)
+        for prov in llm.order:                         # one call; the other provider only if the first fails
+            tok_p = G.PROVIDER.set(prov)
+            try:
+                res, notes = G.proofread_document([base[i] for i in idx_all], groups, ctx, glossary, constructs)
+                llm.used.add(prov)
+                single = True
+                break
+            except G.Unavailable as err:
+                print(f"[proofread] one-call check not possible on {prov}: {err}", flush=True)
+            except Exception as err:
+                print(f"[proofread] one-call check failed on {prov}: {err}", flush=True)
+            finally:
+                G.PROVIDER.reset(tok_p)
         G.REASONING.reset(tok_r)
-    res = [r if r is not None else (None, None) for r in res]
+        if single:
+            unchecked = sum(1 for r in res if r == (None, None))
+            res = [r if r != (None, None) else ("ok", base[i]) for i, r in enumerate(res)]
+            fixed_ids = {i for i, (st, _) in enumerate(res) if st in ("fixed", "repaired")}
+            required = {i for i in work if defect[i] or i in fixed_ids}
+        else:
+            rep(0.2, "The document is too large for one request here: checking it in parts")
+
+    if not single:
+        # 1. grammar audit: Groq's grammar-only correction tells which sentences contain an error (and gives the
+        #    minimal fix for them). Sentences with an error or a defect are REQUIRED fixes: correctness comes first.
+        rep(0.15, "Grammar audit of every sentence")
+        g = llm.run(G.grammar, [base[i] for i in work], glossary, rep, 0.15, 0.35, size=BATCH)
+        unchecked = sum(1 for t in g if t is None)
+        if work and unchecked == len(work):
+            raise G.Unavailable("The proofreading engine has reached its usage limit for now. Please try again in "
+                                "a while; nothing in your document was changed.")
+        cands = [None] * len(jobs)
+        for i, t in zip(work, g):
+            cands[i] = t
+        minimal = _checked(jobs, cands, glossary, minimal=True)
+        required = {i for i in work if defect[i] or i in minimal}
+
+        # 2. full proofread (clarity + grammar + academic wording + flow)
+        tok_r = G.REASONING.set(REASONING)                 # deeper reading for the full proofread
+        try:
+            res = llm.run(lambda ids, *a: G.polish([base[i] for i in ids], [groups[i] for i in ids], glossary,
+                                                    rep, 0.35, 0.65, originals=[ctx[i] for i in ids], size=BATCH),
+                          idx_all, missing=lambda r: r is None or r[1] is None)
+        finally:
+            G.REASONING.reset(tok_r)
+        res = [r if r is not None else (None, None) for r in res]
     asked_repair = {i for i, (st, _) in enumerate(res) if st == "repaired"}
     full = _checked(jobs, [t for _, t in res], glossary, repaired=asked_repair)
     for i in drops:
@@ -380,7 +416,7 @@ def _proofread(path, orig_path=None, rep=None, hi=0.80, mod=0.50):
 
     # 3. verification rounds: every corrected or required sentence is checked again until Groq finds no error
     n_verify = 0
-    for rnd in range(ROUNDS):
+    for rnd in range(0 if single else ROUNDS):
         # a sentence whose text IS the grammar check's own output is not sent again (it was just checked)
         chk = [i for i in work if (i in final or i in required) and final.get(i, base[i]) != minimal.get(i)]
         if not chk:
@@ -466,7 +502,7 @@ def _proofread(path, orig_path=None, rep=None, hi=0.80, mod=0.50):
             verified.setdefault(i, set()).add(t)
 
     n_reworded = 0
-    for rnd in range(AI_ROUNDS):
+    for rnd in range(0 if single else AI_ROUNDS):
         ids = [i for i in work if final.get(i)]
         if not ids:
             break
@@ -539,7 +575,7 @@ def _proofread(path, orig_path=None, rep=None, hi=0.80, mod=0.50):
                 worse.append((pid, was))
         if not worse:
             break
-        if attempt == 1:                               # more correct versions for the sentences in those paragraphs
+        if attempt == 1 and not single:                # more correct versions for the sentences in those paragraphs
             gen_alts([i for pid, _ in worse for i in by_par[pid] if i in final and final[i]], 0.88)
         for pid, was in worse:
             choice = {i: final[i] for i in by_par[pid] if i in final}
@@ -614,7 +650,8 @@ def _proofread(path, orig_path=None, rep=None, hi=0.80, mod=0.50):
         after = D.summarise(S._score_doc(out_doc), hi, mod)
     out = H._save_output(out_doc, S._clean_stem(path), "_proofread")
     who = " + ".join(_NAME.get(p, p) for p in llm.order if p in llm.used) or llm.name
-    note = (f"🔍 Rigorous proofread by {who}: {n} of {len(jobs)} sentences corrected"
+    note = (("🔍 Final check in one request" if single else "🔍 Rigorous proofread")
+            + f" by {who}: {n} of {len(jobs)} sentences corrected"
             + (f" ({len(required)} had a grammar error or defect and were always corrected"
                + (f", {len(repaired)} unclear sentence(s) repaired to the original meaning" if repaired else "")
                + (f", {n_minimal} with the minimal grammar fix" if n_minimal else "")
@@ -634,5 +671,8 @@ def _proofread(path, orig_path=None, rep=None, hi=0.80, mod=0.50):
                "later to finish them." if unchecked else "")
             + ("" if orig_path else " Tip: run it right after humanizing, so your original is used as the meaning "
                                     "reference for unclear sentences."))
+    if notes:                                          # document-level problems that need the author
+        note += " 📝 For the author: " + " · ".join(notes)
     print(f"[proofread] {note}", flush=True)
-    return dict(path=out, before=before, after=after, n=n, total=len(jobs), note=note)
+    return dict(path=out, before=before, after=after, n=n, total=len(jobs), note=note, notes=notes,
+                single=single)