Upload 4 files

clarity.py

@@ -255,8 +255,14 @@ def assess(pairs, use_meaning=True):
         live = [k for k in live if out[k] is None]
     if live and use_meaning:
         sims = MEANING.similarities([pairs[k] for k in live])
+        try:                                            # high vocabulary-change setting: a little more distance
+            import humanizer as _H
+            t = float(_H.VOCAB_TARGET.get() or 0)
+            relax = 0.05 if t >= 50 else 0.03 if t >= 35 else 0.0
+        except Exception:
+            relax = 0.0
         for k, sm in zip(live, sims):
-            if sm < MEANING.min_for(pairs[k][0], MEANING.SOFT_SIM):
+            if sm < MEANING.min_for(pairs[k][0], MEANING.SOFT_SIM) - relax:
                 out[k] = "meaning changed"
                 STATS["meaning"] += 1
     return out

groqai.py

@@ -684,7 +684,10 @@ for the meaning). For every item write {{n}} alternative versions that:
 Answer only with JSON: {"items": [{"id": 1, "versions": ["...", "..."]}, ...]} — one per id."""
 
 
-def _variants_batch(items, glossary, n):
+MIN_CHANGE = _cv.ContextVar("min_change", default=0)   # vocabulary-change target (%) for alternatives
+
+
+def _variants_batch(items, glossary, n, min_change=0):
     sents = [t for t, _ in items]
     terms = _surface_terms(" ".join(sents), glossary)
     lines = []
@@ -694,6 +697,11 @@ def _variants_batch(items, glossary, n):
         else:
             lines.append(f"{i}. SENTENCE: {t}")
     user = "PROTECTED TERMS: " + ("; ".join(terms) if terms else "(none)") + "\n\n" + "\n".join(lines)
+    if min_change:
+        user += (f"\n\nVOCABULARY CHANGE: in every version, at least {int(min_change)}% of the words must differ "
+                 "from the sentence: use different (correct, common) vocabulary and change the clause order or "
+                 "voice where natural. Protected terms, numbers and citations stay exactly as written, and every "
+                 "rule above still holds.")
     max_tokens = min(8000, 500 + int(sum(len(t) for t in sents) * 0.6 * (n + 1)))
     content = _chat([{"role": "system", "content": VARIANTS_SYSTEM.replace("{n}", str(n))},
                      {"role": "user", "content": user}], 0.7, max_tokens)
@@ -718,8 +726,9 @@ def _variants_batch(items, glossary, n):
     return out
 
 
-def variants(sentences, glossary=(), originals=None, n=3, progress=None, lo=0.8, hi=0.85):
-    """Up to n correct, human-sounding alternatives per sentence (list, or None if its batch failed)."""
+def variants(sentences, glossary=(), originals=None, n=3, progress=None, lo=0.8, hi=0.85, min_change=0):
+    """Up to n correct, human-sounding alternatives per sentence (list, or None if its batch failed).
+    min_change: at least this share (%) of the words should differ (vocabulary-change setting)."""
     from concurrent.futures import ThreadPoolExecutor, as_completed
     N = len(sentences)
     out = [None] * N
@@ -733,7 +742,8 @@ def variants(sentences, glossary=(), originals=None, n=3, progress=None, lo=0.8,
         progress(lo, desc=f"Finding natural-sounding correct versions of {N} sentence(s)")
     with ThreadPoolExecutor(max_workers=min(_par(), len(starts))) as pool:
         futs = {pool.submit(_in_provider, PROVIDER.get(), _variants_batch,
-                            list(zip(sentences[st:st + size], originals[st:st + size])), glossary, n): st
+                            list(zip(sentences[st:st + size], originals[st:st + size])), glossary, n,
+                            min_change): st
                 for st in starts}
         for f in as_completed(futs):
             st = futs[f]

humanizer.py

@@ -403,8 +403,12 @@ VOCAB_TARGET = contextvars.ContextVar("vocab_target", default=None)
 
 
 def change_target():
+    """New-wording target per sentence: the customer's Vocabulary change setting when they chose one (up to 90%),
+    otherwise the automatic level (capped at HUM_MAX_VOCAB)."""
     v = VOCAB_TARGET.get()
-    return min(MAX_VOCAB, float(AI_CHANGE_TARGET if v is None else v))
+    if v:
+        return min(90.0, float(v))
+    return min(MAX_VOCAB, float(AI_CHANGE_TARGET))
 FULL_AI = contextvars.ContextVar("full_ai", default=False)
 # Kept for compatibility with smart.py / app.py versions that set it per pass. The 100% AI mode is now
 # "5 Parrot versions + Groq check" (Parrot does all rewriting), so this switch has no effect.
@@ -470,7 +474,7 @@ def _paraphrase_local(texts, temperature, progress=None, lo=0.15, hi=0.85, mode=
 
     full_ai = FULL_AI.get()
     k = CANDIDATES_AI if full_ai else CANDIDATES
-    target = change_target() if full_ai else min(MAX_VOCAB, float(VOCAB_TARGET.get() or 0))
+    target = change_target() if full_ai else min(90.0, float(VOCAB_TARGET.get() or 0))
     er = EXTRA_ROUNDS.get()
     rounds = 1 + ((AI_ROUNDS if er is None else er) if (full_ai or target) else 0)
     temp0 = min(1.5, float(temperature) + (0.2 if full_ai else 0.0))
@@ -548,8 +552,11 @@ def _paraphrase_local(texts, temperature, progress=None, lo=0.15, hi=0.85, mode=
     extra_rounds = meaning_dropped = 0
     qstat = dict(n=0, meaning=0, fluency=0, second=0, sims=[])
     ref = [anchor_of(s) or s for s, _, _ in sent_map]          # the author's original sentence
-    thr = [MEANING.min_for(r) for r in ref]                      # stricter for technical sentences
-    soft = [MEANING.min_for(r, MEANING.SOFT_SIM) for r in ref]
+    # stricter for technical sentences; a high vocabulary-change setting allows a little more distance (the
+    # faithfulness check still rejects changed claims)
+    relax = 0.05 if target >= 50 else 0.03 if target >= 35 else 0.0
+    thr = [MEANING.min_for(r) - relax for r in ref]
+    soft = [MEANING.min_for(r, MEANING.SOFT_SIM) - relax for r in ref]
     # correct sentences with replaced vocabulary: suitable in-context synonyms, structure unchanged (always available
     # as a clear option, so a rejected paraphrase is replaced by new wording instead of the original sentence)
     lex = [[] for _ in sent_map]
@@ -557,7 +564,7 @@ def _paraphrase_local(texts, temperature, progress=None, lo=0.15, hi=0.85, mode=
         if progress is not None:
             progress(lo, desc="Finding suitable replacement vocabulary")
         try:
-            lex = LX.variants(ref, glossary, n=LEX_N)
+            lex = LX.variants(ref, glossary, n=LEX_N, ratios=LX.ratios_for(target, LEX_N + 1) if target else None)
         except Exception as err:
             print(f"[humanizer] vocabulary replacement skipped: {err}", flush=True)
         print(f"[humanizer] vocabulary replacement: {sum(1 for x in lex if x)} of {len(lex)} sentences got "
@@ -742,15 +749,16 @@ def _paraphrase_local(texts, temperature, progress=None, lo=0.15, hi=0.85, mode=
         cur = [chosen[si] or sent_map[si][0] for si in want]
         sc = _detector_scores(cur) or [1.0] * len(cur)
         now = dict(zip(want, sc))
-        todo = sorted([si for si in want if chosen[si] is None or now[si] >= RESCUE_AI],
-                      key=lambda si: -now[si])[:RESCUE_MAX]
+        short = {si for si in want if target and (chosen[si] is None or change(si, chosen[si]) < target)}
+        todo = sorted([si for si in want if chosen[si] is None or now[si] >= RESCUE_AI or si in short],
+                      key=lambda si: (si not in short or chosen[si] is not None, -now[si]))[:RESCUE_MAX]
         if todo:
             vs = None
             for prov in [G.PROVIDER.get()] + (["openai"] if G.PROVIDER.get() != "openai" and G.openai_available()
                                               else []):
                 try:
                     vs = G._in_provider(prov, G.variants, [ref[si] for si in todo], glossary, None, RESCUE_N,
-                                        progress, hi, hi)
+                                        progress, hi, hi, min(80, target) if target else 0)
                     break
                 except G.Unavailable as err:
                     print(f"[humanizer] rescue rewrites unavailable ({prov}): {err}", flush=True)
@@ -775,10 +783,13 @@ def _paraphrase_local(texts, temperature, progress=None, lo=0.15, hi=0.85, mode=
                 csc = _detector_scores([v for _, v in cands]) or [0.5] * len(cands)
                 best = {}
                 for (si, v), d in zip(cands, csc):
-                    if si not in best or d < best[si][0]:
-                        best[si] = (d, v)
-                for si, (d, v) in best.items():
-                    if d < now[si] - 0.05 or chosen[si] is None and d < now[si]:
+                    # with a vocabulary-change setting, versions that reach it come first, then the most human
+                    key = ((0 if not target or change(si, v) >= target else 1), d)
+                    if si not in best or key < best[si][0]:
+                        best[si] = (key, v)
+                for si, ((miss, d), v) in best.items():
+                    reaches = target and not miss and (chosen[si] is None or change(si, chosen[si]) < target)
+                    if d < now[si] - 0.05 or (chosen[si] is None and d < now[si]) or (reaches and d <= now[si] + 0.05):
                         chosen[si] = v
                         rescued += 1
             CL.report()

lexsub.py

@@ -140,6 +140,9 @@ def _same_form(orig, cand, wn=None):
     o, c = orig.lower(), cand.lower()
     if o.endswith("ed") and not c.endswith("ed"):
         return bool(wn) and (wn.morphy(c, "v") or c) != c and not c.endswith(("s", "ing"))
+    if wn and not o.endswith(("ed", "s", "ing")) and set(wn._morphy(o, "v")) - {o}:
+        # irregular past form ('found', 'grew', 'made'): the replacement must be a past form too
+        return c.endswith("ed") or bool(set(wn._morphy(c, "v")) - {c}) and not c.endswith(("s", "ing"))
     for suf in ("ing", "ed", "ly", "s", "est"):
         if o.endswith(suf) != c.endswith(suf):
             return False
@@ -200,9 +203,21 @@ def _protected_words(sentence, glossary):
 
 
 # ------------------------------------------------------------------------------------------------ public API
-def variants(sentences, glossary=(), n=2, ratio=None, predict_fn=None):
-    """Up to n vocabulary-replaced versions of each sentence (list per sentence; [] when nothing to replace)."""
+def ratios_for(target, n=3):
+    """Replacement shares that aim at a vocabulary-change target (%): content words are about half of an academic
+    sentence, so a target of T% needs roughly 2T% of the eligible words replaced; versions straddle that level."""
+    if not target:
+        return [RATIO] * n
+    need = min(1.0, 2.0 * float(target) / 100.0)
+    steps = [0.75, 1.0, 1.3, 1.6][:max(1, n)]
+    return sorted({round(min(1.0, max(0.2, need * f)), 2) for f in steps})
+
+
+def variants(sentences, glossary=(), n=2, ratio=None, predict_fn=None, ratios=None):
+    """Vocabulary-replaced versions of each sentence (list per sentence; [] when nothing to replace): n versions at
+    `ratio`, or one version per share in `ratios` (e.g. from ratios_for(target))."""
     ratio = RATIO if ratio is None else ratio
+    plan = list(ratios) if ratios else [ratio] * n
     out = [[] for _ in sentences]
     if not sentences or not ENABLED:
         return out
@@ -241,15 +256,16 @@ def variants(sentences, glossary=(), n=2, ratio=None, predict_fn=None):
     for k, items in per.items():
         s = sentences[k]
         n_words = len(s.split())
-        limit = max(1, round(ratio * n_elig.get(k, 0))) if n_words >= 6 else 1
         items.sort(key=lambda x: -x[0])                 # most predictable words first
-        for v in range(n):
+        for v, r in enumerate(plan):
+            limit = max(1, round(r * n_elig.get(k, 0))) if n_words >= 6 else 1
+            gap = 0 if r >= 0.6 else 1                  # high targets may replace neighbouring words
             chosen = []
             for p_orig, (a, b, w), good in items:
                 if len(chosen) >= limit:
                     break
-                if any(a <= cb + 1 and ca <= b + 1 for ca, cb, _ in chosen):
-                    continue                            # never two neighbouring words
+                if any(a <= cb + gap and ca <= b + gap for ca, cb, _ in chosen):
+                    continue
                 c = good[min(v, len(good) - 1)][0]
                 if w[0].isupper():
                     c = c.capitalize()