Spaces:
Running
Running
File size: 6,540 Bytes
4a7a056 01f5654 439ec70 01f5654 4a7a056 01f5654 4a7a056 01f5654 4a7a056 439ec70 f66d9ce | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 | """Fast engine (fast.py) end to end on a Word file, with the models replaced by fakes: the detector flags sentences
with AI stock words, the language model offers versions, one of which changes the meaning."""
from docx import Document
import detector as D
import fast as F
import fluency as FL
import groqai as G
import humanizer as H
import meaning as MEANING
AI = ("Furthermore, the proposed controller plays a crucial role in reducing the harmonic distortion of the grid "
"current from 4.8% to 1.9% [14].")
AI2 = "These findings provide valuable insights for engineers seeking to design robust inverter control schemes."
HUMAN = ("We built the test rig in the basement lab and ran it for six weeks before the first usable data came out, "
"mostly because the sensors kept drifting.")
def _ai(t):
"""Share of AI-sounding sentences in the text (a window score behaves like this)."""
sents = [x for x in D.split_sentences(t) if x.strip()] or [t]
bad = sum(any(w in x for w in ("crucial", "valuable", "Furthermore")) for x in sents)
return 0.05 + 0.9 * bad / len(sents)
def test_the_fast_engine_lowers_the_score_and_never_uses_an_unfaithful_version(tmp_path, monkeypatch):
monkeypatch.setattr(D, "score_texts", lambda texts: [_ai(t) for t in texts])
monkeypatch.setattr(H, "_detector_scores", lambda texts: [_ai(t) for t in texts])
monkeypatch.setattr(FL, "acceptable", lambda pairs: [True] * len(pairs))
monkeypatch.setattr(MEANING, "similarities", lambda pairs: [0.95] * len(pairs))
seen = {}
import json
def request(gen, system, user, max_tokens, deadline=None):
seen["faithful"] = "KEEP EVERY DETAIL" in system
seen.setdefault("gens", set()).add(gen)
items = []
for i, line in enumerate([x for x in user.splitlines() if "SENTENCE:" in x], 1):
if "harmonic" in line:
vs = ["The controller lowers the harmonic distortion of the grid current from 4.8% to 1.9% [14].",
"The controller is the main reason the distortion vanished [14]."]
elif "valuable" in line:
vs = ["Engineers who design inverter control schemes can use these results."]
else:
vs = []
items.append({"id": i, "versions": vs})
return json.dumps({"items": items})
monkeypatch.setattr(F, "_request", request)
monkeypatch.setattr(F, "GENERATORS", ["gemini:model-a", "groq:model-b"])
monkeypatch.setenv("GEMINI_API_KEY", "test-key")
monkeypatch.setattr(G, "KEY", "test-key")
monkeypatch.setattr(G, "verify", lambda pairs, **k: [True if "lowers" in b or "Engineers" in b
else "claim changed" for _, b in pairs])
d = Document()
d.add_paragraph(HUMAN + " " + AI + " " + AI2)
d.add_paragraph(HUMAN)
path = tmp_path / "paper.docx"
d.save(str(path))
r = F.humanize_docx(str(path), budget=60)
text = "\n".join(p.text for p in Document(r["path"]).paragraphs)
assert r["before"]["pct"] > 0 and r["after"]["pct"] == 0
assert "lowers the harmonic distortion" in text and "[14]" in text and "4.8% to 1.9%" in text
assert "vanished" not in text # rejected by the faithfulness check
assert HUMAN in text # unflagged sentences are left alone
assert seen["faithful"] is True # keep-every-detail prompt
assert seen["gens"] == {"gemini:model-a", "groq:model-b"} # every model asked
def test_a_rewrite_of_a_sentence_beginning_does_not_end_the_sentence_early():
assert F._fit("The model was trained on 12,500 images", "The model learned from 12,500 images.") \
== "The model learned from 12,500 images"
assert F._fit("A full sentence.", "Another full sentence.") == "Another full sentence."
def test_a_request_still_running_at_the_deadline_is_dropped_and_the_job_ends_on_time(monkeypatch):
"""Whatever the length of the document, the result comes back within the time budget: a slow model's answer is
not waited for."""
import time
import json
def request(gen, system, user, max_tokens, deadline=None):
if gen.startswith("slow"):
time.sleep(3) # answers after the deadline
return json.dumps({"items": [{"id": 1, "versions": ["Too late."]}]})
return json.dumps({"items": [{"id": 1, "versions": ["The controller lowers the distortion [14]."]}]})
monkeypatch.setattr(F, "_request", request)
monkeypatch.setattr(F, "GENERATORS", ["groq:fast-model", "slow:model"])
monkeypatch.setitem(F._URLS, "slow", ("http://unused", "SLOW_KEY"))
monkeypatch.setenv("SLOW_KEY", "k")
monkeypatch.setattr(G, "KEY", "test-key")
t = time.time()
out = F._versions(["The controller reduces the distortion [14]."], ["The controller reduces the distortion [14]."],
frozenset(), 2, 0, deadline=time.time() + 1.0)
assert time.time() - t < 2.0 # did not wait for the slow model
assert out == [["The controller lowers the distortion [14]."]]
def test_an_unchecked_rewrite_is_never_used(tmp_path, monkeypatch):
"""When the faithfulness check cannot run (service busy, spending cap), sentences keep their wording."""
import json
monkeypatch.setattr(D, "score_texts", lambda texts: [_ai(t) for t in texts])
monkeypatch.setattr(H, "_detector_scores", lambda texts: [_ai(t) for t in texts])
monkeypatch.setattr(FL, "acceptable", lambda pairs: [True] * len(pairs))
monkeypatch.setattr(MEANING, "similarities", lambda pairs: [0.95] * len(pairs))
monkeypatch.setattr(F, "_request", lambda gen, system, user, mt, deadline=None: json.dumps(
{"items": [{"id": i, "versions": ["The controller lowers the harmonic distortion from 4.8% to 1.9% [14]."]}
for i in range(1, 4)]}))
monkeypatch.setattr(F, "GENERATORS", ["groq:model-b"])
monkeypatch.setattr(G, "KEY", "test-key")
monkeypatch.setattr(G, "verify", lambda pairs, **k: [None] * len(pairs)) # the check could not run
d = Document()
d.add_paragraph(HUMAN + " " + AI + " " + AI2)
path = tmp_path / "paper.docx"
d.save(str(path))
r = F.humanize_docx(str(path), budget=60)
text = "\n".join(p.text for p in Document(r["path"]).paragraphs)
assert AI in text and AI2 in text # nothing unchecked went in
|