tests/test_fast_engine.py

"""Fast engine (fast.py) end to end on a Word file, with the models replaced by fakes: the detector flags sentences
with AI stock words, the language model offers versions, one of which changes the meaning."""
from docx import Document

import detector as D
import fast as F
import fluency as FL
import groqai as G
import humanizer as H
import meaning as MEANING

AI = ("Furthermore, the proposed controller plays a crucial role in reducing the harmonic distortion of the grid "
      "current from 4.8% to 1.9% [14].")
AI2 = "These findings provide valuable insights for engineers seeking to design robust inverter control schemes."
HUMAN = ("We built the test rig in the basement lab and ran it for six weeks before the first usable data came out, "
         "mostly because the sensors kept drifting.")


def _ai(t):
    """Share of AI-sounding sentences in the text (a window score behaves like this)."""
    sents = [x for x in D.split_sentences(t) if x.strip()] or [t]
    bad = sum(any(w in x for w in ("crucial", "valuable", "Furthermore")) for x in sents)
    return 0.05 + 0.9 * bad / len(sents)


def test_the_fast_engine_lowers_the_score_and_never_uses_an_unfaithful_version(tmp_path, monkeypatch):
    monkeypatch.setattr(D, "score_texts", lambda texts: [_ai(t) for t in texts])
    monkeypatch.setattr(H, "_detector_scores", lambda texts: [_ai(t) for t in texts])
    monkeypatch.setattr(FL, "acceptable", lambda pairs: [True] * len(pairs))
    monkeypatch.setattr(MEANING, "similarities", lambda pairs: [0.95] * len(pairs))
    seen = {}

    import json

    def request(gen, system, user, max_tokens, deadline=None):
        seen["faithful"] = "KEEP EVERY DETAIL" in system
        seen.setdefault("gens", set()).add(gen)
        items = []
        for i, line in enumerate([x for x in user.splitlines() if "SENTENCE:" in x], 1):
            if "harmonic" in line:
                vs = ["The controller lowers the harmonic distortion of the grid current from 4.8% to 1.9% [14].",
                      "The controller is the main reason the distortion vanished [14]."]
            elif "valuable" in line:
                vs = ["Engineers who design inverter control schemes can use these results."]
            else:
                vs = []
            items.append({"id": i, "versions": vs})
        return json.dumps({"items": items})
    monkeypatch.setattr(F, "_request", request)
    monkeypatch.setattr(F, "GENERATORS", ["gemini:model-a", "groq:model-b"])
    monkeypatch.setenv("GEMINI_API_KEY", "test-key")
    monkeypatch.setattr(G, "KEY", "test-key")
    monkeypatch.setattr(G, "verify", lambda pairs, **k: [True if "lowers" in b or "Engineers" in b
                                                        else "claim changed" for _, b in pairs])
    d = Document()
    d.add_paragraph(HUMAN + " " + AI + " " + AI2)
    d.add_paragraph(HUMAN)
    path = tmp_path / "paper.docx"
    d.save(str(path))
    r = F.humanize_docx(str(path), budget=60)
    text = "\n".join(p.text for p in Document(r["path"]).paragraphs)
    assert r["before"]["pct"] > 0 and r["after"]["pct"] == 0
    assert "lowers the harmonic distortion" in text and "[14]" in text and "4.8% to 1.9%" in text
    assert "vanished" not in text                                 # rejected by the faithfulness check
    assert HUMAN in text                                          # unflagged sentences are left alone
    assert seen["faithful"] is True                               # keep-every-detail prompt
    assert seen["gens"] == {"gemini:model-a", "groq:model-b"}     # every model asked


def test_a_rewrite_of_a_sentence_beginning_does_not_end_the_sentence_early():
    assert F._fit("The model was trained on 12,500 images", "The model learned from 12,500 images.") \
        == "The model learned from 12,500 images"
    assert F._fit("A full sentence.", "Another full sentence.") == "Another full sentence."


def test_a_request_still_running_at_the_deadline_is_dropped_and_the_job_ends_on_time(monkeypatch):
    """Whatever the length of the document, the result comes back within the time budget: a slow model's answer is
    not waited for."""
    import time
    import json

    def request(gen, system, user, max_tokens, deadline=None):
        if gen.startswith("slow"):
            time.sleep(3)                              # answers after the deadline
            return json.dumps({"items": [{"id": 1, "versions": ["Too late."]}]})
        return json.dumps({"items": [{"id": 1, "versions": ["The controller lowers the distortion [14]."]}]})
    monkeypatch.setattr(F, "_request", request)
    monkeypatch.setattr(F, "GENERATORS", ["groq:fast-model", "slow:model"])
    monkeypatch.setitem(F._URLS, "slow", ("http://unused", "SLOW_KEY"))
    monkeypatch.setenv("SLOW_KEY", "k")
    monkeypatch.setattr(G, "KEY", "test-key")
    t = time.time()
    out = F._versions(["The controller reduces the distortion [14]."], ["The controller reduces the distortion [14]."],
                      frozenset(), 2, 0, deadline=time.time() + 1.0)
    assert time.time() - t < 2.0                                  # did not wait for the slow model
    assert out == [["The controller lowers the distortion [14]."]]


def test_an_unchecked_rewrite_is_never_used(tmp_path, monkeypatch):
    """When the faithfulness check cannot run (service busy, spending cap), sentences keep their wording."""
    import json
    monkeypatch.setattr(D, "score_texts", lambda texts: [_ai(t) for t in texts])
    monkeypatch.setattr(H, "_detector_scores", lambda texts: [_ai(t) for t in texts])
    monkeypatch.setattr(FL, "acceptable", lambda pairs: [True] * len(pairs))
    monkeypatch.setattr(MEANING, "similarities", lambda pairs: [0.95] * len(pairs))
    monkeypatch.setattr(F, "_request", lambda gen, system, user, mt, deadline=None: json.dumps(
        {"items": [{"id": i, "versions": ["The controller lowers the harmonic distortion from 4.8% to 1.9% [14]."]}
                   for i in range(1, 4)]}))
    monkeypatch.setattr(F, "GENERATORS", ["groq:model-b"])
    monkeypatch.setattr(G, "KEY", "test-key")
    monkeypatch.setattr(G, "verify", lambda pairs, **k: [None] * len(pairs))    # the check could not run
    d = Document()
    d.add_paragraph(HUMAN + " " + AI + " " + AI2)
    path = tmp_path / "paper.docx"
    d.save(str(path))
    r = F.humanize_docx(str(path), budget=60)
    text = "\n".join(p.text for p in Document(r["path"]).paragraphs)
    assert AI in text and AI2 in text                             # nothing unchecked went in