tests/test_textparse.py

import textparse as TP

STAT = ("The findings reveal that perceived organizational support significantly mediates this relationship "
        "(β = 0.34, p < 0.01), suggesting that employees who feel supported are more likely to remain engaged.")
CITED = "The results indicate that communication frequency does not moderate the effect (Smith and Lee, 2019)."


# ---------------------------------------------------------------- pasted text
def test_parse_paste_keeps_breaks_and_joins_soft_wraps():
    text = "Title\n\nA first line that is clearly long enough to be a wrapped line of a PDF\ncontinues here.\nNext."
    parts = TP.parse_paste(text)
    assert [p for p, _ in parts] == ["Title", "A first line that is clearly long enough to be a wrapped line of a PDF "
                                     "continues here.", "Next."]
    assert [s for _, s in parts] == ["\n\n", "\n", ""]
    assert TP.join_paste(parts).count("\n\n") == 1


def test_parse_paste_hyphenated_line_break():
    assert TP.parse_paste("the electro-\nchemical cell")[0][0] == "the electrochemical cell"


def test_split_sentences_abbreviations_and_decimals():
    s = "It rose by 3.5 %. Then it fell (see Fig. 2). Smith et al. reported this. However, cf. Table 1."
    assert TP.split_sentences(s) == ["It rose by 3.5 %.", "Then it fell (see Fig. 2).", "Smith et al. reported this.",
                                     "However, cf. Table 1."]


def test_chunk_and_merge_round_trip():
    long = ("The proposed framework integrates several components, which are trained jointly on the benchmark "
            "dataset, and it achieves strong results across tasks; however, the cost is high, because training "
            "requires many GPUs and days.")
    chunks = TP.chunk_sentence(long)
    assert len(chunks) > 1 and all(len(c.split()) >= 7 for c in chunks)
    assert TP.merge_chunks(chunks, chunks) == long
    assert TP.chunk_sentence("A short sentence stays whole.") == ["A short sentence stays whole."]


# ---------------------------------------------------------------- guard: what a rewrite may never do
def test_guard_accepts_a_faithful_rewrite():
    out = TP.guard(CITED, "The results show that communication frequency does not moderate the effect "
                          "(Smith and Lee, 2019).", frozenset())
    assert out and out.endswith("(Smith and Lee, 2019).")


def test_guard_rejects_lost_relation_sign():
    bad = STAT.replace("p < 0.01", "p 0.01")
    assert TP.guard(STAT, bad, frozenset()) is None
    assert TP.symbol_issue(STAT, bad) == "relation sign '<' changed"


def test_guard_rejects_duplicated_and_invented_citations():
    assert TP.guard(CITED, CITED[:-1] + " (Smith and Lee, 2019).", frozenset()) is None
    assert TP.symbol_issue(CITED, CITED.replace("effect", "effect [7]")) == "citation added"


def test_guard_rejects_unbalanced_bracket():
    assert TP.guard(CITED, CITED.replace("2019).", "2019."), frozenset()) is None


def test_guard_reattaches_dropped_citation():
    out = TP.guard(CITED, "The results show that communication frequency does not moderate the effect.", frozenset())
    assert out == "The results show that communication frequency does not moderate the effect (Smith and Lee, 2019)."


def test_guard_rejects_changed_number_and_lost_acronym():
    o = "The CNN reached an accuracy of 96.3% on 12,500 images."
    assert TP.guard(o, "The CNN reached an accuracy of 93.6% on 12,500 images.", frozenset()) is None
    assert TP.guard(o, "The network reached an accuracy of 96.3% on 12,500 images.", frozenset()) is None
    assert TP.guard(o, "The CNN reached a precision of 96.3% on 12,500 images.", frozenset()) is None   # fragile word
    assert TP.guard(o, "The CNN achieved an accuracy of 96.3% on 12,500 images.", frozenset())


def test_guard_rejects_register_problems():
    o = "The method improves the results of the baseline in most cases."
    for bad in ("I think the method improves the results of the baseline in most cases.",
                "The method doesn't hurt the results of the baseline in most cases.",
                "Does the method improve the results of the baseline in most cases?",
                "You can see the method improves the results of the baseline in most cases."):
        assert TP.guard(o, bad, frozenset()) is None, bad


def test_typography_follows_the_author():
    o = "Lithium-ion batteries are used in the module's cooling loop."
    assert TP.normalize_typography(o, "lithium‑ion cells in the module’s loop") == \
        "lithium-ion cells in the module's loop"
    curly = "The module’s loop is short."                      # the author's own curly apostrophe stays
    assert TP.normalize_typography(curly, "The loop of the module’s pump") == "The loop of the module’s pump"
    assert "‑" not in TP.guard(o, "Lithium‑ion batteries are used in the module's cooling circuit.", frozenset())


def test_invented_names_but_not_forms_of_the_authors_names():
    o = "This study examines 412 knowledge workers in the Indian IT sector."
    assert TP.invented(o, "This study examines 412 knowledge workers in India's IT sector.") is None
    assert TP.invented(o, "This study examines 412 knowledge workers in the Chinese IT sector.") == "name"
    assert TP.invented(o, "This study examines 420 knowledge workers in the Indian IT sector.") == "number"
    assert TP.invented(o, "Dr. Rao examines 412 knowledge workers in the Indian IT sector.") == "title"


# ---------------------------------------------------------------- glossary / term restoration
def test_glossary_and_restore_terms():
    g = TP.build_glossary(["A phase change material (PCM) is used. The state of charge matters. "
                           "State of charge is estimated."])
    assert {"phase change material", "state of charge"} <= set(g)
    o = "The PCM lowers the state of charge error by 5%."
    damaged = "The PCM reduces the charge state error by 5%."
    assert TP.guard(o, damaged, g) is None
    assert TP.restore_terms(o, damaged, g) == "The PCM reduces the state of charge error by 5%."


# ---------------------------------------------------------------- rule-based clean-up
def test_basic_grammar():
    assert TP.basic_grammar("the the model will opening a door , and a apple is an university .") == \
        "The model will open a door, and an apple is a university."
    assert TP.basic_grammar("This is an SVM and a MRI scan with a hour") == "This is an SVM and an MRI scan with an hour."
    assert TP.basic_grammar("Greshake et al. Generalised the attack.") == "Greshake et al. generalised the attack."


def test_clean_artifacts():
    assert TP.clean_artifacts("Certainly! Here is the revised sentence: **The model** works.") == "The model works."
    assert TP.artifact_only("Here is the revised text:")


def test_de_ai_keeps_technical_phrases():
    assert TP.de_ai("It is important to note that robust regression plays a crucial role in this.") == \
        "Robust regression is central to this."


# ---------------------------------------------------------------- strict acceptance (LLM output)
def test_strict_check_reports_the_reason():
    g = frozenset()
    assert TP.strict_check(CITED, CITED, g) == (None, "unchanged")
    assert TP.strict_check(CITED, "It does not.", g)[1] == "length"
    assert TP.strict_check(CITED, CITED.replace("effect", "effect [3]"), g)[1] == "citation added"
    ok, why = TP.strict_check(CITED, "The results show that communication frequency does not moderate the effect "
                                     "(Smith and Lee, 2019).", g)
    assert ok and why is None
    assert TP.strict_accept(CITED, "The results show that communication frequency moderates the effect "
                                   "(Smith and Lee, 2019).", g) is None        # negation removed


def test_statistical_relation_words_are_terms():
    o = "The results indicate that communication frequency does not moderate the effect."
    assert TP.lost_terms(o, "The results indicate that communication frequency does not modify the effect.", frozenset())
    assert not TP.lost_terms(o, "The effect is not moderated by communication frequency.", frozenset())


def test_guard_closes_an_open_sentence():
    o = "The module stays below 2.1 degrees, which delays the onset of failure."
    assert TP.guard(o, "The module remains under 2.1 degrees, which delays the onset of failure;", frozenset()) == \
        "The module remains under 2.1 degrees, which delays the onset of failure."
    assert TP.guard("a fragment without a full stop here", "a piece without a full stop here.", frozenset()) == \
        "a piece without a full stop here"


def test_fragments_do_not_gain_a_full_stop():
    frag = "while the temperature difference across the module remains below 2.1 °C"
    assert TP.match_ending(frag, "while the temperature gap across the module stays under 2.1 °C.") == \
        "while the temperature gap across the module stays under 2.1 °C"
    ok, why = TP.strict_check(frag, "while the temperature gap across the module stays under 2.1 °C.", frozenset())
    assert ok and not ok.endswith(".")
    assert TP.match_ending("A full sentence.", "A complete sentence.") == "A complete sentence."


def test_guard_repairs_or_rejects_damaged_citations():
    o = "The exchange rate channel accounts for approximately 22% of the total pass-through (Mishra and Mishra, 2012)."
    fixed = TP.guard(o, "The exchange rate channel explains approximately 22% of the total pass-through "
                        "(Mishra and Mishri, 2012).", frozenset())
    assert fixed and fixed.count("(Mishra and Mishra, 2012)") == 1 and "Mishri" not in fixed
    spaced = TP.guard(o, "The exchange rate channel explains approximately 22% of the total pass-through "
                         "( Mishra and Mishra, 2012 ).", frozenset())
    assert spaced and spaced.count("2012") == 1                       # not re-attached a second time
    assert TP.guard(o, "The exchange rate channel explains approximately 22% of the total pass-through "
                       "(Mishra and Mishra, 2012) (Mishra, 2012).", frozenset()) is None
    assert TP.guard(o, "The exchange rate channel explains approximately 22% of the total pass-through "
                       "(Taylor and Woodford, 1999).", frozenset()) is None


def test_guard_applies_the_integrity_rules_to_every_version():
    o = "The annual mean PM2.5 concentration was 98 µg/m3, which is nearly 20 times the WHO guideline of 5 µg/m3."
    assert TP.guard(o, "The mean annual PM2.5 concentration was 98 g/m3, more than 20 times the WHO guideline of 5.",
                    frozenset()) is None                              # unit lost, "nearly" dropped
    o2 = "The results indicate that a 100 basis point increase in the repo rate reduces headline inflation."
    assert TP.guard(o2, "The results show that a 100 basis point decrease in the repo rate reduces headline inflation.",
                    frozenset()) is None                              # direction reversed
    o3 = "This study examined whether self-compassion mediates the relationship among 386 undergraduate students."
    assert TP.guard(o3, "In this study we examined whether self-compassion mediates the relationship among 386 "
                        "undergraduate students.", frozenset()) is None
    assert TP.guard(o3, "The study tested whether self-compassion mediates the relationship among 386 undergraduate "
                        "students.", frozenset())


def test_clean_up_does_not_touch_citations_or_names():
    s = "The channel accounts for 22% of the the pass-through (Mishra and Mishra, 2012)."
    assert TP.basic_grammar(s) == "The channel accounts for 22% of the pass-through (Mishra and Mishra, 2012)."
    assert TP.basic_grammar("Johnson and Johnson reported informed and informed consent.") == \
        "Johnson and Johnson reported informed consent."