import detector as D def _blocks(*paras): return D.classify([D.Block(D._norm(p)) for p in paras], D.PASTE_MIN_WORDS) def test_split_keeps_every_character(): for s in ['He said "stop." Then left. The U.S. economy grew.', "The model (trained in 2020.) Was good. It won [3]. [4] shows more. (See Fig. 2.) Next one.", "It rose by 3.5 %. Then it fell (see Fig. 2). Smith et al. reported this."]: assert " ".join(D.split_sentences(s)) == s def test_windows_cover_every_sentence(): lengths = [12, 30, 8, 25, 40, 5, 18, 22, 350, 9] w = D.make_windows(lengths) covered = {k for a, b in w for k in range(a, b)} assert covered == set(range(len(lengths))) assert all(a < b for a, b in w) def test_analyse_and_summarise_with_a_fake_scorer(): paras = ["This sentence is written by a machine for sure. Another machine sentence follows it here.", "A human wrote this one on a slow afternoon. And this one too, with some care."] blocks = _blocks(*paras) D.analyse(blocks, scorer=lambda texts: [1.0 if "machine" in t and "human" not in t else 0.0 for t in texts]) stats = D.summarise(blocks, 0.8, 0.5) assert stats["analysed"] == sum(len(p.split()) for p in paras) assert 0 <= stats["pct"] <= 100 assert all(len(b.scores) == len(b.sents) for b in blocks if b.kind == "para") def test_score_cache_scores_each_text_once(monkeypatch): calls = [] def fake(texts): calls.append(list(texts)) return [len(t) / 100 for t in texts] monkeypatch.setattr(D, "_score_uncached", fake) monkeypatch.setattr(D, "_score_cache", {}) out = D.score_texts(["bbb", "a", "bbb", "cc"]) assert out == [0.03, 0.01, 0.03, 0.02] assert calls == [["a", "cc", "bbb"]] # unique, shortest first assert D.score_texts(["a", "dddd"]) == [0.01, 0.04] assert calls[-1] == ["dddd"] # only the new text reaches the model assert D.score_texts([]) == [] def test_names_and_case_citations_do_not_end_a_sentence(): import textparse as TP cases = { "The Court in Justice K.S. Puttaswamy v. Union of India (2017) recognised privacy. It shapes the law.": 2, "It grew in the U.S. The economy then slowed. In the U.K. it did not.": 3, "The dose for group A. Results were clear. Written by J. Smith and A. Kumar in 2020. Then done.": 4, "The fluid moves at velocity v. The pressure drops. See Roe v. Wade for more.": 3, } for text, n in cases.items(): assert len(TP.split_sentences(text)) == n, text assert D.split_sentences(text) == TP.split_sentences(text)