| import unittest |
|
|
| from hdc.hypervec import sim |
| from hdc.text import ( |
| CORPUS, |
| TextRing, |
| char_ngrams, |
| encode_context, |
| encode_word, |
| evaluate, |
| tokenize, |
| ) |
|
|
| D = 2000 |
|
|
|
|
| class TestTokenizer(unittest.TestCase): |
| def test_splits_words_and_punctuation(self): |
| self.assertEqual(tokenize("The cat, sat."), ["the", "cat", ",", "sat", "."]) |
|
|
| def test_keeps_apostrophes(self): |
| self.assertEqual(tokenize("don't stop"), ["don't", "stop"]) |
|
|
| def test_empty_text(self): |
| self.assertEqual(tokenize(""), []) |
|
|
|
|
| class TestCharNgrams(unittest.TestCase): |
| def test_boundary_markers_pad_the_word(self): |
| grams = char_ngrams("cat", 3) |
| self.assertEqual(len(grams), 5) |
| self.assertEqual(grams[0], "<<c") |
| self.assertEqual(grams[-1], "t>>") |
|
|
| def test_shared_substrings_share_grams(self): |
| self.assertTrue(set(char_ngrams("walked")) & set(char_ngrams("walking"))) |
|
|
|
|
| class TestWordEncoding(unittest.TestCase): |
| def test_word_is_deterministic(self): |
| self.assertEqual(encode_word("cat", D), encode_word("cat", D)) |
|
|
| def test_inflections_are_closer_than_unrelated_words(self): |
| walked, walking = encode_word("walked", D), encode_word("walking", D) |
| unrelated = encode_word("banana", D) |
| self.assertGreater(sim(walked, walking, D), sim(walked, unrelated, D)) |
| self.assertGreater(sim(walked, walking, D), 0.2) |
|
|
| def test_position_binding_distinguishes_anagrams(self): |
| |
| self.assertLess(sim(encode_word("listen", D), encode_word("silent", D), D), 0.9) |
|
|
| def test_context_similarity_tracks_overlap(self): |
| one = encode_context(["the"], D, span=2) |
| two = encode_context(["cat", "the"], D, span=2) |
| other = encode_context(["zebra", "quantum"], D, span=2) |
| self.assertLess(sim(one, two, D), 1.0) |
| self.assertGreater(sim(one, two, D), sim(one, other, D)) |
|
|
|
|
| class TestTextRing(unittest.TestCase): |
| def test_learns_the_corpus(self): |
| tokens = tokenize(CORPUS) |
| ring = TextRing(d=D, span=2) |
| ring.fit(tokens) |
| self.assertGreater(ring.accuracy(tokens), 0.7) |
| self.assertGreater(len(ring.vocabulary), 5) |
|
|
| def test_predicts_a_deterministic_continuation(self): |
| ring = TextRing(d=D, span=2) |
| ring.fit(tokenize(CORPUS)) |
| token, score = ring.predict(tokenize("the cat sat on the")) |
| self.assertEqual(token, "mat") |
| self.assertGreater(score, 0.2) |
|
|
| def test_unseen_history_predicts_nothing(self): |
| ring = TextRing(d=D, span=2) |
| ring.fit(tokenize(CORPUS)) |
| token, score = ring.predict(tokenize("zebra quantum")) |
| self.assertIsNone(token) |
|
|
| def test_order_two_beats_order_one(self): |
| stats = evaluate(D) |
| self.assertGreaterEqual(stats["order2_accuracy"], stats["order1_accuracy"]) |
| self.assertGreater(stats["order1_accuracy"], 0.5) |
|
|
| def test_composition_beats_unrelated(self): |
| stats = evaluate(D) |
| self.assertGreater(stats["similarity_inflections"], stats["similarity_unrelated"]) |
|
|
|
|
| if __name__ == "__main__": |
| unittest.main() |
|
|
