File size: 3,242 Bytes
ad474c9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 | import unittest
from hdc.hypervec import sim
from hdc.text import (
CORPUS,
TextRing,
char_ngrams,
encode_context,
encode_word,
evaluate,
tokenize,
)
D = 2000
class TestTokenizer(unittest.TestCase):
def test_splits_words_and_punctuation(self):
self.assertEqual(tokenize("The cat, sat."), ["the", "cat", ",", "sat", "."])
def test_keeps_apostrophes(self):
self.assertEqual(tokenize("don't stop"), ["don't", "stop"])
def test_empty_text(self):
self.assertEqual(tokenize(""), [])
class TestCharNgrams(unittest.TestCase):
def test_boundary_markers_pad_the_word(self):
grams = char_ngrams("cat", 3)
self.assertEqual(len(grams), 5)
self.assertEqual(grams[0], "<<c")
self.assertEqual(grams[-1], "t>>")
def test_shared_substrings_share_grams(self):
self.assertTrue(set(char_ngrams("walked")) & set(char_ngrams("walking")))
class TestWordEncoding(unittest.TestCase):
def test_word_is_deterministic(self):
self.assertEqual(encode_word("cat", D), encode_word("cat", D))
def test_inflections_are_closer_than_unrelated_words(self):
walked, walking = encode_word("walked", D), encode_word("walking", D)
unrelated = encode_word("banana", D)
self.assertGreater(sim(walked, walking, D), sim(walked, unrelated, D))
self.assertGreater(sim(walked, walking, D), 0.2)
def test_position_binding_distinguishes_anagrams(self):
# same n-gram multiset, different order: must not collapse
self.assertLess(sim(encode_word("listen", D), encode_word("silent", D), D), 0.9)
def test_context_similarity_tracks_overlap(self):
one = encode_context(["the"], D, span=2)
two = encode_context(["cat", "the"], D, span=2)
other = encode_context(["zebra", "quantum"], D, span=2)
self.assertLess(sim(one, two, D), 1.0) # different keys, not aliases
self.assertGreater(sim(one, two, D), sim(one, other, D)) # shared word shows
class TestTextRing(unittest.TestCase):
def test_learns_the_corpus(self):
tokens = tokenize(CORPUS)
ring = TextRing(d=D, span=2)
ring.fit(tokens)
self.assertGreater(ring.accuracy(tokens), 0.7)
self.assertGreater(len(ring.vocabulary), 5)
def test_predicts_a_deterministic_continuation(self):
ring = TextRing(d=D, span=2)
ring.fit(tokenize(CORPUS))
token, score = ring.predict(tokenize("the cat sat on the"))
self.assertEqual(token, "mat")
self.assertGreater(score, 0.2)
def test_unseen_history_predicts_nothing(self):
ring = TextRing(d=D, span=2)
ring.fit(tokenize(CORPUS))
token, score = ring.predict(tokenize("zebra quantum"))
self.assertIsNone(token)
def test_order_two_beats_order_one(self):
stats = evaluate(D)
self.assertGreaterEqual(stats["order2_accuracy"], stats["order1_accuracy"])
self.assertGreater(stats["order1_accuracy"], 0.5)
def test_composition_beats_unrelated(self):
stats = evaluate(D)
self.assertGreater(stats["similarity_inflections"], stats["similarity_unrelated"])
if __name__ == "__main__":
unittest.main()
|