File size: 3,286 Bytes
b999890 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | """Run with: python -m unittest discover -s tests -v"""
import unittest
from compsearch.classify import Classifier, load_learner_factory, log_loss, probabilities
from compsearch.evaluate import PROJECT_ROOT
SPONGE = PROJECT_ROOT / "seeds" / "sponge.py"
ENGLISH = [b"the cat sat on the mat and looked at the dog", b"she said that the weather was fine today",
b"there is a house on the hill with a red door", b"what do you think about the new plan"]
DIGITS = [b"3141592653589793238462643383279502884197", b"2718281828459045235360287471352662497757",
b"1414213562373095048801688724209698078569", b"1732050807568877293527446341505872366942"]
class ClassifierTests(unittest.TestCase):
def setUp(self):
self.classifier = Classifier(load_learner_factory(SPONGE))
for english, digits in zip(ENGLISH, DIGITS):
self.classifier.learn("english", english)
self.classifier.learn("digits", digits)
def test_judges_unseen_text_correctly_with_probabilities_that_sum_to_one(self):
verdict = self.classifier.judge(b"the dog looked at the house on the hill")
self.assertGreater(verdict["english"], 0.9)
self.assertAlmostEqual(sum(verdict.values()), 1.0, places=9)
self.assertGreater(self.classifier.judge(b"8675309112358132134558914423337761098")["digits"], 0.9)
def test_judging_does_not_change_what_was_learned(self):
text = b"some text to judge twice"
self.assertEqual(self.classifier.costs(text), self.classifier.costs(text))
def test_learning_a_new_label_later_works_without_retraining(self):
for _ in range(3):
self.classifier.learn("shouting", b"HELLO THERE!!! WHAT IS GOING ON??? STOP IT NOW!!!")
self.assertEqual(max(self.classifier.judge(b"STOP!!! WHAT NOW???").items(), key=lambda kv: kv[1])[0],
"shouting")
def test_unfamiliar_text_is_more_surprising(self):
familiar = self.classifier.surprise(b"the cat sat on the mat")
strange = self.classifier.surprise(bytes(range(200, 240)))
self.assertGreater(strange, familiar)
def test_calibration_picks_a_temperature_and_lowers_log_loss(self):
held_out = [("english", b"the plan was fine and the cat sat there"), ("digits", b"99887766554433221100123456789")]
scored = [(label, self.classifier.costs(text)) for label, text in held_out]
chosen = self.classifier.calibrate(held_out)
self.assertLessEqual(log_loss(scored, chosen), log_loss(scored, 80))
def test_higher_temperature_softens_probabilities(self):
costs = {"a": 10.0, "b": 20.0}
self.assertGreater(probabilities(costs, 1)["a"], probabilities(costs, 20)["a"])
def test_bad_inputs_are_refused(self):
with self.assertRaises(ValueError):
self.classifier.learn("english", b"")
with self.assertRaises(ValueError):
self.classifier.judge(b"")
with self.assertRaises(ValueError):
Classifier(load_learner_factory(SPONGE)).judge(b"anything")
with self.assertRaises(ValueError):
Classifier(load_learner_factory(SPONGE), temperature=0)
if __name__ == "__main__":
unittest.main()
|