| """Run with: python -m unittest discover -s tests -v""" | |
| import unittest | |
| from compsearch.classify import Classifier, load_learner_factory, log_loss, probabilities | |
| from compsearch.evaluate import PROJECT_ROOT | |
| SPONGE = PROJECT_ROOT / "seeds" / "sponge.py" | |
| ENGLISH = [b"the cat sat on the mat and looked at the dog", b"she said that the weather was fine today", | |
| b"there is a house on the hill with a red door", b"what do you think about the new plan"] | |
| DIGITS = [b"3141592653589793238462643383279502884197", b"2718281828459045235360287471352662497757", | |
| b"1414213562373095048801688724209698078569", b"1732050807568877293527446341505872366942"] | |
| class ClassifierTests(unittest.TestCase): | |
| def setUp(self): | |
| self.classifier = Classifier(load_learner_factory(SPONGE)) | |
| for english, digits in zip(ENGLISH, DIGITS): | |
| self.classifier.learn("english", english) | |
| self.classifier.learn("digits", digits) | |
| def test_judges_unseen_text_correctly_with_probabilities_that_sum_to_one(self): | |
| verdict = self.classifier.judge(b"the dog looked at the house on the hill") | |
| self.assertGreater(verdict["english"], 0.9) | |
| self.assertAlmostEqual(sum(verdict.values()), 1.0, places=9) | |
| self.assertGreater(self.classifier.judge(b"8675309112358132134558914423337761098")["digits"], 0.9) | |
| def test_judging_does_not_change_what_was_learned(self): | |
| text = b"some text to judge twice" | |
| self.assertEqual(self.classifier.costs(text), self.classifier.costs(text)) | |
| def test_learning_a_new_label_later_works_without_retraining(self): | |
| for _ in range(3): | |
| self.classifier.learn("shouting", b"HELLO THERE!!! WHAT IS GOING ON??? STOP IT NOW!!!") | |
| self.assertEqual(max(self.classifier.judge(b"STOP!!! WHAT NOW???").items(), key=lambda kv: kv[1])[0], | |
| "shouting") | |
| def test_unfamiliar_text_is_more_surprising(self): | |
| familiar = self.classifier.surprise(b"the cat sat on the mat") | |
| strange = self.classifier.surprise(bytes(range(200, 240))) | |
| self.assertGreater(strange, familiar) | |
| def test_calibration_picks_a_temperature_and_lowers_log_loss(self): | |
| held_out = [("english", b"the plan was fine and the cat sat there"), ("digits", b"99887766554433221100123456789")] | |
| scored = [(label, self.classifier.costs(text)) for label, text in held_out] | |
| chosen = self.classifier.calibrate(held_out) | |
| self.assertLessEqual(log_loss(scored, chosen), log_loss(scored, 80)) | |
| def test_higher_temperature_softens_probabilities(self): | |
| costs = {"a": 10.0, "b": 20.0} | |
| self.assertGreater(probabilities(costs, 1)["a"], probabilities(costs, 20)["a"]) | |
| def test_bad_inputs_are_refused(self): | |
| with self.assertRaises(ValueError): | |
| self.classifier.learn("english", b"") | |
| with self.assertRaises(ValueError): | |
| self.classifier.judge(b"") | |
| with self.assertRaises(ValueError): | |
| Classifier(load_learner_factory(SPONGE)).judge(b"anything") | |
| with self.assertRaises(ValueError): | |
| Classifier(load_learner_factory(SPONGE), temperature=0) | |
| if __name__ == "__main__": | |
| unittest.main() | |