Spaces:
Running on Zero
Running on Zero
File size: 4,618 Bytes
66ee87e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 | """Normalisation of model answers into one comparable record (app/scoring.py)."""
import math
import unittest
from app.scoring import _lookup, normalise, option_keys
NOUL = {"type": "noul", "instructions": "x"}
CHOICE = {"type": "choice", "instructions": "x", "criteria": {"bug": "b", "how_to": "h", "sales": "s"}}
SCORE = {"type": "score", "instructions": "x", "criteria": ["Low", "Mid", "High", "Critical"]}
class OptionKeysTest(unittest.TestCase):
def test_noul_keys_are_true_false(self):
self.assertEqual(option_keys(NOUL), ["true", "false"])
def test_score_keys_are_level_indices(self):
self.assertEqual(option_keys(SCORE), ["0", "1", "2", "3"])
def test_choice_keys_follow_criteria_order(self):
q = {"type": "choice", "criteria": {"sales": "s", "bug": "b", "how_to": "h"}} # deliberately not alphabetical
self.assertEqual(option_keys(q), ["sales", "bug", "how_to"])
def test_choice_accepts_list_criteria(self):
self.assertEqual(option_keys({"type": "choice", "criteria": ["a", "b"]}), ["a", "b"])
class LookupTest(unittest.TestCase):
def test_finds_true_under_bool_and_word_spellings(self):
for spelling in (True, "True", "yes", 1):
with self.subTest(spelling=spelling):
self.assertEqual(_lookup({spelling: 0.7}, "true"), 0.7)
def test_finds_score_level_under_int_key(self):
self.assertEqual(_lookup({2: 0.4}, "2"), 0.4)
def test_missing_key_returns_none(self):
self.assertIsNone(_lookup({"x": 1}, "true"))
def test_non_dict_returns_none(self):
self.assertIsNone(_lookup(None, "true"))
class NormaliseTest(unittest.TestCase):
def test_noul_builds_probs_from_p_true(self):
rec = normalise(NOUL, None, p_true=0.8)
self.assertAlmostEqual(rec["probs"]["true"], 0.8)
self.assertAlmostEqual(rec["probs"]["false"], 0.2)
self.assertEqual(rec["choice"], "true")
self.assertAlmostEqual(rec["p_true"], 0.8)
def test_noul_laya_conf_is_top_probability(self):
rec = normalise(NOUL, None, p_true=0.3)
self.assertEqual(rec["choice"], "false")
self.assertAlmostEqual(rec["laya_conf"], 0.7)
def test_probabilities_are_renormalised_to_one(self):
rec = normalise(CHOICE, {"bug": 2.0, "how_to": 1.0, "sales": 1.0})
self.assertAlmostEqual(sum(rec["probs"].values()), 1.0)
self.assertAlmostEqual(rec["probs"]["bug"], 0.5)
def test_missing_options_count_as_zero(self):
rec = normalise(CHOICE, {"bug": 1.0})
self.assertEqual(rec["probs"], {"bug": 1.0, "how_to": 0.0, "sales": 0.0})
def test_answer_only_becomes_point_mass_on_choice(self):
rec = normalise(CHOICE, None, choice="SALES")
self.assertEqual(rec["probs"], {"bug": 0.0, "how_to": 0.0, "sales": 1.0})
self.assertEqual(rec["choice"], "sales")
def test_score_answer_only_becomes_point_mass_on_rounded_level(self):
rec = normalise(SCORE, None, level=2.6) # 2.6 rounds to 3 but truncates to 2
self.assertEqual(rec["choice"], "3")
self.assertEqual(rec["probs"]["3"], 1.0)
def test_score_reports_expected_level_and_level_count(self):
rec = normalise(SCORE, {"0": 0.5, "3": 0.5})
self.assertAlmostEqual(rec["expected_level"], 1.5)
self.assertEqual(rec["levels"], 4)
def test_uniform_distribution_has_zero_entropy_confidence(self):
rec = normalise(CHOICE, {"bug": 1, "how_to": 1, "sales": 1})
self.assertAlmostEqual(rec["entropy_conf"], 0.0)
self.assertAlmostEqual(rec["laya_conf"], 0.0)
def test_certain_distribution_has_full_entropy_confidence(self):
rec = normalise(CHOICE, {"bug": 1, "how_to": 0, "sales": 0})
self.assertAlmostEqual(rec["entropy_conf"], 1.0)
self.assertAlmostEqual(rec["top_prob"], 1.0)
def test_entropy_confidence_matches_formula(self):
probs = {"bug": 0.6, "how_to": 0.3, "sales": 0.1}
expected = 1 - (-sum(p * math.log(p) for p in probs.values())) / math.log(3)
self.assertAlmostEqual(normalise(CHOICE, probs)["entropy_conf"], expected)
def test_choice_laya_conf_is_entropy_confidence(self):
rec = normalise(CHOICE, {"bug": 0.6, "how_to": 0.3, "sales": 0.1})
self.assertAlmostEqual(rec["laya_conf"], rec["entropy_conf"])
def test_no_information_gives_uniform_record(self):
rec = normalise(CHOICE, None)
self.assertEqual(sorted(rec["probs"].values()), [0.0, 0.0, 0.0])
self.assertIn(rec["choice"], CHOICE["criteria"])
if __name__ == "__main__":
unittest.main()
|