math-sight-reading / tests /test_core.py
SoulFireMage's picture
Unit tests for bank, engine, judge
82a988d verified
Raw History Blame Contribute Delete
8.19 kB
import os
import random
import sys
import pytest
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
sys.path.insert(0, os.path.dirname(__file__))
import bank
import engine
import judge
from fake_openrouter import Fake
SCORES_GOOD = {"precision": 9, "concept": "Exemplary", "transfer": "A"}
SCORES_BAD = {"precision": 2, "concept": "Superficial", "transfer": "C"}
# ------------------------------------------------------------------ bank ---
def test_bank_is_well_formed():
ids = [f["id"] for f in bank.BANK]
assert len(ids) == len(set(ids))
for f in bank.BANK:
assert f["tier"] in (1, 2, 3, 4)
assert f["latex"].count("{") == f["latex"].count("}"), f["id"]
for k in ("signature", "mechanics", "boundary", "transfer", "pitfalls"):
assert f["ref"][k], (f["id"], k)
assert len(f["ref"]["transfer"]) >= 2
assert f["hint_types"] and f["hint_mech"]
for t in (1, 2, 3, 4):
assert sum(1 for f in bank.BANK if f["tier"] == t) >= 4
def test_names_never_leak_into_latex_or_hints():
giveaways = ["rayleigh", "convolution", "fourier", "laplace", "euler", "lagrange", "stokes",
"riemann", "curl", "resolution of the identity", "lie derivative"]
for f in bank.BANK:
for text in (f["latex"], f["hint_types"], f["hint_mech"]):
low = text.lower()
assert f["name"].lower() not in low, f["id"]
for g in giveaways:
assert g not in low, (f["id"], g)
# ---------------------------------------------------------------- engine ---
def test_composite_range_and_hints():
assert engine.composite(10, "Exemplary", "A") == 1.0
assert engine.composite(0, "Superficial", "C") == 0.0
assert engine.composite(10, "Exemplary", "A", hints=2) == 0.8
assert engine.composite(0, "Superficial", "C", hints=2) == 0.0 # floored
def test_promote_after_three_good_at_tier():
p = engine.new_progress()
f = bank.BY_ID["t1-rayleigh"]
for _ in range(2):
p = engine.record_attempt(p, f, SCORES_GOOD, 0)
assert p["tier"] == 1
p = engine.record_attempt(p, f, SCORES_GOOD, 0)
assert p["tier"] == 2
def test_demote_and_floor():
p = engine.new_progress()
p["tier"] = 2
f = bank.BY_ID["t2-laplace"]
for _ in range(3):
p = engine.record_attempt(p, f, SCORES_BAD, 0)
assert p["tier"] == 1
for _ in range(5):
p = engine.record_attempt(p, bank.BY_ID["t1-rayleigh"], SCORES_BAD, 0)
assert p["tier"] == 1
def test_tier_cap():
p = engine.new_progress()
p["tier"] = 4
for _ in range(5):
p = engine.record_attempt(p, bank.BY_ID["t4-stokes"], SCORES_GOOD, 0)
assert p["tier"] == 4
def test_leitner_boxes():
p = engine.new_progress()
f = bank.BY_ID["t1-rayleigh"]
p = engine.record_attempt(p, f, SCORES_GOOD, 0)
assert p["cards"][f["id"]]["box"] == 1
p = engine.record_attempt(p, f, SCORES_BAD, 0)
assert p["cards"][f["id"]]["box"] == 0
assert p["cards"][f["id"]]["due"] == len(p["history"]) + engine.BOX_INTERVALS[0]
def test_failed_card_comes_due_and_is_prioritised():
p = engine.new_progress()
bad = bank.BY_ID["t1-rayleigh"]
p = engine.record_attempt(p, bad, SCORES_BAD, 0)
p = engine.record_attempt(p, bank.BY_ID["t1-curlcurl"], {"precision": 6, "concept": "Competent", "transfer": "B"}, 0)
p = engine.record_attempt(p, bank.BY_ID["t1-chain-curve"], {"precision": 6, "concept": "Competent", "transfer": "B"}, 0)
assert bad["id"] in engine.due_cards(p, bank.BANK)
class Always: # rng that always takes the review branch
def random(self):
return 0.0
def choice(self, xs):
return xs[0]
assert engine.choose_next(p, bank.BANK, rng=Always())["id"] == bad["id"]
def test_choose_next_avoids_repeat_and_covers_unseen_first():
p = engine.new_progress()
rng = random.Random(1)
seen = set()
last = None
for _ in range(3): # the 4th draw may legitimately be a due review
f = engine.choose_next(p, bank.BANK, rng=rng, exclude=last)
assert f["tier"] == 1 and f["id"] not in seen
seen.add(f["id"])
last = f["id"]
p = engine.record_attempt(p, f, {"precision": 6, "concept": "Competent", "transfer": "B"}, 0)
def test_normalize_survives_garbage():
for junk in (None, 3, "x", [], {"history": "no"}, {"tier": 99, "history": [{"id": 1}], "cards": {"a": 5}}):
p = engine.normalize(junk)
assert p["tier"] == 1 and p["history"] == [] and p["cards"] == {}
def test_stats_by_tag_sorts_weakest_first():
p = engine.new_progress()
p = engine.record_attempt(p, bank.BY_ID["t1-rayleigh"], SCORES_GOOD, 0) # linear-algebra, quadratic-form
p = engine.record_attempt(p, bank.BY_ID["t2-laplace"], SCORES_BAD, 0) # transforms, integration
rows = engine.stats_by_tag(p, bank.BANK)
assert rows[0][0] in ("transforms", "integration")
assert rows[-1][0] in ("linear-algebra", "quadratic-form")
# ----------------------------------------------------------------- judge ---
def test_parse_reply_ok_and_latex_survives():
r = judge.parse_reply(
"<precision>8</precision><concept>competent</concept><transfer>b</transfer>"
"<exemplary>$\\nabla \\times \\tau$</exemplary><missed>- one\n* two</missed>"
)
assert (r["precision"], r["concept"], r["transfer"]) == (8, "Competent", "B")
assert r["exemplary"] == "$\\nabla \\times \\tau$" # backslashes intact
assert r["missed"] == ["one", "two"]
def test_parse_reply_clamps_and_rejects():
assert judge.parse_reply("<precision>42</precision><concept>Exemplary</concept><transfer>A</transfer>")["precision"] == 10
with pytest.raises(judge.JudgeError):
judge.parse_reply("garbage")
with pytest.raises(judge.JudgeError):
judge.parse_reply("<precision>5</precision><concept>Heroic</concept><transfer>A</transfer>")
def test_call_judge_end_to_end_and_prompt_contents():
fake = Fake()
try:
f = bank.BY_ID["t1-rayleigh"]
r = judge.call_judge("sk-or-TESTKEY", "vendor/model-a", f, "formal", "concept", "app", base_url=fake.url)
assert r["precision"] == 7 and r["transfer"] == "A" and r["domains"] == ["mechanics", "statistics"]
req = fake.requests[0]
assert req["path"].endswith("/chat/completions")
assert req["auth"] == "Bearer sk-or-TESTKEY"
user = req["body"]["messages"][1]["content"]
assert f["latex"] in user and "<part1_formal>formal</part1_formal>" in user
assert f["ref"]["signature"] in user # reference goes to the judge
assert f["name"] in user # name only goes to the judge
assert req["body"]["model"] == "vendor/model-a"
finally:
fake.close()
def test_call_judge_errors_never_leak_key():
fake = Fake()
try:
fake.status = 401
with pytest.raises(judge.JudgeError) as e:
judge.call_judge("sk-or-SECRET", "m", bank.BANK[0], "a", "b", "c", base_url=fake.url)
assert "401" in str(e.value) and "SECRET" not in str(e.value)
fake.status = 200
fake.reply = "no tags at all"
n = len(fake.requests)
with pytest.raises(judge.JudgeError):
judge.call_judge("k", "m", bank.BANK[0], "a", "b", "c", base_url=fake.url)
assert len(fake.requests) - n == 2 # retried once
with pytest.raises(judge.JudgeError):
judge.call_judge("", "m", bank.BANK[0], "a", "b", "c", base_url=fake.url)
with pytest.raises(judge.JudgeError):
judge.call_judge("k", "m", bank.BANK[0], "a", "b", "c", base_url="http://127.0.0.1:1/api/v1")
finally:
fake.close()
def test_prompt_truncates_huge_input():
msgs = judge.build_messages(bank.BANK[0], "x" * 10000, "", "")
assert "[truncated]" in msgs[1]["content"] and len(msgs[1]["content"]) < 9000
def test_list_models():
fake = Fake()
try:
assert judge.list_models(fake.url) == ["vendor/model-a", "vendor/model-b"]
finally:
fake.close()
assert judge.list_models("http://127.0.0.1:1/api/v1") == []