import os import random import sys import pytest sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) sys.path.insert(0, os.path.dirname(__file__)) import bank import engine import judge from fake_openrouter import Fake SCORES_GOOD = {"precision": 9, "concept": "Exemplary", "transfer": "A"} SCORES_BAD = {"precision": 2, "concept": "Superficial", "transfer": "C"} # ------------------------------------------------------------------ bank --- def test_bank_is_well_formed(): ids = [f["id"] for f in bank.BANK] assert len(ids) == len(set(ids)) for f in bank.BANK: assert f["tier"] in (1, 2, 3, 4) assert f["latex"].count("{") == f["latex"].count("}"), f["id"] for k in ("signature", "mechanics", "boundary", "transfer", "pitfalls"): assert f["ref"][k], (f["id"], k) assert len(f["ref"]["transfer"]) >= 2 assert f["hint_types"] and f["hint_mech"] for t in (1, 2, 3, 4): assert sum(1 for f in bank.BANK if f["tier"] == t) >= 4 def test_names_never_leak_into_latex_or_hints(): giveaways = ["rayleigh", "convolution", "fourier", "laplace", "euler", "lagrange", "stokes", "riemann", "curl", "resolution of the identity", "lie derivative"] for f in bank.BANK: for text in (f["latex"], f["hint_types"], f["hint_mech"]): low = text.lower() assert f["name"].lower() not in low, f["id"] for g in giveaways: assert g not in low, (f["id"], g) # ---------------------------------------------------------------- engine --- def test_composite_range_and_hints(): assert engine.composite(10, "Exemplary", "A") == 1.0 assert engine.composite(0, "Superficial", "C") == 0.0 assert engine.composite(10, "Exemplary", "A", hints=2) == 0.8 assert engine.composite(0, "Superficial", "C", hints=2) == 0.0 # floored def test_promote_after_three_good_at_tier(): p = engine.new_progress() f = bank.BY_ID["t1-rayleigh"] for _ in range(2): p = engine.record_attempt(p, f, SCORES_GOOD, 0) assert p["tier"] == 1 p = engine.record_attempt(p, f, SCORES_GOOD, 0) assert p["tier"] == 2 def test_demote_and_floor(): p = engine.new_progress() p["tier"] = 2 f = bank.BY_ID["t2-laplace"] for _ in range(3): p = engine.record_attempt(p, f, SCORES_BAD, 0) assert p["tier"] == 1 for _ in range(5): p = engine.record_attempt(p, bank.BY_ID["t1-rayleigh"], SCORES_BAD, 0) assert p["tier"] == 1 def test_tier_cap(): p = engine.new_progress() p["tier"] = 4 for _ in range(5): p = engine.record_attempt(p, bank.BY_ID["t4-stokes"], SCORES_GOOD, 0) assert p["tier"] == 4 def test_leitner_boxes(): p = engine.new_progress() f = bank.BY_ID["t1-rayleigh"] p = engine.record_attempt(p, f, SCORES_GOOD, 0) assert p["cards"][f["id"]]["box"] == 1 p = engine.record_attempt(p, f, SCORES_BAD, 0) assert p["cards"][f["id"]]["box"] == 0 assert p["cards"][f["id"]]["due"] == len(p["history"]) + engine.BOX_INTERVALS[0] def test_failed_card_comes_due_and_is_prioritised(): p = engine.new_progress() bad = bank.BY_ID["t1-rayleigh"] p = engine.record_attempt(p, bad, SCORES_BAD, 0) p = engine.record_attempt(p, bank.BY_ID["t1-curlcurl"], {"precision": 6, "concept": "Competent", "transfer": "B"}, 0) p = engine.record_attempt(p, bank.BY_ID["t1-chain-curve"], {"precision": 6, "concept": "Competent", "transfer": "B"}, 0) assert bad["id"] in engine.due_cards(p, bank.BANK) class Always: # rng that always takes the review branch def random(self): return 0.0 def choice(self, xs): return xs[0] assert engine.choose_next(p, bank.BANK, rng=Always())["id"] == bad["id"] def test_choose_next_avoids_repeat_and_covers_unseen_first(): p = engine.new_progress() rng = random.Random(1) seen = set() last = None for _ in range(3): # the 4th draw may legitimately be a due review f = engine.choose_next(p, bank.BANK, rng=rng, exclude=last) assert f["tier"] == 1 and f["id"] not in seen seen.add(f["id"]) last = f["id"] p = engine.record_attempt(p, f, {"precision": 6, "concept": "Competent", "transfer": "B"}, 0) def test_normalize_survives_garbage(): for junk in (None, 3, "x", [], {"history": "no"}, {"tier": 99, "history": [{"id": 1}], "cards": {"a": 5}}): p = engine.normalize(junk) assert p["tier"] == 1 and p["history"] == [] and p["cards"] == {} def test_stats_by_tag_sorts_weakest_first(): p = engine.new_progress() p = engine.record_attempt(p, bank.BY_ID["t1-rayleigh"], SCORES_GOOD, 0) # linear-algebra, quadratic-form p = engine.record_attempt(p, bank.BY_ID["t2-laplace"], SCORES_BAD, 0) # transforms, integration rows = engine.stats_by_tag(p, bank.BANK) assert rows[0][0] in ("transforms", "integration") assert rows[-1][0] in ("linear-algebra", "quadratic-form") # ----------------------------------------------------------------- judge --- def test_parse_reply_ok_and_latex_survives(): r = judge.parse_reply( "8competentb" "$\\nabla \\times \\tau$- one\n* two" ) assert (r["precision"], r["concept"], r["transfer"]) == (8, "Competent", "B") assert r["exemplary"] == "$\\nabla \\times \\tau$" # backslashes intact assert r["missed"] == ["one", "two"] def test_parse_reply_clamps_and_rejects(): assert judge.parse_reply("42ExemplaryA")["precision"] == 10 with pytest.raises(judge.JudgeError): judge.parse_reply("garbage") with pytest.raises(judge.JudgeError): judge.parse_reply("5HeroicA") def test_call_judge_end_to_end_and_prompt_contents(): fake = Fake() try: f = bank.BY_ID["t1-rayleigh"] r = judge.call_judge("sk-or-TESTKEY", "vendor/model-a", f, "formal", "concept", "app", base_url=fake.url) assert r["precision"] == 7 and r["transfer"] == "A" and r["domains"] == ["mechanics", "statistics"] req = fake.requests[0] assert req["path"].endswith("/chat/completions") assert req["auth"] == "Bearer sk-or-TESTKEY" user = req["body"]["messages"][1]["content"] assert f["latex"] in user and "formal" in user assert f["ref"]["signature"] in user # reference goes to the judge assert f["name"] in user # name only goes to the judge assert req["body"]["model"] == "vendor/model-a" finally: fake.close() def test_call_judge_errors_never_leak_key(): fake = Fake() try: fake.status = 401 with pytest.raises(judge.JudgeError) as e: judge.call_judge("sk-or-SECRET", "m", bank.BANK[0], "a", "b", "c", base_url=fake.url) assert "401" in str(e.value) and "SECRET" not in str(e.value) fake.status = 200 fake.reply = "no tags at all" n = len(fake.requests) with pytest.raises(judge.JudgeError): judge.call_judge("k", "m", bank.BANK[0], "a", "b", "c", base_url=fake.url) assert len(fake.requests) - n == 2 # retried once with pytest.raises(judge.JudgeError): judge.call_judge("", "m", bank.BANK[0], "a", "b", "c", base_url=fake.url) with pytest.raises(judge.JudgeError): judge.call_judge("k", "m", bank.BANK[0], "a", "b", "c", base_url="http://127.0.0.1:1/api/v1") finally: fake.close() def test_prompt_truncates_huge_input(): msgs = judge.build_messages(bank.BANK[0], "x" * 10000, "", "") assert "[truncated]" in msgs[1]["content"] and len(msgs[1]["content"]) < 9000 def test_list_models(): fake = Fake() try: assert judge.list_models(fake.url) == ["vendor/model-a", "vendor/model-b"] finally: fake.close() assert judge.list_models("http://127.0.0.1:1/api/v1") == []