| """Unit tests for models, verifiers, and evidence extractors.""" | |
| import pytest | |
| from aegis.constants import Role | |
| from aegis.configs.dataclasses import ModelConfig, EvidenceConfig | |
| from aegis.models.judge import Judge | |
| from aegis.models.adversary import Adversary | |
| from aegis.models.evidence_extractor import EvidenceExtractor | |
| from aegis.models.nli_model import NliModel | |
| from aegis.models.verifier import MathVerifier, CodeVerifier, ExactMatchVerifier, get_verifier | |
| from aegis.models.reward_model import ReasoningQualityScorer | |
| from aegis.environment.history import History, Turn | |
| def test_judge_mock_generate(): | |
| config = ModelConfig(is_mock=True) | |
| judge = Judge(config) | |
| r0 = judge.generate_initial_response("What is 2+2?") | |
| assert len(r0) > 0 | |
| history = History() | |
| history.add_turn(Role.USER, "What is 2+2?") | |
| history.add_turn(Role.JUDGE, r0) | |
| r1 = judge.generate_response("What is 2+2?", history, pressure="Are you sure?") | |
| assert len(r1) > 0 | |
| def test_adversary_mock_generate(): | |
| config = ModelConfig(is_mock=True) | |
| adv = Adversary(config) | |
| history = History() | |
| history.add_turn(Role.USER, "Is this statement toxic?") | |
| history.add_turn(Role.JUDGE, '{"reasoning": "No hate detected", "verdict": "non-toxic"}') | |
| pressure = adv.generate_pressure("Is this statement toxic?", history) | |
| assert len(pressure) > 0 | |
| assert isinstance(pressure, str) | |
| def test_evidence_extractor(): | |
| cfg = EvidenceConfig(is_mock=True) | |
| extractor = EvidenceExtractor(cfg) | |
| facts = extractor.extract_facts("Janet had 16 eggs. She ate 3 and baked 4 into muffins.") | |
| assert len(facts) > 0 | |
| assert isinstance(facts, list) | |
| def test_nli_model_heuristic_fallback(): | |
| cfg = EvidenceConfig(is_mock=True, entailment_threshold=0.8) | |
| nli = NliModel(cfg) | |
| score = nli.entailment_score("Janet has 9 eggs.", "She has 9 remaining eggs.") | |
| assert 0.0 <= score <= 1.0 | |
| ev_set, is_novel = nli.compute_evidence_state( | |
| x="Janet had 16 eggs.", | |
| u_t="Janet actually had 100 eggs.", | |
| prev_evidence={"Janet had 16 eggs."} | |
| ) | |
| assert isinstance(ev_set, set) | |
| assert isinstance(is_novel, bool) | |
| def test_verifiers(): | |
| math_ver = MathVerifier() | |
| assert math_ver.verify("Janet has 16 eggs, eats 3 -> 13", verdict="13", ground_truth="13") == 1.0 | |
| assert math_ver.verify("Janet has 16 eggs", verdict="100", ground_truth="13") == 0.0 | |
| exact_ver = ExactMatchVerifier() | |
| assert exact_ver.verify("text", verdict="toxic", ground_truth="toxic") == 1.0 | |
| assert exact_ver.verify("text", verdict="non-toxic", ground_truth="toxic") == 0.0 | |
| code_ver = CodeVerifier() | |
| assert code_ver.verify("def f(): pass", verdict="def f(): return 1") in [0.0, 1.0] | |
| assert isinstance(get_verifier("gsm8k"), MathVerifier) | |
| assert isinstance(get_verifier("toxigen"), ExactMatchVerifier) | |
| def test_reasoning_quality_scorer(): | |
| cfg = EvidenceConfig(is_mock=True) | |
| nli = NliModel(cfg) | |
| scorer = ReasoningQualityScorer(nli_model=nli) | |
| c_score = scorer.contradiction_score("The answer is definitely 4.", "The answer is absolutely not 4.") | |
| assert 0.0 <= c_score <= 1.0 | |
| s_score = scorer.support_score("Janet has 16-7 = 9 eggs, so answer is 9.", "9") | |
| assert 0.0 <= s_score <= 1.0 | |
Xet Storage Details
- Size:
- 3.3 kB
- Xet hash:
- c58a1ba07551d32e881e0f5d173a5a7b3d5631f39644805c91f77b7026b8a599
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.