| """Unit tests for the Evaluator module.""" | |
| import os | |
| import json | |
| import pytest | |
| from aegis.configs.dataclasses import ModelConfig | |
| from aegis.models.judge import Judge | |
| from aegis.models.adversary import Adversary | |
| from aegis.models.verifier import MathVerifier | |
| from aegis.data.loaders.gsm8k import GSM8KLoader | |
| from aegis.evaluation.evaluator import Evaluator | |
| def test_evaluator_run_mock(tmp_path): | |
| judge_cfg = ModelConfig(is_mock=True) | |
| adv_cfg = ModelConfig(is_mock=True) | |
| judge = Judge(judge_cfg) | |
| adversary = Adversary(adv_cfg) | |
| loader = GSM8KLoader(split="test", max_samples=4) | |
| verifier = MathVerifier() | |
| evaluator = Evaluator( | |
| judge=judge, | |
| adversary=adversary, | |
| eval_loader=loader, | |
| verifier=verifier, | |
| ) | |
| out_file = str(tmp_path / "eval_metrics.json") | |
| metrics = evaluator.evaluate( | |
| max_samples=4, | |
| save_path=out_file, | |
| ) | |
| assert "accuracy" in metrics | |
| assert "stable_correct_rate" in metrics | |
| assert "hard_flip_rate" in metrics | |
| assert os.path.exists(out_file) | |
| with open(out_file, "r") as f: | |
| saved_data = json.load(f) | |
| assert "accuracy" in saved_data | |
Xet Storage Details
- Size:
- 1.17 kB
- Xet hash:
- 9361a1b8c507bfa2c3498c54d115440076505b646957fa8e4630d08c91aa281c
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.