Buckets:

Hooshaai/RL_AEGIS / tests /test_evaluator.py
tahamajs's picture
download
raw
1.17 kB
"""Unit tests for the Evaluator module."""
import os
import json
import pytest
from aegis.configs.dataclasses import ModelConfig
from aegis.models.judge import Judge
from aegis.models.adversary import Adversary
from aegis.models.verifier import MathVerifier
from aegis.data.loaders.gsm8k import GSM8KLoader
from aegis.evaluation.evaluator import Evaluator
def test_evaluator_run_mock(tmp_path):
judge_cfg = ModelConfig(is_mock=True)
adv_cfg = ModelConfig(is_mock=True)
judge = Judge(judge_cfg)
adversary = Adversary(adv_cfg)
loader = GSM8KLoader(split="test", max_samples=4)
verifier = MathVerifier()
evaluator = Evaluator(
judge=judge,
adversary=adversary,
eval_loader=loader,
verifier=verifier,
)
out_file = str(tmp_path / "eval_metrics.json")
metrics = evaluator.evaluate(
max_samples=4,
save_path=out_file,
)
assert "accuracy" in metrics
assert "stable_correct_rate" in metrics
assert "hard_flip_rate" in metrics
assert os.path.exists(out_file)
with open(out_file, "r") as f:
saved_data = json.load(f)
assert "accuracy" in saved_data

Xet Storage Details

Size:
1.17 kB
·
Xet hash:
9361a1b8c507bfa2c3498c54d115440076505b646957fa8e4630d08c91aa281c

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.