| from pathlib import Path |
|
|
| import pytest |
|
|
| from cache import ResultStore |
| from evaluation import run_evaluation, submission_answers |
|
|
|
|
| class FakeClient: |
| def get_questions(self): |
| return [ |
| {"task_id": "ok", "question": "q", "file_name": ""}, |
| {"task_id": "bad", "question": "q", "file_name": "x.py"}, |
| ] |
|
|
| def download_attachment(self, task): |
| if task["task_id"] == "bad": |
| raise FileNotFoundError("missing") |
|
|
|
|
| class FakeAgent: |
| class settings: |
| worker_limit = 2 |
|
|
| def cached_answer(self, task): |
| return None |
|
|
| def solve_detailed(self, task, attachment, *, force=False): |
| from agent import SolveResult |
|
|
| return SolveResult("answer", "general_reasoning") |
|
|
|
|
| def test_per_task_isolation_and_submission_guard(tmp_path): |
| results = run_evaluation( |
| FakeClient(), FakeAgent(), ResultStore(tmp_path / "run.json") |
| ) |
| assert [row["status"] for row in results] == ["ok", "agent_error"] |
| with pytest.raises(ValueError, match="incomplete"): |
| submission_answers(results, expected_count=None) |
|
|
|
|
| def test_official_answer_shape(): |
| payload = submission_answers( |
| [{"task_id": "abc", "answer": "42", "status": "ok"}], expected_count=None |
| ) |
| assert payload == [{"task_id": "abc", "submitted_answer": "42"}] |
|
|
|
|
| def test_submission_requires_exactly_twenty_unique_nonempty_answers(): |
| rows = [ |
| {"task_id": str(index), "answer": str(index), "status": "ok"} |
| for index in range(20) |
| ] |
| assert len(submission_answers(rows)) == 20 |
| with pytest.raises(ValueError, match="exactly 20"): |
| submission_answers(rows[:-1]) |
| rows[-1]["answer"] = "" |
| with pytest.raises(ValueError, match="empty"): |
| submission_answers(rows) |
|
|
|
|
| def test_dry_run_modules_have_no_submission_call(): |
| root = Path(__file__).resolve().parents[1] |
| for filename in ("evaluation.py", "run_local_eval.py"): |
| source = (root / filename).read_text(encoding="utf-8") |
| assert "/submit" not in source |
| assert ".submit_answers(" not in source |
|
|
