| """Code-execution verify loop (M1c) — with a scripted runner (no model/network).""" |
| from mathcompose.eval.code_verify import verify_with_code |
| from mathcompose.eval.parse import PARSE_FAIL |
|
|
|
|
| class _ScriptedRunner: |
| """Emits a code block first, then boxes an index after seeing execution output.""" |
| def chat(self, messages, n=1, temperature=0.0, max_new_tokens=512): |
| saw_exec = any("Execution results" in m["content"] for m in messages if m["role"] == "user") |
| if saw_exec: |
| return ["The output shows 5, contradicting the step. \\boxed{2}"] |
| return ["Check paragraph 2.\n```python\nprint(2+3)\n```"] |
|
|
|
|
| class _ImmediateRunner: |
| def chat(self, messages, n=1, temperature=0.0, max_new_tokens=512): |
| return ["All steps verify. \\boxed{-1}"] |
|
|
|
|
| class _GarbageRunner: |
| def chat(self, messages, n=1, temperature=0.0, max_new_tokens=512): |
| return ["hmm no answer here"] |
|
|
|
|
| def test_executes_then_boxes(): |
| assert verify_with_code(_ScriptedRunner(), "2+3", ["a", "2+3=4", "c"], max_rounds=3) == 2 |
|
|
|
|
| def test_immediate_box_no_code(): |
| assert verify_with_code(_ImmediateRunner(), "p", ["a", "b"]) == -1 |
|
|
|
|
| def test_unparseable_returns_sentinel(): |
| assert verify_with_code(_GarbageRunner(), "p", ["a", "b"], max_rounds=2) == PARSE_FAIL |
|
|
