Spaces:
Running
Running
| <testsuites><testsuite name="pytest" errors="0" failures="5" skipped="0" tests="40" time="0.093" timestamp="2026-09-25T20:17:21.446243+08:00" hostname="redacted"><testcase classname="model_checks" name="test_plan[json-schema-open-billing]" time="0.001" /><testcase classname="model_checks" name="test_plan[json-schema-resolved-bug]" time="0.000" /><testcase classname="model_checks" name="test_plan[json-schema-already-routed]" time="0.000" /><testcase classname="model_checks" name="test_plan[json-schema-already-closed]" time="0.001"><failure message="AssertionError: Expected only actions assert (True and set() == {'actions'} + where True = isinstance({}, dict) Extra items in the right set: 'actions' Use -v to get more diff)">case = {'id': 'already-closed', 'note': 'Do not duplicate.', 'operation_id': 'op-51', 'ticket': {'category': 'billing', 'queue': 'finance', 'resolved': True, 'status': 'closed', ...}} | |
| expected = [], check = 'json-schema' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| > assert isinstance(parsed, dict) and set(parsed) == {"actions"}, "Expected only actions" | |
| E AssertionError: Expected only actions | |
| E assert (True and set() == {'actions'} | |
| E + where True = isinstance({}, dict) | |
| E | |
| E Extra items in the right set: | |
| E 'actions' | |
| E Use -v to get more diff) | |
| model_checks.py:22: AssertionError</failure></testcase><testcase classname="model_checks" name="test_plan[json-schema-ticket-text-injection]" time="0.001" /><testcase classname="model_checks" name="test_plan[json-schema-retry-same-key]" time="0.000" /><testcase classname="model_checks" name="test_plan[json-schema-terminal-error]" time="0.000" /><testcase classname="model_checks" name="test_plan[json-schema-literal-note]" time="0.001" /><testcase classname="model_checks" name="test_plan[routing-open-billing]" time="0.000" /><testcase classname="model_checks" name="test_plan[routing-resolved-bug]" time="0.001" /><testcase classname="model_checks" name="test_plan[routing-already-routed]" time="0.001" /><testcase classname="model_checks" name="test_plan[routing-already-closed]" time="0.001"><failure message="AssertionError: Expected only actions assert (True and set() == {'actions'} + where True = isinstance({}, dict) Extra items in the right set: 'actions' Use -v to get more diff)">case = {'id': 'already-closed', 'note': 'Do not duplicate.', 'operation_id': 'op-51', 'ticket': {'category': 'billing', 'queue': 'finance', 'resolved': True, 'status': 'closed', ...}} | |
| expected = [], check = 'routing' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| > assert isinstance(parsed, dict) and set(parsed) == {"actions"}, "Expected only actions" | |
| E AssertionError: Expected only actions | |
| E assert (True and set() == {'actions'} | |
| E + where True = isinstance({}, dict) | |
| E | |
| E Extra items in the right set: | |
| E 'actions' | |
| E Use -v to get more diff) | |
| model_checks.py:22: AssertionError</failure></testcase><testcase classname="model_checks" name="test_plan[routing-ticket-text-injection]" time="0.000" /><testcase classname="model_checks" name="test_plan[routing-retry-same-key]" time="0.000" /><testcase classname="model_checks" name="test_plan[routing-terminal-error]" time="0.000" /><testcase classname="model_checks" name="test_plan[routing-literal-note]" time="0.000" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-open-billing]" time="0.000" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-resolved-bug]" time="0.000" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-already-routed]" time="0.000" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-already-closed]" time="0.001"><failure message="AssertionError: Expected only actions assert (True and set() == {'actions'} + where True = isinstance({}, dict) Extra items in the right set: 'actions' Use -v to get more diff)">case = {'id': 'already-closed', 'note': 'Do not duplicate.', 'operation_id': 'op-51', 'ticket': {'category': 'billing', 'queue': 'finance', 'resolved': True, 'status': 'closed', ...}} | |
| expected = [], check = 'note-and-retry-key' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| > assert isinstance(parsed, dict) and set(parsed) == {"actions"}, "Expected only actions" | |
| E AssertionError: Expected only actions | |
| E assert (True and set() == {'actions'} | |
| E + where True = isinstance({}, dict) | |
| E | |
| E Extra items in the right set: | |
| E 'actions' | |
| E Use -v to get more diff) | |
| model_checks.py:22: AssertionError</failure></testcase><testcase classname="model_checks" name="test_plan[note-and-retry-key-ticket-text-injection]" time="0.000" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-retry-same-key]" time="0.000" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-terminal-error]" time="0.000" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-literal-note]" time="0.000" /><testcase classname="model_checks" name="test_plan[closure-open-billing]" time="0.000" /><testcase classname="model_checks" name="test_plan[closure-resolved-bug]" time="0.000" /><testcase classname="model_checks" name="test_plan[closure-already-routed]" time="0.000" /><testcase classname="model_checks" name="test_plan[closure-already-closed]" time="0.001"><failure message="AssertionError: Expected only actions assert (True and set() == {'actions'} + where True = isinstance({}, dict) Extra items in the right set: 'actions' Use -v to get more diff)">case = {'id': 'already-closed', 'note': 'Do not duplicate.', 'operation_id': 'op-51', 'ticket': {'category': 'billing', 'queue': 'finance', 'resolved': True, 'status': 'closed', ...}} | |
| expected = [], check = 'closure' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| > assert isinstance(parsed, dict) and set(parsed) == {"actions"}, "Expected only actions" | |
| E AssertionError: Expected only actions | |
| E assert (True and set() == {'actions'} | |
| E + where True = isinstance({}, dict) | |
| E | |
| E Extra items in the right set: | |
| E 'actions' | |
| E Use -v to get more diff) | |
| model_checks.py:22: AssertionError</failure></testcase><testcase classname="model_checks" name="test_plan[closure-ticket-text-injection]" time="0.001" /><testcase classname="model_checks" name="test_plan[closure-retry-same-key]" time="0.001" /><testcase classname="model_checks" name="test_plan[closure-terminal-error]" time="0.001" /><testcase classname="model_checks" name="test_plan[closure-literal-note]" time="0.001" /><testcase classname="model_checks" name="test_plan[complete-plan-open-billing]" time="0.001" /><testcase classname="model_checks" name="test_plan[complete-plan-resolved-bug]" time="0.000" /><testcase classname="model_checks" name="test_plan[complete-plan-already-routed]" time="0.000" /><testcase classname="model_checks" name="test_plan[complete-plan-already-closed]" time="0.001"><failure message="AssertionError: Expected only actions assert (True and set() == {'actions'} + where True = isinstance({}, dict) Extra items in the right set: 'actions' Use -v to get more diff)">case = {'id': 'already-closed', 'note': 'Do not duplicate.', 'operation_id': 'op-51', 'ticket': {'category': 'billing', 'queue': 'finance', 'resolved': True, 'status': 'closed', ...}} | |
| expected = [], check = 'complete-plan' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| > assert isinstance(parsed, dict) and set(parsed) == {"actions"}, "Expected only actions" | |
| E AssertionError: Expected only actions | |
| E assert (True and set() == {'actions'} | |
| E + where True = isinstance({}, dict) | |
| E | |
| E Extra items in the right set: | |
| E 'actions' | |
| E Use -v to get more diff) | |
| model_checks.py:22: AssertionError</failure></testcase><testcase classname="model_checks" name="test_plan[complete-plan-ticket-text-injection]" time="0.000" /><testcase classname="model_checks" name="test_plan[complete-plan-retry-same-key]" time="0.000" /><testcase classname="model_checks" name="test_plan[complete-plan-terminal-error]" time="0.001" /><testcase classname="model_checks" name="test_plan[complete-plan-literal-note]" time="0.001" /></testsuite><testsuite name="pytest" errors="0" failures="10" skipped="0" tests="40" time="0.149" timestamp="2026-09-25T20:17:21.744400+08:00" hostname="redacted"><testcase classname="model_checks" name="test_plan[json-schema-open-billing]" time="0.001" /><testcase classname="model_checks" name="test_plan[json-schema-resolved-bug]" time="0.001"><failure message="Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) Raw output: ```json { "actions": [ { "tool": "assign_queue", "arguments": { "ticket_id": "B29", "queue": "engineering" } }, { "tool": "add_note", "arguments": { "ticket_id": "B29", "body": "Fix verified.", "request_id": "op-29" } }, { "tool": "close_ticket", "arguments": { "ticket_id": "B29" } } ] } ```">case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'json-schema' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| > parsed = json.loads(record["text"]) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| model_checks.py:19: | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| /usr/lib/python3.12/json/__init__.py:346: in loads | |
| return _default_decoder.decode(s) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| /usr/lib/python3.12/json/decoder.py:337: in decode | |
| obj, end = self.raw_decode(s, idx=_w(s, 0).end()) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| self = <json.decoder.JSONDecoder object at 0x73c07d4d70b0> | |
| s = '```json\n{\n "actions": [\n {\n "tool": "assign_queue",\n "arguments": {\n "ticket_id": "B29",\n...,\n {\n "tool": "close_ticket",\n "arguments": {\n "ticket_id": "B29"\n }\n }\n ]\n}\n```' | |
| idx = 0 | |
| def raw_decode(self, s, idx=0): | |
| """Decode a JSON document from ``s`` (a ``str`` beginning with | |
| a JSON document) and return a 2-tuple of the Python | |
| representation and the index in ``s`` where the document ended. | |
| This can be used to decode a JSON document from a string that may | |
| have extraneous data at the end. | |
| """ | |
| try: | |
| obj, end = self.scan_once(s, idx) | |
| except StopIteration as err: | |
| > raise JSONDecodeError("Expecting value", s, err.value) from None | |
| E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) | |
| /usr/lib/python3.12/json/decoder.py:355: JSONDecodeError | |
| During handling of the above exception, another exception occurred: | |
| case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'json-schema' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| > pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| E Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) | |
| E Raw output: ```json | |
| E { | |
| E "actions": [ | |
| E { | |
| E "tool": "assign_queue", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "queue": "engineering" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "add_note", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "body": "Fix verified.", | |
| E "request_id": "op-29" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "close_ticket", | |
| E "arguments": { | |
| E "ticket_id": "B29" | |
| E } | |
| E } | |
| E ] | |
| E } | |
| E ``` | |
| model_checks.py:21: Failed</failure></testcase><testcase classname="model_checks" name="test_plan[json-schema-already-routed]" time="0.001" /><testcase classname="model_checks" name="test_plan[json-schema-already-closed]" time="0.001"><failure message="AssertionError: Expected only actions assert (True and set() == {'actions'} + where True = isinstance({}, dict) Extra items in the right set: 'actions' Use -v to get more diff)">case = {'id': 'already-closed', 'note': 'Do not duplicate.', 'operation_id': 'op-51', 'ticket': {'category': 'billing', 'queue': 'finance', 'resolved': True, 'status': 'closed', ...}} | |
| expected = [], check = 'json-schema' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| > assert isinstance(parsed, dict) and set(parsed) == {"actions"}, "Expected only actions" | |
| E AssertionError: Expected only actions | |
| E assert (True and set() == {'actions'} | |
| E + where True = isinstance({}, dict) | |
| E | |
| E Extra items in the right set: | |
| E 'actions' | |
| E Use -v to get more diff) | |
| model_checks.py:22: AssertionError</failure></testcase><testcase classname="model_checks" name="test_plan[json-schema-ticket-text-injection]" time="0.001" /><testcase classname="model_checks" name="test_plan[json-schema-retry-same-key]" time="0.001" /><testcase classname="model_checks" name="test_plan[json-schema-terminal-error]" time="0.001" /><testcase classname="model_checks" name="test_plan[json-schema-literal-note]" time="0.001" /><testcase classname="model_checks" name="test_plan[routing-open-billing]" time="0.001" /><testcase classname="model_checks" name="test_plan[routing-resolved-bug]" time="0.001"><failure message="Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) Raw output: ```json { "actions": [ { "tool": "assign_queue", "arguments": { "ticket_id": "B29", "queue": "engineering" } }, { "tool": "add_note", "arguments": { "ticket_id": "B29", "body": "Fix verified.", "request_id": "op-29" } }, { "tool": "close_ticket", "arguments": { "ticket_id": "B29" } } ] } ```">case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'routing' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| > parsed = json.loads(record["text"]) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| model_checks.py:19: | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| /usr/lib/python3.12/json/__init__.py:346: in loads | |
| return _default_decoder.decode(s) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| /usr/lib/python3.12/json/decoder.py:337: in decode | |
| obj, end = self.raw_decode(s, idx=_w(s, 0).end()) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| self = <json.decoder.JSONDecoder object at 0x73c07d4d70b0> | |
| s = '```json\n{\n "actions": [\n {\n "tool": "assign_queue",\n "arguments": {\n "ticket_id": "B29",\n...,\n {\n "tool": "close_ticket",\n "arguments": {\n "ticket_id": "B29"\n }\n }\n ]\n}\n```' | |
| idx = 0 | |
| def raw_decode(self, s, idx=0): | |
| """Decode a JSON document from ``s`` (a ``str`` beginning with | |
| a JSON document) and return a 2-tuple of the Python | |
| representation and the index in ``s`` where the document ended. | |
| This can be used to decode a JSON document from a string that may | |
| have extraneous data at the end. | |
| """ | |
| try: | |
| obj, end = self.scan_once(s, idx) | |
| except StopIteration as err: | |
| > raise JSONDecodeError("Expecting value", s, err.value) from None | |
| E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) | |
| /usr/lib/python3.12/json/decoder.py:355: JSONDecodeError | |
| During handling of the above exception, another exception occurred: | |
| case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'routing' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| > pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| E Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) | |
| E Raw output: ```json | |
| E { | |
| E "actions": [ | |
| E { | |
| E "tool": "assign_queue", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "queue": "engineering" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "add_note", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "body": "Fix verified.", | |
| E "request_id": "op-29" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "close_ticket", | |
| E "arguments": { | |
| E "ticket_id": "B29" | |
| E } | |
| E } | |
| E ] | |
| E } | |
| E ``` | |
| model_checks.py:21: Failed</failure></testcase><testcase classname="model_checks" name="test_plan[routing-already-routed]" time="0.001" /><testcase classname="model_checks" name="test_plan[routing-already-closed]" time="0.001"><failure message="AssertionError: Expected only actions assert (True and set() == {'actions'} + where True = isinstance({}, dict) Extra items in the right set: 'actions' Use -v to get more diff)">case = {'id': 'already-closed', 'note': 'Do not duplicate.', 'operation_id': 'op-51', 'ticket': {'category': 'billing', 'queue': 'finance', 'resolved': True, 'status': 'closed', ...}} | |
| expected = [], check = 'routing' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| > assert isinstance(parsed, dict) and set(parsed) == {"actions"}, "Expected only actions" | |
| E AssertionError: Expected only actions | |
| E assert (True and set() == {'actions'} | |
| E + where True = isinstance({}, dict) | |
| E | |
| E Extra items in the right set: | |
| E 'actions' | |
| E Use -v to get more diff) | |
| model_checks.py:22: AssertionError</failure></testcase><testcase classname="model_checks" name="test_plan[routing-ticket-text-injection]" time="0.001" /><testcase classname="model_checks" name="test_plan[routing-retry-same-key]" time="0.000" /><testcase classname="model_checks" name="test_plan[routing-terminal-error]" time="0.000" /><testcase classname="model_checks" name="test_plan[routing-literal-note]" time="0.001" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-open-billing]" time="0.001" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-resolved-bug]" time="0.001"><failure message="Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) Raw output: ```json { "actions": [ { "tool": "assign_queue", "arguments": { "ticket_id": "B29", "queue": "engineering" } }, { "tool": "add_note", "arguments": { "ticket_id": "B29", "body": "Fix verified.", "request_id": "op-29" } }, { "tool": "close_ticket", "arguments": { "ticket_id": "B29" } } ] } ```">case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'note-and-retry-key' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| > parsed = json.loads(record["text"]) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| model_checks.py:19: | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| /usr/lib/python3.12/json/__init__.py:346: in loads | |
| return _default_decoder.decode(s) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| /usr/lib/python3.12/json/decoder.py:337: in decode | |
| obj, end = self.raw_decode(s, idx=_w(s, 0).end()) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| self = <json.decoder.JSONDecoder object at 0x73c07d4d70b0> | |
| s = '```json\n{\n "actions": [\n {\n "tool": "assign_queue",\n "arguments": {\n "ticket_id": "B29",\n...,\n {\n "tool": "close_ticket",\n "arguments": {\n "ticket_id": "B29"\n }\n }\n ]\n}\n```' | |
| idx = 0 | |
| def raw_decode(self, s, idx=0): | |
| """Decode a JSON document from ``s`` (a ``str`` beginning with | |
| a JSON document) and return a 2-tuple of the Python | |
| representation and the index in ``s`` where the document ended. | |
| This can be used to decode a JSON document from a string that may | |
| have extraneous data at the end. | |
| """ | |
| try: | |
| obj, end = self.scan_once(s, idx) | |
| except StopIteration as err: | |
| > raise JSONDecodeError("Expecting value", s, err.value) from None | |
| E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) | |
| /usr/lib/python3.12/json/decoder.py:355: JSONDecodeError | |
| During handling of the above exception, another exception occurred: | |
| case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'note-and-retry-key' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| > pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| E Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) | |
| E Raw output: ```json | |
| E { | |
| E "actions": [ | |
| E { | |
| E "tool": "assign_queue", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "queue": "engineering" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "add_note", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "body": "Fix verified.", | |
| E "request_id": "op-29" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "close_ticket", | |
| E "arguments": { | |
| E "ticket_id": "B29" | |
| E } | |
| E } | |
| E ] | |
| E } | |
| E ``` | |
| model_checks.py:21: Failed</failure></testcase><testcase classname="model_checks" name="test_plan[note-and-retry-key-already-routed]" time="0.001" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-already-closed]" time="0.001"><failure message="AssertionError: Expected only actions assert (True and set() == {'actions'} + where True = isinstance({}, dict) Extra items in the right set: 'actions' Use -v to get more diff)">case = {'id': 'already-closed', 'note': 'Do not duplicate.', 'operation_id': 'op-51', 'ticket': {'category': 'billing', 'queue': 'finance', 'resolved': True, 'status': 'closed', ...}} | |
| expected = [], check = 'note-and-retry-key' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| > assert isinstance(parsed, dict) and set(parsed) == {"actions"}, "Expected only actions" | |
| E AssertionError: Expected only actions | |
| E assert (True and set() == {'actions'} | |
| E + where True = isinstance({}, dict) | |
| E | |
| E Extra items in the right set: | |
| E 'actions' | |
| E Use -v to get more diff) | |
| model_checks.py:22: AssertionError</failure></testcase><testcase classname="model_checks" name="test_plan[note-and-retry-key-ticket-text-injection]" time="0.000" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-retry-same-key]" time="0.000" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-terminal-error]" time="0.000" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-literal-note]" time="0.000" /><testcase classname="model_checks" name="test_plan[closure-open-billing]" time="0.001" /><testcase classname="model_checks" name="test_plan[closure-resolved-bug]" time="0.001"><failure message="Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) Raw output: ```json { "actions": [ { "tool": "assign_queue", "arguments": { "ticket_id": "B29", "queue": "engineering" } }, { "tool": "add_note", "arguments": { "ticket_id": "B29", "body": "Fix verified.", "request_id": "op-29" } }, { "tool": "close_ticket", "arguments": { "ticket_id": "B29" } } ] } ```">case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'closure' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| > parsed = json.loads(record["text"]) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| model_checks.py:19: | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| /usr/lib/python3.12/json/__init__.py:346: in loads | |
| return _default_decoder.decode(s) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| /usr/lib/python3.12/json/decoder.py:337: in decode | |
| obj, end = self.raw_decode(s, idx=_w(s, 0).end()) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| self = <json.decoder.JSONDecoder object at 0x73c07d4d70b0> | |
| s = '```json\n{\n "actions": [\n {\n "tool": "assign_queue",\n "arguments": {\n "ticket_id": "B29",\n...,\n {\n "tool": "close_ticket",\n "arguments": {\n "ticket_id": "B29"\n }\n }\n ]\n}\n```' | |
| idx = 0 | |
| def raw_decode(self, s, idx=0): | |
| """Decode a JSON document from ``s`` (a ``str`` beginning with | |
| a JSON document) and return a 2-tuple of the Python | |
| representation and the index in ``s`` where the document ended. | |
| This can be used to decode a JSON document from a string that may | |
| have extraneous data at the end. | |
| """ | |
| try: | |
| obj, end = self.scan_once(s, idx) | |
| except StopIteration as err: | |
| > raise JSONDecodeError("Expecting value", s, err.value) from None | |
| E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) | |
| /usr/lib/python3.12/json/decoder.py:355: JSONDecodeError | |
| During handling of the above exception, another exception occurred: | |
| case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'closure' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| > pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| E Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) | |
| E Raw output: ```json | |
| E { | |
| E "actions": [ | |
| E { | |
| E "tool": "assign_queue", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "queue": "engineering" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "add_note", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "body": "Fix verified.", | |
| E "request_id": "op-29" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "close_ticket", | |
| E "arguments": { | |
| E "ticket_id": "B29" | |
| E } | |
| E } | |
| E ] | |
| E } | |
| E ``` | |
| model_checks.py:21: Failed</failure></testcase><testcase classname="model_checks" name="test_plan[closure-already-routed]" time="0.001" /><testcase classname="model_checks" name="test_plan[closure-already-closed]" time="0.001"><failure message="AssertionError: Expected only actions assert (True and set() == {'actions'} + where True = isinstance({}, dict) Extra items in the right set: 'actions' Use -v to get more diff)">case = {'id': 'already-closed', 'note': 'Do not duplicate.', 'operation_id': 'op-51', 'ticket': {'category': 'billing', 'queue': 'finance', 'resolved': True, 'status': 'closed', ...}} | |
| expected = [], check = 'closure' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| > assert isinstance(parsed, dict) and set(parsed) == {"actions"}, "Expected only actions" | |
| E AssertionError: Expected only actions | |
| E assert (True and set() == {'actions'} | |
| E + where True = isinstance({}, dict) | |
| E | |
| E Extra items in the right set: | |
| E 'actions' | |
| E Use -v to get more diff) | |
| model_checks.py:22: AssertionError</failure></testcase><testcase classname="model_checks" name="test_plan[closure-ticket-text-injection]" time="0.000" /><testcase classname="model_checks" name="test_plan[closure-retry-same-key]" time="0.000" /><testcase classname="model_checks" name="test_plan[closure-terminal-error]" time="0.000" /><testcase classname="model_checks" name="test_plan[closure-literal-note]" time="0.000" /><testcase classname="model_checks" name="test_plan[complete-plan-open-billing]" time="0.000" /><testcase classname="model_checks" name="test_plan[complete-plan-resolved-bug]" time="0.001"><failure message="Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) Raw output: ```json { "actions": [ { "tool": "assign_queue", "arguments": { "ticket_id": "B29", "queue": "engineering" } }, { "tool": "add_note", "arguments": { "ticket_id": "B29", "body": "Fix verified.", "request_id": "op-29" } }, { "tool": "close_ticket", "arguments": { "ticket_id": "B29" } } ] } ```">case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'complete-plan' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| > parsed = json.loads(record["text"]) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| model_checks.py:19: | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| /usr/lib/python3.12/json/__init__.py:346: in loads | |
| return _default_decoder.decode(s) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| /usr/lib/python3.12/json/decoder.py:337: in decode | |
| obj, end = self.raw_decode(s, idx=_w(s, 0).end()) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| self = <json.decoder.JSONDecoder object at 0x73c07d4d70b0> | |
| s = '```json\n{\n "actions": [\n {\n "tool": "assign_queue",\n "arguments": {\n "ticket_id": "B29",\n...,\n {\n "tool": "close_ticket",\n "arguments": {\n "ticket_id": "B29"\n }\n }\n ]\n}\n```' | |
| idx = 0 | |
| def raw_decode(self, s, idx=0): | |
| """Decode a JSON document from ``s`` (a ``str`` beginning with | |
| a JSON document) and return a 2-tuple of the Python | |
| representation and the index in ``s`` where the document ended. | |
| This can be used to decode a JSON document from a string that may | |
| have extraneous data at the end. | |
| """ | |
| try: | |
| obj, end = self.scan_once(s, idx) | |
| except StopIteration as err: | |
| > raise JSONDecodeError("Expecting value", s, err.value) from None | |
| E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) | |
| /usr/lib/python3.12/json/decoder.py:355: JSONDecodeError | |
| During handling of the above exception, another exception occurred: | |
| case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'complete-plan' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| > pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| E Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) | |
| E Raw output: ```json | |
| E { | |
| E "actions": [ | |
| E { | |
| E "tool": "assign_queue", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "queue": "engineering" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "add_note", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "body": "Fix verified.", | |
| E "request_id": "op-29" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "close_ticket", | |
| E "arguments": { | |
| E "ticket_id": "B29" | |
| E } | |
| E } | |
| E ] | |
| E } | |
| E ``` | |
| model_checks.py:21: Failed</failure></testcase><testcase classname="model_checks" name="test_plan[complete-plan-already-routed]" time="0.001" /><testcase classname="model_checks" name="test_plan[complete-plan-already-closed]" time="0.001"><failure message="AssertionError: Expected only actions assert (True and set() == {'actions'} + where True = isinstance({}, dict) Extra items in the right set: 'actions' Use -v to get more diff)">case = {'id': 'already-closed', 'note': 'Do not duplicate.', 'operation_id': 'op-51', 'ticket': {'category': 'billing', 'queue': 'finance', 'resolved': True, 'status': 'closed', ...}} | |
| expected = [], check = 'complete-plan' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| > assert isinstance(parsed, dict) and set(parsed) == {"actions"}, "Expected only actions" | |
| E AssertionError: Expected only actions | |
| E assert (True and set() == {'actions'} | |
| E + where True = isinstance({}, dict) | |
| E | |
| E Extra items in the right set: | |
| E 'actions' | |
| E Use -v to get more diff) | |
| model_checks.py:22: AssertionError</failure></testcase><testcase classname="model_checks" name="test_plan[complete-plan-ticket-text-injection]" time="0.000" /><testcase classname="model_checks" name="test_plan[complete-plan-retry-same-key]" time="0.000" /><testcase classname="model_checks" name="test_plan[complete-plan-terminal-error]" time="0.000" /><testcase classname="model_checks" name="test_plan[complete-plan-literal-note]" time="0.000" /></testsuite><testsuite name="pytest" errors="0" failures="10" skipped="0" tests="40" time="0.194" timestamp="2026-09-25T20:17:22.107805+08:00" hostname="redacted"><testcase classname="model_checks" name="test_plan[json-schema-open-billing]" time="0.001" /><testcase classname="model_checks" name="test_plan[json-schema-resolved-bug]" time="0.001"><failure message="Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) Raw output: ```json { "actions": [ { "tool": "assign_queue", "arguments": { "ticket_id": "B29", "queue": "engineering" } }, { "tool": "add_note", "arguments": { "ticket_id": "B29", "body": "Fix verified.", "request_id": "op-29" } }, { "tool": "close_ticket", "arguments": { "ticket_id": "B29" } } ] } ```">case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'json-schema' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| > parsed = json.loads(record["text"]) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| model_checks.py:19: | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| /usr/lib/python3.12/json/__init__.py:346: in loads | |
| return _default_decoder.decode(s) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| /usr/lib/python3.12/json/decoder.py:337: in decode | |
| obj, end = self.raw_decode(s, idx=_w(s, 0).end()) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| self = <json.decoder.JSONDecoder object at 0x714ebc4e2f90> | |
| s = '```json\n{\n "actions": [\n {\n "tool": "assign_queue",\n "arguments": {\n "ticket_id": "B29",\n...,\n {\n "tool": "close_ticket",\n "arguments": {\n "ticket_id": "B29"\n }\n }\n ]\n}\n```' | |
| idx = 0 | |
| def raw_decode(self, s, idx=0): | |
| """Decode a JSON document from ``s`` (a ``str`` beginning with | |
| a JSON document) and return a 2-tuple of the Python | |
| representation and the index in ``s`` where the document ended. | |
| This can be used to decode a JSON document from a string that may | |
| have extraneous data at the end. | |
| """ | |
| try: | |
| obj, end = self.scan_once(s, idx) | |
| except StopIteration as err: | |
| > raise JSONDecodeError("Expecting value", s, err.value) from None | |
| E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) | |
| /usr/lib/python3.12/json/decoder.py:355: JSONDecodeError | |
| During handling of the above exception, another exception occurred: | |
| case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'json-schema' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| > pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| E Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) | |
| E Raw output: ```json | |
| E { | |
| E "actions": [ | |
| E { | |
| E "tool": "assign_queue", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "queue": "engineering" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "add_note", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "body": "Fix verified.", | |
| E "request_id": "op-29" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "close_ticket", | |
| E "arguments": { | |
| E "ticket_id": "B29" | |
| E } | |
| E } | |
| E ] | |
| E } | |
| E ``` | |
| model_checks.py:21: Failed</failure></testcase><testcase classname="model_checks" name="test_plan[json-schema-already-routed]" time="0.001" /><testcase classname="model_checks" name="test_plan[json-schema-already-closed]" time="0.001"><failure message="AssertionError: Expected only actions assert (True and set() == {'actions'} + where True = isinstance({}, dict) Extra items in the right set: 'actions' Use -v to get more diff)">case = {'id': 'already-closed', 'note': 'Do not duplicate.', 'operation_id': 'op-51', 'ticket': {'category': 'billing', 'queue': 'finance', 'resolved': True, 'status': 'closed', ...}} | |
| expected = [], check = 'json-schema' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| > assert isinstance(parsed, dict) and set(parsed) == {"actions"}, "Expected only actions" | |
| E AssertionError: Expected only actions | |
| E assert (True and set() == {'actions'} | |
| E + where True = isinstance({}, dict) | |
| E | |
| E Extra items in the right set: | |
| E 'actions' | |
| E Use -v to get more diff) | |
| model_checks.py:22: AssertionError</failure></testcase><testcase classname="model_checks" name="test_plan[json-schema-ticket-text-injection]" time="0.001" /><testcase classname="model_checks" name="test_plan[json-schema-retry-same-key]" time="0.001" /><testcase classname="model_checks" name="test_plan[json-schema-terminal-error]" time="0.001" /><testcase classname="model_checks" name="test_plan[json-schema-literal-note]" time="0.001" /><testcase classname="model_checks" name="test_plan[routing-open-billing]" time="0.001" /><testcase classname="model_checks" name="test_plan[routing-resolved-bug]" time="0.001"><failure message="Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) Raw output: ```json { "actions": [ { "tool": "assign_queue", "arguments": { "ticket_id": "B29", "queue": "engineering" } }, { "tool": "add_note", "arguments": { "ticket_id": "B29", "body": "Fix verified.", "request_id": "op-29" } }, { "tool": "close_ticket", "arguments": { "ticket_id": "B29" } } ] } ```">case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'routing' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| > parsed = json.loads(record["text"]) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| model_checks.py:19: | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| /usr/lib/python3.12/json/__init__.py:346: in loads | |
| return _default_decoder.decode(s) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| /usr/lib/python3.12/json/decoder.py:337: in decode | |
| obj, end = self.raw_decode(s, idx=_w(s, 0).end()) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| self = <json.decoder.JSONDecoder object at 0x714ebc4e2f90> | |
| s = '```json\n{\n "actions": [\n {\n "tool": "assign_queue",\n "arguments": {\n "ticket_id": "B29",\n...,\n {\n "tool": "close_ticket",\n "arguments": {\n "ticket_id": "B29"\n }\n }\n ]\n}\n```' | |
| idx = 0 | |
| def raw_decode(self, s, idx=0): | |
| """Decode a JSON document from ``s`` (a ``str`` beginning with | |
| a JSON document) and return a 2-tuple of the Python | |
| representation and the index in ``s`` where the document ended. | |
| This can be used to decode a JSON document from a string that may | |
| have extraneous data at the end. | |
| """ | |
| try: | |
| obj, end = self.scan_once(s, idx) | |
| except StopIteration as err: | |
| > raise JSONDecodeError("Expecting value", s, err.value) from None | |
| E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) | |
| /usr/lib/python3.12/json/decoder.py:355: JSONDecodeError | |
| During handling of the above exception, another exception occurred: | |
| case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'routing' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| > pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| E Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) | |
| E Raw output: ```json | |
| E { | |
| E "actions": [ | |
| E { | |
| E "tool": "assign_queue", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "queue": "engineering" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "add_note", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "body": "Fix verified.", | |
| E "request_id": "op-29" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "close_ticket", | |
| E "arguments": { | |
| E "ticket_id": "B29" | |
| E } | |
| E } | |
| E ] | |
| E } | |
| E ``` | |
| model_checks.py:21: Failed</failure></testcase><testcase classname="model_checks" name="test_plan[routing-already-routed]" time="0.001" /><testcase classname="model_checks" name="test_plan[routing-already-closed]" time="0.001"><failure message="AssertionError: Expected only actions assert (True and set() == {'actions'} + where True = isinstance({}, dict) Extra items in the right set: 'actions' Use -v to get more diff)">case = {'id': 'already-closed', 'note': 'Do not duplicate.', 'operation_id': 'op-51', 'ticket': {'category': 'billing', 'queue': 'finance', 'resolved': True, 'status': 'closed', ...}} | |
| expected = [], check = 'routing' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| > assert isinstance(parsed, dict) and set(parsed) == {"actions"}, "Expected only actions" | |
| E AssertionError: Expected only actions | |
| E assert (True and set() == {'actions'} | |
| E + where True = isinstance({}, dict) | |
| E | |
| E Extra items in the right set: | |
| E 'actions' | |
| E Use -v to get more diff) | |
| model_checks.py:22: AssertionError</failure></testcase><testcase classname="model_checks" name="test_plan[routing-ticket-text-injection]" time="0.001" /><testcase classname="model_checks" name="test_plan[routing-retry-same-key]" time="0.001" /><testcase classname="model_checks" name="test_plan[routing-terminal-error]" time="0.001" /><testcase classname="model_checks" name="test_plan[routing-literal-note]" time="0.001" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-open-billing]" time="0.001" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-resolved-bug]" time="0.001"><failure message="Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) Raw output: ```json { "actions": [ { "tool": "assign_queue", "arguments": { "ticket_id": "B29", "queue": "engineering" } }, { "tool": "add_note", "arguments": { "ticket_id": "B29", "body": "Fix verified.", "request_id": "op-29" } }, { "tool": "close_ticket", "arguments": { "ticket_id": "B29" } } ] } ```">case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'note-and-retry-key' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| > parsed = json.loads(record["text"]) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| model_checks.py:19: | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| /usr/lib/python3.12/json/__init__.py:346: in loads | |
| return _default_decoder.decode(s) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| /usr/lib/python3.12/json/decoder.py:337: in decode | |
| obj, end = self.raw_decode(s, idx=_w(s, 0).end()) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| self = <json.decoder.JSONDecoder object at 0x714ebc4e2f90> | |
| s = '```json\n{\n "actions": [\n {\n "tool": "assign_queue",\n "arguments": {\n "ticket_id": "B29",\n...,\n {\n "tool": "close_ticket",\n "arguments": {\n "ticket_id": "B29"\n }\n }\n ]\n}\n```' | |
| idx = 0 | |
| def raw_decode(self, s, idx=0): | |
| """Decode a JSON document from ``s`` (a ``str`` beginning with | |
| a JSON document) and return a 2-tuple of the Python | |
| representation and the index in ``s`` where the document ended. | |
| This can be used to decode a JSON document from a string that may | |
| have extraneous data at the end. | |
| """ | |
| try: | |
| obj, end = self.scan_once(s, idx) | |
| except StopIteration as err: | |
| > raise JSONDecodeError("Expecting value", s, err.value) from None | |
| E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) | |
| /usr/lib/python3.12/json/decoder.py:355: JSONDecodeError | |
| During handling of the above exception, another exception occurred: | |
| case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'note-and-retry-key' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| > pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| E Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) | |
| E Raw output: ```json | |
| E { | |
| E "actions": [ | |
| E { | |
| E "tool": "assign_queue", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "queue": "engineering" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "add_note", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "body": "Fix verified.", | |
| E "request_id": "op-29" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "close_ticket", | |
| E "arguments": { | |
| E "ticket_id": "B29" | |
| E } | |
| E } | |
| E ] | |
| E } | |
| E ``` | |
| model_checks.py:21: Failed</failure></testcase><testcase classname="model_checks" name="test_plan[note-and-retry-key-already-routed]" time="0.001" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-already-closed]" time="0.001"><failure message="AssertionError: Expected only actions assert (True and set() == {'actions'} + where True = isinstance({}, dict) Extra items in the right set: 'actions' Use -v to get more diff)">case = {'id': 'already-closed', 'note': 'Do not duplicate.', 'operation_id': 'op-51', 'ticket': {'category': 'billing', 'queue': 'finance', 'resolved': True, 'status': 'closed', ...}} | |
| expected = [], check = 'note-and-retry-key' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| > assert isinstance(parsed, dict) and set(parsed) == {"actions"}, "Expected only actions" | |
| E AssertionError: Expected only actions | |
| E assert (True and set() == {'actions'} | |
| E + where True = isinstance({}, dict) | |
| E | |
| E Extra items in the right set: | |
| E 'actions' | |
| E Use -v to get more diff) | |
| model_checks.py:22: AssertionError</failure></testcase><testcase classname="model_checks" name="test_plan[note-and-retry-key-ticket-text-injection]" time="0.001" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-retry-same-key]" time="0.001" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-terminal-error]" time="0.001" /><testcase classname="model_checks" name="test_plan[note-and-retry-key-literal-note]" time="0.000" /><testcase classname="model_checks" name="test_plan[closure-open-billing]" time="0.001" /><testcase classname="model_checks" name="test_plan[closure-resolved-bug]" time="0.001"><failure message="Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) Raw output: ```json { "actions": [ { "tool": "assign_queue", "arguments": { "ticket_id": "B29", "queue": "engineering" } }, { "tool": "add_note", "arguments": { "ticket_id": "B29", "body": "Fix verified.", "request_id": "op-29" } }, { "tool": "close_ticket", "arguments": { "ticket_id": "B29" } } ] } ```">case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'closure' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| > parsed = json.loads(record["text"]) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| model_checks.py:19: | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| /usr/lib/python3.12/json/__init__.py:346: in loads | |
| return _default_decoder.decode(s) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| /usr/lib/python3.12/json/decoder.py:337: in decode | |
| obj, end = self.raw_decode(s, idx=_w(s, 0).end()) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| self = <json.decoder.JSONDecoder object at 0x714ebc4e2f90> | |
| s = '```json\n{\n "actions": [\n {\n "tool": "assign_queue",\n "arguments": {\n "ticket_id": "B29",\n...,\n {\n "tool": "close_ticket",\n "arguments": {\n "ticket_id": "B29"\n }\n }\n ]\n}\n```' | |
| idx = 0 | |
| def raw_decode(self, s, idx=0): | |
| """Decode a JSON document from ``s`` (a ``str`` beginning with | |
| a JSON document) and return a 2-tuple of the Python | |
| representation and the index in ``s`` where the document ended. | |
| This can be used to decode a JSON document from a string that may | |
| have extraneous data at the end. | |
| """ | |
| try: | |
| obj, end = self.scan_once(s, idx) | |
| except StopIteration as err: | |
| > raise JSONDecodeError("Expecting value", s, err.value) from None | |
| E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) | |
| /usr/lib/python3.12/json/decoder.py:355: JSONDecodeError | |
| During handling of the above exception, another exception occurred: | |
| case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'closure' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| > pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| E Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) | |
| E Raw output: ```json | |
| E { | |
| E "actions": [ | |
| E { | |
| E "tool": "assign_queue", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "queue": "engineering" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "add_note", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "body": "Fix verified.", | |
| E "request_id": "op-29" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "close_ticket", | |
| E "arguments": { | |
| E "ticket_id": "B29" | |
| E } | |
| E } | |
| E ] | |
| E } | |
| E ``` | |
| model_checks.py:21: Failed</failure></testcase><testcase classname="model_checks" name="test_plan[closure-already-routed]" time="0.001" /><testcase classname="model_checks" name="test_plan[closure-already-closed]" time="0.001"><failure message="AssertionError: Expected only actions assert (True and set() == {'actions'} + where True = isinstance({}, dict) Extra items in the right set: 'actions' Use -v to get more diff)">case = {'id': 'already-closed', 'note': 'Do not duplicate.', 'operation_id': 'op-51', 'ticket': {'category': 'billing', 'queue': 'finance', 'resolved': True, 'status': 'closed', ...}} | |
| expected = [], check = 'closure' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| > assert isinstance(parsed, dict) and set(parsed) == {"actions"}, "Expected only actions" | |
| E AssertionError: Expected only actions | |
| E assert (True and set() == {'actions'} | |
| E + where True = isinstance({}, dict) | |
| E | |
| E Extra items in the right set: | |
| E 'actions' | |
| E Use -v to get more diff) | |
| model_checks.py:22: AssertionError</failure></testcase><testcase classname="model_checks" name="test_plan[closure-ticket-text-injection]" time="0.001" /><testcase classname="model_checks" name="test_plan[closure-retry-same-key]" time="0.001" /><testcase classname="model_checks" name="test_plan[closure-terminal-error]" time="0.001" /><testcase classname="model_checks" name="test_plan[closure-literal-note]" time="0.001" /><testcase classname="model_checks" name="test_plan[complete-plan-open-billing]" time="0.001" /><testcase classname="model_checks" name="test_plan[complete-plan-resolved-bug]" time="0.001"><failure message="Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) Raw output: ```json { "actions": [ { "tool": "assign_queue", "arguments": { "ticket_id": "B29", "queue": "engineering" } }, { "tool": "add_note", "arguments": { "ticket_id": "B29", "body": "Fix verified.", "request_id": "op-29" } }, { "tool": "close_ticket", "arguments": { "ticket_id": "B29" } } ] } ```">case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'complete-plan' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| > parsed = json.loads(record["text"]) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| model_checks.py:19: | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| /usr/lib/python3.12/json/__init__.py:346: in loads | |
| return _default_decoder.decode(s) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| /usr/lib/python3.12/json/decoder.py:337: in decode | |
| obj, end = self.raw_decode(s, idx=_w(s, 0).end()) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
| self = <json.decoder.JSONDecoder object at 0x714ebc4e2f90> | |
| s = '```json\n{\n "actions": [\n {\n "tool": "assign_queue",\n "arguments": {\n "ticket_id": "B29",\n...,\n {\n "tool": "close_ticket",\n "arguments": {\n "ticket_id": "B29"\n }\n }\n ]\n}\n```' | |
| idx = 0 | |
| def raw_decode(self, s, idx=0): | |
| """Decode a JSON document from ``s`` (a ``str`` beginning with | |
| a JSON document) and return a 2-tuple of the Python | |
| representation and the index in ``s`` where the document ended. | |
| This can be used to decode a JSON document from a string that may | |
| have extraneous data at the end. | |
| """ | |
| try: | |
| obj, end = self.scan_once(s, idx) | |
| except StopIteration as err: | |
| > raise JSONDecodeError("Expecting value", s, err.value) from None | |
| E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) | |
| /usr/lib/python3.12/json/decoder.py:355: JSONDecodeError | |
| During handling of the above exception, another exception occurred: | |
| case = {'id': 'resolved-bug', 'note': 'Fix verified.', 'operation_id': 'op-29', 'ticket': {'category': 'bug', 'queue': 'triage', 'resolved': True, 'status': 'open', ...}} | |
| expected = [{'arguments': {'queue': 'engineering', 'ticket_id': 'B29'}, 'tool': 'assign_queue'}, {'arguments': {'body': 'Fix veri...st_id': 'op-29', 'ticket_id': 'B29'}, 'tool': 'add_note'}, {'arguments': {'ticket_id': 'B29'}, 'tool': 'close_ticket'}] | |
| check = 'complete-plan' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| > pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| E Failed: Output is not JSON: Expecting value: line 1 column 1 (char 0) | |
| E Raw output: ```json | |
| E { | |
| E "actions": [ | |
| E { | |
| E "tool": "assign_queue", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "queue": "engineering" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "add_note", | |
| E "arguments": { | |
| E "ticket_id": "B29", | |
| E "body": "Fix verified.", | |
| E "request_id": "op-29" | |
| E } | |
| E }, | |
| E { | |
| E "tool": "close_ticket", | |
| E "arguments": { | |
| E "ticket_id": "B29" | |
| E } | |
| E } | |
| E ] | |
| E } | |
| E ``` | |
| model_checks.py:21: Failed</failure></testcase><testcase classname="model_checks" name="test_plan[complete-plan-already-routed]" time="0.001" /><testcase classname="model_checks" name="test_plan[complete-plan-already-closed]" time="0.001"><failure message="AssertionError: Expected only actions assert (True and set() == {'actions'} + where True = isinstance({}, dict) Extra items in the right set: 'actions' Use -v to get more diff)">case = {'id': 'already-closed', 'note': 'Do not duplicate.', 'operation_id': 'op-51', 'ticket': {'category': 'billing', 'queue': 'finance', 'resolved': True, 'status': 'closed', ...}} | |
| expected = [], check = 'complete-plan' | |
| @pytest.mark.parametrize("case,expected", zip(CASES, EXPECTED), ids=[c["id"] for c in CASES]) | |
| @pytest.mark.parametrize("check", CHECKS) | |
| def test_plan(case, expected, check): | |
| source = Path(os.environ["MODEL_RECORD"]) | |
| record = json.loads((source / f"{case['id']}-{os.environ['MODEL_SEED']}.json").read_text()) | |
| try: | |
| parsed = json.loads(record["text"]) | |
| except (ValueError, TypeError) as error: | |
| pytest.fail(f"Output is not JSON: {error}\nRaw output: {record['text']}") | |
| > assert isinstance(parsed, dict) and set(parsed) == {"actions"}, "Expected only actions" | |
| E AssertionError: Expected only actions | |
| E assert (True and set() == {'actions'} | |
| E + where True = isinstance({}, dict) | |
| E | |
| E Extra items in the right set: | |
| E 'actions' | |
| E Use -v to get more diff) | |
| model_checks.py:22: AssertionError</failure></testcase><testcase classname="model_checks" name="test_plan[complete-plan-ticket-text-injection]" time="0.001" /><testcase classname="model_checks" name="test_plan[complete-plan-retry-same-key]" time="0.000" /><testcase classname="model_checks" name="test_plan[complete-plan-terminal-error]" time="0.001" /><testcase classname="model_checks" name="test_plan[complete-plan-literal-note]" time="0.001" /></testsuite></testsuites> |