Hugging Face's logo

Spaces:
glayguo
/
evalarc
Running

App Files Community
1
evalarc / support /audit.json
glayguo's picture
glayguo
Publish verified EvalArc 0941fde4d452
c0e0aa1 verified
Raw History Blame Contribute Delete
209 kB
{
"schema_version": "evalarc.audit.v2",
"valid": true,
"reference_passed": true,
"reference": {
"schema_version": "evalarc.evaluation.v2",
"created_at": "2026-09-14T01:29:39.246926+00:00",
"evalarc_version": "0.2.0",
"task": {
"id": "support-routing",
"version": "0.1.0",
"domain": "business-tools",
"split": "public-development"
},
"candidate_sha256": "18822c8e9e7cf7466d6c5c8c3437bbc73abc43f07236c754980269b3a026c779",
"grader_sha256": "2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255",
"cases_sha256": "b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df",
"runtime": {
"backend": "docker",
"image": "python:3.12-slim",
"image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea",
"command": [
"{python}",
"-I",
"-B",
"main.py"
],
"response_timeout_seconds": 10.0,
"session_output_limit_bytes": 1048576,
"python": "3.12.3",
"platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39"
},
"seeds": [
17
],
"valid": true,
"status": "passed",
"score": 1.0,
"resolved": true,
"dimensions": {
"routing": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.3,
"score": 1.0
},
"notes": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.25,
"score": 1.0
},
"closure": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.2,
"score": 1.0
},
"scope": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.15,
"score": 1.0
},
"protocol": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.1,
"score": 1.0
}
},
"cases": [
{
"seed": 17,
"case_id": "route-open-ticket",
"checks": {
"routing": true,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": true,
"status": "passed",
"error": null,
"duration_seconds": 0.802482,
"transcript_sha256": "7b5b4973e934a60c55b95bda6eedf79fcff62764fbc1b81723c5126608caf4bd",
"initial_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-85a0bcc1"
}
},
"result": {
"ok": true,
"ticket": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-85a0bcc1",
"queue": "queue-0-2"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-85a0bcc1": {
"before": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-85a0bcc1",
"body": "Reviewed request T-85a0bcc1.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-85a0bcc1": {
"before": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1."
]
}
}
}
},
{
"step": 3,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 3
},
{
"seed": 17,
"case_id": "close-resolved-ticket",
"checks": {
"routing": true,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": true,
"status": "passed",
"error": null,
"duration_seconds": 0.704571,
"transcript_sha256": "a260768c11b589ce3f58658cac6aae8a01d36e388a47fc47c7995f8d5c36708e",
"initial_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-c41f9dfd"
}
},
"result": {
"ok": true,
"ticket": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-c41f9dfd",
"queue": "queue-1-0"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-c41f9dfd",
"body": "Reviewed request T-c41f9dfd.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "close_ticket",
"arguments": {
"ticket_id": "T-c41f9dfd"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
}
}
}
},
{
"step": 4,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 4
},
{
"seed": 17,
"case_id": "retry-before-commit",
"checks": {
"routing": true,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": true,
"status": "passed",
"error": null,
"duration_seconds": 0.710904,
"transcript_sha256": "b1d1d91dc01fc4bfb3243aa3739e953a40aa3920caf63a4b2ab73c87ecabda66",
"initial_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-3fb81d27."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-3fb81d27"
}
},
"result": {
"ok": true,
"ticket": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-3fb81d27",
"queue": "queue-2-1"
}
},
"result": {
"ok": false,
"error": {
"code": "temporarily_unavailable",
"retryable": true
}
},
"status": "tool_error",
"error_source": "injected",
"changes": {}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-3fb81d27",
"queue": "queue-2-1"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-3fb81d27": {
"before": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-3fb81d27",
"body": "Reviewed request T-3fb81d27.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-3fb81d27": {
"before": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-3fb81d27."
]
}
}
}
},
{
"step": 4,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 4
},
{
"seed": 17,
"case_id": "retry-after-commit",
"checks": {
"routing": true,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": true,
"status": "passed",
"error": null,
"duration_seconds": 0.736961,
"transcript_sha256": "c97b016921c8d8fdcdeffd1ebf1f438d5a29be06732c31c86f99fd087bc0b58f",
"initial_state": {
"T-cd68615c": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-cd68615c": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-cd68615c"
}
},
"result": {
"ok": true,
"ticket": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-cd68615c",
"queue": "queue-3-0"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-cd68615c",
"body": "Reviewed request T-cd68615c.",
"request_id": "review"
}
},
"result": {
"ok": false,
"error": {
"code": "temporarily_unavailable",
"retryable": true
}
},
"status": "tool_error",
"error_source": "injected",
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-cd68615c",
"body": "Reviewed request T-cd68615c.",
"request_id": "review"
}
},
"result": {
"ok": true,
"replayed": true
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 4,
"type": "tool_call",
"action": {
"tool": "close_ticket",
"arguments": {
"ticket_id": "T-cd68615c"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 5,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 5
}
],
"evaluation_seconds": 2.963313,
"agent_cost_usd": null,
"agent_tokens": null
},
"mutants": [
{
"name": "claim-without-actions",
"target_dimension": "routing",
"killed": true,
"valid": true,
"score": 0.35,
"failing_cases": [
"close-resolved-ticket",
"retry-after-commit",
"retry-before-commit",
"route-open-ticket"
],
"candidate_sha256": "d4412a9c3e8c47c744dd397aa582edec83b7ae1a6436b75a905fb4bb63faa3cd",
"evaluation": {
"schema_version": "evalarc.evaluation.v2",
"created_at": "2026-09-14T01:29:41.862960+00:00",
"evalarc_version": "0.2.0",
"task": {
"id": "support-routing",
"version": "0.1.0",
"domain": "business-tools",
"split": "public-development"
},
"candidate_sha256": "d4412a9c3e8c47c744dd397aa582edec83b7ae1a6436b75a905fb4bb63faa3cd",
"grader_sha256": "2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255",
"cases_sha256": "b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df",
"runtime": {
"backend": "docker",
"image": "python:3.12-slim",
"image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea",
"command": [
"{python}",
"-I",
"-B",
"main.py"
],
"response_timeout_seconds": 10.0,
"session_output_limit_bytes": 1048576,
"python": "3.12.3",
"platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39"
},
"seeds": [
17
],
"valid": true,
"status": "failed",
"score": 0.35,
"resolved": false,
"dimensions": {
"routing": {
"passed": 0,
"total": 4,
"assessed": 4,
"weight": 0.3,
"score": 0.0
},
"notes": {
"passed": 0,
"total": 4,
"assessed": 4,
"weight": 0.25,
"score": 0.0
},
"closure": {
"passed": 2,
"total": 4,
"assessed": 4,
"weight": 0.2,
"score": 0.5
},
"scope": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.15,
"score": 1.0
},
"protocol": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.1,
"score": 1.0
}
},
"cases": [
{
"seed": 17,
"case_id": "route-open-ticket",
"checks": {
"routing": false,
"notes": false,
"closure": true,
"scope": true,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.713709,
"transcript_sha256": "dc89878d8618444cc445bec619924daccdde74b03fca5826ccd7ef1d02d4245e",
"initial_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "finish",
"action": {
"finish": true,
"message": "Completed all requested work."
}
}
],
"tool_calls": 0
},
{
"seed": 17,
"case_id": "close-resolved-ticket",
"checks": {
"routing": false,
"notes": false,
"closure": false,
"scope": true,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.617806,
"transcript_sha256": "dc89878d8618444cc445bec619924daccdde74b03fca5826ccd7ef1d02d4245e",
"initial_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "finish",
"action": {
"finish": true,
"message": "Completed all requested work."
}
}
],
"tool_calls": 0
},
{
"seed": 17,
"case_id": "retry-before-commit",
"checks": {
"routing": false,
"notes": false,
"closure": true,
"scope": true,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.635735,
"transcript_sha256": "dc89878d8618444cc445bec619924daccdde74b03fca5826ccd7ef1d02d4245e",
"initial_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "finish",
"action": {
"finish": true,
"message": "Completed all requested work."
}
}
],
"tool_calls": 0
},
{
"seed": 17,
"case_id": "retry-after-commit",
"checks": {
"routing": false,
"notes": false,
"closure": false,
"scope": true,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.642983,
"transcript_sha256": "dc89878d8618444cc445bec619924daccdde74b03fca5826ccd7ef1d02d4245e",
"initial_state": {
"T-cd68615c": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-cd68615c": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "finish",
"action": {
"finish": true,
"message": "Completed all requested work."
}
}
],
"tool_calls": 0
}
],
"evaluation_seconds": 2.613067,
"agent_cost_usd": null,
"agent_tokens": null
}
},
{
"name": "wrong-ticket",
"target_dimension": "scope",
"killed": true,
"valid": true,
"score": 0.2,
"failing_cases": [
"close-resolved-ticket",
"retry-after-commit",
"retry-before-commit",
"route-open-ticket"
],
"candidate_sha256": "183da04ff8f34d6d8640176ab9ea40882985b568163399c7add81971e782c761",
"evaluation": {
"schema_version": "evalarc.evaluation.v2",
"created_at": "2026-09-14T01:29:44.442973+00:00",
"evalarc_version": "0.2.0",
"task": {
"id": "support-routing",
"version": "0.1.0",
"domain": "business-tools",
"split": "public-development"
},
"candidate_sha256": "183da04ff8f34d6d8640176ab9ea40882985b568163399c7add81971e782c761",
"grader_sha256": "2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255",
"cases_sha256": "b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df",
"runtime": {
"backend": "docker",
"image": "python:3.12-slim",
"image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea",
"command": [
"{python}",
"-I",
"-B",
"main.py"
],
"response_timeout_seconds": 10.0,
"session_output_limit_bytes": 1048576,
"python": "3.12.3",
"platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39"
},
"seeds": [
17
],
"valid": true,
"status": "failed",
"score": 0.2,
"resolved": false,
"dimensions": {
"routing": {
"passed": 0,
"total": 4,
"assessed": 4,
"weight": 0.3,
"score": 0.0
},
"notes": {
"passed": 0,
"total": 4,
"assessed": 4,
"weight": 0.25,
"score": 0.0
},
"closure": {
"passed": 2,
"total": 4,
"assessed": 4,
"weight": 0.2,
"score": 0.5
},
"scope": {
"passed": 0,
"total": 4,
"assessed": 4,
"weight": 0.15,
"score": 0.0
},
"protocol": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.1,
"score": 1.0
}
},
"cases": [
{
"seed": 17,
"case_id": "route-open-ticket",
"checks": {
"routing": false,
"notes": false,
"closure": true,
"scope": false,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.66702,
"transcript_sha256": "13bb6318b8c0c1e61289c5ad6878ae37d2cb7a2591279528eea61699f94cd047",
"initial_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "queue-0-0",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket.",
"Reviewed request T-85a0bcc1."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "U-6a06e9ab"
}
},
"result": {
"ok": true,
"ticket": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "U-6a06e9ab",
"queue": "queue-0-0"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"U-6a06e9ab": {
"before": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
},
"after": {
"category": "billing",
"queue": "queue-0-0",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "U-6a06e9ab",
"body": "Reviewed request T-85a0bcc1.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"U-6a06e9ab": {
"before": {
"category": "billing",
"queue": "queue-0-0",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
},
"after": {
"category": "billing",
"queue": "queue-0-0",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket.",
"Reviewed request T-85a0bcc1."
]
}
}
}
},
{
"step": 3,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 3
},
{
"seed": 17,
"case_id": "close-resolved-ticket",
"checks": {
"routing": false,
"notes": false,
"closure": false,
"scope": false,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.688977,
"transcript_sha256": "af34c4df1f605b18475bb88594ec02c95aa692cdedc06945af120e864d7b20d1",
"initial_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "queue-1-0",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket.",
"Reviewed request T-c41f9dfd."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "U-b48438b5"
}
},
"result": {
"ok": true,
"ticket": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "U-b48438b5",
"queue": "queue-1-0"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"U-b48438b5": {
"before": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "U-b48438b5",
"body": "Reviewed request T-c41f9dfd.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"U-b48438b5": {
"before": {
"category": "billing",
"queue": "queue-1-0",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket.",
"Reviewed request T-c41f9dfd."
]
}
}
}
},
{
"step": 3,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 3
},
{
"seed": 17,
"case_id": "retry-before-commit",
"checks": {
"routing": false,
"notes": false,
"closure": true,
"scope": false,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.661649,
"transcript_sha256": "dce95ae090de1ec11795608277ea5d9d4ea46545741bdd9f89e8b351d2221909",
"initial_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-62590992": {
"category": "account",
"queue": "queue-2-2",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket.",
"Reviewed request T-3fb81d27."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "U-62590992"
}
},
"result": {
"ok": true,
"ticket": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "U-62590992",
"queue": "queue-2-2"
}
},
"result": {
"ok": false,
"error": {
"code": "temporarily_unavailable",
"retryable": true
}
},
"status": "tool_error",
"error_source": "injected",
"changes": {}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "U-62590992",
"queue": "queue-2-2"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"U-62590992": {
"before": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
},
"after": {
"category": "account",
"queue": "queue-2-2",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "U-62590992",
"body": "Reviewed request T-3fb81d27.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"U-62590992": {
"before": {
"category": "account",
"queue": "queue-2-2",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
},
"after": {
"category": "account",
"queue": "queue-2-2",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket.",
"Reviewed request T-3fb81d27."
]
}
}
}
},
{
"step": 4,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 4
},
{
"seed": 17,
"case_id": "retry-after-commit",
"checks": {
"routing": false,
"notes": false,
"closure": false,
"scope": false,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.558768,
"transcript_sha256": "cfc5d50e5cf0c9e319b39869d751e334459cfa948ea4693b7a24d9652ee8d30e",
"initial_state": {
"T-cd68615c": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-cd68615c": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-51436d1f": {
"category": "account",
"queue": "queue-3-2",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket.",
"Reviewed request T-cd68615c."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "U-51436d1f"
}
},
"result": {
"ok": true,
"ticket": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "U-51436d1f",
"queue": "queue-3-2"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"U-51436d1f": {
"before": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
},
"after": {
"category": "account",
"queue": "queue-3-2",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "U-51436d1f",
"body": "Reviewed request T-cd68615c.",
"request_id": "review"
}
},
"result": {
"ok": false,
"error": {
"code": "temporarily_unavailable",
"retryable": true
}
},
"status": "tool_error",
"error_source": "injected",
"changes": {
"U-51436d1f": {
"before": {
"category": "account",
"queue": "queue-3-2",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
},
"after": {
"category": "account",
"queue": "queue-3-2",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "U-51436d1f",
"body": "Reviewed request T-cd68615c.",
"request_id": "review"
}
},
"result": {
"ok": true,
"replayed": true
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 4,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 4
}
],
"evaluation_seconds": 2.579827,
"agent_cost_usd": null,
"agent_tokens": null
}
},
{
"name": "wrong-queue",
"target_dimension": "routing",
"killed": true,
"valid": true,
"score": 0.7,
"failing_cases": [
"close-resolved-ticket",
"retry-after-commit",
"retry-before-commit",
"route-open-ticket"
],
"candidate_sha256": "c4d20e7a258872b9555f703264a14e3e775c03563d08ca7f6c11fb0eb73253fa",
"evaluation": {
"schema_version": "evalarc.evaluation.v2",
"created_at": "2026-09-14T01:29:46.670548+00:00",
"evalarc_version": "0.2.0",
"task": {
"id": "support-routing",
"version": "0.1.0",
"domain": "business-tools",
"split": "public-development"
},
"candidate_sha256": "c4d20e7a258872b9555f703264a14e3e775c03563d08ca7f6c11fb0eb73253fa",
"grader_sha256": "2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255",
"cases_sha256": "b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df",
"runtime": {
"backend": "docker",
"image": "python:3.12-slim",
"image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea",
"command": [
"{python}",
"-I",
"-B",
"main.py"
],
"response_timeout_seconds": 10.0,
"session_output_limit_bytes": 1048576,
"python": "3.12.3",
"platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39"
},
"seeds": [
17
],
"valid": true,
"status": "failed",
"score": 0.7,
"resolved": false,
"dimensions": {
"routing": {
"passed": 0,
"total": 4,
"assessed": 4,
"weight": 0.3,
"score": 0.0
},
"notes": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.25,
"score": 1.0
},
"closure": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.2,
"score": 1.0
},
"scope": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.15,
"score": 1.0
},
"protocol": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.1,
"score": 1.0
}
},
"cases": [
{
"seed": 17,
"case_id": "route-open-ticket",
"checks": {
"routing": false,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.576444,
"transcript_sha256": "06bfdd936f8a75a1f79fac629e71818d6f8750e756c87b7318481642641860f9",
"initial_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "queue-0-0",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-85a0bcc1"
}
},
"result": {
"ok": true,
"ticket": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-85a0bcc1",
"queue": "queue-0-0"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-85a0bcc1": {
"before": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-0-0",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-85a0bcc1",
"body": "Reviewed request T-85a0bcc1.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-85a0bcc1": {
"before": {
"category": "technical",
"queue": "queue-0-0",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-0-0",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1."
]
}
}
}
},
{
"step": 3,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 3
},
{
"seed": 17,
"case_id": "close-resolved-ticket",
"checks": {
"routing": false,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.552313,
"transcript_sha256": "18b3105a6a7f077107630156bf9124018c8271feaf28f6fd778cad881d67e0b2",
"initial_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "queue-1-1",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-c41f9dfd"
}
},
"result": {
"ok": true,
"ticket": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-c41f9dfd",
"queue": "queue-1-1"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-1-1",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-c41f9dfd",
"body": "Reviewed request T-c41f9dfd.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "queue-1-1",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-1-1",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "close_ticket",
"arguments": {
"ticket_id": "T-c41f9dfd"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "queue-1-1",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
},
"after": {
"category": "billing",
"queue": "queue-1-1",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
}
}
}
},
{
"step": 4,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 4
},
{
"seed": 17,
"case_id": "retry-before-commit",
"checks": {
"routing": false,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.55564,
"transcript_sha256": "8bb84e30d03030ac20e834eae63758c4041341b0325cebc35e1aeca01d59ccfe",
"initial_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "queue-2-0",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-3fb81d27."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-3fb81d27"
}
},
"result": {
"ok": true,
"ticket": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-3fb81d27",
"queue": "queue-2-0"
}
},
"result": {
"ok": false,
"error": {
"code": "temporarily_unavailable",
"retryable": true
}
},
"status": "tool_error",
"error_source": "injected",
"changes": {}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-3fb81d27",
"queue": "queue-2-0"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-3fb81d27": {
"before": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-2-0",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-3fb81d27",
"body": "Reviewed request T-3fb81d27.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-3fb81d27": {
"before": {
"category": "technical",
"queue": "queue-2-0",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-2-0",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-3fb81d27."
]
}
}
}
},
{
"step": 4,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 4
},
{
"seed": 17,
"case_id": "retry-after-commit",
"checks": {
"routing": false,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.538827,
"transcript_sha256": "3076729f9cfaeec8c787fb9216588135070ea5aee843c853840be4436629ef92",
"initial_state": {
"T-cd68615c": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-cd68615c": {
"category": "billing",
"queue": "queue-3-1",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-cd68615c"
}
},
"result": {
"ok": true,
"ticket": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-cd68615c",
"queue": "queue-3-1"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-3-1",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-cd68615c",
"body": "Reviewed request T-cd68615c.",
"request_id": "review"
}
},
"result": {
"ok": false,
"error": {
"code": "temporarily_unavailable",
"retryable": true
}
},
"status": "tool_error",
"error_source": "injected",
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "queue-3-1",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-3-1",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-cd68615c",
"body": "Reviewed request T-cd68615c.",
"request_id": "review"
}
},
"result": {
"ok": true,
"replayed": true
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 4,
"type": "tool_call",
"action": {
"tool": "close_ticket",
"arguments": {
"ticket_id": "T-cd68615c"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "queue-3-1",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
},
"after": {
"category": "billing",
"queue": "queue-3-1",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 5,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 5
}
],
"evaluation_seconds": 2.22727,
"agent_cost_usd": null,
"agent_tokens": null
}
},
{
"name": "duplicate-note",
"target_dimension": "notes",
"killed": true,
"valid": true,
"score": 0.75,
"failing_cases": [
"close-resolved-ticket",
"retry-after-commit",
"retry-before-commit",
"route-open-ticket"
],
"candidate_sha256": "56c5cb4be6ffd347dc0a382a6a7b9bacde1c308ce1538faae18c002affc36213",
"evaluation": {
"schema_version": "evalarc.evaluation.v2",
"created_at": "2026-09-14T01:29:48.962131+00:00",
"evalarc_version": "0.2.0",
"task": {
"id": "support-routing",
"version": "0.1.0",
"domain": "business-tools",
"split": "public-development"
},
"candidate_sha256": "56c5cb4be6ffd347dc0a382a6a7b9bacde1c308ce1538faae18c002affc36213",
"grader_sha256": "2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255",
"cases_sha256": "b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df",
"runtime": {
"backend": "docker",
"image": "python:3.12-slim",
"image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea",
"command": [
"{python}",
"-I",
"-B",
"main.py"
],
"response_timeout_seconds": 10.0,
"session_output_limit_bytes": 1048576,
"python": "3.12.3",
"platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39"
},
"seeds": [
17
],
"valid": true,
"status": "failed",
"score": 0.75,
"resolved": false,
"dimensions": {
"routing": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.3,
"score": 1.0
},
"notes": {
"passed": 0,
"total": 4,
"assessed": 4,
"weight": 0.25,
"score": 0.0
},
"closure": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.2,
"score": 1.0
},
"scope": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.15,
"score": 1.0
},
"protocol": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.1,
"score": 1.0
}
},
"cases": [
{
"seed": 17,
"case_id": "route-open-ticket",
"checks": {
"routing": true,
"notes": false,
"closure": true,
"scope": true,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.585063,
"transcript_sha256": "eba30506565d7d2797e622ec54845d26c0b3a9fc9e323c298cbc76dc390e082c",
"initial_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1.",
"Reviewed request T-85a0bcc1."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-85a0bcc1"
}
},
"result": {
"ok": true,
"ticket": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-85a0bcc1",
"queue": "queue-0-2"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-85a0bcc1": {
"before": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-85a0bcc1",
"body": "Reviewed request T-85a0bcc1.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-85a0bcc1": {
"before": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-85a0bcc1",
"body": "Reviewed request T-85a0bcc1.",
"request_id": "duplicate"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-85a0bcc1": {
"before": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1."
]
},
"after": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1.",
"Reviewed request T-85a0bcc1."
]
}
}
}
},
{
"step": 4,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 4
},
{
"seed": 17,
"case_id": "close-resolved-ticket",
"checks": {
"routing": true,
"notes": false,
"closure": true,
"scope": true,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.568052,
"transcript_sha256": "7b2dd2bb2f48dc19e31cd072abca3fd5d43d9d7369a1a19077e3bcf0c45ae229",
"initial_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd.",
"Reviewed request T-c41f9dfd."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-c41f9dfd"
}
},
"result": {
"ok": true,
"ticket": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-c41f9dfd",
"queue": "queue-1-0"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-c41f9dfd",
"body": "Reviewed request T-c41f9dfd.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-c41f9dfd",
"body": "Reviewed request T-c41f9dfd.",
"request_id": "duplicate"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd.",
"Reviewed request T-c41f9dfd."
]
}
}
}
},
{
"step": 4,
"type": "tool_call",
"action": {
"tool": "close_ticket",
"arguments": {
"ticket_id": "T-c41f9dfd"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd.",
"Reviewed request T-c41f9dfd."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd.",
"Reviewed request T-c41f9dfd."
]
}
}
}
},
{
"step": 5,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 5
},
{
"seed": 17,
"case_id": "retry-before-commit",
"checks": {
"routing": true,
"notes": false,
"closure": true,
"scope": true,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.538005,
"transcript_sha256": "34dc374c3e62cf1911796b91f4c3d7b6d1f7c14a4ac997c8bcd8d7ed2c7f6fdd",
"initial_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-3fb81d27.",
"Reviewed request T-3fb81d27."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-3fb81d27"
}
},
"result": {
"ok": true,
"ticket": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-3fb81d27",
"queue": "queue-2-1"
}
},
"result": {
"ok": false,
"error": {
"code": "temporarily_unavailable",
"retryable": true
}
},
"status": "tool_error",
"error_source": "injected",
"changes": {}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-3fb81d27",
"queue": "queue-2-1"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-3fb81d27": {
"before": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-3fb81d27",
"body": "Reviewed request T-3fb81d27.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-3fb81d27": {
"before": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-3fb81d27."
]
}
}
}
},
{
"step": 4,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-3fb81d27",
"body": "Reviewed request T-3fb81d27.",
"request_id": "duplicate"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-3fb81d27": {
"before": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-3fb81d27."
]
},
"after": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-3fb81d27.",
"Reviewed request T-3fb81d27."
]
}
}
}
},
{
"step": 5,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 5
},
{
"seed": 17,
"case_id": "retry-after-commit",
"checks": {
"routing": true,
"notes": false,
"closure": true,
"scope": true,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.595631,
"transcript_sha256": "877a29b2016a4c4dd8565f1fb9987faf298ad23e39683d63b32762d7abfcadf2",
"initial_state": {
"T-cd68615c": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-cd68615c": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c.",
"Reviewed request T-cd68615c."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-cd68615c"
}
},
"result": {
"ok": true,
"ticket": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-cd68615c",
"queue": "queue-3-0"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-cd68615c",
"body": "Reviewed request T-cd68615c.",
"request_id": "review"
}
},
"result": {
"ok": false,
"error": {
"code": "temporarily_unavailable",
"retryable": true
}
},
"status": "tool_error",
"error_source": "injected",
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-cd68615c",
"body": "Reviewed request T-cd68615c.",
"request_id": "review"
}
},
"result": {
"ok": true,
"replayed": true
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 4,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-cd68615c",
"body": "Reviewed request T-cd68615c.",
"request_id": "duplicate"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 5,
"type": "tool_call",
"action": {
"tool": "close_ticket",
"arguments": {
"ticket_id": "T-cd68615c"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c.",
"Reviewed request T-cd68615c."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 6,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 6
}
],
"evaluation_seconds": 2.291397,
"agent_cost_usd": null,
"agent_tokens": null
}
},
{
"name": "close-unresolved",
"target_dimension": "closure",
"killed": true,
"valid": true,
"score": 0.9,
"failing_cases": [
"retry-before-commit",
"route-open-ticket"
],
"candidate_sha256": "a34429419d7c794420d43f7873ce5b1474f73821329f414609d5e4d18fc174c3",
"evaluation": {
"schema_version": "evalarc.evaluation.v2",
"created_at": "2026-09-14T01:29:51.180933+00:00",
"evalarc_version": "0.2.0",
"task": {
"id": "support-routing",
"version": "0.1.0",
"domain": "business-tools",
"split": "public-development"
},
"candidate_sha256": "a34429419d7c794420d43f7873ce5b1474f73821329f414609d5e4d18fc174c3",
"grader_sha256": "2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255",
"cases_sha256": "b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df",
"runtime": {
"backend": "docker",
"image": "python:3.12-slim",
"image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea",
"command": [
"{python}",
"-I",
"-B",
"main.py"
],
"response_timeout_seconds": 10.0,
"session_output_limit_bytes": 1048576,
"python": "3.12.3",
"platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39"
},
"seeds": [
17
],
"valid": true,
"status": "failed",
"score": 0.9,
"resolved": false,
"dimensions": {
"routing": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.3,
"score": 1.0
},
"notes": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.25,
"score": 1.0
},
"closure": {
"passed": 2,
"total": 4,
"assessed": 4,
"weight": 0.2,
"score": 0.5
},
"scope": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.15,
"score": 1.0
},
"protocol": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.1,
"score": 1.0
}
},
"cases": [
{
"seed": 17,
"case_id": "route-open-ticket",
"checks": {
"routing": true,
"notes": true,
"closure": false,
"scope": true,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.554959,
"transcript_sha256": "3b8ea18d1468afe56a69b06b884469141aa5aed9ab211a2c60378f64aaf456e5",
"initial_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-85a0bcc1"
}
},
"result": {
"ok": true,
"ticket": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-85a0bcc1",
"queue": "queue-0-2"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-85a0bcc1": {
"before": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-85a0bcc1",
"body": "Reviewed request T-85a0bcc1.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-85a0bcc1": {
"before": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "close_ticket",
"arguments": {
"ticket_id": "T-85a0bcc1"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-85a0bcc1": {
"before": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1."
]
},
"after": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1."
]
}
}
}
},
{
"step": 4,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 4
},
{
"seed": 17,
"case_id": "close-resolved-ticket",
"checks": {
"routing": true,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": true,
"status": "passed",
"error": null,
"duration_seconds": 0.567618,
"transcript_sha256": "a260768c11b589ce3f58658cac6aae8a01d36e388a47fc47c7995f8d5c36708e",
"initial_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-c41f9dfd"
}
},
"result": {
"ok": true,
"ticket": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-c41f9dfd",
"queue": "queue-1-0"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-c41f9dfd",
"body": "Reviewed request T-c41f9dfd.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "close_ticket",
"arguments": {
"ticket_id": "T-c41f9dfd"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
}
}
}
},
{
"step": 4,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 4
},
{
"seed": 17,
"case_id": "retry-before-commit",
"checks": {
"routing": true,
"notes": true,
"closure": false,
"scope": true,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.540602,
"transcript_sha256": "4d7abd25df6f1a223a236570e6b1d0f5d31f8f0343a6497a27b12f396f65e2ca",
"initial_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-3fb81d27."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-3fb81d27"
}
},
"result": {
"ok": true,
"ticket": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-3fb81d27",
"queue": "queue-2-1"
}
},
"result": {
"ok": false,
"error": {
"code": "temporarily_unavailable",
"retryable": true
}
},
"status": "tool_error",
"error_source": "injected",
"changes": {}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-3fb81d27",
"queue": "queue-2-1"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-3fb81d27": {
"before": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-3fb81d27",
"body": "Reviewed request T-3fb81d27.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-3fb81d27": {
"before": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-3fb81d27."
]
}
}
}
},
{
"step": 4,
"type": "tool_call",
"action": {
"tool": "close_ticket",
"arguments": {
"ticket_id": "T-3fb81d27"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-3fb81d27": {
"before": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-3fb81d27."
]
},
"after": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-3fb81d27."
]
}
}
}
},
{
"step": 5,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 5
},
{
"seed": 17,
"case_id": "retry-after-commit",
"checks": {
"routing": true,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": true,
"status": "passed",
"error": null,
"duration_seconds": 0.5517,
"transcript_sha256": "c97b016921c8d8fdcdeffd1ebf1f438d5a29be06732c31c86f99fd087bc0b58f",
"initial_state": {
"T-cd68615c": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-cd68615c": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-cd68615c"
}
},
"result": {
"ok": true,
"ticket": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-cd68615c",
"queue": "queue-3-0"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-cd68615c",
"body": "Reviewed request T-cd68615c.",
"request_id": "review"
}
},
"result": {
"ok": false,
"error": {
"code": "temporarily_unavailable",
"retryable": true
}
},
"status": "tool_error",
"error_source": "injected",
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-cd68615c",
"body": "Reviewed request T-cd68615c.",
"request_id": "review"
}
},
"result": {
"ok": true,
"replayed": true
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 4,
"type": "tool_call",
"action": {
"tool": "close_ticket",
"arguments": {
"ticket_id": "T-cd68615c"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 5,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 5
}
],
"evaluation_seconds": 2.218637,
"agent_cost_usd": null,
"agent_tokens": null
}
},
{
"name": "skip-retry",
"target_dimension": "notes",
"killed": true,
"valid": true,
"score": 0.8125,
"failing_cases": [
"retry-before-commit"
],
"candidate_sha256": "ad5718fa57624480f1f6a28c1fbf055d5164d89c35bc0ca3ce28be1753ae0f54",
"evaluation": {
"schema_version": "evalarc.evaluation.v2",
"created_at": "2026-09-14T01:29:53.496163+00:00",
"evalarc_version": "0.2.0",
"task": {
"id": "support-routing",
"version": "0.1.0",
"domain": "business-tools",
"split": "public-development"
},
"candidate_sha256": "ad5718fa57624480f1f6a28c1fbf055d5164d89c35bc0ca3ce28be1753ae0f54",
"grader_sha256": "2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255",
"cases_sha256": "b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df",
"runtime": {
"backend": "docker",
"image": "python:3.12-slim",
"image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea",
"command": [
"{python}",
"-I",
"-B",
"main.py"
],
"response_timeout_seconds": 10.0,
"session_output_limit_bytes": 1048576,
"python": "3.12.3",
"platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39"
},
"seeds": [
17
],
"valid": true,
"status": "failed",
"score": 0.8125,
"resolved": false,
"dimensions": {
"routing": {
"passed": 3,
"total": 4,
"assessed": 4,
"weight": 0.3,
"score": 0.75
},
"notes": {
"passed": 3,
"total": 4,
"assessed": 4,
"weight": 0.25,
"score": 0.75
},
"closure": {
"passed": 3,
"total": 4,
"assessed": 4,
"weight": 0.2,
"score": 0.75
},
"scope": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.15,
"score": 1.0
},
"protocol": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.1,
"score": 1.0
}
},
"cases": [
{
"seed": 17,
"case_id": "route-open-ticket",
"checks": {
"routing": true,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": true,
"status": "passed",
"error": null,
"duration_seconds": 0.604977,
"transcript_sha256": "7b5b4973e934a60c55b95bda6eedf79fcff62764fbc1b81723c5126608caf4bd",
"initial_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-85a0bcc1"
}
},
"result": {
"ok": true,
"ticket": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-85a0bcc1",
"queue": "queue-0-2"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-85a0bcc1": {
"before": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-85a0bcc1",
"body": "Reviewed request T-85a0bcc1.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-85a0bcc1": {
"before": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1."
]
}
}
}
},
{
"step": 3,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 3
},
{
"seed": 17,
"case_id": "close-resolved-ticket",
"checks": {
"routing": true,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": true,
"status": "passed",
"error": null,
"duration_seconds": 0.588704,
"transcript_sha256": "a260768c11b589ce3f58658cac6aae8a01d36e388a47fc47c7995f8d5c36708e",
"initial_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-c41f9dfd"
}
},
"result": {
"ok": true,
"ticket": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-c41f9dfd",
"queue": "queue-1-0"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-c41f9dfd",
"body": "Reviewed request T-c41f9dfd.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "close_ticket",
"arguments": {
"ticket_id": "T-c41f9dfd"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
}
}
}
},
{
"step": 4,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 4
},
{
"seed": 17,
"case_id": "retry-before-commit",
"checks": {
"routing": false,
"notes": false,
"closure": true,
"scope": true,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.566178,
"transcript_sha256": "7700d40c0510e616f031164599fb05256c512aae5f280105701d533ec030f826",
"initial_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-3fb81d27"
}
},
"result": {
"ok": true,
"ticket": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-3fb81d27",
"queue": "queue-2-1"
}
},
"result": {
"ok": false,
"error": {
"code": "temporarily_unavailable",
"retryable": true
}
},
"status": "tool_error",
"error_source": "injected",
"changes": {}
},
{
"step": 2,
"type": "finish",
"action": {
"finish": true,
"message": "Could not complete the operation."
}
}
],
"tool_calls": 2
},
{
"seed": 17,
"case_id": "retry-after-commit",
"checks": {
"routing": true,
"notes": true,
"closure": false,
"scope": true,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.550785,
"transcript_sha256": "9cb4cd29bf17742367de9a0d73987d95440b2f23e38674ef6e770535ba953b09",
"initial_state": {
"T-cd68615c": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-cd68615c": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-cd68615c"
}
},
"result": {
"ok": true,
"ticket": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-cd68615c",
"queue": "queue-3-0"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-cd68615c",
"body": "Reviewed request T-cd68615c.",
"request_id": "review"
}
},
"result": {
"ok": false,
"error": {
"code": "temporarily_unavailable",
"retryable": true
}
},
"status": "tool_error",
"error_source": "injected",
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 3,
"type": "finish",
"action": {
"finish": true,
"message": "Could not complete the operation."
}
}
],
"tool_calls": 3
}
],
"evaluation_seconds": 2.314765,
"agent_cost_usd": null,
"agent_tokens": null
}
},
{
"name": "new-key-on-retry",
"target_dimension": "notes",
"killed": true,
"valid": true,
"score": 0.9375,
"failing_cases": [
"retry-after-commit"
],
"candidate_sha256": "b89a99819c03e8290096ca268f637c5517d465b3ec5f0b5fe2b21c35509ff202",
"evaluation": {
"schema_version": "evalarc.evaluation.v2",
"created_at": "2026-09-14T01:29:55.858010+00:00",
"evalarc_version": "0.2.0",
"task": {
"id": "support-routing",
"version": "0.1.0",
"domain": "business-tools",
"split": "public-development"
},
"candidate_sha256": "b89a99819c03e8290096ca268f637c5517d465b3ec5f0b5fe2b21c35509ff202",
"grader_sha256": "2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255",
"cases_sha256": "b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df",
"runtime": {
"backend": "docker",
"image": "python:3.12-slim",
"image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea",
"command": [
"{python}",
"-I",
"-B",
"main.py"
],
"response_timeout_seconds": 10.0,
"session_output_limit_bytes": 1048576,
"python": "3.12.3",
"platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39"
},
"seeds": [
17
],
"valid": true,
"status": "failed",
"score": 0.9375,
"resolved": false,
"dimensions": {
"routing": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.3,
"score": 1.0
},
"notes": {
"passed": 3,
"total": 4,
"assessed": 4,
"weight": 0.25,
"score": 0.75
},
"closure": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.2,
"score": 1.0
},
"scope": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.15,
"score": 1.0
},
"protocol": {
"passed": 4,
"total": 4,
"assessed": 4,
"weight": 0.1,
"score": 1.0
}
},
"cases": [
{
"seed": 17,
"case_id": "route-open-ticket",
"checks": {
"routing": true,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": true,
"status": "passed",
"error": null,
"duration_seconds": 0.615903,
"transcript_sha256": "7b5b4973e934a60c55b95bda6eedf79fcff62764fbc1b81723c5126608caf4bd",
"initial_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-85a0bcc1"
}
},
"result": {
"ok": true,
"ticket": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-85a0bcc1",
"queue": "queue-0-2"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-85a0bcc1": {
"before": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-85a0bcc1",
"body": "Reviewed request T-85a0bcc1.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-85a0bcc1": {
"before": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1."
]
}
}
}
},
{
"step": 3,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 3
},
{
"seed": 17,
"case_id": "close-resolved-ticket",
"checks": {
"routing": true,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": true,
"status": "passed",
"error": null,
"duration_seconds": 0.579634,
"transcript_sha256": "a260768c11b589ce3f58658cac6aae8a01d36e388a47fc47c7995f8d5c36708e",
"initial_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-c41f9dfd"
}
},
"result": {
"ok": true,
"ticket": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-c41f9dfd",
"queue": "queue-1-0"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-c41f9dfd",
"body": "Reviewed request T-c41f9dfd.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "close_ticket",
"arguments": {
"ticket_id": "T-c41f9dfd"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
}
}
}
},
{
"step": 4,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 4
},
{
"seed": 17,
"case_id": "retry-before-commit",
"checks": {
"routing": true,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": true,
"status": "passed",
"error": null,
"duration_seconds": 0.574111,
"transcript_sha256": "b1d1d91dc01fc4bfb3243aa3739e953a40aa3920caf63a4b2ab73c87ecabda66",
"initial_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-3fb81d27."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-3fb81d27"
}
},
"result": {
"ok": true,
"ticket": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-3fb81d27",
"queue": "queue-2-1"
}
},
"result": {
"ok": false,
"error": {
"code": "temporarily_unavailable",
"retryable": true
}
},
"status": "tool_error",
"error_source": "injected",
"changes": {}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-3fb81d27",
"queue": "queue-2-1"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-3fb81d27": {
"before": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-3fb81d27",
"body": "Reviewed request T-3fb81d27.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-3fb81d27": {
"before": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-3fb81d27."
]
}
}
}
},
{
"step": 4,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 4
},
{
"seed": 17,
"case_id": "retry-after-commit",
"checks": {
"routing": true,
"notes": false,
"closure": true,
"scope": true,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.58689,
"transcript_sha256": "4d1123f9ef2ebc43d80c84e4f2054b3ed62a22db7d6c565f0f7f3a9f28c662b4",
"initial_state": {
"T-cd68615c": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-cd68615c": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c.",
"Reviewed request T-cd68615c."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-cd68615c"
}
},
"result": {
"ok": true,
"ticket": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-cd68615c",
"queue": "queue-3-0"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-cd68615c",
"body": "Reviewed request T-cd68615c.",
"request_id": "review"
}
},
"result": {
"ok": false,
"error": {
"code": "temporarily_unavailable",
"retryable": true
}
},
"status": "tool_error",
"error_source": "injected",
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-cd68615c",
"body": "Reviewed request T-cd68615c.",
"request_id": "review-retry"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 4,
"type": "tool_call",
"action": {
"tool": "close_ticket",
"arguments": {
"ticket_id": "T-cd68615c"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c.",
"Reviewed request T-cd68615c."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 5,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 5
}
],
"evaluation_seconds": 2.361685,
"agent_cost_usd": null,
"agent_tokens": null
}
}
],
"mutation_score": 1.0,
"killed": 7,
"total": 7,
"passed": true,
"interpretation": "Coverage of 7 declared behavioral fault models only. This is not a bound on reward hacking or a model benchmark."
}