Spaces:
Running
Running
| { | |
| "schema_version": "evalarc.evaluation.v2", | |
| "created_at": "2026-09-14T01:58:21.160793+00:00", | |
| "evalarc_version": "0.3.0", | |
| "task": { | |
| "id": "support-routing", | |
| "version": "0.1.0", | |
| "domain": "business-tools", | |
| "split": "public-development" | |
| }, | |
| "candidate_sha256": "b89a99819c03e8290096ca268f637c5517d465b3ec5f0b5fe2b21c35509ff202", | |
| "grader_sha256": "2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255", | |
| "cases_sha256": "b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df", | |
| "runtime": { | |
| "backend": "docker", | |
| "image": "python:3.12-slim", | |
| "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", | |
| "command": [ | |
| "{python}", | |
| "-I", | |
| "-B", | |
| "main.py" | |
| ], | |
| "response_timeout_seconds": 10.0, | |
| "session_output_limit_bytes": 1048576, | |
| "python": "3.12.3", | |
| "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" | |
| }, | |
| "seeds": [ | |
| 17 | |
| ], | |
| "valid": true, | |
| "status": "failed", | |
| "score": 0.9375, | |
| "resolved": false, | |
| "dimensions": { | |
| "routing": { | |
| "passed": 4, | |
| "total": 4, | |
| "assessed": 4, | |
| "weight": 0.3, | |
| "score": 1.0 | |
| }, | |
| "notes": { | |
| "passed": 3, | |
| "total": 4, | |
| "assessed": 4, | |
| "weight": 0.25, | |
| "score": 0.75 | |
| }, | |
| "closure": { | |
| "passed": 4, | |
| "total": 4, | |
| "assessed": 4, | |
| "weight": 0.2, | |
| "score": 1.0 | |
| }, | |
| "scope": { | |
| "passed": 4, | |
| "total": 4, | |
| "assessed": 4, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "protocol": { | |
| "passed": 4, | |
| "total": 4, | |
| "assessed": 4, | |
| "weight": 0.1, | |
| "score": 1.0 | |
| } | |
| }, | |
| "cases": [ | |
| { | |
| "seed": 17, | |
| "case_id": "route-open-ticket", | |
| "checks": { | |
| "routing": true, | |
| "notes": true, | |
| "closure": true, | |
| "scope": true, | |
| "protocol": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "error": null, | |
| "duration_seconds": 0.461764, | |
| "transcript_sha256": "7b5b4973e934a60c55b95bda6eedf79fcff62764fbc1b81723c5126608caf4bd", | |
| "initial_state": { | |
| "T-85a0bcc1": { | |
| "category": "technical", | |
| "queue": "inbox", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| }, | |
| "U-6a06e9ab": { | |
| "category": "billing", | |
| "queue": "inbox", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Unrelated request; preserve this ticket." | |
| ] | |
| } | |
| }, | |
| "final_state": { | |
| "T-85a0bcc1": { | |
| "category": "technical", | |
| "queue": "queue-0-2", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support.", | |
| "Reviewed request T-85a0bcc1." | |
| ] | |
| }, | |
| "U-6a06e9ab": { | |
| "category": "billing", | |
| "queue": "inbox", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Unrelated request; preserve this ticket." | |
| ] | |
| } | |
| }, | |
| "trace": [ | |
| { | |
| "step": 0, | |
| "type": "tool_call", | |
| "action": { | |
| "tool": "get_ticket", | |
| "arguments": { | |
| "ticket_id": "T-85a0bcc1" | |
| } | |
| }, | |
| "result": { | |
| "ok": true, | |
| "ticket": { | |
| "category": "technical", | |
| "queue": "inbox", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| } | |
| }, | |
| "status": "ok", | |
| "error_source": null, | |
| "changes": {} | |
| }, | |
| { | |
| "step": 1, | |
| "type": "tool_call", | |
| "action": { | |
| "tool": "assign_queue", | |
| "arguments": { | |
| "ticket_id": "T-85a0bcc1", | |
| "queue": "queue-0-2" | |
| } | |
| }, | |
| "result": { | |
| "ok": true | |
| }, | |
| "status": "ok", | |
| "error_source": null, | |
| "changes": { | |
| "T-85a0bcc1": { | |
| "before": { | |
| "category": "technical", | |
| "queue": "inbox", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| }, | |
| "after": { | |
| "category": "technical", | |
| "queue": "queue-0-2", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "step": 2, | |
| "type": "tool_call", | |
| "action": { | |
| "tool": "add_note", | |
| "arguments": { | |
| "ticket_id": "T-85a0bcc1", | |
| "body": "Reviewed request T-85a0bcc1.", | |
| "request_id": "review" | |
| } | |
| }, | |
| "result": { | |
| "ok": true | |
| }, | |
| "status": "ok", | |
| "error_source": null, | |
| "changes": { | |
| "T-85a0bcc1": { | |
| "before": { | |
| "category": "technical", | |
| "queue": "queue-0-2", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| }, | |
| "after": { | |
| "category": "technical", | |
| "queue": "queue-0-2", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support.", | |
| "Reviewed request T-85a0bcc1." | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "step": 3, | |
| "type": "finish", | |
| "action": { | |
| "finish": true | |
| } | |
| } | |
| ], | |
| "tool_calls": 3 | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "close-resolved-ticket", | |
| "checks": { | |
| "routing": true, | |
| "notes": true, | |
| "closure": true, | |
| "scope": true, | |
| "protocol": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "error": null, | |
| "duration_seconds": 0.480463, | |
| "transcript_sha256": "a260768c11b589ce3f58658cac6aae8a01d36e388a47fc47c7995f8d5c36708e", | |
| "initial_state": { | |
| "T-c41f9dfd": { | |
| "category": "billing", | |
| "queue": "inbox", | |
| "resolved": true, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| }, | |
| "U-b48438b5": { | |
| "category": "billing", | |
| "queue": "inbox", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Unrelated request; preserve this ticket." | |
| ] | |
| } | |
| }, | |
| "final_state": { | |
| "T-c41f9dfd": { | |
| "category": "billing", | |
| "queue": "queue-1-0", | |
| "resolved": true, | |
| "status": "closed", | |
| "notes": [ | |
| "Customer contacted support.", | |
| "Reviewed request T-c41f9dfd." | |
| ] | |
| }, | |
| "U-b48438b5": { | |
| "category": "billing", | |
| "queue": "inbox", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Unrelated request; preserve this ticket." | |
| ] | |
| } | |
| }, | |
| "trace": [ | |
| { | |
| "step": 0, | |
| "type": "tool_call", | |
| "action": { | |
| "tool": "get_ticket", | |
| "arguments": { | |
| "ticket_id": "T-c41f9dfd" | |
| } | |
| }, | |
| "result": { | |
| "ok": true, | |
| "ticket": { | |
| "category": "billing", | |
| "queue": "inbox", | |
| "resolved": true, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| } | |
| }, | |
| "status": "ok", | |
| "error_source": null, | |
| "changes": {} | |
| }, | |
| { | |
| "step": 1, | |
| "type": "tool_call", | |
| "action": { | |
| "tool": "assign_queue", | |
| "arguments": { | |
| "ticket_id": "T-c41f9dfd", | |
| "queue": "queue-1-0" | |
| } | |
| }, | |
| "result": { | |
| "ok": true | |
| }, | |
| "status": "ok", | |
| "error_source": null, | |
| "changes": { | |
| "T-c41f9dfd": { | |
| "before": { | |
| "category": "billing", | |
| "queue": "inbox", | |
| "resolved": true, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| }, | |
| "after": { | |
| "category": "billing", | |
| "queue": "queue-1-0", | |
| "resolved": true, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "step": 2, | |
| "type": "tool_call", | |
| "action": { | |
| "tool": "add_note", | |
| "arguments": { | |
| "ticket_id": "T-c41f9dfd", | |
| "body": "Reviewed request T-c41f9dfd.", | |
| "request_id": "review" | |
| } | |
| }, | |
| "result": { | |
| "ok": true | |
| }, | |
| "status": "ok", | |
| "error_source": null, | |
| "changes": { | |
| "T-c41f9dfd": { | |
| "before": { | |
| "category": "billing", | |
| "queue": "queue-1-0", | |
| "resolved": true, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| }, | |
| "after": { | |
| "category": "billing", | |
| "queue": "queue-1-0", | |
| "resolved": true, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support.", | |
| "Reviewed request T-c41f9dfd." | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "step": 3, | |
| "type": "tool_call", | |
| "action": { | |
| "tool": "close_ticket", | |
| "arguments": { | |
| "ticket_id": "T-c41f9dfd" | |
| } | |
| }, | |
| "result": { | |
| "ok": true | |
| }, | |
| "status": "ok", | |
| "error_source": null, | |
| "changes": { | |
| "T-c41f9dfd": { | |
| "before": { | |
| "category": "billing", | |
| "queue": "queue-1-0", | |
| "resolved": true, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support.", | |
| "Reviewed request T-c41f9dfd." | |
| ] | |
| }, | |
| "after": { | |
| "category": "billing", | |
| "queue": "queue-1-0", | |
| "resolved": true, | |
| "status": "closed", | |
| "notes": [ | |
| "Customer contacted support.", | |
| "Reviewed request T-c41f9dfd." | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "step": 4, | |
| "type": "finish", | |
| "action": { | |
| "finish": true | |
| } | |
| } | |
| ], | |
| "tool_calls": 4 | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "retry-before-commit", | |
| "checks": { | |
| "routing": true, | |
| "notes": true, | |
| "closure": true, | |
| "scope": true, | |
| "protocol": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "error": null, | |
| "duration_seconds": 0.458016, | |
| "transcript_sha256": "b1d1d91dc01fc4bfb3243aa3739e953a40aa3920caf63a4b2ab73c87ecabda66", | |
| "initial_state": { | |
| "T-3fb81d27": { | |
| "category": "technical", | |
| "queue": "inbox", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| }, | |
| "U-62590992": { | |
| "category": "account", | |
| "queue": "inbox", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Unrelated request; preserve this ticket." | |
| ] | |
| } | |
| }, | |
| "final_state": { | |
| "T-3fb81d27": { | |
| "category": "technical", | |
| "queue": "queue-2-1", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support.", | |
| "Reviewed request T-3fb81d27." | |
| ] | |
| }, | |
| "U-62590992": { | |
| "category": "account", | |
| "queue": "inbox", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Unrelated request; preserve this ticket." | |
| ] | |
| } | |
| }, | |
| "trace": [ | |
| { | |
| "step": 0, | |
| "type": "tool_call", | |
| "action": { | |
| "tool": "get_ticket", | |
| "arguments": { | |
| "ticket_id": "T-3fb81d27" | |
| } | |
| }, | |
| "result": { | |
| "ok": true, | |
| "ticket": { | |
| "category": "technical", | |
| "queue": "inbox", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| } | |
| }, | |
| "status": "ok", | |
| "error_source": null, | |
| "changes": {} | |
| }, | |
| { | |
| "step": 1, | |
| "type": "tool_call", | |
| "action": { | |
| "tool": "assign_queue", | |
| "arguments": { | |
| "ticket_id": "T-3fb81d27", | |
| "queue": "queue-2-1" | |
| } | |
| }, | |
| "result": { | |
| "ok": false, | |
| "error": { | |
| "code": "temporarily_unavailable", | |
| "retryable": true | |
| } | |
| }, | |
| "status": "tool_error", | |
| "error_source": "injected", | |
| "changes": {} | |
| }, | |
| { | |
| "step": 2, | |
| "type": "tool_call", | |
| "action": { | |
| "tool": "assign_queue", | |
| "arguments": { | |
| "ticket_id": "T-3fb81d27", | |
| "queue": "queue-2-1" | |
| } | |
| }, | |
| "result": { | |
| "ok": true | |
| }, | |
| "status": "ok", | |
| "error_source": null, | |
| "changes": { | |
| "T-3fb81d27": { | |
| "before": { | |
| "category": "technical", | |
| "queue": "inbox", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| }, | |
| "after": { | |
| "category": "technical", | |
| "queue": "queue-2-1", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "step": 3, | |
| "type": "tool_call", | |
| "action": { | |
| "tool": "add_note", | |
| "arguments": { | |
| "ticket_id": "T-3fb81d27", | |
| "body": "Reviewed request T-3fb81d27.", | |
| "request_id": "review" | |
| } | |
| }, | |
| "result": { | |
| "ok": true | |
| }, | |
| "status": "ok", | |
| "error_source": null, | |
| "changes": { | |
| "T-3fb81d27": { | |
| "before": { | |
| "category": "technical", | |
| "queue": "queue-2-1", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| }, | |
| "after": { | |
| "category": "technical", | |
| "queue": "queue-2-1", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support.", | |
| "Reviewed request T-3fb81d27." | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "step": 4, | |
| "type": "finish", | |
| "action": { | |
| "finish": true | |
| } | |
| } | |
| ], | |
| "tool_calls": 4 | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "retry-after-commit", | |
| "checks": { | |
| "routing": true, | |
| "notes": false, | |
| "closure": true, | |
| "scope": true, | |
| "protocol": true | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "error": null, | |
| "duration_seconds": 0.501206, | |
| "transcript_sha256": "4d1123f9ef2ebc43d80c84e4f2054b3ed62a22db7d6c565f0f7f3a9f28c662b4", | |
| "initial_state": { | |
| "T-cd68615c": { | |
| "category": "billing", | |
| "queue": "inbox", | |
| "resolved": true, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| }, | |
| "U-51436d1f": { | |
| "category": "account", | |
| "queue": "inbox", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Unrelated request; preserve this ticket." | |
| ] | |
| } | |
| }, | |
| "final_state": { | |
| "T-cd68615c": { | |
| "category": "billing", | |
| "queue": "queue-3-0", | |
| "resolved": true, | |
| "status": "closed", | |
| "notes": [ | |
| "Customer contacted support.", | |
| "Reviewed request T-cd68615c.", | |
| "Reviewed request T-cd68615c." | |
| ] | |
| }, | |
| "U-51436d1f": { | |
| "category": "account", | |
| "queue": "inbox", | |
| "resolved": false, | |
| "status": "open", | |
| "notes": [ | |
| "Unrelated request; preserve this ticket." | |
| ] | |
| } | |
| }, | |
| "trace": [ | |
| { | |
| "step": 0, | |
| "type": "tool_call", | |
| "action": { | |
| "tool": "get_ticket", | |
| "arguments": { | |
| "ticket_id": "T-cd68615c" | |
| } | |
| }, | |
| "result": { | |
| "ok": true, | |
| "ticket": { | |
| "category": "billing", | |
| "queue": "inbox", | |
| "resolved": true, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| } | |
| }, | |
| "status": "ok", | |
| "error_source": null, | |
| "changes": {} | |
| }, | |
| { | |
| "step": 1, | |
| "type": "tool_call", | |
| "action": { | |
| "tool": "assign_queue", | |
| "arguments": { | |
| "ticket_id": "T-cd68615c", | |
| "queue": "queue-3-0" | |
| } | |
| }, | |
| "result": { | |
| "ok": true | |
| }, | |
| "status": "ok", | |
| "error_source": null, | |
| "changes": { | |
| "T-cd68615c": { | |
| "before": { | |
| "category": "billing", | |
| "queue": "inbox", | |
| "resolved": true, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| }, | |
| "after": { | |
| "category": "billing", | |
| "queue": "queue-3-0", | |
| "resolved": true, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "step": 2, | |
| "type": "tool_call", | |
| "action": { | |
| "tool": "add_note", | |
| "arguments": { | |
| "ticket_id": "T-cd68615c", | |
| "body": "Reviewed request T-cd68615c.", | |
| "request_id": "review" | |
| } | |
| }, | |
| "result": { | |
| "ok": false, | |
| "error": { | |
| "code": "temporarily_unavailable", | |
| "retryable": true | |
| } | |
| }, | |
| "status": "tool_error", | |
| "error_source": "injected", | |
| "changes": { | |
| "T-cd68615c": { | |
| "before": { | |
| "category": "billing", | |
| "queue": "queue-3-0", | |
| "resolved": true, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support." | |
| ] | |
| }, | |
| "after": { | |
| "category": "billing", | |
| "queue": "queue-3-0", | |
| "resolved": true, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support.", | |
| "Reviewed request T-cd68615c." | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "step": 3, | |
| "type": "tool_call", | |
| "action": { | |
| "tool": "add_note", | |
| "arguments": { | |
| "ticket_id": "T-cd68615c", | |
| "body": "Reviewed request T-cd68615c.", | |
| "request_id": "review-retry" | |
| } | |
| }, | |
| "result": { | |
| "ok": true | |
| }, | |
| "status": "ok", | |
| "error_source": null, | |
| "changes": { | |
| "T-cd68615c": { | |
| "before": { | |
| "category": "billing", | |
| "queue": "queue-3-0", | |
| "resolved": true, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support.", | |
| "Reviewed request T-cd68615c." | |
| ] | |
| }, | |
| "after": { | |
| "category": "billing", | |
| "queue": "queue-3-0", | |
| "resolved": true, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support.", | |
| "Reviewed request T-cd68615c.", | |
| "Reviewed request T-cd68615c." | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "step": 4, | |
| "type": "tool_call", | |
| "action": { | |
| "tool": "close_ticket", | |
| "arguments": { | |
| "ticket_id": "T-cd68615c" | |
| } | |
| }, | |
| "result": { | |
| "ok": true | |
| }, | |
| "status": "ok", | |
| "error_source": null, | |
| "changes": { | |
| "T-cd68615c": { | |
| "before": { | |
| "category": "billing", | |
| "queue": "queue-3-0", | |
| "resolved": true, | |
| "status": "open", | |
| "notes": [ | |
| "Customer contacted support.", | |
| "Reviewed request T-cd68615c.", | |
| "Reviewed request T-cd68615c." | |
| ] | |
| }, | |
| "after": { | |
| "category": "billing", | |
| "queue": "queue-3-0", | |
| "resolved": true, | |
| "status": "closed", | |
| "notes": [ | |
| "Customer contacted support.", | |
| "Reviewed request T-cd68615c.", | |
| "Reviewed request T-cd68615c." | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "step": 5, | |
| "type": "finish", | |
| "action": { | |
| "finish": true | |
| } | |
| } | |
| ], | |
| "tool_calls": 5 | |
| } | |
| ], | |
| "evaluation_seconds": 1.907783, | |
| "agent_cost_usd": null, | |
| "agent_tokens": null | |
| } | |