{
  "schema_version": "evalarc.evaluation.v2",
  "created_at": "2026-09-14T01:58:21.160793+00:00",
  "evalarc_version": "0.3.0",
  "task": {
    "id": "support-routing",
    "version": "0.1.0",
    "domain": "business-tools",
    "split": "public-development"
  },
  "candidate_sha256": "b89a99819c03e8290096ca268f637c5517d465b3ec5f0b5fe2b21c35509ff202",
  "grader_sha256": "2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255",
  "cases_sha256": "b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df",
  "runtime": {
    "backend": "docker",
    "image": "python:3.12-slim",
    "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea",
    "command": [
      "{python}",
      "-I",
      "-B",
      "main.py"
    ],
    "response_timeout_seconds": 10.0,
    "session_output_limit_bytes": 1048576,
    "python": "3.12.3",
    "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39"
  },
  "seeds": [
    17
  ],
  "valid": true,
  "status": "failed",
  "score": 0.9375,
  "resolved": false,
  "dimensions": {
    "routing": {
      "passed": 4,
      "total": 4,
      "assessed": 4,
      "weight": 0.3,
      "score": 1.0
    },
    "notes": {
      "passed": 3,
      "total": 4,
      "assessed": 4,
      "weight": 0.25,
      "score": 0.75
    },
    "closure": {
      "passed": 4,
      "total": 4,
      "assessed": 4,
      "weight": 0.2,
      "score": 1.0
    },
    "scope": {
      "passed": 4,
      "total": 4,
      "assessed": 4,
      "weight": 0.15,
      "score": 1.0
    },
    "protocol": {
      "passed": 4,
      "total": 4,
      "assessed": 4,
      "weight": 0.1,
      "score": 1.0
    }
  },
  "cases": [
    {
      "seed": 17,
      "case_id": "route-open-ticket",
      "checks": {
        "routing": true,
        "notes": true,
        "closure": true,
        "scope": true,
        "protocol": true
      },
      "passed": true,
      "status": "passed",
      "error": null,
      "duration_seconds": 0.461764,
      "transcript_sha256": "7b5b4973e934a60c55b95bda6eedf79fcff62764fbc1b81723c5126608caf4bd",
      "initial_state": {
        "T-85a0bcc1": {
          "category": "technical",
          "queue": "inbox",
          "resolved": false,
          "status": "open",
          "notes": [
            "Customer contacted support."
          ]
        },
        "U-6a06e9ab": {
          "category": "billing",
          "queue": "inbox",
          "resolved": false,
          "status": "open",
          "notes": [
            "Unrelated request; preserve this ticket."
          ]
        }
      },
      "final_state": {
        "T-85a0bcc1": {
          "category": "technical",
          "queue": "queue-0-2",
          "resolved": false,
          "status": "open",
          "notes": [
            "Customer contacted support.",
            "Reviewed request T-85a0bcc1."
          ]
        },
        "U-6a06e9ab": {
          "category": "billing",
          "queue": "inbox",
          "resolved": false,
          "status": "open",
          "notes": [
            "Unrelated request; preserve this ticket."
          ]
        }
      },
      "trace": [
        {
          "step": 0,
          "type": "tool_call",
          "action": {
            "tool": "get_ticket",
            "arguments": {
              "ticket_id": "T-85a0bcc1"
            }
          },
          "result": {
            "ok": true,
            "ticket": {
              "category": "technical",
              "queue": "inbox",
              "resolved": false,
              "status": "open",
              "notes": [
                "Customer contacted support."
              ]
            }
          },
          "status": "ok",
          "error_source": null,
          "changes": {}
        },
        {
          "step": 1,
          "type": "tool_call",
          "action": {
            "tool": "assign_queue",
            "arguments": {
              "ticket_id": "T-85a0bcc1",
              "queue": "queue-0-2"
            }
          },
          "result": {
            "ok": true
          },
          "status": "ok",
          "error_source": null,
          "changes": {
            "T-85a0bcc1": {
              "before": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "after": {
                "category": "technical",
                "queue": "queue-0-2",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              }
            }
          }
        },
        {
          "step": 2,
          "type": "tool_call",
          "action": {
            "tool": "add_note",
            "arguments": {
              "ticket_id": "T-85a0bcc1",
              "body": "Reviewed request T-85a0bcc1.",
              "request_id": "review"
            }
          },
          "result": {
            "ok": true
          },
          "status": "ok",
          "error_source": null,
          "changes": {
            "T-85a0bcc1": {
              "before": {
                "category": "technical",
                "queue": "queue-0-2",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "after": {
                "category": "technical",
                "queue": "queue-0-2",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-85a0bcc1."
                ]
              }
            }
          }
        },
        {
          "step": 3,
          "type": "finish",
          "action": {
            "finish": true
          }
        }
      ],
      "tool_calls": 3
    },
    {
      "seed": 17,
      "case_id": "close-resolved-ticket",
      "checks": {
        "routing": true,
        "notes": true,
        "closure": true,
        "scope": true,
        "protocol": true
      },
      "passed": true,
      "status": "passed",
      "error": null,
      "duration_seconds": 0.480463,
      "transcript_sha256": "a260768c11b589ce3f58658cac6aae8a01d36e388a47fc47c7995f8d5c36708e",
      "initial_state": {
        "T-c41f9dfd": {
          "category": "billing",
          "queue": "inbox",
          "resolved": true,
          "status": "open",
          "notes": [
            "Customer contacted support."
          ]
        },
        "U-b48438b5": {
          "category": "billing",
          "queue": "inbox",
          "resolved": false,
          "status": "open",
          "notes": [
            "Unrelated request; preserve this ticket."
          ]
        }
      },
      "final_state": {
        "T-c41f9dfd": {
          "category": "billing",
          "queue": "queue-1-0",
          "resolved": true,
          "status": "closed",
          "notes": [
            "Customer contacted support.",
            "Reviewed request T-c41f9dfd."
          ]
        },
        "U-b48438b5": {
          "category": "billing",
          "queue": "inbox",
          "resolved": false,
          "status": "open",
          "notes": [
            "Unrelated request; preserve this ticket."
          ]
        }
      },
      "trace": [
        {
          "step": 0,
          "type": "tool_call",
          "action": {
            "tool": "get_ticket",
            "arguments": {
              "ticket_id": "T-c41f9dfd"
            }
          },
          "result": {
            "ok": true,
            "ticket": {
              "category": "billing",
              "queue": "inbox",
              "resolved": true,
              "status": "open",
              "notes": [
                "Customer contacted support."
              ]
            }
          },
          "status": "ok",
          "error_source": null,
          "changes": {}
        },
        {
          "step": 1,
          "type": "tool_call",
          "action": {
            "tool": "assign_queue",
            "arguments": {
              "ticket_id": "T-c41f9dfd",
              "queue": "queue-1-0"
            }
          },
          "result": {
            "ok": true
          },
          "status": "ok",
          "error_source": null,
          "changes": {
            "T-c41f9dfd": {
              "before": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "after": {
                "category": "billing",
                "queue": "queue-1-0",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              }
            }
          }
        },
        {
          "step": 2,
          "type": "tool_call",
          "action": {
            "tool": "add_note",
            "arguments": {
              "ticket_id": "T-c41f9dfd",
              "body": "Reviewed request T-c41f9dfd.",
              "request_id": "review"
            }
          },
          "result": {
            "ok": true
          },
          "status": "ok",
          "error_source": null,
          "changes": {
            "T-c41f9dfd": {
              "before": {
                "category": "billing",
                "queue": "queue-1-0",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "after": {
                "category": "billing",
                "queue": "queue-1-0",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-c41f9dfd."
                ]
              }
            }
          }
        },
        {
          "step": 3,
          "type": "tool_call",
          "action": {
            "tool": "close_ticket",
            "arguments": {
              "ticket_id": "T-c41f9dfd"
            }
          },
          "result": {
            "ok": true
          },
          "status": "ok",
          "error_source": null,
          "changes": {
            "T-c41f9dfd": {
              "before": {
                "category": "billing",
                "queue": "queue-1-0",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-c41f9dfd."
                ]
              },
              "after": {
                "category": "billing",
                "queue": "queue-1-0",
                "resolved": true,
                "status": "closed",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-c41f9dfd."
                ]
              }
            }
          }
        },
        {
          "step": 4,
          "type": "finish",
          "action": {
            "finish": true
          }
        }
      ],
      "tool_calls": 4
    },
    {
      "seed": 17,
      "case_id": "retry-before-commit",
      "checks": {
        "routing": true,
        "notes": true,
        "closure": true,
        "scope": true,
        "protocol": true
      },
      "passed": true,
      "status": "passed",
      "error": null,
      "duration_seconds": 0.458016,
      "transcript_sha256": "b1d1d91dc01fc4bfb3243aa3739e953a40aa3920caf63a4b2ab73c87ecabda66",
      "initial_state": {
        "T-3fb81d27": {
          "category": "technical",
          "queue": "inbox",
          "resolved": false,
          "status": "open",
          "notes": [
            "Customer contacted support."
          ]
        },
        "U-62590992": {
          "category": "account",
          "queue": "inbox",
          "resolved": false,
          "status": "open",
          "notes": [
            "Unrelated request; preserve this ticket."
          ]
        }
      },
      "final_state": {
        "T-3fb81d27": {
          "category": "technical",
          "queue": "queue-2-1",
          "resolved": false,
          "status": "open",
          "notes": [
            "Customer contacted support.",
            "Reviewed request T-3fb81d27."
          ]
        },
        "U-62590992": {
          "category": "account",
          "queue": "inbox",
          "resolved": false,
          "status": "open",
          "notes": [
            "Unrelated request; preserve this ticket."
          ]
        }
      },
      "trace": [
        {
          "step": 0,
          "type": "tool_call",
          "action": {
            "tool": "get_ticket",
            "arguments": {
              "ticket_id": "T-3fb81d27"
            }
          },
          "result": {
            "ok": true,
            "ticket": {
              "category": "technical",
              "queue": "inbox",
              "resolved": false,
              "status": "open",
              "notes": [
                "Customer contacted support."
              ]
            }
          },
          "status": "ok",
          "error_source": null,
          "changes": {}
        },
        {
          "step": 1,
          "type": "tool_call",
          "action": {
            "tool": "assign_queue",
            "arguments": {
              "ticket_id": "T-3fb81d27",
              "queue": "queue-2-1"
            }
          },
          "result": {
            "ok": false,
            "error": {
              "code": "temporarily_unavailable",
              "retryable": true
            }
          },
          "status": "tool_error",
          "error_source": "injected",
          "changes": {}
        },
        {
          "step": 2,
          "type": "tool_call",
          "action": {
            "tool": "assign_queue",
            "arguments": {
              "ticket_id": "T-3fb81d27",
              "queue": "queue-2-1"
            }
          },
          "result": {
            "ok": true
          },
          "status": "ok",
          "error_source": null,
          "changes": {
            "T-3fb81d27": {
              "before": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "after": {
                "category": "technical",
                "queue": "queue-2-1",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              }
            }
          }
        },
        {
          "step": 3,
          "type": "tool_call",
          "action": {
            "tool": "add_note",
            "arguments": {
              "ticket_id": "T-3fb81d27",
              "body": "Reviewed request T-3fb81d27.",
              "request_id": "review"
            }
          },
          "result": {
            "ok": true
          },
          "status": "ok",
          "error_source": null,
          "changes": {
            "T-3fb81d27": {
              "before": {
                "category": "technical",
                "queue": "queue-2-1",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "after": {
                "category": "technical",
                "queue": "queue-2-1",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-3fb81d27."
                ]
              }
            }
          }
        },
        {
          "step": 4,
          "type": "finish",
          "action": {
            "finish": true
          }
        }
      ],
      "tool_calls": 4
    },
    {
      "seed": 17,
      "case_id": "retry-after-commit",
      "checks": {
        "routing": true,
        "notes": false,
        "closure": true,
        "scope": true,
        "protocol": true
      },
      "passed": false,
      "status": "failed",
      "error": null,
      "duration_seconds": 0.501206,
      "transcript_sha256": "4d1123f9ef2ebc43d80c84e4f2054b3ed62a22db7d6c565f0f7f3a9f28c662b4",
      "initial_state": {
        "T-cd68615c": {
          "category": "billing",
          "queue": "inbox",
          "resolved": true,
          "status": "open",
          "notes": [
            "Customer contacted support."
          ]
        },
        "U-51436d1f": {
          "category": "account",
          "queue": "inbox",
          "resolved": false,
          "status": "open",
          "notes": [
            "Unrelated request; preserve this ticket."
          ]
        }
      },
      "final_state": {
        "T-cd68615c": {
          "category": "billing",
          "queue": "queue-3-0",
          "resolved": true,
          "status": "closed",
          "notes": [
            "Customer contacted support.",
            "Reviewed request T-cd68615c.",
            "Reviewed request T-cd68615c."
          ]
        },
        "U-51436d1f": {
          "category": "account",
          "queue": "inbox",
          "resolved": false,
          "status": "open",
          "notes": [
            "Unrelated request; preserve this ticket."
          ]
        }
      },
      "trace": [
        {
          "step": 0,
          "type": "tool_call",
          "action": {
            "tool": "get_ticket",
            "arguments": {
              "ticket_id": "T-cd68615c"
            }
          },
          "result": {
            "ok": true,
            "ticket": {
              "category": "billing",
              "queue": "inbox",
              "resolved": true,
              "status": "open",
              "notes": [
                "Customer contacted support."
              ]
            }
          },
          "status": "ok",
          "error_source": null,
          "changes": {}
        },
        {
          "step": 1,
          "type": "tool_call",
          "action": {
            "tool": "assign_queue",
            "arguments": {
              "ticket_id": "T-cd68615c",
              "queue": "queue-3-0"
            }
          },
          "result": {
            "ok": true
          },
          "status": "ok",
          "error_source": null,
          "changes": {
            "T-cd68615c": {
              "before": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "after": {
                "category": "billing",
                "queue": "queue-3-0",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              }
            }
          }
        },
        {
          "step": 2,
          "type": "tool_call",
          "action": {
            "tool": "add_note",
            "arguments": {
              "ticket_id": "T-cd68615c",
              "body": "Reviewed request T-cd68615c.",
              "request_id": "review"
            }
          },
          "result": {
            "ok": false,
            "error": {
              "code": "temporarily_unavailable",
              "retryable": true
            }
          },
          "status": "tool_error",
          "error_source": "injected",
          "changes": {
            "T-cd68615c": {
              "before": {
                "category": "billing",
                "queue": "queue-3-0",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "after": {
                "category": "billing",
                "queue": "queue-3-0",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-cd68615c."
                ]
              }
            }
          }
        },
        {
          "step": 3,
          "type": "tool_call",
          "action": {
            "tool": "add_note",
            "arguments": {
              "ticket_id": "T-cd68615c",
              "body": "Reviewed request T-cd68615c.",
              "request_id": "review-retry"
            }
          },
          "result": {
            "ok": true
          },
          "status": "ok",
          "error_source": null,
          "changes": {
            "T-cd68615c": {
              "before": {
                "category": "billing",
                "queue": "queue-3-0",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-cd68615c."
                ]
              },
              "after": {
                "category": "billing",
                "queue": "queue-3-0",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-cd68615c.",
                  "Reviewed request T-cd68615c."
                ]
              }
            }
          }
        },
        {
          "step": 4,
          "type": "tool_call",
          "action": {
            "tool": "close_ticket",
            "arguments": {
              "ticket_id": "T-cd68615c"
            }
          },
          "result": {
            "ok": true
          },
          "status": "ok",
          "error_source": null,
          "changes": {
            "T-cd68615c": {
              "before": {
                "category": "billing",
                "queue": "queue-3-0",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-cd68615c.",
                  "Reviewed request T-cd68615c."
                ]
              },
              "after": {
                "category": "billing",
                "queue": "queue-3-0",
                "resolved": true,
                "status": "closed",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-cd68615c.",
                  "Reviewed request T-cd68615c."
                ]
              }
            }
          }
        },
        {
          "step": 5,
          "type": "finish",
          "action": {
            "finish": true
          }
        }
      ],
      "tool_calls": 5
    }
  ],
  "evaluation_seconds": 1.907783,
  "agent_cost_usd": null,
  "agent_tokens": null
}