{
  "schema_version": "evalarc.audit.v2",
  "valid": true,
  "reference_passed": true,
  "reference": {
    "schema_version": "evalarc.evaluation.v2",
    "created_at": "2026-09-14T01:29:39.246926+00:00",
    "evalarc_version": "0.2.0",
    "task": {
      "id": "support-routing",
      "version": "0.1.0",
      "domain": "business-tools",
      "split": "public-development"
    },
    "candidate_sha256": "18822c8e9e7cf7466d6c5c8c3437bbc73abc43f07236c754980269b3a026c779",
    "grader_sha256": "2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255",
    "cases_sha256": "b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df",
    "runtime": {
      "backend": "docker",
      "image": "python:3.12-slim",
      "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea",
      "command": [
        "{python}",
        "-I",
        "-B",
        "main.py"
      ],
      "response_timeout_seconds": 10.0,
      "session_output_limit_bytes": 1048576,
      "python": "3.12.3",
      "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39"
    },
    "seeds": [
      17
    ],
    "valid": true,
    "status": "passed",
    "score": 1.0,
    "resolved": true,
    "dimensions": {
      "routing": {
        "passed": 4,
        "total": 4,
        "assessed": 4,
        "weight": 0.3,
        "score": 1.0
      },
      "notes": {
        "passed": 4,
        "total": 4,
        "assessed": 4,
        "weight": 0.25,
        "score": 1.0
      },
      "closure": {
        "passed": 4,
        "total": 4,
        "assessed": 4,
        "weight": 0.2,
        "score": 1.0
      },
      "scope": {
        "passed": 4,
        "total": 4,
        "assessed": 4,
        "weight": 0.15,
        "score": 1.0
      },
      "protocol": {
        "passed": 4,
        "total": 4,
        "assessed": 4,
        "weight": 0.1,
        "score": 1.0
      }
    },
    "cases": [
      {
        "seed": 17,
        "case_id": "route-open-ticket",
        "checks": {
          "routing": true,
          "notes": true,
          "closure": true,
          "scope": true,
          "protocol": true
        },
        "passed": true,
        "status": "passed",
        "error": null,
        "duration_seconds": 0.802482,
        "transcript_sha256": "7b5b4973e934a60c55b95bda6eedf79fcff62764fbc1b81723c5126608caf4bd",
        "initial_state": {
          "T-85a0bcc1": {
            "category": "technical",
            "queue": "inbox",
            "resolved": false,
            "status": "open",
            "notes": [
              "Customer contacted support."
            ]
          },
          "U-6a06e9ab": {
            "category": "billing",
            "queue": "inbox",
            "resolved": false,
            "status": "open",
            "notes": [
              "Unrelated request; preserve this ticket."
            ]
          }
        },
        "final_state": {
          "T-85a0bcc1": {
            "category": "technical",
            "queue": "queue-0-2",
            "resolved": false,
            "status": "open",
            "notes": [
              "Customer contacted support.",
              "Reviewed request T-85a0bcc1."
            ]
          },
          "U-6a06e9ab": {
            "category": "billing",
            "queue": "inbox",
            "resolved": false,
            "status": "open",
            "notes": [
              "Unrelated request; preserve this ticket."
            ]
          }
        },
        "trace": [
          {
            "step": 0,
            "type": "tool_call",
            "action": {
              "tool": "get_ticket",
              "arguments": {
                "ticket_id": "T-85a0bcc1"
              }
            },
            "result": {
              "ok": true,
              "ticket": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              }
            },
            "status": "ok",
            "error_source": null,
            "changes": {}
          },
          {
            "step": 1,
            "type": "tool_call",
            "action": {
              "tool": "assign_queue",
              "arguments": {
                "ticket_id": "T-85a0bcc1",
                "queue": "queue-0-2"
              }
            },
            "result": {
              "ok": true
            },
            "status": "ok",
            "error_source": null,
            "changes": {
              "T-85a0bcc1": {
                "before": {
                  "category": "technical",
                  "queue": "inbox",
                  "resolved": false,
                  "status": "open",
                  "notes": [
                    "Customer contacted support."
                  ]
                },
                "after": {
                  "category": "technical",
                  "queue": "queue-0-2",
                  "resolved": false,
                  "status": "open",
                  "notes": [
                    "Customer contacted support."
                  ]
                }
              }
            }
          },
          {
            "step": 2,
            "type": "tool_call",
            "action": {
              "tool": "add_note",
              "arguments": {
                "ticket_id": "T-85a0bcc1",
                "body": "Reviewed request T-85a0bcc1.",
                "request_id": "review"
              }
            },
            "result": {
              "ok": true
            },
            "status": "ok",
            "error_source": null,
            "changes": {
              "T-85a0bcc1": {
                "before": {
                  "category": "technical",
                  "queue": "queue-0-2",
                  "resolved": false,
                  "status": "open",
                  "notes": [
                    "Customer contacted support."
                  ]
                },
                "after": {
                  "category": "technical",
                  "queue": "queue-0-2",
                  "resolved": false,
                  "status": "open",
                  "notes": [
                    "Customer contacted support.",
                    "Reviewed request T-85a0bcc1."
                  ]
                }
              }
            }
          },
          {
            "step": 3,
            "type": "finish",
            "action": {
              "finish": true
            }
          }
        ],
        "tool_calls": 3
      },
      {
        "seed": 17,
        "case_id": "close-resolved-ticket",
        "checks": {
          "routing": true,
          "notes": true,
          "closure": true,
          "scope": true,
          "protocol": true
        },
        "passed": true,
        "status": "passed",
        "error": null,
        "duration_seconds": 0.704571,
        "transcript_sha256": "a260768c11b589ce3f58658cac6aae8a01d36e388a47fc47c7995f8d5c36708e",
        "initial_state": {
          "T-c41f9dfd": {
            "category": "billing",
            "queue": "inbox",
            "resolved": true,
            "status": "open",
            "notes": [
              "Customer contacted support."
            ]
          },
          "U-b48438b5": {
            "category": "billing",
            "queue": "inbox",
            "resolved": false,
            "status": "open",
            "notes": [
              "Unrelated request; preserve this ticket."
            ]
          }
        },
        "final_state": {
          "T-c41f9dfd": {
            "category": "billing",
            "queue": "queue-1-0",
            "resolved": true,
            "status": "closed",
            "notes": [
              "Customer contacted support.",
              "Reviewed request T-c41f9dfd."
            ]
          },
          "U-b48438b5": {
            "category": "billing",
            "queue": "inbox",
            "resolved": false,
            "status": "open",
            "notes": [
              "Unrelated request; preserve this ticket."
            ]
          }
        },
        "trace": [
          {
            "step": 0,
            "type": "tool_call",
            "action": {
              "tool": "get_ticket",
              "arguments": {
                "ticket_id": "T-c41f9dfd"
              }
            },
            "result": {
              "ok": true,
              "ticket": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              }
            },
            "status": "ok",
            "error_source": null,
            "changes": {}
          },
          {
            "step": 1,
            "type": "tool_call",
            "action": {
              "tool": "assign_queue",
              "arguments": {
                "ticket_id": "T-c41f9dfd",
                "queue": "queue-1-0"
              }
            },
            "result": {
              "ok": true
            },
            "status": "ok",
            "error_source": null,
            "changes": {
              "T-c41f9dfd": {
                "before": {
                  "category": "billing",
                  "queue": "inbox",
                  "resolved": true,
                  "status": "open",
                  "notes": [
                    "Customer contacted support."
                  ]
                },
                "after": {
                  "category": "billing",
                  "queue": "queue-1-0",
                  "resolved": true,
                  "status": "open",
                  "notes": [
                    "Customer contacted support."
                  ]
                }
              }
            }
          },
          {
            "step": 2,
            "type": "tool_call",
            "action": {
              "tool": "add_note",
              "arguments": {
                "ticket_id": "T-c41f9dfd",
                "body": "Reviewed request T-c41f9dfd.",
                "request_id": "review"
              }
            },
            "result": {
              "ok": true
            },
            "status": "ok",
            "error_source": null,
            "changes": {
              "T-c41f9dfd": {
                "before": {
                  "category": "billing",
                  "queue": "queue-1-0",
                  "resolved": true,
                  "status": "open",
                  "notes": [
                    "Customer contacted support."
                  ]
                },
                "after": {
                  "category": "billing",
                  "queue": "queue-1-0",
                  "resolved": true,
                  "status": "open",
                  "notes": [
                    "Customer contacted support.",
                    "Reviewed request T-c41f9dfd."
                  ]
                }
              }
            }
          },
          {
            "step": 3,
            "type": "tool_call",
            "action": {
              "tool": "close_ticket",
              "arguments": {
                "ticket_id": "T-c41f9dfd"
              }
            },
            "result": {
              "ok": true
            },
            "status": "ok",
            "error_source": null,
            "changes": {
              "T-c41f9dfd": {
                "before": {
                  "category": "billing",
                  "queue": "queue-1-0",
                  "resolved": true,
                  "status": "open",
                  "notes": [
                    "Customer contacted support.",
                    "Reviewed request T-c41f9dfd."
                  ]
                },
                "after": {
                  "category": "billing",
                  "queue": "queue-1-0",
                  "resolved": true,
                  "status": "closed",
                  "notes": [
                    "Customer contacted support.",
                    "Reviewed request T-c41f9dfd."
                  ]
                }
              }
            }
          },
          {
            "step": 4,
            "type": "finish",
            "action": {
              "finish": true
            }
          }
        ],
        "tool_calls": 4
      },
      {
        "seed": 17,
        "case_id": "retry-before-commit",
        "checks": {
          "routing": true,
          "notes": true,
          "closure": true,
          "scope": true,
          "protocol": true
        },
        "passed": true,
        "status": "passed",
        "error": null,
        "duration_seconds": 0.710904,
        "transcript_sha256": "b1d1d91dc01fc4bfb3243aa3739e953a40aa3920caf63a4b2ab73c87ecabda66",
        "initial_state": {
          "T-3fb81d27": {
            "category": "technical",
            "queue": "inbox",
            "resolved": false,
            "status": "open",
            "notes": [
              "Customer contacted support."
            ]
          },
          "U-62590992": {
            "category": "account",
            "queue": "inbox",
            "resolved": false,
            "status": "open",
            "notes": [
              "Unrelated request; preserve this ticket."
            ]
          }
        },
        "final_state": {
          "T-3fb81d27": {
            "category": "technical",
            "queue": "queue-2-1",
            "resolved": false,
            "status": "open",
            "notes": [
              "Customer contacted support.",
              "Reviewed request T-3fb81d27."
            ]
          },
          "U-62590992": {
            "category": "account",
            "queue": "inbox",
            "resolved": false,
            "status": "open",
            "notes": [
              "Unrelated request; preserve this ticket."
            ]
          }
        },
        "trace": [
          {
            "step": 0,
            "type": "tool_call",
            "action": {
              "tool": "get_ticket",
              "arguments": {
                "ticket_id": "T-3fb81d27"
              }
            },
            "result": {
              "ok": true,
              "ticket": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              }
            },
            "status": "ok",
            "error_source": null,
            "changes": {}
          },
          {
            "step": 1,
            "type": "tool_call",
            "action": {
              "tool": "assign_queue",
              "arguments": {
                "ticket_id": "T-3fb81d27",
                "queue": "queue-2-1"
              }
            },
            "result": {
              "ok": false,
              "error": {
                "code": "temporarily_unavailable",
                "retryable": true
              }
            },
            "status": "tool_error",
            "error_source": "injected",
            "changes": {}
          },
          {
            "step": 2,
            "type": "tool_call",
            "action": {
              "tool": "assign_queue",
              "arguments": {
                "ticket_id": "T-3fb81d27",
                "queue": "queue-2-1"
              }
            },
            "result": {
              "ok": true
            },
            "status": "ok",
            "error_source": null,
            "changes": {
              "T-3fb81d27": {
                "before": {
                  "category": "technical",
                  "queue": "inbox",
                  "resolved": false,
                  "status": "open",
                  "notes": [
                    "Customer contacted support."
                  ]
                },
                "after": {
                  "category": "technical",
                  "queue": "queue-2-1",
                  "resolved": false,
                  "status": "open",
                  "notes": [
                    "Customer contacted support."
                  ]
                }
              }
            }
          },
          {
            "step": 3,
            "type": "tool_call",
            "action": {
              "tool": "add_note",
              "arguments": {
                "ticket_id": "T-3fb81d27",
                "body": "Reviewed request T-3fb81d27.",
                "request_id": "review"
              }
            },
            "result": {
              "ok": true
            },
            "status": "ok",
            "error_source": null,
            "changes": {
              "T-3fb81d27": {
                "before": {
                  "category": "technical",
                  "queue": "queue-2-1",
                  "resolved": false,
                  "status": "open",
                  "notes": [
                    "Customer contacted support."
                  ]
                },
                "after": {
                  "category": "technical",
                  "queue": "queue-2-1",
                  "resolved": false,
                  "status": "open",
                  "notes": [
                    "Customer contacted support.",
                    "Reviewed request T-3fb81d27."
                  ]
                }
              }
            }
          },
          {
            "step": 4,
            "type": "finish",
            "action": {
              "finish": true
            }
          }
        ],
        "tool_calls": 4
      },
      {
        "seed": 17,
        "case_id": "retry-after-commit",
        "checks": {
          "routing": true,
          "notes": true,
          "closure": true,
          "scope": true,
          "protocol": true
        },
        "passed": true,
        "status": "passed",
        "error": null,
        "duration_seconds": 0.736961,
        "transcript_sha256": "c97b016921c8d8fdcdeffd1ebf1f438d5a29be06732c31c86f99fd087bc0b58f",
        "initial_state": {
          "T-cd68615c": {
            "category": "billing",
            "queue": "inbox",
            "resolved": true,
            "status": "open",
            "notes": [
              "Customer contacted support."
            ]
          },
          "U-51436d1f": {
            "category": "account",
            "queue": "inbox",
            "resolved": false,
            "status": "open",
            "notes": [
              "Unrelated request; preserve this ticket."
            ]
          }
        },
        "final_state": {
          "T-cd68615c": {
            "category": "billing",
            "queue": "queue-3-0",
            "resolved": true,
            "status": "closed",
            "notes": [
              "Customer contacted support.",
              "Reviewed request T-cd68615c."
            ]
          },
          "U-51436d1f": {
            "category": "account",
            "queue": "inbox",
            "resolved": false,
            "status": "open",
            "notes": [
              "Unrelated request; preserve this ticket."
            ]
          }
        },
        "trace": [
          {
            "step": 0,
            "type": "tool_call",
            "action": {
              "tool": "get_ticket",
              "arguments": {
                "ticket_id": "T-cd68615c"
              }
            },
            "result": {
              "ok": true,
              "ticket": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              }
            },
            "status": "ok",
            "error_source": null,
            "changes": {}
          },
          {
            "step": 1,
            "type": "tool_call",
            "action": {
              "tool": "assign_queue",
              "arguments": {
                "ticket_id": "T-cd68615c",
                "queue": "queue-3-0"
              }
            },
            "result": {
              "ok": true
            },
            "status": "ok",
            "error_source": null,
            "changes": {
              "T-cd68615c": {
                "before": {
                  "category": "billing",
                  "queue": "inbox",
                  "resolved": true,
                  "status": "open",
                  "notes": [
                    "Customer contacted support."
                  ]
                },
                "after": {
                  "category": "billing",
                  "queue": "queue-3-0",
                  "resolved": true,
                  "status": "open",
                  "notes": [
                    "Customer contacted support."
                  ]
                }
              }
            }
          },
          {
            "step": 2,
            "type": "tool_call",
            "action": {
              "tool": "add_note",
              "arguments": {
                "ticket_id": "T-cd68615c",
                "body": "Reviewed request T-cd68615c.",
                "request_id": "review"
              }
            },
            "result": {
              "ok": false,
              "error": {
                "code": "temporarily_unavailable",
                "retryable": true
              }
            },
            "status": "tool_error",
            "error_source": "injected",
            "changes": {
              "T-cd68615c": {
                "before": {
                  "category": "billing",
                  "queue": "queue-3-0",
                  "resolved": true,
                  "status": "open",
                  "notes": [
                    "Customer contacted support."
                  ]
                },
                "after": {
                  "category": "billing",
                  "queue": "queue-3-0",
                  "resolved": true,
                  "status": "open",
                  "notes": [
                    "Customer contacted support.",
                    "Reviewed request T-cd68615c."
                  ]
                }
              }
            }
          },
          {
            "step": 3,
            "type": "tool_call",
            "action": {
              "tool": "add_note",
              "arguments": {
                "ticket_id": "T-cd68615c",
                "body": "Reviewed request T-cd68615c.",
                "request_id": "review"
              }
            },
            "result": {
              "ok": true,
              "replayed": true
            },
            "status": "ok",
            "error_source": null,
            "changes": {}
          },
          {
            "step": 4,
            "type": "tool_call",
            "action": {
              "tool": "close_ticket",
              "arguments": {
                "ticket_id": "T-cd68615c"
              }
            },
            "result": {
              "ok": true
            },
            "status": "ok",
            "error_source": null,
            "changes": {
              "T-cd68615c": {
                "before": {
                  "category": "billing",
                  "queue": "queue-3-0",
                  "resolved": true,
                  "status": "open",
                  "notes": [
                    "Customer contacted support.",
                    "Reviewed request T-cd68615c."
                  ]
                },
                "after": {
                  "category": "billing",
                  "queue": "queue-3-0",
                  "resolved": true,
                  "status": "closed",
                  "notes": [
                    "Customer contacted support.",
                    "Reviewed request T-cd68615c."
                  ]
                }
              }
            }
          },
          {
            "step": 5,
            "type": "finish",
            "action": {
              "finish": true
            }
          }
        ],
        "tool_calls": 5
      }
    ],
    "evaluation_seconds": 2.963313,
    "agent_cost_usd": null,
    "agent_tokens": null
  },
  "mutants": [
    {
      "name": "claim-without-actions",
      "target_dimension": "routing",
      "killed": true,
      "valid": true,
      "score": 0.35,
      "failing_cases": [
        "close-resolved-ticket",
        "retry-after-commit",
        "retry-before-commit",
        "route-open-ticket"
      ],
      "candidate_sha256": "d4412a9c3e8c47c744dd397aa582edec83b7ae1a6436b75a905fb4bb63faa3cd",
      "evaluation": {
        "schema_version": "evalarc.evaluation.v2",
        "created_at": "2026-09-14T01:29:41.862960+00:00",
        "evalarc_version": "0.2.0",
        "task": {
          "id": "support-routing",
          "version": "0.1.0",
          "domain": "business-tools",
          "split": "public-development"
        },
        "candidate_sha256": "d4412a9c3e8c47c744dd397aa582edec83b7ae1a6436b75a905fb4bb63faa3cd",
        "grader_sha256": "2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255",
        "cases_sha256": "b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df",
        "runtime": {
          "backend": "docker",
          "image": "python:3.12-slim",
          "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea",
          "command": [
            "{python}",
            "-I",
            "-B",
            "main.py"
          ],
          "response_timeout_seconds": 10.0,
          "session_output_limit_bytes": 1048576,
          "python": "3.12.3",
          "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39"
        },
        "seeds": [
          17
        ],
        "valid": true,
        "status": "failed",
        "score": 0.35,
        "resolved": false,
        "dimensions": {
          "routing": {
            "passed": 0,
            "total": 4,
            "assessed": 4,
            "weight": 0.3,
            "score": 0.0
          },
          "notes": {
            "passed": 0,
            "total": 4,
            "assessed": 4,
            "weight": 0.25,
            "score": 0.0
          },
          "closure": {
            "passed": 2,
            "total": 4,
            "assessed": 4,
            "weight": 0.2,
            "score": 0.5
          },
          "scope": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.15,
            "score": 1.0
          },
          "protocol": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.1,
            "score": 1.0
          }
        },
        "cases": [
          {
            "seed": 17,
            "case_id": "route-open-ticket",
            "checks": {
              "routing": false,
              "notes": false,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.713709,
            "transcript_sha256": "dc89878d8618444cc445bec619924daccdde74b03fca5826ccd7ef1d02d4245e",
            "initial_state": {
              "T-85a0bcc1": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-6a06e9ab": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-85a0bcc1": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-6a06e9ab": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "finish",
                "action": {
                  "finish": true,
                  "message": "Completed all requested work."
                }
              }
            ],
            "tool_calls": 0
          },
          {
            "seed": 17,
            "case_id": "close-resolved-ticket",
            "checks": {
              "routing": false,
              "notes": false,
              "closure": false,
              "scope": true,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.617806,
            "transcript_sha256": "dc89878d8618444cc445bec619924daccdde74b03fca5826ccd7ef1d02d4245e",
            "initial_state": {
              "T-c41f9dfd": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-b48438b5": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-c41f9dfd": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-b48438b5": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "finish",
                "action": {
                  "finish": true,
                  "message": "Completed all requested work."
                }
              }
            ],
            "tool_calls": 0
          },
          {
            "seed": 17,
            "case_id": "retry-before-commit",
            "checks": {
              "routing": false,
              "notes": false,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.635735,
            "transcript_sha256": "dc89878d8618444cc445bec619924daccdde74b03fca5826ccd7ef1d02d4245e",
            "initial_state": {
              "T-3fb81d27": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-62590992": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-3fb81d27": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-62590992": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "finish",
                "action": {
                  "finish": true,
                  "message": "Completed all requested work."
                }
              }
            ],
            "tool_calls": 0
          },
          {
            "seed": 17,
            "case_id": "retry-after-commit",
            "checks": {
              "routing": false,
              "notes": false,
              "closure": false,
              "scope": true,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.642983,
            "transcript_sha256": "dc89878d8618444cc445bec619924daccdde74b03fca5826ccd7ef1d02d4245e",
            "initial_state": {
              "T-cd68615c": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-51436d1f": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-cd68615c": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-51436d1f": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "finish",
                "action": {
                  "finish": true,
                  "message": "Completed all requested work."
                }
              }
            ],
            "tool_calls": 0
          }
        ],
        "evaluation_seconds": 2.613067,
        "agent_cost_usd": null,
        "agent_tokens": null
      }
    },
    {
      "name": "wrong-ticket",
      "target_dimension": "scope",
      "killed": true,
      "valid": true,
      "score": 0.2,
      "failing_cases": [
        "close-resolved-ticket",
        "retry-after-commit",
        "retry-before-commit",
        "route-open-ticket"
      ],
      "candidate_sha256": "183da04ff8f34d6d8640176ab9ea40882985b568163399c7add81971e782c761",
      "evaluation": {
        "schema_version": "evalarc.evaluation.v2",
        "created_at": "2026-09-14T01:29:44.442973+00:00",
        "evalarc_version": "0.2.0",
        "task": {
          "id": "support-routing",
          "version": "0.1.0",
          "domain": "business-tools",
          "split": "public-development"
        },
        "candidate_sha256": "183da04ff8f34d6d8640176ab9ea40882985b568163399c7add81971e782c761",
        "grader_sha256": "2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255",
        "cases_sha256": "b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df",
        "runtime": {
          "backend": "docker",
          "image": "python:3.12-slim",
          "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea",
          "command": [
            "{python}",
            "-I",
            "-B",
            "main.py"
          ],
          "response_timeout_seconds": 10.0,
          "session_output_limit_bytes": 1048576,
          "python": "3.12.3",
          "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39"
        },
        "seeds": [
          17
        ],
        "valid": true,
        "status": "failed",
        "score": 0.2,
        "resolved": false,
        "dimensions": {
          "routing": {
            "passed": 0,
            "total": 4,
            "assessed": 4,
            "weight": 0.3,
            "score": 0.0
          },
          "notes": {
            "passed": 0,
            "total": 4,
            "assessed": 4,
            "weight": 0.25,
            "score": 0.0
          },
          "closure": {
            "passed": 2,
            "total": 4,
            "assessed": 4,
            "weight": 0.2,
            "score": 0.5
          },
          "scope": {
            "passed": 0,
            "total": 4,
            "assessed": 4,
            "weight": 0.15,
            "score": 0.0
          },
          "protocol": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.1,
            "score": 1.0
          }
        },
        "cases": [
          {
            "seed": 17,
            "case_id": "route-open-ticket",
            "checks": {
              "routing": false,
              "notes": false,
              "closure": true,
              "scope": false,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.66702,
            "transcript_sha256": "13bb6318b8c0c1e61289c5ad6878ae37d2cb7a2591279528eea61699f94cd047",
            "initial_state": {
              "T-85a0bcc1": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-6a06e9ab": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-85a0bcc1": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-6a06e9ab": {
                "category": "billing",
                "queue": "queue-0-0",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket.",
                  "Reviewed request T-85a0bcc1."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "U-6a06e9ab"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "billing",
                    "queue": "inbox",
                    "resolved": false,
                    "status": "open",
                    "notes": [
                      "Unrelated request; preserve this ticket."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "U-6a06e9ab",
                    "queue": "queue-0-0"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "U-6a06e9ab": {
                    "before": {
                      "category": "billing",
                      "queue": "inbox",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Unrelated request; preserve this ticket."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-0-0",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Unrelated request; preserve this ticket."
                      ]
                    }
                  }
                }
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "U-6a06e9ab",
                    "body": "Reviewed request T-85a0bcc1.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "U-6a06e9ab": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-0-0",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Unrelated request; preserve this ticket."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-0-0",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Unrelated request; preserve this ticket.",
                        "Reviewed request T-85a0bcc1."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 3
          },
          {
            "seed": 17,
            "case_id": "close-resolved-ticket",
            "checks": {
              "routing": false,
              "notes": false,
              "closure": false,
              "scope": false,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.688977,
            "transcript_sha256": "af34c4df1f605b18475bb88594ec02c95aa692cdedc06945af120e864d7b20d1",
            "initial_state": {
              "T-c41f9dfd": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-b48438b5": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-c41f9dfd": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-b48438b5": {
                "category": "billing",
                "queue": "queue-1-0",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket.",
                  "Reviewed request T-c41f9dfd."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "U-b48438b5"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "billing",
                    "queue": "inbox",
                    "resolved": false,
                    "status": "open",
                    "notes": [
                      "Unrelated request; preserve this ticket."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "U-b48438b5",
                    "queue": "queue-1-0"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "U-b48438b5": {
                    "before": {
                      "category": "billing",
                      "queue": "inbox",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Unrelated request; preserve this ticket."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Unrelated request; preserve this ticket."
                      ]
                    }
                  }
                }
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "U-b48438b5",
                    "body": "Reviewed request T-c41f9dfd.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "U-b48438b5": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Unrelated request; preserve this ticket."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Unrelated request; preserve this ticket.",
                        "Reviewed request T-c41f9dfd."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 3
          },
          {
            "seed": 17,
            "case_id": "retry-before-commit",
            "checks": {
              "routing": false,
              "notes": false,
              "closure": true,
              "scope": false,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.661649,
            "transcript_sha256": "dce95ae090de1ec11795608277ea5d9d4ea46545741bdd9f89e8b351d2221909",
            "initial_state": {
              "T-3fb81d27": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-62590992": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-3fb81d27": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-62590992": {
                "category": "account",
                "queue": "queue-2-2",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket.",
                  "Reviewed request T-3fb81d27."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "U-62590992"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "account",
                    "queue": "inbox",
                    "resolved": false,
                    "status": "open",
                    "notes": [
                      "Unrelated request; preserve this ticket."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "U-62590992",
                    "queue": "queue-2-2"
                  }
                },
                "result": {
                  "ok": false,
                  "error": {
                    "code": "temporarily_unavailable",
                    "retryable": true
                  }
                },
                "status": "tool_error",
                "error_source": "injected",
                "changes": {}
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "U-62590992",
                    "queue": "queue-2-2"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "U-62590992": {
                    "before": {
                      "category": "account",
                      "queue": "inbox",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Unrelated request; preserve this ticket."
                      ]
                    },
                    "after": {
                      "category": "account",
                      "queue": "queue-2-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Unrelated request; preserve this ticket."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "U-62590992",
                    "body": "Reviewed request T-3fb81d27.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "U-62590992": {
                    "before": {
                      "category": "account",
                      "queue": "queue-2-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Unrelated request; preserve this ticket."
                      ]
                    },
                    "after": {
                      "category": "account",
                      "queue": "queue-2-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Unrelated request; preserve this ticket.",
                        "Reviewed request T-3fb81d27."
                      ]
                    }
                  }
                }
              },
              {
                "step": 4,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 4
          },
          {
            "seed": 17,
            "case_id": "retry-after-commit",
            "checks": {
              "routing": false,
              "notes": false,
              "closure": false,
              "scope": false,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.558768,
            "transcript_sha256": "cfc5d50e5cf0c9e319b39869d751e334459cfa948ea4693b7a24d9652ee8d30e",
            "initial_state": {
              "T-cd68615c": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-51436d1f": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-cd68615c": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-51436d1f": {
                "category": "account",
                "queue": "queue-3-2",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket.",
                  "Reviewed request T-cd68615c."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "U-51436d1f"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "account",
                    "queue": "inbox",
                    "resolved": false,
                    "status": "open",
                    "notes": [
                      "Unrelated request; preserve this ticket."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "U-51436d1f",
                    "queue": "queue-3-2"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "U-51436d1f": {
                    "before": {
                      "category": "account",
                      "queue": "inbox",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Unrelated request; preserve this ticket."
                      ]
                    },
                    "after": {
                      "category": "account",
                      "queue": "queue-3-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Unrelated request; preserve this ticket."
                      ]
                    }
                  }
                }
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "U-51436d1f",
                    "body": "Reviewed request T-cd68615c.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": false,
                  "error": {
                    "code": "temporarily_unavailable",
                    "retryable": true
                  }
                },
                "status": "tool_error",
                "error_source": "injected",
                "changes": {
                  "U-51436d1f": {
                    "before": {
                      "category": "account",
                      "queue": "queue-3-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Unrelated request; preserve this ticket."
                      ]
                    },
                    "after": {
                      "category": "account",
                      "queue": "queue-3-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Unrelated request; preserve this ticket.",
                        "Reviewed request T-cd68615c."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "U-51436d1f",
                    "body": "Reviewed request T-cd68615c.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true,
                  "replayed": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 4,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 4
          }
        ],
        "evaluation_seconds": 2.579827,
        "agent_cost_usd": null,
        "agent_tokens": null
      }
    },
    {
      "name": "wrong-queue",
      "target_dimension": "routing",
      "killed": true,
      "valid": true,
      "score": 0.7,
      "failing_cases": [
        "close-resolved-ticket",
        "retry-after-commit",
        "retry-before-commit",
        "route-open-ticket"
      ],
      "candidate_sha256": "c4d20e7a258872b9555f703264a14e3e775c03563d08ca7f6c11fb0eb73253fa",
      "evaluation": {
        "schema_version": "evalarc.evaluation.v2",
        "created_at": "2026-09-14T01:29:46.670548+00:00",
        "evalarc_version": "0.2.0",
        "task": {
          "id": "support-routing",
          "version": "0.1.0",
          "domain": "business-tools",
          "split": "public-development"
        },
        "candidate_sha256": "c4d20e7a258872b9555f703264a14e3e775c03563d08ca7f6c11fb0eb73253fa",
        "grader_sha256": "2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255",
        "cases_sha256": "b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df",
        "runtime": {
          "backend": "docker",
          "image": "python:3.12-slim",
          "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea",
          "command": [
            "{python}",
            "-I",
            "-B",
            "main.py"
          ],
          "response_timeout_seconds": 10.0,
          "session_output_limit_bytes": 1048576,
          "python": "3.12.3",
          "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39"
        },
        "seeds": [
          17
        ],
        "valid": true,
        "status": "failed",
        "score": 0.7,
        "resolved": false,
        "dimensions": {
          "routing": {
            "passed": 0,
            "total": 4,
            "assessed": 4,
            "weight": 0.3,
            "score": 0.0
          },
          "notes": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.25,
            "score": 1.0
          },
          "closure": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.2,
            "score": 1.0
          },
          "scope": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.15,
            "score": 1.0
          },
          "protocol": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.1,
            "score": 1.0
          }
        },
        "cases": [
          {
            "seed": 17,
            "case_id": "route-open-ticket",
            "checks": {
              "routing": false,
              "notes": true,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.576444,
            "transcript_sha256": "06bfdd936f8a75a1f79fac629e71818d6f8750e756c87b7318481642641860f9",
            "initial_state": {
              "T-85a0bcc1": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-6a06e9ab": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-85a0bcc1": {
                "category": "technical",
                "queue": "queue-0-0",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-85a0bcc1."
                ]
              },
              "U-6a06e9ab": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-85a0bcc1"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "technical",
                    "queue": "inbox",
                    "resolved": false,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-85a0bcc1",
                    "queue": "queue-0-0"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-85a0bcc1": {
                    "before": {
                      "category": "technical",
                      "queue": "inbox",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-0-0",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-85a0bcc1",
                    "body": "Reviewed request T-85a0bcc1.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-85a0bcc1": {
                    "before": {
                      "category": "technical",
                      "queue": "queue-0-0",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-0-0",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-85a0bcc1."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 3
          },
          {
            "seed": 17,
            "case_id": "close-resolved-ticket",
            "checks": {
              "routing": false,
              "notes": true,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.552313,
            "transcript_sha256": "18b3105a6a7f077107630156bf9124018c8271feaf28f6fd778cad881d67e0b2",
            "initial_state": {
              "T-c41f9dfd": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-b48438b5": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-c41f9dfd": {
                "category": "billing",
                "queue": "queue-1-1",
                "resolved": true,
                "status": "closed",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-c41f9dfd."
                ]
              },
              "U-b48438b5": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "billing",
                    "queue": "inbox",
                    "resolved": true,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd",
                    "queue": "queue-1-1"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-c41f9dfd": {
                    "before": {
                      "category": "billing",
                      "queue": "inbox",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-1-1",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd",
                    "body": "Reviewed request T-c41f9dfd.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-c41f9dfd": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-1-1",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-1-1",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-c41f9dfd."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "tool_call",
                "action": {
                  "tool": "close_ticket",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-c41f9dfd": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-1-1",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-c41f9dfd."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-1-1",
                      "resolved": true,
                      "status": "closed",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-c41f9dfd."
                      ]
                    }
                  }
                }
              },
              {
                "step": 4,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 4
          },
          {
            "seed": 17,
            "case_id": "retry-before-commit",
            "checks": {
              "routing": false,
              "notes": true,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.55564,
            "transcript_sha256": "8bb84e30d03030ac20e834eae63758c4041341b0325cebc35e1aeca01d59ccfe",
            "initial_state": {
              "T-3fb81d27": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-62590992": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-3fb81d27": {
                "category": "technical",
                "queue": "queue-2-0",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-3fb81d27."
                ]
              },
              "U-62590992": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-3fb81d27"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "technical",
                    "queue": "inbox",
                    "resolved": false,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-3fb81d27",
                    "queue": "queue-2-0"
                  }
                },
                "result": {
                  "ok": false,
                  "error": {
                    "code": "temporarily_unavailable",
                    "retryable": true
                  }
                },
                "status": "tool_error",
                "error_source": "injected",
                "changes": {}
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-3fb81d27",
                    "queue": "queue-2-0"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-3fb81d27": {
                    "before": {
                      "category": "technical",
                      "queue": "inbox",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-2-0",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-3fb81d27",
                    "body": "Reviewed request T-3fb81d27.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-3fb81d27": {
                    "before": {
                      "category": "technical",
                      "queue": "queue-2-0",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-2-0",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-3fb81d27."
                      ]
                    }
                  }
                }
              },
              {
                "step": 4,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 4
          },
          {
            "seed": 17,
            "case_id": "retry-after-commit",
            "checks": {
              "routing": false,
              "notes": true,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.538827,
            "transcript_sha256": "3076729f9cfaeec8c787fb9216588135070ea5aee843c853840be4436629ef92",
            "initial_state": {
              "T-cd68615c": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-51436d1f": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-cd68615c": {
                "category": "billing",
                "queue": "queue-3-1",
                "resolved": true,
                "status": "closed",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-cd68615c."
                ]
              },
              "U-51436d1f": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-cd68615c"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "billing",
                    "queue": "inbox",
                    "resolved": true,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-cd68615c",
                    "queue": "queue-3-1"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-cd68615c": {
                    "before": {
                      "category": "billing",
                      "queue": "inbox",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-3-1",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-cd68615c",
                    "body": "Reviewed request T-cd68615c.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": false,
                  "error": {
                    "code": "temporarily_unavailable",
                    "retryable": true
                  }
                },
                "status": "tool_error",
                "error_source": "injected",
                "changes": {
                  "T-cd68615c": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-3-1",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-3-1",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-cd68615c."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-cd68615c",
                    "body": "Reviewed request T-cd68615c.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true,
                  "replayed": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 4,
                "type": "tool_call",
                "action": {
                  "tool": "close_ticket",
                  "arguments": {
                    "ticket_id": "T-cd68615c"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-cd68615c": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-3-1",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-cd68615c."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-3-1",
                      "resolved": true,
                      "status": "closed",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-cd68615c."
                      ]
                    }
                  }
                }
              },
              {
                "step": 5,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 5
          }
        ],
        "evaluation_seconds": 2.22727,
        "agent_cost_usd": null,
        "agent_tokens": null
      }
    },
    {
      "name": "duplicate-note",
      "target_dimension": "notes",
      "killed": true,
      "valid": true,
      "score": 0.75,
      "failing_cases": [
        "close-resolved-ticket",
        "retry-after-commit",
        "retry-before-commit",
        "route-open-ticket"
      ],
      "candidate_sha256": "56c5cb4be6ffd347dc0a382a6a7b9bacde1c308ce1538faae18c002affc36213",
      "evaluation": {
        "schema_version": "evalarc.evaluation.v2",
        "created_at": "2026-09-14T01:29:48.962131+00:00",
        "evalarc_version": "0.2.0",
        "task": {
          "id": "support-routing",
          "version": "0.1.0",
          "domain": "business-tools",
          "split": "public-development"
        },
        "candidate_sha256": "56c5cb4be6ffd347dc0a382a6a7b9bacde1c308ce1538faae18c002affc36213",
        "grader_sha256": "2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255",
        "cases_sha256": "b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df",
        "runtime": {
          "backend": "docker",
          "image": "python:3.12-slim",
          "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea",
          "command": [
            "{python}",
            "-I",
            "-B",
            "main.py"
          ],
          "response_timeout_seconds": 10.0,
          "session_output_limit_bytes": 1048576,
          "python": "3.12.3",
          "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39"
        },
        "seeds": [
          17
        ],
        "valid": true,
        "status": "failed",
        "score": 0.75,
        "resolved": false,
        "dimensions": {
          "routing": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.3,
            "score": 1.0
          },
          "notes": {
            "passed": 0,
            "total": 4,
            "assessed": 4,
            "weight": 0.25,
            "score": 0.0
          },
          "closure": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.2,
            "score": 1.0
          },
          "scope": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.15,
            "score": 1.0
          },
          "protocol": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.1,
            "score": 1.0
          }
        },
        "cases": [
          {
            "seed": 17,
            "case_id": "route-open-ticket",
            "checks": {
              "routing": true,
              "notes": false,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.585063,
            "transcript_sha256": "eba30506565d7d2797e622ec54845d26c0b3a9fc9e323c298cbc76dc390e082c",
            "initial_state": {
              "T-85a0bcc1": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-6a06e9ab": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-85a0bcc1": {
                "category": "technical",
                "queue": "queue-0-2",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-85a0bcc1.",
                  "Reviewed request T-85a0bcc1."
                ]
              },
              "U-6a06e9ab": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-85a0bcc1"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "technical",
                    "queue": "inbox",
                    "resolved": false,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-85a0bcc1",
                    "queue": "queue-0-2"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-85a0bcc1": {
                    "before": {
                      "category": "technical",
                      "queue": "inbox",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-0-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-85a0bcc1",
                    "body": "Reviewed request T-85a0bcc1.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-85a0bcc1": {
                    "before": {
                      "category": "technical",
                      "queue": "queue-0-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-0-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-85a0bcc1."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-85a0bcc1",
                    "body": "Reviewed request T-85a0bcc1.",
                    "request_id": "duplicate"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-85a0bcc1": {
                    "before": {
                      "category": "technical",
                      "queue": "queue-0-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-85a0bcc1."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-0-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-85a0bcc1.",
                        "Reviewed request T-85a0bcc1."
                      ]
                    }
                  }
                }
              },
              {
                "step": 4,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 4
          },
          {
            "seed": 17,
            "case_id": "close-resolved-ticket",
            "checks": {
              "routing": true,
              "notes": false,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.568052,
            "transcript_sha256": "7b2dd2bb2f48dc19e31cd072abca3fd5d43d9d7369a1a19077e3bcf0c45ae229",
            "initial_state": {
              "T-c41f9dfd": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-b48438b5": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-c41f9dfd": {
                "category": "billing",
                "queue": "queue-1-0",
                "resolved": true,
                "status": "closed",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-c41f9dfd.",
                  "Reviewed request T-c41f9dfd."
                ]
              },
              "U-b48438b5": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "billing",
                    "queue": "inbox",
                    "resolved": true,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd",
                    "queue": "queue-1-0"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-c41f9dfd": {
                    "before": {
                      "category": "billing",
                      "queue": "inbox",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd",
                    "body": "Reviewed request T-c41f9dfd.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-c41f9dfd": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-c41f9dfd."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd",
                    "body": "Reviewed request T-c41f9dfd.",
                    "request_id": "duplicate"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-c41f9dfd": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-c41f9dfd."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-c41f9dfd.",
                        "Reviewed request T-c41f9dfd."
                      ]
                    }
                  }
                }
              },
              {
                "step": 4,
                "type": "tool_call",
                "action": {
                  "tool": "close_ticket",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-c41f9dfd": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-c41f9dfd.",
                        "Reviewed request T-c41f9dfd."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "closed",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-c41f9dfd.",
                        "Reviewed request T-c41f9dfd."
                      ]
                    }
                  }
                }
              },
              {
                "step": 5,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 5
          },
          {
            "seed": 17,
            "case_id": "retry-before-commit",
            "checks": {
              "routing": true,
              "notes": false,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.538005,
            "transcript_sha256": "34dc374c3e62cf1911796b91f4c3d7b6d1f7c14a4ac997c8bcd8d7ed2c7f6fdd",
            "initial_state": {
              "T-3fb81d27": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-62590992": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-3fb81d27": {
                "category": "technical",
                "queue": "queue-2-1",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-3fb81d27.",
                  "Reviewed request T-3fb81d27."
                ]
              },
              "U-62590992": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-3fb81d27"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "technical",
                    "queue": "inbox",
                    "resolved": false,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-3fb81d27",
                    "queue": "queue-2-1"
                  }
                },
                "result": {
                  "ok": false,
                  "error": {
                    "code": "temporarily_unavailable",
                    "retryable": true
                  }
                },
                "status": "tool_error",
                "error_source": "injected",
                "changes": {}
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-3fb81d27",
                    "queue": "queue-2-1"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-3fb81d27": {
                    "before": {
                      "category": "technical",
                      "queue": "inbox",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-2-1",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-3fb81d27",
                    "body": "Reviewed request T-3fb81d27.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-3fb81d27": {
                    "before": {
                      "category": "technical",
                      "queue": "queue-2-1",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-2-1",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-3fb81d27."
                      ]
                    }
                  }
                }
              },
              {
                "step": 4,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-3fb81d27",
                    "body": "Reviewed request T-3fb81d27.",
                    "request_id": "duplicate"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-3fb81d27": {
                    "before": {
                      "category": "technical",
                      "queue": "queue-2-1",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-3fb81d27."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-2-1",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-3fb81d27.",
                        "Reviewed request T-3fb81d27."
                      ]
                    }
                  }
                }
              },
              {
                "step": 5,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 5
          },
          {
            "seed": 17,
            "case_id": "retry-after-commit",
            "checks": {
              "routing": true,
              "notes": false,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.595631,
            "transcript_sha256": "877a29b2016a4c4dd8565f1fb9987faf298ad23e39683d63b32762d7abfcadf2",
            "initial_state": {
              "T-cd68615c": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-51436d1f": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-cd68615c": {
                "category": "billing",
                "queue": "queue-3-0",
                "resolved": true,
                "status": "closed",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-cd68615c.",
                  "Reviewed request T-cd68615c."
                ]
              },
              "U-51436d1f": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-cd68615c"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "billing",
                    "queue": "inbox",
                    "resolved": true,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-cd68615c",
                    "queue": "queue-3-0"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-cd68615c": {
                    "before": {
                      "category": "billing",
                      "queue": "inbox",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-cd68615c",
                    "body": "Reviewed request T-cd68615c.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": false,
                  "error": {
                    "code": "temporarily_unavailable",
                    "retryable": true
                  }
                },
                "status": "tool_error",
                "error_source": "injected",
                "changes": {
                  "T-cd68615c": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-cd68615c."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-cd68615c",
                    "body": "Reviewed request T-cd68615c.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true,
                  "replayed": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 4,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-cd68615c",
                    "body": "Reviewed request T-cd68615c.",
                    "request_id": "duplicate"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-cd68615c": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-cd68615c."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-cd68615c.",
                        "Reviewed request T-cd68615c."
                      ]
                    }
                  }
                }
              },
              {
                "step": 5,
                "type": "tool_call",
                "action": {
                  "tool": "close_ticket",
                  "arguments": {
                    "ticket_id": "T-cd68615c"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-cd68615c": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-cd68615c.",
                        "Reviewed request T-cd68615c."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "closed",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-cd68615c.",
                        "Reviewed request T-cd68615c."
                      ]
                    }
                  }
                }
              },
              {
                "step": 6,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 6
          }
        ],
        "evaluation_seconds": 2.291397,
        "agent_cost_usd": null,
        "agent_tokens": null
      }
    },
    {
      "name": "close-unresolved",
      "target_dimension": "closure",
      "killed": true,
      "valid": true,
      "score": 0.9,
      "failing_cases": [
        "retry-before-commit",
        "route-open-ticket"
      ],
      "candidate_sha256": "a34429419d7c794420d43f7873ce5b1474f73821329f414609d5e4d18fc174c3",
      "evaluation": {
        "schema_version": "evalarc.evaluation.v2",
        "created_at": "2026-09-14T01:29:51.180933+00:00",
        "evalarc_version": "0.2.0",
        "task": {
          "id": "support-routing",
          "version": "0.1.0",
          "domain": "business-tools",
          "split": "public-development"
        },
        "candidate_sha256": "a34429419d7c794420d43f7873ce5b1474f73821329f414609d5e4d18fc174c3",
        "grader_sha256": "2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255",
        "cases_sha256": "b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df",
        "runtime": {
          "backend": "docker",
          "image": "python:3.12-slim",
          "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea",
          "command": [
            "{python}",
            "-I",
            "-B",
            "main.py"
          ],
          "response_timeout_seconds": 10.0,
          "session_output_limit_bytes": 1048576,
          "python": "3.12.3",
          "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39"
        },
        "seeds": [
          17
        ],
        "valid": true,
        "status": "failed",
        "score": 0.9,
        "resolved": false,
        "dimensions": {
          "routing": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.3,
            "score": 1.0
          },
          "notes": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.25,
            "score": 1.0
          },
          "closure": {
            "passed": 2,
            "total": 4,
            "assessed": 4,
            "weight": 0.2,
            "score": 0.5
          },
          "scope": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.15,
            "score": 1.0
          },
          "protocol": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.1,
            "score": 1.0
          }
        },
        "cases": [
          {
            "seed": 17,
            "case_id": "route-open-ticket",
            "checks": {
              "routing": true,
              "notes": true,
              "closure": false,
              "scope": true,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.554959,
            "transcript_sha256": "3b8ea18d1468afe56a69b06b884469141aa5aed9ab211a2c60378f64aaf456e5",
            "initial_state": {
              "T-85a0bcc1": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-6a06e9ab": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-85a0bcc1": {
                "category": "technical",
                "queue": "queue-0-2",
                "resolved": false,
                "status": "closed",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-85a0bcc1."
                ]
              },
              "U-6a06e9ab": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-85a0bcc1"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "technical",
                    "queue": "inbox",
                    "resolved": false,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-85a0bcc1",
                    "queue": "queue-0-2"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-85a0bcc1": {
                    "before": {
                      "category": "technical",
                      "queue": "inbox",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-0-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-85a0bcc1",
                    "body": "Reviewed request T-85a0bcc1.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-85a0bcc1": {
                    "before": {
                      "category": "technical",
                      "queue": "queue-0-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-0-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-85a0bcc1."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "tool_call",
                "action": {
                  "tool": "close_ticket",
                  "arguments": {
                    "ticket_id": "T-85a0bcc1"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-85a0bcc1": {
                    "before": {
                      "category": "technical",
                      "queue": "queue-0-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-85a0bcc1."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-0-2",
                      "resolved": false,
                      "status": "closed",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-85a0bcc1."
                      ]
                    }
                  }
                }
              },
              {
                "step": 4,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 4
          },
          {
            "seed": 17,
            "case_id": "close-resolved-ticket",
            "checks": {
              "routing": true,
              "notes": true,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": true,
            "status": "passed",
            "error": null,
            "duration_seconds": 0.567618,
            "transcript_sha256": "a260768c11b589ce3f58658cac6aae8a01d36e388a47fc47c7995f8d5c36708e",
            "initial_state": {
              "T-c41f9dfd": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-b48438b5": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-c41f9dfd": {
                "category": "billing",
                "queue": "queue-1-0",
                "resolved": true,
                "status": "closed",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-c41f9dfd."
                ]
              },
              "U-b48438b5": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "billing",
                    "queue": "inbox",
                    "resolved": true,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd",
                    "queue": "queue-1-0"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-c41f9dfd": {
                    "before": {
                      "category": "billing",
                      "queue": "inbox",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd",
                    "body": "Reviewed request T-c41f9dfd.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-c41f9dfd": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-c41f9dfd."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "tool_call",
                "action": {
                  "tool": "close_ticket",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-c41f9dfd": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-c41f9dfd."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "closed",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-c41f9dfd."
                      ]
                    }
                  }
                }
              },
              {
                "step": 4,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 4
          },
          {
            "seed": 17,
            "case_id": "retry-before-commit",
            "checks": {
              "routing": true,
              "notes": true,
              "closure": false,
              "scope": true,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.540602,
            "transcript_sha256": "4d7abd25df6f1a223a236570e6b1d0f5d31f8f0343a6497a27b12f396f65e2ca",
            "initial_state": {
              "T-3fb81d27": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-62590992": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-3fb81d27": {
                "category": "technical",
                "queue": "queue-2-1",
                "resolved": false,
                "status": "closed",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-3fb81d27."
                ]
              },
              "U-62590992": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-3fb81d27"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "technical",
                    "queue": "inbox",
                    "resolved": false,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-3fb81d27",
                    "queue": "queue-2-1"
                  }
                },
                "result": {
                  "ok": false,
                  "error": {
                    "code": "temporarily_unavailable",
                    "retryable": true
                  }
                },
                "status": "tool_error",
                "error_source": "injected",
                "changes": {}
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-3fb81d27",
                    "queue": "queue-2-1"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-3fb81d27": {
                    "before": {
                      "category": "technical",
                      "queue": "inbox",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-2-1",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-3fb81d27",
                    "body": "Reviewed request T-3fb81d27.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-3fb81d27": {
                    "before": {
                      "category": "technical",
                      "queue": "queue-2-1",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-2-1",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-3fb81d27."
                      ]
                    }
                  }
                }
              },
              {
                "step": 4,
                "type": "tool_call",
                "action": {
                  "tool": "close_ticket",
                  "arguments": {
                    "ticket_id": "T-3fb81d27"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-3fb81d27": {
                    "before": {
                      "category": "technical",
                      "queue": "queue-2-1",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-3fb81d27."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-2-1",
                      "resolved": false,
                      "status": "closed",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-3fb81d27."
                      ]
                    }
                  }
                }
              },
              {
                "step": 5,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 5
          },
          {
            "seed": 17,
            "case_id": "retry-after-commit",
            "checks": {
              "routing": true,
              "notes": true,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": true,
            "status": "passed",
            "error": null,
            "duration_seconds": 0.5517,
            "transcript_sha256": "c97b016921c8d8fdcdeffd1ebf1f438d5a29be06732c31c86f99fd087bc0b58f",
            "initial_state": {
              "T-cd68615c": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-51436d1f": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-cd68615c": {
                "category": "billing",
                "queue": "queue-3-0",
                "resolved": true,
                "status": "closed",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-cd68615c."
                ]
              },
              "U-51436d1f": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-cd68615c"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "billing",
                    "queue": "inbox",
                    "resolved": true,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-cd68615c",
                    "queue": "queue-3-0"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-cd68615c": {
                    "before": {
                      "category": "billing",
                      "queue": "inbox",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-cd68615c",
                    "body": "Reviewed request T-cd68615c.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": false,
                  "error": {
                    "code": "temporarily_unavailable",
                    "retryable": true
                  }
                },
                "status": "tool_error",
                "error_source": "injected",
                "changes": {
                  "T-cd68615c": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-cd68615c."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-cd68615c",
                    "body": "Reviewed request T-cd68615c.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true,
                  "replayed": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 4,
                "type": "tool_call",
                "action": {
                  "tool": "close_ticket",
                  "arguments": {
                    "ticket_id": "T-cd68615c"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-cd68615c": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-cd68615c."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "closed",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-cd68615c."
                      ]
                    }
                  }
                }
              },
              {
                "step": 5,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 5
          }
        ],
        "evaluation_seconds": 2.218637,
        "agent_cost_usd": null,
        "agent_tokens": null
      }
    },
    {
      "name": "skip-retry",
      "target_dimension": "notes",
      "killed": true,
      "valid": true,
      "score": 0.8125,
      "failing_cases": [
        "retry-before-commit"
      ],
      "candidate_sha256": "ad5718fa57624480f1f6a28c1fbf055d5164d89c35bc0ca3ce28be1753ae0f54",
      "evaluation": {
        "schema_version": "evalarc.evaluation.v2",
        "created_at": "2026-09-14T01:29:53.496163+00:00",
        "evalarc_version": "0.2.0",
        "task": {
          "id": "support-routing",
          "version": "0.1.0",
          "domain": "business-tools",
          "split": "public-development"
        },
        "candidate_sha256": "ad5718fa57624480f1f6a28c1fbf055d5164d89c35bc0ca3ce28be1753ae0f54",
        "grader_sha256": "2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255",
        "cases_sha256": "b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df",
        "runtime": {
          "backend": "docker",
          "image": "python:3.12-slim",
          "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea",
          "command": [
            "{python}",
            "-I",
            "-B",
            "main.py"
          ],
          "response_timeout_seconds": 10.0,
          "session_output_limit_bytes": 1048576,
          "python": "3.12.3",
          "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39"
        },
        "seeds": [
          17
        ],
        "valid": true,
        "status": "failed",
        "score": 0.8125,
        "resolved": false,
        "dimensions": {
          "routing": {
            "passed": 3,
            "total": 4,
            "assessed": 4,
            "weight": 0.3,
            "score": 0.75
          },
          "notes": {
            "passed": 3,
            "total": 4,
            "assessed": 4,
            "weight": 0.25,
            "score": 0.75
          },
          "closure": {
            "passed": 3,
            "total": 4,
            "assessed": 4,
            "weight": 0.2,
            "score": 0.75
          },
          "scope": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.15,
            "score": 1.0
          },
          "protocol": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.1,
            "score": 1.0
          }
        },
        "cases": [
          {
            "seed": 17,
            "case_id": "route-open-ticket",
            "checks": {
              "routing": true,
              "notes": true,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": true,
            "status": "passed",
            "error": null,
            "duration_seconds": 0.604977,
            "transcript_sha256": "7b5b4973e934a60c55b95bda6eedf79fcff62764fbc1b81723c5126608caf4bd",
            "initial_state": {
              "T-85a0bcc1": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-6a06e9ab": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-85a0bcc1": {
                "category": "technical",
                "queue": "queue-0-2",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-85a0bcc1."
                ]
              },
              "U-6a06e9ab": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-85a0bcc1"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "technical",
                    "queue": "inbox",
                    "resolved": false,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-85a0bcc1",
                    "queue": "queue-0-2"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-85a0bcc1": {
                    "before": {
                      "category": "technical",
                      "queue": "inbox",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-0-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-85a0bcc1",
                    "body": "Reviewed request T-85a0bcc1.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-85a0bcc1": {
                    "before": {
                      "category": "technical",
                      "queue": "queue-0-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-0-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-85a0bcc1."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 3
          },
          {
            "seed": 17,
            "case_id": "close-resolved-ticket",
            "checks": {
              "routing": true,
              "notes": true,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": true,
            "status": "passed",
            "error": null,
            "duration_seconds": 0.588704,
            "transcript_sha256": "a260768c11b589ce3f58658cac6aae8a01d36e388a47fc47c7995f8d5c36708e",
            "initial_state": {
              "T-c41f9dfd": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-b48438b5": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-c41f9dfd": {
                "category": "billing",
                "queue": "queue-1-0",
                "resolved": true,
                "status": "closed",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-c41f9dfd."
                ]
              },
              "U-b48438b5": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "billing",
                    "queue": "inbox",
                    "resolved": true,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd",
                    "queue": "queue-1-0"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-c41f9dfd": {
                    "before": {
                      "category": "billing",
                      "queue": "inbox",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd",
                    "body": "Reviewed request T-c41f9dfd.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-c41f9dfd": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-c41f9dfd."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "tool_call",
                "action": {
                  "tool": "close_ticket",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-c41f9dfd": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-c41f9dfd."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "closed",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-c41f9dfd."
                      ]
                    }
                  }
                }
              },
              {
                "step": 4,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 4
          },
          {
            "seed": 17,
            "case_id": "retry-before-commit",
            "checks": {
              "routing": false,
              "notes": false,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.566178,
            "transcript_sha256": "7700d40c0510e616f031164599fb05256c512aae5f280105701d533ec030f826",
            "initial_state": {
              "T-3fb81d27": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-62590992": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-3fb81d27": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-62590992": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-3fb81d27"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "technical",
                    "queue": "inbox",
                    "resolved": false,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-3fb81d27",
                    "queue": "queue-2-1"
                  }
                },
                "result": {
                  "ok": false,
                  "error": {
                    "code": "temporarily_unavailable",
                    "retryable": true
                  }
                },
                "status": "tool_error",
                "error_source": "injected",
                "changes": {}
              },
              {
                "step": 2,
                "type": "finish",
                "action": {
                  "finish": true,
                  "message": "Could not complete the operation."
                }
              }
            ],
            "tool_calls": 2
          },
          {
            "seed": 17,
            "case_id": "retry-after-commit",
            "checks": {
              "routing": true,
              "notes": true,
              "closure": false,
              "scope": true,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.550785,
            "transcript_sha256": "9cb4cd29bf17742367de9a0d73987d95440b2f23e38674ef6e770535ba953b09",
            "initial_state": {
              "T-cd68615c": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-51436d1f": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-cd68615c": {
                "category": "billing",
                "queue": "queue-3-0",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-cd68615c."
                ]
              },
              "U-51436d1f": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-cd68615c"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "billing",
                    "queue": "inbox",
                    "resolved": true,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-cd68615c",
                    "queue": "queue-3-0"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-cd68615c": {
                    "before": {
                      "category": "billing",
                      "queue": "inbox",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-cd68615c",
                    "body": "Reviewed request T-cd68615c.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": false,
                  "error": {
                    "code": "temporarily_unavailable",
                    "retryable": true
                  }
                },
                "status": "tool_error",
                "error_source": "injected",
                "changes": {
                  "T-cd68615c": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-cd68615c."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "finish",
                "action": {
                  "finish": true,
                  "message": "Could not complete the operation."
                }
              }
            ],
            "tool_calls": 3
          }
        ],
        "evaluation_seconds": 2.314765,
        "agent_cost_usd": null,
        "agent_tokens": null
      }
    },
    {
      "name": "new-key-on-retry",
      "target_dimension": "notes",
      "killed": true,
      "valid": true,
      "score": 0.9375,
      "failing_cases": [
        "retry-after-commit"
      ],
      "candidate_sha256": "b89a99819c03e8290096ca268f637c5517d465b3ec5f0b5fe2b21c35509ff202",
      "evaluation": {
        "schema_version": "evalarc.evaluation.v2",
        "created_at": "2026-09-14T01:29:55.858010+00:00",
        "evalarc_version": "0.2.0",
        "task": {
          "id": "support-routing",
          "version": "0.1.0",
          "domain": "business-tools",
          "split": "public-development"
        },
        "candidate_sha256": "b89a99819c03e8290096ca268f637c5517d465b3ec5f0b5fe2b21c35509ff202",
        "grader_sha256": "2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255",
        "cases_sha256": "b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df",
        "runtime": {
          "backend": "docker",
          "image": "python:3.12-slim",
          "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea",
          "command": [
            "{python}",
            "-I",
            "-B",
            "main.py"
          ],
          "response_timeout_seconds": 10.0,
          "session_output_limit_bytes": 1048576,
          "python": "3.12.3",
          "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39"
        },
        "seeds": [
          17
        ],
        "valid": true,
        "status": "failed",
        "score": 0.9375,
        "resolved": false,
        "dimensions": {
          "routing": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.3,
            "score": 1.0
          },
          "notes": {
            "passed": 3,
            "total": 4,
            "assessed": 4,
            "weight": 0.25,
            "score": 0.75
          },
          "closure": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.2,
            "score": 1.0
          },
          "scope": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.15,
            "score": 1.0
          },
          "protocol": {
            "passed": 4,
            "total": 4,
            "assessed": 4,
            "weight": 0.1,
            "score": 1.0
          }
        },
        "cases": [
          {
            "seed": 17,
            "case_id": "route-open-ticket",
            "checks": {
              "routing": true,
              "notes": true,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": true,
            "status": "passed",
            "error": null,
            "duration_seconds": 0.615903,
            "transcript_sha256": "7b5b4973e934a60c55b95bda6eedf79fcff62764fbc1b81723c5126608caf4bd",
            "initial_state": {
              "T-85a0bcc1": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-6a06e9ab": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-85a0bcc1": {
                "category": "technical",
                "queue": "queue-0-2",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-85a0bcc1."
                ]
              },
              "U-6a06e9ab": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-85a0bcc1"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "technical",
                    "queue": "inbox",
                    "resolved": false,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-85a0bcc1",
                    "queue": "queue-0-2"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-85a0bcc1": {
                    "before": {
                      "category": "technical",
                      "queue": "inbox",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-0-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-85a0bcc1",
                    "body": "Reviewed request T-85a0bcc1.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-85a0bcc1": {
                    "before": {
                      "category": "technical",
                      "queue": "queue-0-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-0-2",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-85a0bcc1."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 3
          },
          {
            "seed": 17,
            "case_id": "close-resolved-ticket",
            "checks": {
              "routing": true,
              "notes": true,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": true,
            "status": "passed",
            "error": null,
            "duration_seconds": 0.579634,
            "transcript_sha256": "a260768c11b589ce3f58658cac6aae8a01d36e388a47fc47c7995f8d5c36708e",
            "initial_state": {
              "T-c41f9dfd": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-b48438b5": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-c41f9dfd": {
                "category": "billing",
                "queue": "queue-1-0",
                "resolved": true,
                "status": "closed",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-c41f9dfd."
                ]
              },
              "U-b48438b5": {
                "category": "billing",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "billing",
                    "queue": "inbox",
                    "resolved": true,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd",
                    "queue": "queue-1-0"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-c41f9dfd": {
                    "before": {
                      "category": "billing",
                      "queue": "inbox",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd",
                    "body": "Reviewed request T-c41f9dfd.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-c41f9dfd": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-c41f9dfd."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "tool_call",
                "action": {
                  "tool": "close_ticket",
                  "arguments": {
                    "ticket_id": "T-c41f9dfd"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-c41f9dfd": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-c41f9dfd."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-1-0",
                      "resolved": true,
                      "status": "closed",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-c41f9dfd."
                      ]
                    }
                  }
                }
              },
              {
                "step": 4,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 4
          },
          {
            "seed": 17,
            "case_id": "retry-before-commit",
            "checks": {
              "routing": true,
              "notes": true,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": true,
            "status": "passed",
            "error": null,
            "duration_seconds": 0.574111,
            "transcript_sha256": "b1d1d91dc01fc4bfb3243aa3739e953a40aa3920caf63a4b2ab73c87ecabda66",
            "initial_state": {
              "T-3fb81d27": {
                "category": "technical",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-62590992": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-3fb81d27": {
                "category": "technical",
                "queue": "queue-2-1",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-3fb81d27."
                ]
              },
              "U-62590992": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-3fb81d27"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "technical",
                    "queue": "inbox",
                    "resolved": false,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-3fb81d27",
                    "queue": "queue-2-1"
                  }
                },
                "result": {
                  "ok": false,
                  "error": {
                    "code": "temporarily_unavailable",
                    "retryable": true
                  }
                },
                "status": "tool_error",
                "error_source": "injected",
                "changes": {}
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-3fb81d27",
                    "queue": "queue-2-1"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-3fb81d27": {
                    "before": {
                      "category": "technical",
                      "queue": "inbox",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-2-1",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-3fb81d27",
                    "body": "Reviewed request T-3fb81d27.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-3fb81d27": {
                    "before": {
                      "category": "technical",
                      "queue": "queue-2-1",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "technical",
                      "queue": "queue-2-1",
                      "resolved": false,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-3fb81d27."
                      ]
                    }
                  }
                }
              },
              {
                "step": 4,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 4
          },
          {
            "seed": 17,
            "case_id": "retry-after-commit",
            "checks": {
              "routing": true,
              "notes": false,
              "closure": true,
              "scope": true,
              "protocol": true
            },
            "passed": false,
            "status": "failed",
            "error": null,
            "duration_seconds": 0.58689,
            "transcript_sha256": "4d1123f9ef2ebc43d80c84e4f2054b3ed62a22db7d6c565f0f7f3a9f28c662b4",
            "initial_state": {
              "T-cd68615c": {
                "category": "billing",
                "queue": "inbox",
                "resolved": true,
                "status": "open",
                "notes": [
                  "Customer contacted support."
                ]
              },
              "U-51436d1f": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "final_state": {
              "T-cd68615c": {
                "category": "billing",
                "queue": "queue-3-0",
                "resolved": true,
                "status": "closed",
                "notes": [
                  "Customer contacted support.",
                  "Reviewed request T-cd68615c.",
                  "Reviewed request T-cd68615c."
                ]
              },
              "U-51436d1f": {
                "category": "account",
                "queue": "inbox",
                "resolved": false,
                "status": "open",
                "notes": [
                  "Unrelated request; preserve this ticket."
                ]
              }
            },
            "trace": [
              {
                "step": 0,
                "type": "tool_call",
                "action": {
                  "tool": "get_ticket",
                  "arguments": {
                    "ticket_id": "T-cd68615c"
                  }
                },
                "result": {
                  "ok": true,
                  "ticket": {
                    "category": "billing",
                    "queue": "inbox",
                    "resolved": true,
                    "status": "open",
                    "notes": [
                      "Customer contacted support."
                    ]
                  }
                },
                "status": "ok",
                "error_source": null,
                "changes": {}
              },
              {
                "step": 1,
                "type": "tool_call",
                "action": {
                  "tool": "assign_queue",
                  "arguments": {
                    "ticket_id": "T-cd68615c",
                    "queue": "queue-3-0"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-cd68615c": {
                    "before": {
                      "category": "billing",
                      "queue": "inbox",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    }
                  }
                }
              },
              {
                "step": 2,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-cd68615c",
                    "body": "Reviewed request T-cd68615c.",
                    "request_id": "review"
                  }
                },
                "result": {
                  "ok": false,
                  "error": {
                    "code": "temporarily_unavailable",
                    "retryable": true
                  }
                },
                "status": "tool_error",
                "error_source": "injected",
                "changes": {
                  "T-cd68615c": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-cd68615c."
                      ]
                    }
                  }
                }
              },
              {
                "step": 3,
                "type": "tool_call",
                "action": {
                  "tool": "add_note",
                  "arguments": {
                    "ticket_id": "T-cd68615c",
                    "body": "Reviewed request T-cd68615c.",
                    "request_id": "review-retry"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-cd68615c": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-cd68615c."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-cd68615c.",
                        "Reviewed request T-cd68615c."
                      ]
                    }
                  }
                }
              },
              {
                "step": 4,
                "type": "tool_call",
                "action": {
                  "tool": "close_ticket",
                  "arguments": {
                    "ticket_id": "T-cd68615c"
                  }
                },
                "result": {
                  "ok": true
                },
                "status": "ok",
                "error_source": null,
                "changes": {
                  "T-cd68615c": {
                    "before": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "open",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-cd68615c.",
                        "Reviewed request T-cd68615c."
                      ]
                    },
                    "after": {
                      "category": "billing",
                      "queue": "queue-3-0",
                      "resolved": true,
                      "status": "closed",
                      "notes": [
                        "Customer contacted support.",
                        "Reviewed request T-cd68615c.",
                        "Reviewed request T-cd68615c."
                      ]
                    }
                  }
                }
              },
              {
                "step": 5,
                "type": "finish",
                "action": {
                  "finish": true
                }
              }
            ],
            "tool_calls": 5
          }
        ],
        "evaluation_seconds": 2.361685,
        "agent_cost_usd": null,
        "agent_tokens": null
      }
    }
  ],
  "mutation_score": 1.0,
  "killed": 7,
  "total": 7,
  "passed": true,
  "interpretation": "Coverage of 7 declared behavioral fault models only. This is not a bound on reward hacking or a model benchmark."
}