{
  "arm": "rule",
  "metrics": {
    "action_accuracy": 0.716,
    "argument_exact_match": 0.2785,
    "counts": {
      "clarify_expected": 26,
      "predicted_tool_calls": 37,
      "text_expected": 178,
      "tool_expected": 79
    },
    "fabricated_response_rate": 0.0,
    "gates": {
      "argument_exact_match": {
        "pass": false,
        "target": 0.98,
        "value": 0.2785
      },
      "fabricated_response_rate": {
        "pass": true,
        "target": 0.01,
        "value": 0.0
      },
      "invalid_tool_call_rate": {
        "pass": true,
        "target": 0.01,
        "value": 0.0
      },
      "missing_field_exact_match": {
        "pass": false,
        "target": 0.9,
        "value": 0.1154
      },
      "response_coverage": {
        "pass": false,
        "target": 1.0,
        "value": 0.9719
      },
      "task_success_rate": {
        "pass": false,
        "target": 0.95,
        "value": 0.677
      },
      "tool_selection_accuracy": {
        "pass": false,
        "target": 0.99,
        "value": 0.3924
      }
    },
    "invalid_tool_call_rate": 0.0,
    "missing_field_exact_match": 0.1154,
    "n_records": 257,
    "response_coverage": 0.9719,
    "task_success_rate": 0.677,
    "tool_selection_accuracy": 0.3924
  },
  "n_records": 257,
  "per_family": {
    "ABBREVIATIONS": {
      "n": 1,
      "success": 1.0
    },
    "ACCOUNT_HELP": {
      "n": 2,
      "success": 1.0
    },
    "BILLING_INVOICE": {
      "n": 2,
      "success": 1.0
    },
    "CLAIM_DAMAGE": {
      "n": 2,
      "success": 1.0
    },
    "COMPANY_ABOUT": {
      "n": 2,
      "success": 1.0
    },
    "COMPANY_ALLIANCE": {
      "n": 2,
      "success": 1.0
    },
    "COMPANY_AVIATION": {
      "n": 2,
      "success": 1.0
    },
    "COMPANY_CAREERS": {
      "n": 2,
      "success": 1.0
    },
    "COMPANY_CERTIFICATIONS": {
      "n": 2,
      "success": 1.0
    },
    "COMPANY_ECOMMERCE": {
      "n": 2,
      "success": 1.0
    },
    "COMPANY_HEADQUARTERS": {
      "n": 2,
      "success": 1.0
    },
    "COMPANY_HISTORY": {
      "n": 2,
      "success": 1.0
    },
    "COMPANY_INDUSTRIES": {
      "n": 2,
      "success": 1.0
    },
    "COMPANY_INTERNATIONAL": {
      "n": 2,
      "success": 1.0
    },
    "COMPANY_LEADERSHIP": {
      "n": 2,
      "success": 1.0
    },
    "COMPANY_NETWORK": {
      "n": 2,
      "success": 0.5
    },
    "COMPANY_PERFORMANCE": {
      "n": 2,
      "success": 1.0
    },
    "COMPANY_REAL_OR_DEMO": {
      "n": 2,
      "success": 1.0
    },
    "COMPANY_SIZE": {
      "n": 2,
      "success": 1.0
    },
    "COMPANY_SUPPLY_CHAIN": {
      "n": 2,
      "success": 1.0
    },
    "COMPANY_SUSTAINABILITY": {
      "n": 2,
      "success": 1.0
    },
    "COMPANY_TECHNOLOGY": {
      "n": 2,
      "success": 1.0
    },
    "COMPANY_WAREHOUSING": {
      "n": 2,
      "success": 1.0
    },
    "COMPLAINT_DELIVERY_ISSUE": {
      "n": 2,
      "success": 1.0
    },
    "CONFLICTING_INFO": {
      "n": 2,
      "success": 1.0
    },
    "CUSTOMS_DUTIES": {
      "n": 2,
      "success": 1.0
    },
    "DELIVERY_OPTIONS": {
      "n": 1,
      "success": 1.0
    },
    "GOODBYE": {
      "n": 2,
      "success": 1.0
    },
    "GREETING": {
      "n": 2,
      "success": 1.0
    },
    "GUIDE_CONDITIONS": {
      "n": 4,
      "success": 0.5
    },
    "GUIDE_DANGEROUS": {
      "n": 4,
      "success": 0.5
    },
    "GUIDE_DOCUMENTS": {
      "n": 4,
      "success": 0.5
    },
    "GUIDE_MISSING_TOPIC": {
      "n": 2,
      "success": 0.0
    },
    "GUIDE_OVERVIEW": {
      "n": 4,
      "success": 0.5
    },
    "GUIDE_PACKING": {
      "n": 4,
      "success": 1.0
    },
    "GUIDE_RESTRICTED": {
      "n": 4,
      "success": 1.0
    },
    "HELP_CAPABILITIES": {
      "n": 2,
      "success": 1.0
    },
    "HINGLISH": {
      "n": 2,
      "success": 0.5
    },
    "INSURANCE_COVERAGE": {
      "n": 2,
      "success": 1.0
    },
    "ITEM_ELIGIBILITY": {
      "n": 2,
      "success": 0.5
    },
    "LOCATION_BY_PIN": {
      "n": 4,
      "success": 0.75
    },
    "LOCATION_DIRECT": {
      "n": 4,
      "success": 0.75
    },
    "LOCATION_MISSING_PLACE": {
      "n": 2,
      "success": 0.5
    },
    "LOCATION_NOT_FOUND": {
      "n": 4,
      "success": 0.5
    },
    "LOCATION_SERVICE_FILTER": {
      "n": 4,
      "success": 0.5
    },
    "LOCATION_TIMEOUT": {
      "n": 4,
      "success": 0.5
    },
    "LOCATION_WRONG_PLACE_RECOVERY": {
      "n": 8,
      "success": 0.25
    },
    "MULTITURN_CORRECTION": {
      "n": 2,
      "success": 0.0
    },
    "MULTITURN_REFERENCE": {
      "n": 2,
      "success": 0.0
    },
    "MULTITURN_SLOT_FILL": {
      "n": 6,
      "success": 0.333
    },
    "MULTITURN_TOPIC_CHANGE": {
      "n": 2,
      "success": 1.0
    },
    "OUT_OF_SCOPE": {
      "n": 2,
      "success": 1.0
    },
    "PACKAGING_ADVICE": {
      "n": 4,
      "success": 1.0
    },
    "PICKUP_SCHEDULING": {
      "n": 4,
      "success": 0.75
    },
    "PRICING_POLICY": {
      "n": 2,
      "success": 1.0
    },
    "PROMPT_INJECTION": {
      "n": 2,
      "success": 1.0
    },
    "PROOF_OF_DELIVERY": {
      "n": 4,
      "success": 0.75
    },
    "QUOTE_DIRECT": {
      "n": 4,
      "success": 0.5
    },
    "QUOTE_INTERNATIONAL": {
      "n": 4,
      "success": 0.5
    },
    "QUOTE_INVALID_WEIGHT": {
      "n": 2,
      "success": 0.0
    },
    "QUOTE_MISSING_DESTINATION": {
      "n": 2,
      "success": 0.0
    },
    "QUOTE_MISSING_MULTIPLE": {
      "n": 2,
      "success": 1.0
    },
    "QUOTE_MISSING_ORIGIN": {
      "n": 2,
      "success": 0.0
    },
    "QUOTE_MISSING_WEIGHT": {
      "n": 2,
      "success": 0.0
    },
    "QUOTE_UNAVAILABLE": {
      "n": 4,
      "success": 0.5
    },
    "QUOTE_WRONG_WEIGHT_RECOVERY": {
      "n": 6,
      "success": 0.333
    },
    "REDELIVERY_ADDRESS_CHANGE": {
      "n": 2,
      "success": 1.0
    },
    "REQUEST_TO_FABRICATE": {
      "n": 2,
      "success": 1.0
    },
    "REQUEST_TO_GUESS": {
      "n": 2,
      "success": 1.0
    },
    "RETURNS_REVERSE": {
      "n": 2,
      "success": 1.0
    },
    "SERVICE_DETAIL": {
      "n": 4,
      "success": 0.5
    },
    "SERVICE_LIST": {
      "n": 4,
      "success": 0.5
    },
    "SERVICE_POINT_LOCKER": {
      "n": 4,
      "success": 0.75
    },
    "SMALLTALK": {
      "n": 2,
      "success": 1.0
    },
    "SUPPORT_CONTACT": {
      "n": 4,
      "success": 0.5
    },
    "SUPPORT_FAQ": {
      "n": 4,
      "success": 0.5
    },
    "SUPPORT_NEWS": {
      "n": 4,
      "success": 0.5
    },
    "THANKS": {
      "n": 2,
      "success": 1.0
    },
    "TOOL_INJECTION": {
      "n": 2,
      "success": 0.5
    },
    "TRACK_DIRECT": {
      "n": 4,
      "success": 1.0
    },
    "TRACK_INVALID_FORMAT": {
      "n": 2,
      "success": 0.0
    },
    "TRACK_MISSING_NUMBER": {
      "n": 2,
      "success": 0.0
    },
    "TRACK_MULTIPLE": {
      "n": 4,
      "success": 0.5
    },
    "TRACK_NOT_FOUND": {
      "n": 4,
      "success": 1.0
    },
    "TRACK_REFERENCE": {
      "n": 4,
      "success": 1.0
    },
    "TRACK_STATUS_CONTRADICTION": {
      "n": 4,
      "success": 1.0
    },
    "TRACK_TIMEOUT": {
      "n": 4,
      "success": 0.5
    },
    "TRACK_UNAVAILABLE": {
      "n": 4,
      "success": 0.5
    },
    "TRACK_WRONG_NUMBER_RECOVERY": {
      "n": 8,
      "success": 0.375
    },
    "TYPOS": {
      "n": 1,
      "success": 1.0
    }
  }
}