| { |
| "arm": "rule", |
| "metrics": { |
| "action_accuracy": 0.716, |
| "argument_exact_match": 0.2785, |
| "counts": { |
| "clarify_expected": 26, |
| "predicted_tool_calls": 37, |
| "text_expected": 178, |
| "tool_expected": 79 |
| }, |
| "fabricated_response_rate": 0.0, |
| "gates": { |
| "argument_exact_match": { |
| "pass": false, |
| "target": 0.98, |
| "value": 0.2785 |
| }, |
| "fabricated_response_rate": { |
| "pass": true, |
| "target": 0.01, |
| "value": 0.0 |
| }, |
| "invalid_tool_call_rate": { |
| "pass": true, |
| "target": 0.01, |
| "value": 0.0 |
| }, |
| "missing_field_exact_match": { |
| "pass": false, |
| "target": 0.9, |
| "value": 0.1154 |
| }, |
| "response_coverage": { |
| "pass": false, |
| "target": 1.0, |
| "value": 0.9719 |
| }, |
| "task_success_rate": { |
| "pass": false, |
| "target": 0.95, |
| "value": 0.677 |
| }, |
| "tool_selection_accuracy": { |
| "pass": false, |
| "target": 0.99, |
| "value": 0.3924 |
| } |
| }, |
| "invalid_tool_call_rate": 0.0, |
| "missing_field_exact_match": 0.1154, |
| "n_records": 257, |
| "response_coverage": 0.9719, |
| "task_success_rate": 0.677, |
| "tool_selection_accuracy": 0.3924 |
| }, |
| "n_records": 257, |
| "per_family": { |
| "ABBREVIATIONS": { |
| "n": 1, |
| "success": 1.0 |
| }, |
| "ACCOUNT_HELP": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "BILLING_INVOICE": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "CLAIM_DAMAGE": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPANY_ABOUT": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPANY_ALLIANCE": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPANY_AVIATION": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPANY_CAREERS": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPANY_CERTIFICATIONS": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPANY_ECOMMERCE": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPANY_HEADQUARTERS": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPANY_HISTORY": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPANY_INDUSTRIES": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPANY_INTERNATIONAL": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPANY_LEADERSHIP": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPANY_NETWORK": { |
| "n": 2, |
| "success": 0.5 |
| }, |
| "COMPANY_PERFORMANCE": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPANY_REAL_OR_DEMO": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPANY_SIZE": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPANY_SUPPLY_CHAIN": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPANY_SUSTAINABILITY": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPANY_TECHNOLOGY": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPANY_WAREHOUSING": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "COMPLAINT_DELIVERY_ISSUE": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "CONFLICTING_INFO": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "CUSTOMS_DUTIES": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "DELIVERY_OPTIONS": { |
| "n": 1, |
| "success": 1.0 |
| }, |
| "GOODBYE": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "GREETING": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "GUIDE_CONDITIONS": { |
| "n": 4, |
| "success": 0.5 |
| }, |
| "GUIDE_DANGEROUS": { |
| "n": 4, |
| "success": 0.5 |
| }, |
| "GUIDE_DOCUMENTS": { |
| "n": 4, |
| "success": 0.5 |
| }, |
| "GUIDE_MISSING_TOPIC": { |
| "n": 2, |
| "success": 0.0 |
| }, |
| "GUIDE_OVERVIEW": { |
| "n": 4, |
| "success": 0.5 |
| }, |
| "GUIDE_PACKING": { |
| "n": 4, |
| "success": 1.0 |
| }, |
| "GUIDE_RESTRICTED": { |
| "n": 4, |
| "success": 1.0 |
| }, |
| "HELP_CAPABILITIES": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "HINGLISH": { |
| "n": 2, |
| "success": 0.5 |
| }, |
| "INSURANCE_COVERAGE": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "ITEM_ELIGIBILITY": { |
| "n": 2, |
| "success": 0.5 |
| }, |
| "LOCATION_BY_PIN": { |
| "n": 4, |
| "success": 0.75 |
| }, |
| "LOCATION_DIRECT": { |
| "n": 4, |
| "success": 0.75 |
| }, |
| "LOCATION_MISSING_PLACE": { |
| "n": 2, |
| "success": 0.5 |
| }, |
| "LOCATION_NOT_FOUND": { |
| "n": 4, |
| "success": 0.5 |
| }, |
| "LOCATION_SERVICE_FILTER": { |
| "n": 4, |
| "success": 0.5 |
| }, |
| "LOCATION_TIMEOUT": { |
| "n": 4, |
| "success": 0.5 |
| }, |
| "LOCATION_WRONG_PLACE_RECOVERY": { |
| "n": 8, |
| "success": 0.25 |
| }, |
| "MULTITURN_CORRECTION": { |
| "n": 2, |
| "success": 0.0 |
| }, |
| "MULTITURN_REFERENCE": { |
| "n": 2, |
| "success": 0.0 |
| }, |
| "MULTITURN_SLOT_FILL": { |
| "n": 6, |
| "success": 0.333 |
| }, |
| "MULTITURN_TOPIC_CHANGE": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "OUT_OF_SCOPE": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "PACKAGING_ADVICE": { |
| "n": 4, |
| "success": 1.0 |
| }, |
| "PICKUP_SCHEDULING": { |
| "n": 4, |
| "success": 0.75 |
| }, |
| "PRICING_POLICY": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "PROMPT_INJECTION": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "PROOF_OF_DELIVERY": { |
| "n": 4, |
| "success": 0.75 |
| }, |
| "QUOTE_DIRECT": { |
| "n": 4, |
| "success": 0.5 |
| }, |
| "QUOTE_INTERNATIONAL": { |
| "n": 4, |
| "success": 0.5 |
| }, |
| "QUOTE_INVALID_WEIGHT": { |
| "n": 2, |
| "success": 0.0 |
| }, |
| "QUOTE_MISSING_DESTINATION": { |
| "n": 2, |
| "success": 0.0 |
| }, |
| "QUOTE_MISSING_MULTIPLE": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "QUOTE_MISSING_ORIGIN": { |
| "n": 2, |
| "success": 0.0 |
| }, |
| "QUOTE_MISSING_WEIGHT": { |
| "n": 2, |
| "success": 0.0 |
| }, |
| "QUOTE_UNAVAILABLE": { |
| "n": 4, |
| "success": 0.5 |
| }, |
| "QUOTE_WRONG_WEIGHT_RECOVERY": { |
| "n": 6, |
| "success": 0.333 |
| }, |
| "REDELIVERY_ADDRESS_CHANGE": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "REQUEST_TO_FABRICATE": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "REQUEST_TO_GUESS": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "RETURNS_REVERSE": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "SERVICE_DETAIL": { |
| "n": 4, |
| "success": 0.5 |
| }, |
| "SERVICE_LIST": { |
| "n": 4, |
| "success": 0.5 |
| }, |
| "SERVICE_POINT_LOCKER": { |
| "n": 4, |
| "success": 0.75 |
| }, |
| "SMALLTALK": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "SUPPORT_CONTACT": { |
| "n": 4, |
| "success": 0.5 |
| }, |
| "SUPPORT_FAQ": { |
| "n": 4, |
| "success": 0.5 |
| }, |
| "SUPPORT_NEWS": { |
| "n": 4, |
| "success": 0.5 |
| }, |
| "THANKS": { |
| "n": 2, |
| "success": 1.0 |
| }, |
| "TOOL_INJECTION": { |
| "n": 2, |
| "success": 0.5 |
| }, |
| "TRACK_DIRECT": { |
| "n": 4, |
| "success": 1.0 |
| }, |
| "TRACK_INVALID_FORMAT": { |
| "n": 2, |
| "success": 0.0 |
| }, |
| "TRACK_MISSING_NUMBER": { |
| "n": 2, |
| "success": 0.0 |
| }, |
| "TRACK_MULTIPLE": { |
| "n": 4, |
| "success": 0.5 |
| }, |
| "TRACK_NOT_FOUND": { |
| "n": 4, |
| "success": 1.0 |
| }, |
| "TRACK_REFERENCE": { |
| "n": 4, |
| "success": 1.0 |
| }, |
| "TRACK_STATUS_CONTRADICTION": { |
| "n": 4, |
| "success": 1.0 |
| }, |
| "TRACK_TIMEOUT": { |
| "n": 4, |
| "success": 0.5 |
| }, |
| "TRACK_UNAVAILABLE": { |
| "n": 4, |
| "success": 0.5 |
| }, |
| "TRACK_WRONG_NUMBER_RECOVERY": { |
| "n": 8, |
| "success": 0.375 |
| }, |
| "TYPOS": { |
| "n": 1, |
| "success": 1.0 |
| } |
| } |
| } |
