{
  "benchmark": "ape_compliance",
  "eval_set": "C:\\Users\\jassi\\Desktop\\ProGPT\\models\\meltdown_q8\\benchmarks\\data\\meltdown_ape_eval50.jsonl",
  "scoring": "harness-aware (partial tool_call tags tolerated per PARSING_REFERENCE.md)",
  "n": 20,
  "passes": 12,
  "accuracy": 60.0,
  "by_category": {
    "action_first": {
      "passes": 0,
      "total": 1,
      "accuracy": 0.0
    },
    "chat_direct": {
      "passes": 0,
      "total": 4,
      "accuracy": 0.0
    },
    "chat_tool": {
      "passes": 0,
      "total": 1,
      "accuracy": 0.0
    },
    "code_edit": {
      "passes": 2,
      "total": 2,
      "accuracy": 100.0
    },
    "edge_case": {
      "passes": 1,
      "total": 2,
      "accuracy": 50.0
    },
    "file_edit": {
      "passes": 1,
      "total": 2,
      "accuracy": 50.0
    },
    "file_response": {
      "passes": 1,
      "total": 1,
      "accuracy": 100.0
    },
    "file_system": {
      "passes": 2,
      "total": 2,
      "accuracy": 100.0
    },
    "image_response": {
      "passes": 1,
      "total": 1,
      "accuracy": 100.0
    },
    "info_retrieval": {
      "passes": 2,
      "total": 2,
      "accuracy": 100.0
    },
    "multi_step": {
      "passes": 2,
      "total": 2,
      "accuracy": 100.0
    }
  },
  "results": [
    {
      "index": 0,
      "category": "file_system",
      "user": "On Mac, create /Users/developer/Documents/iRun/Projects/scratch_21/output.txt",
      "expect_tool": true,
      "has_assess": true,
      "has_plan": true,
      "has_execute": true,
      "assess_closed": true,
      "plan_closed": true,
      "execute_closed": false,
      "ape_protocol": true,
      "tool_call_detected": true,
      "tool_call_complete": false,
      "tool_name_valid": true,
      "tool_json_parseable": true,
      "path_match": true,
      "pass": true,
      "tool_name": "wf",
      "note": "tool_call truncated by server stop (harness should continue)"
    },
    {
      "index": 1,
      "category": "file_system",
      "user": "Replace OLD_API_KEY with NEW_API_KEY in C:\\Users\\jassi\\Documents\\ProGPT\\Billa\\.env",
      "expect_tool": true,
      "has_assess": true,
      "has_plan": true,
      "has_execute": true,
      "assess_closed": true,
      "plan_closed": true,
      "execute_closed": false,
      "ape_protocol": true,
      "tool_call_detected": true,
      "tool_call_complete": false,
      "tool_name_valid": true,
      "tool_json_parseable": true,
      "path_match": true,
      "pass": true,
      "tool_name": "ef",
      "note": "tool_call truncated by server stop (harness should continue)"
    },
    {
      "index": 2,
      "category": "info_retrieval",
      "user": "What's the weather in Seattle right now?",
      "expect_tool": true,
      "has_assess": true,
      "has_plan": true,
      "has_execute": true,
      "assess_closed": true,
      "plan_closed": true,
      "execute_closed": false,
      "ape_protocol": true,
      "tool_call_detected": true,
      "tool_call_complete": false,
      "tool_name_valid": true,
      "tool_json_parseable": true,
      "path_match": true,
      "pass": true,
      "tool_name": "ws",
      "note": "tool_call truncated by server stop (harness should continue)"
    },
    {
      "index": 3,
      "category": "info_retrieval",
      "user": "Read what's on my clipboard (Windows).",
      "expect_tool": true,
      "has_assess": true,
      "has_plan": true,
      "has_execute": true,
      "assess_closed": true,
      "plan_closed": true,
      "execute_closed": false,
      "ape_protocol": true,
      "tool_call_detected": true,
      "tool_call_complete": false,
      "tool_name_valid": true,
      "tool_json_parseable": true,
      "path_match": true,
      "pass": true,
      "tool_name": "cb",
      "note": "tool_call truncated by server stop (harness should continue)"
    },
    {
      "index": 4,
      "category": "multi_step",
      "user": "Code check 9: py compute square, memo result, read memo back.",
      "expect_tool": true,
      "has_assess": true,
      "has_plan": true,
      "has_execute": true,
      "assess_closed": true,
      "plan_closed": true,
      "execute_closed": false,
      "ape_protocol": true,
      "tool_call_detected": true,
      "tool_call_complete": false,
      "tool_name_valid": true,
      "tool_json_parseable": true,
      "path_match": true,
      "pass": true,
      "tool_name": "py",
      "note": "tool_call truncated by server stop (harness should continue)"
    },
    {
      "index": 5,
      "category": "multi_step",
      "user": "Backup workflow 8: timestamp, write backup_8.log, append completion line.",
      "expect_tool": true,
      "has_assess": true,
      "has_plan": true,
      "has_execute": true,
      "assess_closed": true,
      "plan_closed": true,
      "execute_closed": false,
      "ape_protocol": true,
      "tool_call_detected": true,
      "tool_call_complete": false,
      "tool_name_valid": true,
      "tool_json_parseable": true,
      "path_match": true,
      "pass": true,
      "tool_name": "wf",
      "note": "tool_call truncated by server stop (harness should continue)"
    },
    {
      "index": 6,
      "category": "edge_case",
      "user": "Call unknown_tool with param foo=bar.",
      "expect_tool": true,
      "has_assess": true,
      "has_plan": true,
      "has_execute": true,
      "assess_closed": true,
      "plan_closed": true,
      "execute_closed": false,
      "ape_protocol": true,
      "tool_call_detected": false,
      "tool_call_complete": false,
      "tool_name_valid": false,
      "tool_json_parseable": false,
      "path_match": false,
      "pass": false,
      "tool_name": null
    },
    {
      "index": 7,
      "category": "edge_case",
      "user": "Read C:\\Users\\jassi\\Documents\\ProGPT\\Billa\\secret_vault\\credentials.txt",
      "expect_tool": true,
      "has_assess": true,
      "has_plan": true,
      "has_execute": true,
      "assess_closed": true,
      "plan_closed": true,
      "execute_closed": false,
      "ape_protocol": true,
      "tool_call_detected": true,
      "tool_call_complete": false,
      "tool_name_valid": true,
      "tool_json_parseable": true,
      "path_match": true,
      "pass": true,
      "tool_name": "rf",
      "note": "tool_call truncated by server stop (harness should continue)"
    },
    {
      "index": 8,
      "category": "code_edit",
      "user": "In C:\\Users\\jassi\\Documents\\ProGPT\\Billa\\model_24.py swap class name OldModel for NewModel.",
      "expect_tool": true,
      "has_assess": true,
      "has_plan": true,
      "has_execute": true,
      "assess_closed": true,
      "plan_closed": true,
      "execute_closed": false,
      "ape_protocol": true,
      "tool_call_detected": true,
      "tool_call_complete": false,
      "tool_name_valid": true,
      "tool_json_parseable": true,
      "path_match": true,
      "pass": true,
      "tool_name": "ef",
      "note": "tool_call truncated by server stop (harness should continue)"
    },
    {
      "index": 9,
      "category": "code_edit",
      "user": "In C:\\Users\\jassi\\Documents\\ProGPT\\Billa\\src\\api_4.py, point api to production url.",
      "expect_tool": true,
      "has_assess": true,
      "has_plan": true,
      "has_execute": true,
      "assess_closed": true,
      "plan_closed": true,
      "execute_closed": false,
      "ape_protocol": true,
      "tool_call_detected": true,
      "tool_call_complete": false,
      "tool_name_valid": true,
      "tool_json_parseable": true,
      "path_match": true,
      "pass": true,
      "tool_name": "ef",
      "note": "tool_call truncated by server stop (harness should continue)"
    },
    {
      "index": 10,
      "category": "file_edit",
      "user": "Overwrite C:\\Users\\jassi\\Documents\\ProGPT\\Billa\\app_18.config \u2014 set debug=true only.",
      "expect_tool": true,
      "has_assess": true,
      "has_plan": true,
      "has_execute": true,
      "assess_closed": true,
      "plan_closed": true,
      "execute_closed": false,
      "ape_protocol": true,
      "tool_call_detected": true,
      "tool_call_complete": false,
      "tool_name_valid": true,
      "tool_json_parseable": true,
      "path_match": true,
      "pass": true,
      "tool_name": "ef",
      "note": "tool_call truncated by server stop (harness should continue)"
    },
    {
      "index": 11,
      "category": "file_edit",
      "user": "Mark doc section complete. in C:\\Users\\jassi\\Documents\\ProGPT\\Billa\\docs\\guide_3.md",
      "expect_tool": true,
      "has_assess": true,
      "has_plan": true,
      "has_execute": true,
      "assess_closed": true,
      "plan_closed": true,
      "execute_closed": false,
      "ape_protocol": true,
      "tool_call_detected": true,
      "tool_call_complete": false,
      "tool_name_valid": true,
      "tool_json_parseable": false,
      "path_match": true,
      "pass": false,
      "tool_name": "wf",
      "note": "tool_call truncated by server stop (harness should continue)"
    },
    {
      "index": 12,
      "category": "file_response",
      "user": "Analyze C:\\Users\\jassi\\Documents\\ProGPT\\Billa\\metrics_10.csv \u2014 what's the trend?",
      "expect_tool": true,
      "has_assess": true,
      "has_plan": true,
      "has_execute": true,
      "assess_closed": true,
      "plan_closed": true,
      "execute_closed": false,
      "ape_protocol": true,
      "tool_call_detected": true,
      "tool_call_complete": false,
      "tool_name_valid": true,
      "tool_json_parseable": true,
      "path_match": true,
      "pass": true,
      "tool_name": "rc",
      "note": "tool_call truncated by server stop (harness should continue)"
    },
    {
      "index": 13,
      "category": "image_response",
      "user": "[Attached image context]\nScreenshot of failing test: Expected '200' got '500' on /api/health.\n\nCheck the health handler.",
      "expect_tool": true,
      "has_assess": true,
      "has_plan": true,
      "has_execute": true,
      "assess_closed": true,
      "plan_closed": true,
      "execute_closed": false,
      "ape_protocol": true,
      "tool_call_detected": true,
      "tool_call_complete": false,
      "tool_name_valid": true,
      "tool_json_parseable": true,
      "path_match": true,
      "pass": true,
      "tool_name": "rc",
      "note": "tool_call truncated by server stop (harness should continue)"
    },
    {
      "index": 14,
      "category": "action_first",
      "user": "Schedule a meeting tomorrow 3pm with the team.",
      "expect_tool": true,
      "has_assess": true,
      "has_plan": true,
      "has_execute": false,
      "assess_closed": true,
      "plan_closed": false,
      "execute_closed": false,
      "ape_protocol": false,
      "tool_call_detected": false,
      "tool_call_complete": false,
      "tool_name_valid": false,
      "tool_json_parseable": false,
      "path_match": false,
      "pass": false,
      "tool_name": null
    },
    {
      "index": 15,
      "category": "chat_direct",
      "user": "Thanks for fixing that!",
      "expect_tool": false,
      "has_assess": false,
      "has_plan": false,
      "has_execute": true,
      "assess_closed": false,
      "plan_closed": false,
      "execute_closed": true,
      "ape_protocol": false,
      "tool_call_detected": false,
      "tool_call_complete": false,
      "tool_name_valid": false,
      "tool_json_parseable": false,
      "path_match": true,
      "pass": false,
      "tool_name": null
    },
    {
      "index": 16,
      "category": "chat_direct",
      "user": "What does Response path mean in Meltdown?",
      "expect_tool": false,
      "has_assess": false,
      "has_plan": false,
      "has_execute": false,
      "assess_closed": false,
      "plan_closed": false,
      "execute_closed": false,
      "ape_protocol": false,
      "tool_call_detected": false,
      "tool_call_complete": false,
      "tool_name_valid": false,
      "tool_json_parseable": false,
      "path_match": true,
      "pass": false,
      "tool_name": null
    },
    {
      "index": 17,
      "category": "chat_direct",
      "user": "Is my data sent to the cloud?",
      "expect_tool": false,
      "has_assess": true,
      "has_plan": true,
      "has_execute": true,
      "assess_closed": true,
      "plan_closed": true,
      "execute_closed": false,
      "ape_protocol": true,
      "tool_call_detected": true,
      "tool_call_complete": false,
      "tool_name_valid": true,
      "tool_json_parseable": true,
      "path_match": false,
      "pass": false,
      "tool_name": "mp"
    },
    {
      "index": 18,
      "category": "chat_direct",
      "user": "Can you append to journals?",
      "expect_tool": false,
      "has_assess": true,
      "has_plan": true,
      "has_execute": true,
      "assess_closed": true,
      "plan_closed": true,
      "execute_closed": false,
      "ape_protocol": true,
      "tool_call_detected": true,
      "tool_call_complete": false,
      "tool_name_valid": true,
      "tool_json_parseable": true,
      "path_match": false,
      "pass": false,
      "tool_name": "wf"
    },
    {
      "index": 19,
      "category": "chat_tool",
      "user": "Append line run_24 to C:\\Users\\jassi\\Documents\\ProGPT\\Billa\\journal.txt",
      "expect_tool": true,
      "has_assess": false,
      "has_plan": false,
      "has_execute": false,
      "assess_closed": false,
      "plan_closed": false,
      "execute_closed": false,
      "ape_protocol": false,
      "tool_call_detected": true,
      "tool_call_complete": false,
      "tool_name_valid": true,
      "tool_json_parseable": true,
      "path_match": true,
      "pass": false,
      "tool_name": "wf"
    }
  ]
}