| { | |
| "benchmark": "ape_compliance", | |
| "eval_set": "C:\\Users\\jassi\\Desktop\\ProGPT\\models\\meltdown_q8\\benchmarks\\data\\meltdown_ape_eval50.jsonl", | |
| "scoring": "harness-aware (partial tool_call tags tolerated per PARSING_REFERENCE.md)", | |
| "n": 20, | |
| "passes": 12, | |
| "accuracy": 60.0, | |
| "by_category": { | |
| "action_first": { | |
| "passes": 0, | |
| "total": 1, | |
| "accuracy": 0.0 | |
| }, | |
| "chat_direct": { | |
| "passes": 0, | |
| "total": 4, | |
| "accuracy": 0.0 | |
| }, | |
| "chat_tool": { | |
| "passes": 0, | |
| "total": 1, | |
| "accuracy": 0.0 | |
| }, | |
| "code_edit": { | |
| "passes": 2, | |
| "total": 2, | |
| "accuracy": 100.0 | |
| }, | |
| "edge_case": { | |
| "passes": 1, | |
| "total": 2, | |
| "accuracy": 50.0 | |
| }, | |
| "file_edit": { | |
| "passes": 1, | |
| "total": 2, | |
| "accuracy": 50.0 | |
| }, | |
| "file_response": { | |
| "passes": 1, | |
| "total": 1, | |
| "accuracy": 100.0 | |
| }, | |
| "file_system": { | |
| "passes": 2, | |
| "total": 2, | |
| "accuracy": 100.0 | |
| }, | |
| "image_response": { | |
| "passes": 1, | |
| "total": 1, | |
| "accuracy": 100.0 | |
| }, | |
| "info_retrieval": { | |
| "passes": 2, | |
| "total": 2, | |
| "accuracy": 100.0 | |
| }, | |
| "multi_step": { | |
| "passes": 2, | |
| "total": 2, | |
| "accuracy": 100.0 | |
| } | |
| }, | |
| "results": [ | |
| { | |
| "index": 0, | |
| "category": "file_system", | |
| "user": "On Mac, create /Users/developer/Documents/iRun/Projects/scratch_21/output.txt", | |
| "expect_tool": true, | |
| "has_assess": true, | |
| "has_plan": true, | |
| "has_execute": true, | |
| "assess_closed": true, | |
| "plan_closed": true, | |
| "execute_closed": false, | |
| "ape_protocol": true, | |
| "tool_call_detected": true, | |
| "tool_call_complete": false, | |
| "tool_name_valid": true, | |
| "tool_json_parseable": true, | |
| "path_match": true, | |
| "pass": true, | |
| "tool_name": "wf", | |
| "note": "tool_call truncated by server stop (harness should continue)" | |
| }, | |
| { | |
| "index": 1, | |
| "category": "file_system", | |
| "user": "Replace OLD_API_KEY with NEW_API_KEY in C:\\Users\\jassi\\Documents\\ProGPT\\Billa\\.env", | |
| "expect_tool": true, | |
| "has_assess": true, | |
| "has_plan": true, | |
| "has_execute": true, | |
| "assess_closed": true, | |
| "plan_closed": true, | |
| "execute_closed": false, | |
| "ape_protocol": true, | |
| "tool_call_detected": true, | |
| "tool_call_complete": false, | |
| "tool_name_valid": true, | |
| "tool_json_parseable": true, | |
| "path_match": true, | |
| "pass": true, | |
| "tool_name": "ef", | |
| "note": "tool_call truncated by server stop (harness should continue)" | |
| }, | |
| { | |
| "index": 2, | |
| "category": "info_retrieval", | |
| "user": "What's the weather in Seattle right now?", | |
| "expect_tool": true, | |
| "has_assess": true, | |
| "has_plan": true, | |
| "has_execute": true, | |
| "assess_closed": true, | |
| "plan_closed": true, | |
| "execute_closed": false, | |
| "ape_protocol": true, | |
| "tool_call_detected": true, | |
| "tool_call_complete": false, | |
| "tool_name_valid": true, | |
| "tool_json_parseable": true, | |
| "path_match": true, | |
| "pass": true, | |
| "tool_name": "ws", | |
| "note": "tool_call truncated by server stop (harness should continue)" | |
| }, | |
| { | |
| "index": 3, | |
| "category": "info_retrieval", | |
| "user": "Read what's on my clipboard (Windows).", | |
| "expect_tool": true, | |
| "has_assess": true, | |
| "has_plan": true, | |
| "has_execute": true, | |
| "assess_closed": true, | |
| "plan_closed": true, | |
| "execute_closed": false, | |
| "ape_protocol": true, | |
| "tool_call_detected": true, | |
| "tool_call_complete": false, | |
| "tool_name_valid": true, | |
| "tool_json_parseable": true, | |
| "path_match": true, | |
| "pass": true, | |
| "tool_name": "cb", | |
| "note": "tool_call truncated by server stop (harness should continue)" | |
| }, | |
| { | |
| "index": 4, | |
| "category": "multi_step", | |
| "user": "Code check 9: py compute square, memo result, read memo back.", | |
| "expect_tool": true, | |
| "has_assess": true, | |
| "has_plan": true, | |
| "has_execute": true, | |
| "assess_closed": true, | |
| "plan_closed": true, | |
| "execute_closed": false, | |
| "ape_protocol": true, | |
| "tool_call_detected": true, | |
| "tool_call_complete": false, | |
| "tool_name_valid": true, | |
| "tool_json_parseable": true, | |
| "path_match": true, | |
| "pass": true, | |
| "tool_name": "py", | |
| "note": "tool_call truncated by server stop (harness should continue)" | |
| }, | |
| { | |
| "index": 5, | |
| "category": "multi_step", | |
| "user": "Backup workflow 8: timestamp, write backup_8.log, append completion line.", | |
| "expect_tool": true, | |
| "has_assess": true, | |
| "has_plan": true, | |
| "has_execute": true, | |
| "assess_closed": true, | |
| "plan_closed": true, | |
| "execute_closed": false, | |
| "ape_protocol": true, | |
| "tool_call_detected": true, | |
| "tool_call_complete": false, | |
| "tool_name_valid": true, | |
| "tool_json_parseable": true, | |
| "path_match": true, | |
| "pass": true, | |
| "tool_name": "wf", | |
| "note": "tool_call truncated by server stop (harness should continue)" | |
| }, | |
| { | |
| "index": 6, | |
| "category": "edge_case", | |
| "user": "Call unknown_tool with param foo=bar.", | |
| "expect_tool": true, | |
| "has_assess": true, | |
| "has_plan": true, | |
| "has_execute": true, | |
| "assess_closed": true, | |
| "plan_closed": true, | |
| "execute_closed": false, | |
| "ape_protocol": true, | |
| "tool_call_detected": false, | |
| "tool_call_complete": false, | |
| "tool_name_valid": false, | |
| "tool_json_parseable": false, | |
| "path_match": false, | |
| "pass": false, | |
| "tool_name": null | |
| }, | |
| { | |
| "index": 7, | |
| "category": "edge_case", | |
| "user": "Read C:\\Users\\jassi\\Documents\\ProGPT\\Billa\\secret_vault\\credentials.txt", | |
| "expect_tool": true, | |
| "has_assess": true, | |
| "has_plan": true, | |
| "has_execute": true, | |
| "assess_closed": true, | |
| "plan_closed": true, | |
| "execute_closed": false, | |
| "ape_protocol": true, | |
| "tool_call_detected": true, | |
| "tool_call_complete": false, | |
| "tool_name_valid": true, | |
| "tool_json_parseable": true, | |
| "path_match": true, | |
| "pass": true, | |
| "tool_name": "rf", | |
| "note": "tool_call truncated by server stop (harness should continue)" | |
| }, | |
| { | |
| "index": 8, | |
| "category": "code_edit", | |
| "user": "In C:\\Users\\jassi\\Documents\\ProGPT\\Billa\\model_24.py swap class name OldModel for NewModel.", | |
| "expect_tool": true, | |
| "has_assess": true, | |
| "has_plan": true, | |
| "has_execute": true, | |
| "assess_closed": true, | |
| "plan_closed": true, | |
| "execute_closed": false, | |
| "ape_protocol": true, | |
| "tool_call_detected": true, | |
| "tool_call_complete": false, | |
| "tool_name_valid": true, | |
| "tool_json_parseable": true, | |
| "path_match": true, | |
| "pass": true, | |
| "tool_name": "ef", | |
| "note": "tool_call truncated by server stop (harness should continue)" | |
| }, | |
| { | |
| "index": 9, | |
| "category": "code_edit", | |
| "user": "In C:\\Users\\jassi\\Documents\\ProGPT\\Billa\\src\\api_4.py, point api to production url.", | |
| "expect_tool": true, | |
| "has_assess": true, | |
| "has_plan": true, | |
| "has_execute": true, | |
| "assess_closed": true, | |
| "plan_closed": true, | |
| "execute_closed": false, | |
| "ape_protocol": true, | |
| "tool_call_detected": true, | |
| "tool_call_complete": false, | |
| "tool_name_valid": true, | |
| "tool_json_parseable": true, | |
| "path_match": true, | |
| "pass": true, | |
| "tool_name": "ef", | |
| "note": "tool_call truncated by server stop (harness should continue)" | |
| }, | |
| { | |
| "index": 10, | |
| "category": "file_edit", | |
| "user": "Overwrite C:\\Users\\jassi\\Documents\\ProGPT\\Billa\\app_18.config \u2014 set debug=true only.", | |
| "expect_tool": true, | |
| "has_assess": true, | |
| "has_plan": true, | |
| "has_execute": true, | |
| "assess_closed": true, | |
| "plan_closed": true, | |
| "execute_closed": false, | |
| "ape_protocol": true, | |
| "tool_call_detected": true, | |
| "tool_call_complete": false, | |
| "tool_name_valid": true, | |
| "tool_json_parseable": true, | |
| "path_match": true, | |
| "pass": true, | |
| "tool_name": "ef", | |
| "note": "tool_call truncated by server stop (harness should continue)" | |
| }, | |
| { | |
| "index": 11, | |
| "category": "file_edit", | |
| "user": "Mark doc section complete. in C:\\Users\\jassi\\Documents\\ProGPT\\Billa\\docs\\guide_3.md", | |
| "expect_tool": true, | |
| "has_assess": true, | |
| "has_plan": true, | |
| "has_execute": true, | |
| "assess_closed": true, | |
| "plan_closed": true, | |
| "execute_closed": false, | |
| "ape_protocol": true, | |
| "tool_call_detected": true, | |
| "tool_call_complete": false, | |
| "tool_name_valid": true, | |
| "tool_json_parseable": false, | |
| "path_match": true, | |
| "pass": false, | |
| "tool_name": "wf", | |
| "note": "tool_call truncated by server stop (harness should continue)" | |
| }, | |
| { | |
| "index": 12, | |
| "category": "file_response", | |
| "user": "Analyze C:\\Users\\jassi\\Documents\\ProGPT\\Billa\\metrics_10.csv \u2014 what's the trend?", | |
| "expect_tool": true, | |
| "has_assess": true, | |
| "has_plan": true, | |
| "has_execute": true, | |
| "assess_closed": true, | |
| "plan_closed": true, | |
| "execute_closed": false, | |
| "ape_protocol": true, | |
| "tool_call_detected": true, | |
| "tool_call_complete": false, | |
| "tool_name_valid": true, | |
| "tool_json_parseable": true, | |
| "path_match": true, | |
| "pass": true, | |
| "tool_name": "rc", | |
| "note": "tool_call truncated by server stop (harness should continue)" | |
| }, | |
| { | |
| "index": 13, | |
| "category": "image_response", | |
| "user": "[Attached image context]\nScreenshot of failing test: Expected '200' got '500' on /api/health.\n\nCheck the health handler.", | |
| "expect_tool": true, | |
| "has_assess": true, | |
| "has_plan": true, | |
| "has_execute": true, | |
| "assess_closed": true, | |
| "plan_closed": true, | |
| "execute_closed": false, | |
| "ape_protocol": true, | |
| "tool_call_detected": true, | |
| "tool_call_complete": false, | |
| "tool_name_valid": true, | |
| "tool_json_parseable": true, | |
| "path_match": true, | |
| "pass": true, | |
| "tool_name": "rc", | |
| "note": "tool_call truncated by server stop (harness should continue)" | |
| }, | |
| { | |
| "index": 14, | |
| "category": "action_first", | |
| "user": "Schedule a meeting tomorrow 3pm with the team.", | |
| "expect_tool": true, | |
| "has_assess": true, | |
| "has_plan": true, | |
| "has_execute": false, | |
| "assess_closed": true, | |
| "plan_closed": false, | |
| "execute_closed": false, | |
| "ape_protocol": false, | |
| "tool_call_detected": false, | |
| "tool_call_complete": false, | |
| "tool_name_valid": false, | |
| "tool_json_parseable": false, | |
| "path_match": false, | |
| "pass": false, | |
| "tool_name": null | |
| }, | |
| { | |
| "index": 15, | |
| "category": "chat_direct", | |
| "user": "Thanks for fixing that!", | |
| "expect_tool": false, | |
| "has_assess": false, | |
| "has_plan": false, | |
| "has_execute": true, | |
| "assess_closed": false, | |
| "plan_closed": false, | |
| "execute_closed": true, | |
| "ape_protocol": false, | |
| "tool_call_detected": false, | |
| "tool_call_complete": false, | |
| "tool_name_valid": false, | |
| "tool_json_parseable": false, | |
| "path_match": true, | |
| "pass": false, | |
| "tool_name": null | |
| }, | |
| { | |
| "index": 16, | |
| "category": "chat_direct", | |
| "user": "What does Response path mean in Meltdown?", | |
| "expect_tool": false, | |
| "has_assess": false, | |
| "has_plan": false, | |
| "has_execute": false, | |
| "assess_closed": false, | |
| "plan_closed": false, | |
| "execute_closed": false, | |
| "ape_protocol": false, | |
| "tool_call_detected": false, | |
| "tool_call_complete": false, | |
| "tool_name_valid": false, | |
| "tool_json_parseable": false, | |
| "path_match": true, | |
| "pass": false, | |
| "tool_name": null | |
| }, | |
| { | |
| "index": 17, | |
| "category": "chat_direct", | |
| "user": "Is my data sent to the cloud?", | |
| "expect_tool": false, | |
| "has_assess": true, | |
| "has_plan": true, | |
| "has_execute": true, | |
| "assess_closed": true, | |
| "plan_closed": true, | |
| "execute_closed": false, | |
| "ape_protocol": true, | |
| "tool_call_detected": true, | |
| "tool_call_complete": false, | |
| "tool_name_valid": true, | |
| "tool_json_parseable": true, | |
| "path_match": false, | |
| "pass": false, | |
| "tool_name": "mp" | |
| }, | |
| { | |
| "index": 18, | |
| "category": "chat_direct", | |
| "user": "Can you append to journals?", | |
| "expect_tool": false, | |
| "has_assess": true, | |
| "has_plan": true, | |
| "has_execute": true, | |
| "assess_closed": true, | |
| "plan_closed": true, | |
| "execute_closed": false, | |
| "ape_protocol": true, | |
| "tool_call_detected": true, | |
| "tool_call_complete": false, | |
| "tool_name_valid": true, | |
| "tool_json_parseable": true, | |
| "path_match": false, | |
| "pass": false, | |
| "tool_name": "wf" | |
| }, | |
| { | |
| "index": 19, | |
| "category": "chat_tool", | |
| "user": "Append line run_24 to C:\\Users\\jassi\\Documents\\ProGPT\\Billa\\journal.txt", | |
| "expect_tool": true, | |
| "has_assess": false, | |
| "has_plan": false, | |
| "has_execute": false, | |
| "assess_closed": false, | |
| "plan_closed": false, | |
| "execute_closed": false, | |
| "ape_protocol": false, | |
| "tool_call_detected": true, | |
| "tool_call_complete": false, | |
| "tool_name_valid": true, | |
| "tool_json_parseable": true, | |
| "path_match": true, | |
| "pass": false, | |
| "tool_name": "wf" | |
| } | |
| ] | |
| } |