File size: 25,348 Bytes
18a1638 | 1 2 | {"schema_version":"clw-reliability-lab/1","fixture_version":"2026-10-14.1","generated_at":"2026-10-15T05:46:52.225Z","source_sha256":"e35d7973d8b581c1fe3a1c9c70923c8a089e59b05861e1cf2c2110a7c6a1ae01","runtime":{"node":"v24.22.0","os":"linux","arch":"arm64","platform":"local","platform_evidence":"Runtime environment declaration; cloud deployment receipt is required to corroborate it."},"scope":"Synthetic official-SDK reference-client experiments. No language model, production agent, network fault injection, or cross-cloud performance ranking.","duration_ms":1714,"experiments":[{"id":"transient-recovery","title":"Give a transient failure a bounded recovery window","change":"One attempt → three attempts. The same tool fails twice, then succeeds.","tradeoff":"Extra attempts help this fixture, not permanent or non-idempotent failures.","before":{"schema_version":"1.0","scenario":"transient-error","client":"official-sdk-reference-client","connected":true,"outcome":"tool_error","policy":{"timeout_ms":2000,"max_attempts":1,"retry":"tool_error_only"},"attempts":[{"attempt":1,"outcome":"tool_error","elapsed_ms":5}],"transport_errors":0,"trace":{"messages":[{"role":"user","content":"Synthetic reference-client probe. No language model participated."},{"role":"assistant","content":null,"tool_calls":[{"id":"probe-1","type":"function","function":{"name":"lab_probe","arguments":"{\"message\":\"probe\"}"}}]},{"role":"tool","tool_call_id":"probe-1","content":"Injected temporary tool failure","is_error":true}]},"trace_sha256":"14806ab6057f6c8ee28a864697ebe38a634032f184ac1abc3288a21a48e1ceaa","clinic":{"schema_version":"1.0","format":"openai","container":"json","steps":[{"i":0,"role":"user","kind":"message","tool":null,"args":null,"text":"Synthetic reference-client probe. No language model participated.","tokens_in":null,"tokens_out":null,"error":false},{"i":1,"role":"assistant","kind":"tool_call","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"","tokens_in":null,"tokens_out":null,"error":false},{"i":2,"role":"tool","kind":"tool_result","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"Injected temporary tool failure","tokens_in":null,"tokens_out":null,"error":true}],"instructions":[],"findings":[{"id":"unhandled_tool_error","severity":"medium","steps":[2],"evidence":"lab_probe({\"message\":\"probe\"}) -> Injected temporary tool failure","explain":"lab_probe failed and the next tool call was the identical call, or the trace ended on the error. Retrying unchanged input rarely changes the outcome; read the error, then change the arguments, switch tools, or stop and report it."}],"usage":{"per_task":{"tokens_in":0,"tokens_out":0,"total":0,"model_calls":0,"complete":false},"per_day":[{"tasks":10,"tokens_in":0,"tokens_out":0,"total":0},{"tasks":1000,"tokens_in":0,"tokens_out":0,"total":0}],"price":null,"cost_per_task":null}}},"after":{"schema_version":"1.0","scenario":"transient-error","client":"official-sdk-reference-client","connected":true,"outcome":"success","policy":{"timeout_ms":2000,"max_attempts":3,"retry":"tool_error_only"},"attempts":[{"attempt":1,"outcome":"tool_error","elapsed_ms":3},{"attempt":2,"outcome":"tool_error","elapsed_ms":2},{"attempt":3,"outcome":"success","elapsed_ms":1}],"transport_errors":0,"trace":{"messages":[{"role":"user","content":"Synthetic reference-client probe. No language model participated."},{"role":"assistant","content":null,"tool_calls":[{"id":"probe-1","type":"function","function":{"name":"lab_probe","arguments":"{\"message\":\"probe\"}"}}]},{"role":"tool","tool_call_id":"probe-1","content":"Injected temporary tool failure","is_error":true},{"role":"assistant","content":null,"tool_calls":[{"id":"probe-2","type":"function","function":{"name":"lab_probe","arguments":"{\"message\":\"probe\"}"}}]},{"role":"tool","tool_call_id":"probe-2","content":"Injected temporary tool failure","is_error":true},{"role":"assistant","content":null,"tool_calls":[{"id":"probe-3","type":"function","function":{"name":"lab_probe","arguments":"{\"message\":\"probe\"}"}}]},{"role":"tool","tool_call_id":"probe-3","content":"Probe response: probe","is_error":false}]},"trace_sha256":"ce922a08104db02be5625a86a53afad9a5739c94b55b0d0c6da3ee3776b00fbc","clinic":{"schema_version":"1.0","format":"openai","container":"json","steps":[{"i":0,"role":"user","kind":"message","tool":null,"args":null,"text":"Synthetic reference-client probe. No language model participated.","tokens_in":null,"tokens_out":null,"error":false},{"i":1,"role":"assistant","kind":"tool_call","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"","tokens_in":null,"tokens_out":null,"error":false},{"i":2,"role":"tool","kind":"tool_result","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"Injected temporary tool failure","tokens_in":null,"tokens_out":null,"error":true},{"i":3,"role":"assistant","kind":"tool_call","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"","tokens_in":null,"tokens_out":null,"error":false},{"i":4,"role":"tool","kind":"tool_result","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"Injected temporary tool failure","tokens_in":null,"tokens_out":null,"error":true},{"i":5,"role":"assistant","kind":"tool_call","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"","tokens_in":null,"tokens_out":null,"error":false},{"i":6,"role":"tool","kind":"tool_result","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"Probe response: probe","tokens_in":null,"tokens_out":null,"error":false}],"instructions":[],"findings":[{"id":"tool_loop","severity":"high","steps":[1,3,5],"evidence":"lab_probe({\"message\":\"probe\"}) called 3 times","explain":"The agent called lab_probe with identical arguments 3 times. Repeating the same call expects a different answer from the same input; the usual fix is to feed the result back into the plan, or to stop after a set number of identical calls."},{"id":"unhandled_tool_error","severity":"medium","steps":[2,4],"evidence":"lab_probe({\"message\":\"probe\"}) -> Injected temporary tool failure","explain":"lab_probe failed (2 times here) and the next tool call was the identical call, or the trace ended on the error. Retrying unchanged input rarely changes the outcome; read the error, then change the arguments, switch tools, or stop and report it."}],"usage":{"per_task":{"tokens_in":0,"tokens_out":0,"total":0,"model_calls":0,"complete":false},"per_day":[{"tasks":10,"tokens_in":0,"tokens_out":0,"total":0},{"tasks":1000,"tokens_in":0,"tokens_out":0,"total":0}],"price":null,"cost_per_task":null}}},"expected":{"before":["tool_error",1],"after":["success",3]},"verification":{"pass":true,"checks":[{"name":"before outcome","pass":true,"expected":"tool_error","observed":"tool_error"},{"name":"before attempt count","pass":true,"expected":1,"observed":1},{"name":"before trace completeness","pass":true},{"name":"after outcome","pass":true,"expected":"success","observed":"success"},{"name":"after attempt count","pass":true,"expected":3,"observed":3},{"name":"after trace completeness","pass":true}]}},{"id":"permanent-failure","title":"Stop wasting retries on a permanently failing tool","change":"Five attempts → two attempts. The tool remains broken; the client stops earlier.","tradeoff":"This is bounded failure handling, not a repaired downstream service.","before":{"schema_version":"1.0","scenario":"tool-error","client":"official-sdk-reference-client","connected":true,"outcome":"tool_error","policy":{"timeout_ms":2000,"max_attempts":5,"retry":"tool_error_only"},"attempts":[{"attempt":1,"outcome":"tool_error","elapsed_ms":3},{"attempt":2,"outcome":"tool_error","elapsed_ms":3},{"attempt":3,"outcome":"tool_error","elapsed_ms":0},{"attempt":4,"outcome":"tool_error","elapsed_ms":0},{"attempt":5,"outcome":"tool_error","elapsed_ms":0}],"transport_errors":0,"trace":{"messages":[{"role":"user","content":"Synthetic reference-client probe. No language model participated."},{"role":"assistant","content":null,"tool_calls":[{"id":"probe-1","type":"function","function":{"name":"lab_probe","arguments":"{\"message\":\"probe\"}"}}]},{"role":"tool","tool_call_id":"probe-1","content":"Injected temporary tool failure","is_error":true},{"role":"assistant","content":null,"tool_calls":[{"id":"probe-2","type":"function","function":{"name":"lab_probe","arguments":"{\"message\":\"probe\"}"}}]},{"role":"tool","tool_call_id":"probe-2","content":"Injected temporary tool failure","is_error":true},{"role":"assistant","content":null,"tool_calls":[{"id":"probe-3","type":"function","function":{"name":"lab_probe","arguments":"{\"message\":\"probe\"}"}}]},{"role":"tool","tool_call_id":"probe-3","content":"Injected temporary tool failure","is_error":true},{"role":"assistant","content":null,"tool_calls":[{"id":"probe-4","type":"function","function":{"name":"lab_probe","arguments":"{\"message\":\"probe\"}"}}]},{"role":"tool","tool_call_id":"probe-4","content":"Injected temporary tool failure","is_error":true},{"role":"assistant","content":null,"tool_calls":[{"id":"probe-5","type":"function","function":{"name":"lab_probe","arguments":"{\"message\":\"probe\"}"}}]},{"role":"tool","tool_call_id":"probe-5","content":"Injected temporary tool failure","is_error":true}]},"trace_sha256":"6fe85ae2dbc3d2d2a2ca15b4513e471174d53e4c5fc202f3ae69865140dfda2e","clinic":{"schema_version":"1.0","format":"openai","container":"json","steps":[{"i":0,"role":"user","kind":"message","tool":null,"args":null,"text":"Synthetic reference-client probe. No language model participated.","tokens_in":null,"tokens_out":null,"error":false},{"i":1,"role":"assistant","kind":"tool_call","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"","tokens_in":null,"tokens_out":null,"error":false},{"i":2,"role":"tool","kind":"tool_result","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"Injected temporary tool failure","tokens_in":null,"tokens_out":null,"error":true},{"i":3,"role":"assistant","kind":"tool_call","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"","tokens_in":null,"tokens_out":null,"error":false},{"i":4,"role":"tool","kind":"tool_result","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"Injected temporary tool failure","tokens_in":null,"tokens_out":null,"error":true},{"i":5,"role":"assistant","kind":"tool_call","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"","tokens_in":null,"tokens_out":null,"error":false},{"i":6,"role":"tool","kind":"tool_result","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"Injected temporary tool failure","tokens_in":null,"tokens_out":null,"error":true},{"i":7,"role":"assistant","kind":"tool_call","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"","tokens_in":null,"tokens_out":null,"error":false},{"i":8,"role":"tool","kind":"tool_result","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"Injected temporary tool failure","tokens_in":null,"tokens_out":null,"error":true},{"i":9,"role":"assistant","kind":"tool_call","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"","tokens_in":null,"tokens_out":null,"error":false},{"i":10,"role":"tool","kind":"tool_result","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"Injected temporary tool failure","tokens_in":null,"tokens_out":null,"error":true}],"instructions":[],"findings":[{"id":"tool_loop","severity":"high","steps":[1,3,5,7,9],"evidence":"lab_probe({\"message\":\"probe\"}) called 5 times","explain":"The agent called lab_probe with identical arguments 5 times. Repeating the same call expects a different answer from the same input; the usual fix is to feed the result back into the plan, or to stop after a set number of identical calls."},{"id":"retry_storm","severity":"high","steps":[2,4,6,8,10],"evidence":"lab_probe failed 5 times; first error: Injected temporary tool failure","explain":"lab_probe returned an error 5 times in one task. Retries without backoff, a cap, or a change of input burn tokens and can hammer the service behind the tool."},{"id":"unhandled_tool_error","severity":"medium","steps":[2,4,6,8,10],"evidence":"lab_probe({\"message\":\"probe\"}) -> Injected temporary tool failure","explain":"lab_probe failed (5 times here) and the next tool call was the identical call, or the trace ended on the error. Retrying unchanged input rarely changes the outcome; read the error, then change the arguments, switch tools, or stop and report it."}],"usage":{"per_task":{"tokens_in":0,"tokens_out":0,"total":0,"model_calls":0,"complete":false},"per_day":[{"tasks":10,"tokens_in":0,"tokens_out":0,"total":0},{"tasks":1000,"tokens_in":0,"tokens_out":0,"total":0}],"price":null,"cost_per_task":null}}},"after":{"schema_version":"1.0","scenario":"tool-error","client":"official-sdk-reference-client","connected":true,"outcome":"tool_error","policy":{"timeout_ms":2000,"max_attempts":2,"retry":"tool_error_only"},"attempts":[{"attempt":1,"outcome":"tool_error","elapsed_ms":3},{"attempt":2,"outcome":"tool_error","elapsed_ms":2}],"transport_errors":0,"trace":{"messages":[{"role":"user","content":"Synthetic reference-client probe. No language model participated."},{"role":"assistant","content":null,"tool_calls":[{"id":"probe-1","type":"function","function":{"name":"lab_probe","arguments":"{\"message\":\"probe\"}"}}]},{"role":"tool","tool_call_id":"probe-1","content":"Injected temporary tool failure","is_error":true},{"role":"assistant","content":null,"tool_calls":[{"id":"probe-2","type":"function","function":{"name":"lab_probe","arguments":"{\"message\":\"probe\"}"}}]},{"role":"tool","tool_call_id":"probe-2","content":"Injected temporary tool failure","is_error":true}]},"trace_sha256":"d08501886fa2fe3eb0d29916f39e6fbc6b1fb418b29cca0dfd2f5a71ede6e81a","clinic":{"schema_version":"1.0","format":"openai","container":"json","steps":[{"i":0,"role":"user","kind":"message","tool":null,"args":null,"text":"Synthetic reference-client probe. No language model participated.","tokens_in":null,"tokens_out":null,"error":false},{"i":1,"role":"assistant","kind":"tool_call","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"","tokens_in":null,"tokens_out":null,"error":false},{"i":2,"role":"tool","kind":"tool_result","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"Injected temporary tool failure","tokens_in":null,"tokens_out":null,"error":true},{"i":3,"role":"assistant","kind":"tool_call","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"","tokens_in":null,"tokens_out":null,"error":false},{"i":4,"role":"tool","kind":"tool_result","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"Injected temporary tool failure","tokens_in":null,"tokens_out":null,"error":true}],"instructions":[],"findings":[{"id":"unhandled_tool_error","severity":"medium","steps":[2,4],"evidence":"lab_probe({\"message\":\"probe\"}) -> Injected temporary tool failure","explain":"lab_probe failed (2 times here) and the next tool call was the identical call, or the trace ended on the error. Retrying unchanged input rarely changes the outcome; read the error, then change the arguments, switch tools, or stop and report it."}],"usage":{"per_task":{"tokens_in":0,"tokens_out":0,"total":0,"model_calls":0,"complete":false},"per_day":[{"tasks":10,"tokens_in":0,"tokens_out":0,"total":0},{"tasks":1000,"tokens_in":0,"tokens_out":0,"total":0}],"price":null,"cost_per_task":null}}},"expected":{"before":["tool_error",5],"after":["tool_error",2]},"verification":{"pass":true,"checks":[{"name":"before outcome","pass":true,"expected":"tool_error","observed":"tool_error"},{"name":"before attempt count","pass":true,"expected":5,"observed":5},{"name":"before trace completeness","pass":true},{"name":"after outcome","pass":true,"expected":"tool_error","observed":"tool_error"},{"name":"after attempt count","pass":true,"expected":2,"observed":2},{"name":"after trace completeness","pass":true}]}},{"id":"deadline-fit","title":"Fit the deadline to the declared tool latency","change":"100 ms deadline → 2000 ms deadline for a tool with a 400 ms injected delay.","tradeoff":"The longer deadline increases the time spent waiting if the tool really hangs.","before":{"schema_version":"1.0","scenario":"delay","client":"official-sdk-reference-client","connected":true,"outcome":"timeout","policy":{"timeout_ms":100,"max_attempts":1,"retry":"tool_error_only"},"attempts":[{"attempt":1,"outcome":"timeout","elapsed_ms":102,"error_code":"REQUEST_TIMEOUT"}],"transport_errors":0,"trace":{"messages":[{"role":"user","content":"Synthetic reference-client probe. No language model participated."},{"role":"assistant","content":null,"tool_calls":[{"id":"probe-1","type":"function","function":{"name":"lab_probe","arguments":"{\"message\":\"probe\"}"}}]},{"role":"tool","tool_call_id":"probe-1","content":"Error: reference client observed timeout","is_error":true}]},"trace_sha256":"ca8d0b945d71b0c1814ae234d73f877fdb46b3ebc3a440e95d6c2114ecafbe2a","clinic":{"schema_version":"1.0","format":"openai","container":"json","steps":[{"i":0,"role":"user","kind":"message","tool":null,"args":null,"text":"Synthetic reference-client probe. No language model participated.","tokens_in":null,"tokens_out":null,"error":false},{"i":1,"role":"assistant","kind":"tool_call","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"","tokens_in":null,"tokens_out":null,"error":false},{"i":2,"role":"tool","kind":"tool_result","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"Error: reference client observed timeout","tokens_in":null,"tokens_out":null,"error":true}],"instructions":[],"findings":[{"id":"unhandled_tool_error","severity":"medium","steps":[2],"evidence":"lab_probe({\"message\":\"probe\"}) -> Error: reference client observed timeout","explain":"lab_probe failed and the next tool call was the identical call, or the trace ended on the error. Retrying unchanged input rarely changes the outcome; read the error, then change the arguments, switch tools, or stop and report it."}],"usage":{"per_task":{"tokens_in":0,"tokens_out":0,"total":0,"model_calls":0,"complete":false},"per_day":[{"tasks":10,"tokens_in":0,"tokens_out":0,"total":0},{"tasks":1000,"tokens_in":0,"tokens_out":0,"total":0}],"price":null,"cost_per_task":null}}},"after":{"schema_version":"1.0","scenario":"delay","client":"official-sdk-reference-client","connected":true,"outcome":"success","policy":{"timeout_ms":2000,"max_attempts":1,"retry":"tool_error_only"},"attempts":[{"attempt":1,"outcome":"success","elapsed_ms":404}],"transport_errors":0,"trace":{"messages":[{"role":"user","content":"Synthetic reference-client probe. No language model participated."},{"role":"assistant","content":null,"tool_calls":[{"id":"probe-1","type":"function","function":{"name":"lab_probe","arguments":"{\"message\":\"probe\"}"}}]},{"role":"tool","tool_call_id":"probe-1","content":"Probe response: probe","is_error":false}]},"trace_sha256":"45543eb60cd9bf030322aa827426f549c995b208427aa6168c0bc8c37390cbb9","clinic":{"schema_version":"1.0","format":"openai","container":"json","steps":[{"i":0,"role":"user","kind":"message","tool":null,"args":null,"text":"Synthetic reference-client probe. No language model participated.","tokens_in":null,"tokens_out":null,"error":false},{"i":1,"role":"assistant","kind":"tool_call","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"","tokens_in":null,"tokens_out":null,"error":false},{"i":2,"role":"tool","kind":"tool_result","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"Probe response: probe","tokens_in":null,"tokens_out":null,"error":false}],"instructions":[],"findings":[],"usage":{"per_task":{"tokens_in":0,"tokens_out":0,"total":0,"model_calls":0,"complete":false},"per_day":[{"tasks":10,"tokens_in":0,"tokens_out":0,"total":0},{"tasks":1000,"tokens_in":0,"tokens_out":0,"total":0}],"price":null,"cost_per_task":null}}},"expected":{"before":["timeout",1],"after":["success",1]},"verification":{"pass":true,"checks":[{"name":"before outcome","pass":true,"expected":"timeout","observed":"timeout"},{"name":"before attempt count","pass":true,"expected":1,"observed":1},{"name":"before trace completeness","pass":true},{"name":"after outcome","pass":true,"expected":"success","observed":"success"},{"name":"after attempt count","pass":true,"expected":1,"observed":1},{"name":"after trace completeness","pass":true}]}},{"id":"malformed-control","title":"Keep protocol failures terminal","change":"One-attempt budget → five-attempt budget; protocol errors must still stop after one call.","tradeoff":"Increasing a retry budget must not cause malformed responses to be accepted.","before":{"schema_version":"1.0","scenario":"malformed","client":"official-sdk-reference-client","connected":true,"outcome":"protocol_error","policy":{"timeout_ms":2000,"max_attempts":1,"retry":"tool_error_only"},"attempts":[{"attempt":1,"outcome":"protocol_error","elapsed_ms":7,"error_code":"INVALID_RESULT"}],"transport_errors":0,"trace":{"messages":[{"role":"user","content":"Synthetic reference-client probe. No language model participated."},{"role":"assistant","content":null,"tool_calls":[{"id":"probe-1","type":"function","function":{"name":"lab_probe","arguments":"{\"message\":\"probe\"}"}}]},{"role":"tool","tool_call_id":"probe-1","content":"Error: reference client observed protocol_error","is_error":true}]},"trace_sha256":"b01d8e246ec581b6d332074ed83420c64eddb36eaff010f3f056bf4dc52fec9d","clinic":{"schema_version":"1.0","format":"openai","container":"json","steps":[{"i":0,"role":"user","kind":"message","tool":null,"args":null,"text":"Synthetic reference-client probe. No language model participated.","tokens_in":null,"tokens_out":null,"error":false},{"i":1,"role":"assistant","kind":"tool_call","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"","tokens_in":null,"tokens_out":null,"error":false},{"i":2,"role":"tool","kind":"tool_result","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"Error: reference client observed protocol_error","tokens_in":null,"tokens_out":null,"error":true}],"instructions":[],"findings":[{"id":"unhandled_tool_error","severity":"medium","steps":[2],"evidence":"lab_probe({\"message\":\"probe\"}) -> Error: reference client observed protocol_e...","explain":"lab_probe failed and the next tool call was the identical call, or the trace ended on the error. Retrying unchanged input rarely changes the outcome; read the error, then change the arguments, switch tools, or stop and report it."}],"usage":{"per_task":{"tokens_in":0,"tokens_out":0,"total":0,"model_calls":0,"complete":false},"per_day":[{"tasks":10,"tokens_in":0,"tokens_out":0,"total":0},{"tasks":1000,"tokens_in":0,"tokens_out":0,"total":0}],"price":null,"cost_per_task":null}}},"after":{"schema_version":"1.0","scenario":"malformed","client":"official-sdk-reference-client","connected":true,"outcome":"protocol_error","policy":{"timeout_ms":2000,"max_attempts":5,"retry":"tool_error_only"},"attempts":[{"attempt":1,"outcome":"protocol_error","elapsed_ms":3,"error_code":"INVALID_RESULT"}],"transport_errors":0,"trace":{"messages":[{"role":"user","content":"Synthetic reference-client probe. No language model participated."},{"role":"assistant","content":null,"tool_calls":[{"id":"probe-1","type":"function","function":{"name":"lab_probe","arguments":"{\"message\":\"probe\"}"}}]},{"role":"tool","tool_call_id":"probe-1","content":"Error: reference client observed protocol_error","is_error":true}]},"trace_sha256":"b01d8e246ec581b6d332074ed83420c64eddb36eaff010f3f056bf4dc52fec9d","clinic":{"schema_version":"1.0","format":"openai","container":"json","steps":[{"i":0,"role":"user","kind":"message","tool":null,"args":null,"text":"Synthetic reference-client probe. No language model participated.","tokens_in":null,"tokens_out":null,"error":false},{"i":1,"role":"assistant","kind":"tool_call","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"","tokens_in":null,"tokens_out":null,"error":false},{"i":2,"role":"tool","kind":"tool_result","tool":"lab_probe","args":"{\"message\":\"probe\"}","text":"Error: reference client observed protocol_error","tokens_in":null,"tokens_out":null,"error":true}],"instructions":[],"findings":[{"id":"unhandled_tool_error","severity":"medium","steps":[2],"evidence":"lab_probe({\"message\":\"probe\"}) -> Error: reference client observed protocol_e...","explain":"lab_probe failed and the next tool call was the identical call, or the trace ended on the error. Retrying unchanged input rarely changes the outcome; read the error, then change the arguments, switch tools, or stop and report it."}],"usage":{"per_task":{"tokens_in":0,"tokens_out":0,"total":0,"model_calls":0,"complete":false},"per_day":[{"tasks":10,"tokens_in":0,"tokens_out":0,"total":0},{"tasks":1000,"tokens_in":0,"tokens_out":0,"total":0}],"price":null,"cost_per_task":null}}},"expected":{"before":["protocol_error",1],"after":["protocol_error",1]},"verification":{"pass":true,"checks":[{"name":"before outcome","pass":true,"expected":"protocol_error","observed":"protocol_error"},{"name":"before attempt count","pass":true,"expected":1,"observed":1},{"name":"before trace completeness","pass":true},{"name":"after outcome","pass":true,"expected":"protocol_error","observed":"protocol_error"},{"name":"after attempt count","pass":true,"expected":1,"observed":1},{"name":"after trace completeness","pass":true}]}}],"pass":true}
|