File size: 4,231 Bytes
12496fc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 | {
"model": ".cache/Qwen3.5-0.8B",
"total_parameters": 852985920,
"modified_weights": false,
"device": "cpu",
"cases": [
{
"category": "arithmetic",
"prompt": "Return only the integer: 17 * 23.",
"answer": "17 * 23 = 391\n",
"passed": false,
"input_tokens": 25,
"output_tokens": 14,
"seconds": 4.555993000045419,
"tokens_per_second": 3.07287566066507,
"ttft": null,
"note": "Non-streaming adapter; TTFT not measured"
},
{
"category": "json",
"prompt": "Return only a JSON object with key \"ready\" and boolean value true.",
"answer": "{\"ready\": true}\n",
"passed": true,
"input_tokens": 27,
"output_tokens": 8,
"seconds": 2.8135307000484318,
"tokens_per_second": 2.8434024195514516,
"ttft": null,
"note": "Non-streaming adapter; TTFT not measured"
},
{
"category": "ordering",
"prompt": "Sort these integers ascending. Return only a JSON array: 19, -3, 7, 0.",
"answer": "[\"-3\", \"0\", \"7\", \"19\"]\n",
"passed": false,
"input_tokens": 36,
"output_tokens": 17,
"seconds": 5.810469900025055,
"tokens_per_second": 2.9257530445044893,
"ttft": null,
"note": "Non-streaming adapter; TTFT not measured"
},
{
"category": "reasoning",
"prompt": "Every raven is a bird. Some birds swim. Must every raven swim? Answer only yes or no.",
"answer": "No\n",
"passed": true,
"input_tokens": 35,
"output_tokens": 4,
"seconds": 1.9602427000645548,
"tokens_per_second": 2.0405636505460634,
"ttft": null,
"note": "Non-streaming adapter; TTFT not measured"
}
],
"pass_at_1": 0.5,
"wilson_95": [
0.15003570882017148,
0.8499642911798285
],
"latency_seconds": {
"p50": 2.8135307000484318,
"p95": 5.810469900025055,
"p99": 5.810469900025055
},
"agent": {
"status": "failed",
"reason": "failure_budget",
"trace": [
{
"state": "PLAN",
"plan": "Read note.txt to understand the context for the summary."
},
{
"state": "ACT_OBSERVE",
"receipt": {
"id": "e64f3ae7e2dd41fbaba3649677629f18-0",
"tool": "shell.exec",
"ok": false,
"output": "",
"elapsed_ms": 2.9220000142231584,
"truncated": false,
"exit_code": null,
"error": "PermissionError: Required permission is disabled",
"output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
}
},
{
"state": "PLAN",
"plan": "Attempt to read note.txt to understand the context for the summary."
},
{
"state": "ACT_OBSERVE",
"receipt": {
"id": "e64f3ae7e2dd41fbaba3649677629f18-1",
"tool": "shell.exec",
"ok": false,
"output": "",
"elapsed_ms": 1.3812999241054058,
"truncated": false,
"exit_code": null,
"error": "PermissionError: Required permission is disabled",
"output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
}
},
{
"state": "PLAN",
"plan": "Attempt to read note.txt to understand the context for the summary."
},
{
"state": "ACT_OBSERVE",
"receipt": {
"id": "e64f3ae7e2dd41fbaba3649677629f18-2",
"tool": "shell.exec",
"ok": false,
"output": "",
"elapsed_ms": 2.3052000906318426,
"truncated": false,
"exit_code": null,
"error": "PermissionError: Required permission is disabled",
"output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
}
}
]
},
"limitations": [
"Public hand-authored smoke tests; contamination unknown",
"Four cases cannot rank models",
"Agent file task is not repository-level coding evaluation",
"Non-streaming latency only",
"No private data used"
]
} |