reports/qwen35.json

{
  "model": ".cache/Qwen3.5-0.8B",
  "total_parameters": 852985920,
  "modified_weights": false,
  "device": "cpu",
  "cases": [
    {
      "category": "arithmetic",
      "prompt": "Return only the integer: 17 * 23.",
      "answer": "17 * 23 = 391\n",
      "passed": false,
      "input_tokens": 25,
      "output_tokens": 14,
      "seconds": 4.555993000045419,
      "tokens_per_second": 3.07287566066507,
      "ttft": null,
      "note": "Non-streaming adapter; TTFT not measured"
    },
    {
      "category": "json",
      "prompt": "Return only a JSON object with key \"ready\" and boolean value true.",
      "answer": "{\"ready\": true}\n",
      "passed": true,
      "input_tokens": 27,
      "output_tokens": 8,
      "seconds": 2.8135307000484318,
      "tokens_per_second": 2.8434024195514516,
      "ttft": null,
      "note": "Non-streaming adapter; TTFT not measured"
    },
    {
      "category": "ordering",
      "prompt": "Sort these integers ascending. Return only a JSON array: 19, -3, 7, 0.",
      "answer": "[\"-3\", \"0\", \"7\", \"19\"]\n",
      "passed": false,
      "input_tokens": 36,
      "output_tokens": 17,
      "seconds": 5.810469900025055,
      "tokens_per_second": 2.9257530445044893,
      "ttft": null,
      "note": "Non-streaming adapter; TTFT not measured"
    },
    {
      "category": "reasoning",
      "prompt": "Every raven is a bird. Some birds swim. Must every raven swim? Answer only yes or no.",
      "answer": "No\n",
      "passed": true,
      "input_tokens": 35,
      "output_tokens": 4,
      "seconds": 1.9602427000645548,
      "tokens_per_second": 2.0405636505460634,
      "ttft": null,
      "note": "Non-streaming adapter; TTFT not measured"
    }
  ],
  "pass_at_1": 0.5,
  "wilson_95": [
    0.15003570882017148,
    0.8499642911798285
  ],
  "latency_seconds": {
    "p50": 2.8135307000484318,
    "p95": 5.810469900025055,
    "p99": 5.810469900025055
  },
  "agent": {
    "status": "failed",
    "reason": "failure_budget",
    "trace": [
      {
        "state": "PLAN",
        "plan": "Read note.txt to understand the context for the summary."
      },
      {
        "state": "ACT_OBSERVE",
        "receipt": {
          "id": "e64f3ae7e2dd41fbaba3649677629f18-0",
          "tool": "shell.exec",
          "ok": false,
          "output": "",
          "elapsed_ms": 2.9220000142231584,
          "truncated": false,
          "exit_code": null,
          "error": "PermissionError: Required permission is disabled",
          "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        }
      },
      {
        "state": "PLAN",
        "plan": "Attempt to read note.txt to understand the context for the summary."
      },
      {
        "state": "ACT_OBSERVE",
        "receipt": {
          "id": "e64f3ae7e2dd41fbaba3649677629f18-1",
          "tool": "shell.exec",
          "ok": false,
          "output": "",
          "elapsed_ms": 1.3812999241054058,
          "truncated": false,
          "exit_code": null,
          "error": "PermissionError: Required permission is disabled",
          "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        }
      },
      {
        "state": "PLAN",
        "plan": "Attempt to read note.txt to understand the context for the summary."
      },
      {
        "state": "ACT_OBSERVE",
        "receipt": {
          "id": "e64f3ae7e2dd41fbaba3649677629f18-2",
          "tool": "shell.exec",
          "ok": false,
          "output": "",
          "elapsed_ms": 2.3052000906318426,
          "truncated": false,
          "exit_code": null,
          "error": "PermissionError: Required permission is disabled",
          "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        }
      }
    ]
  },
  "limitations": [
    "Public hand-authored smoke tests; contamination unknown",
    "Four cases cannot rank models",
    "Agent file task is not repository-level coding evaluation",
    "Non-streaming latency only",
    "No private data used"
  ]
}