evaluation.json

{
  "scope": "Tiny sanity check on operation families used in the original completion templates. NOT an unseen-task benchmark or a general coding accuracy claim. JavaScript/TypeScript tests check extracted arithmetic return expressions, not full-program correctness. HTML checks only balanced tags. No generated code was executed.",
  "summary": {
    "python": {
      "passed": 6,
      "total": 11
    },
    "javascript": {
      "passed": 7,
      "total": 11
    },
    "typescript": {
      "passed": 9,
      "total": 11
    },
    "html": {
      "balanced_tag_samples": 2,
      "total": 4
    }
  },
  "results": [
    {
      "language": "python",
      "task": "add",
      "passed_safe_arithmetic_checks": true,
      "prefix": "def add(a, b):\n",
      "continuation": "    return a + b\n\n",
      "error": ""
    },
    {
      "language": "python",
      "task": "subtract",
      "passed_safe_arithmetic_checks": true,
      "prefix": "def subtract(a, b):\n",
      "continuation": "    return a - b\n\n",
      "error": ""
    },
    {
      "language": "python",
      "task": "multiply",
      "passed_safe_arithmetic_checks": true,
      "prefix": "def multiply(a, b):\n",
      "continuation": "    return a * b\n\n",
      "error": ""
    },
    {
      "language": "python",
      "task": "average",
      "passed_safe_arithmetic_checks": true,
      "prefix": "def average(a, b):\n",
      "continuation": "    return (a + b) / 2\n\n",
      "error": ""
    },
    {
      "language": "python",
      "task": "maximum",
      "passed_safe_arithmetic_checks": true,
      "prefix": "def maximum(a, b):\n",
      "continuation": "    return a if a > b else b\n\n",
      "error": ""
    },
    {
      "language": "python",
      "task": "minimum",
      "passed_safe_arithmetic_checks": true,
      "prefix": "def minimum(a, b):\n",
      "continuation": "    return a if a < b else b\n\n",
      "error": ""
    },
    {
      "language": "python",
      "task": "square",
      "passed_safe_arithmetic_checks": false,
      "prefix": "def square(a):\n",
      "continuation": "    return a * b\n\n",
      "error": "Expression outside the safe arithmetic subset"
    },
    {
      "language": "python",
      "task": "absolute",
      "passed_safe_arithmetic_checks": false,
      "prefix": "def absolute(a):\n",
      "continuation": "    return abs(n)\n\n",
      "error": "Expression outside the safe arithmetic subset"
    },
    {
      "language": "python",
      "task": "increment",
      "passed_safe_arithmetic_checks": false,
      "prefix": "def increment(a):\n",
      "continuation": "    return a + b\n\n",
      "error": "Expression outside the safe arithmetic subset"
    },
    {
      "language": "python",
      "task": "decrement",
      "passed_safe_arithmetic_checks": false,
      "prefix": "def decrement(a):\n",
      "continuation": "    return a - b\n\n",
      "error": "Expression outside the safe arithmetic subset"
    },
    {
      "language": "python",
      "task": "is_even",
      "passed_safe_arithmetic_checks": false,
      "prefix": "def is_even(a):\n",
      "continuation": "    return a + b\n\n",
      "error": "Expression outside the safe arithmetic subset"
    },
    {
      "language": "javascript",
      "task": "add",
      "passed_safe_arithmetic_checks": true,
      "prefix": "function add(a, b) {\n",
      "continuation": "  return a + b;\n}\n\n",
      "error": ""
    },
    {
      "language": "javascript",
      "task": "subtract",
      "passed_safe_arithmetic_checks": true,
      "prefix": "function subtract(a, b) {\n",
      "continuation": "  return a - b;\n}\n\n",
      "error": ""
    },
    {
      "language": "javascript",
      "task": "multiply",
      "passed_safe_arithmetic_checks": true,
      "prefix": "function multiply(a, b) {\n",
      "continuation": "  return a * b;\n}\n\n",
      "error": ""
    },
    {
      "language": "javascript",
      "task": "average",
      "passed_safe_arithmetic_checks": true,
      "prefix": "function average(a, b) {\n",
      "continuation": "  return (a + b) / 2;\n}\n\n",
      "error": ""
    },
    {
      "language": "javascript",
      "task": "maximum",
      "passed_safe_arithmetic_checks": true,
      "prefix": "function maximum(a, b) {\n",
      "continuation": "  return a > b ? a : b;\n}\n\n",
      "error": ""
    },
    {
      "language": "javascript",
      "task": "minimum",
      "passed_safe_arithmetic_checks": true,
      "prefix": "function minimum(a, b) {\n",
      "continuation": "  return a < b ? a : b;\n}\n\n",
      "error": ""
    },
    {
      "language": "javascript",
      "task": "square",
      "passed_safe_arithmetic_checks": false,
      "prefix": "function square(a) {\n",
      "continuation": "  return a * y;\n}\n\n",
      "error": "Expression outside the safe arithmetic subset"
    },
    {
      "language": "javascript",
      "task": "absolute",
      "passed_safe_arithmetic_checks": false,
      "prefix": "function absolute(a) {\n",
      "continuation": "  return Math.abs(n);\n}\n\n",
      "error": "Expression outside the safe arithmetic subset"
    },
    {
      "language": "javascript",
      "task": "increment",
      "passed_safe_arithmetic_checks": false,
      "prefix": "function increment(a) {\n",
      "continuation": "  return a + b;\n}\n\n",
      "error": "Expression outside the safe arithmetic subset"
    },
    {
      "language": "javascript",
      "task": "decrement",
      "passed_safe_arithmetic_checks": true,
      "prefix": "function decrement(a) {\n",
      "continuation": "  return a - 1;\n}\n\n",
      "error": ""
    },
    {
      "language": "javascript",
      "task": "is_even",
      "passed_safe_arithmetic_checks": false,
      "prefix": "function is_even(a) {\n",
      "continuation": "  return a - left;\n}\n\n",
      "error": "Expression outside the safe arithmetic subset"
    },
    {
      "language": "typescript",
      "task": "add",
      "passed_safe_arithmetic_checks": true,
      "prefix": "function add(a: number, b: number): number {\n",
      "continuation": "  return a + b;\n}\n\n",
      "error": ""
    },
    {
      "language": "typescript",
      "task": "subtract",
      "passed_safe_arithmetic_checks": true,
      "prefix": "function subtract(a: number, b: number): number {\n",
      "continuation": "  return a - b;\n}\n\n",
      "error": ""
    },
    {
      "language": "typescript",
      "task": "multiply",
      "passed_safe_arithmetic_checks": true,
      "prefix": "function multiply(a: number, b: number): number {\n",
      "continuation": "  return a * b;\n}\n\n",
      "error": ""
    },
    {
      "language": "typescript",
      "task": "average",
      "passed_safe_arithmetic_checks": true,
      "prefix": "function average(a: number, b: number): number {\n",
      "continuation": "  return (a + b) / 2;\n}\n\n",
      "error": ""
    },
    {
      "language": "typescript",
      "task": "maximum",
      "passed_safe_arithmetic_checks": true,
      "prefix": "function maximum(a: number, b: number): number {\n",
      "continuation": "  return a > b ? a : b;\n}\n\n",
      "error": ""
    },
    {
      "language": "typescript",
      "task": "minimum",
      "passed_safe_arithmetic_checks": true,
      "prefix": "function minimum(a: number, b: number): number {\n",
      "continuation": "  return a < b ? a : b;\n}\n\n",
      "error": ""
    },
    {
      "language": "typescript",
      "task": "square",
      "passed_safe_arithmetic_checks": false,
      "prefix": "function square(a: number): number {\n",
      "continuation": "  return a * left;\n}\n\n",
      "error": "Expression outside the safe arithmetic subset"
    },
    {
      "language": "typescript",
      "task": "absolute",
      "passed_safe_arithmetic_checks": false,
      "prefix": "function absolute(a: number): number {\n",
      "continuation": "  return Math.abs(number);\n}\n\n",
      "error": "Expression outside the safe arithmetic subset"
    },
    {
      "language": "typescript",
      "task": "increment",
      "passed_safe_arithmetic_checks": true,
      "prefix": "function increment(a: number): number {\n",
      "continuation": "  return a + 1;\n}\n\n",
      "error": ""
    },
    {
      "language": "typescript",
      "task": "decrement",
      "passed_safe_arithmetic_checks": true,
      "prefix": "function decrement(a: number): number {\n",
      "continuation": "  return a - 1;\n}\n\n",
      "error": ""
    },
    {
      "language": "typescript",
      "task": "is_even",
      "passed_safe_arithmetic_checks": true,
      "prefix": "function is_even(a: number): boolean {\n",
      "continuation": "  return a % 2 === 0;\n}\n\n",
      "error": ""
    }
  ],
  "html_samples": [
    {
      "prefix": "<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n",
      "continuation": "  <meta charset=\"utf-8\">\n  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n  <title>Gallery 38</title>\n</head>\n<body>\n  <main>\n    <form action=\"/submit\" method=\"post\">\n      <label for=\"name\">Name</label>\n      <input id=\"name\" name=\"name\" type=\"text\" required>\n      <button type=\"submit\">Send</button>\n    </form>\n  </main>\n</body>\n</html>\n\n",
      "balanced_tags": true,
      "unclosed_tags": [],
      "errors": []
    },
    {
      "prefix": "<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n  <meta charset=\"utf-8\">\n  <title>Demo</title>\n</head>\n<body>\n",
      "continuation": "  <main>\n    <form action=\"/submit\" method=\"post\">\n      <label for=\"name\">Name</label>\n      <input id=\"name\" name=\"name\" type=\"text\" required>\n      <button type=\"submit\">Send</button>\n    </form>\n  </main>\n</body>\n</html>\n\n",
      "balanced_tags": true,
      "unclosed_tags": [],
      "errors": []
    },
    {
      "prefix": "<form action=\"/submit\" method=\"post\">\n",
      "continuation": "      <label for=\"name\">Name</label>\n      <input id=\"name\" name=\"name\" type=\"text\" required>\n      <button type=\"submit\">Send</button>\n    </form>\n  </main>\n</body>\n</html>\n\n",
      "balanced_tags": false,
      "unclosed_tags": [],
      "errors": [
        "Unexpected closing tag main",
        "Unexpected closing tag body",
        "Unexpected closing tag html"
      ]
    },
    {
      "prefix": "<ul>\n",
      "continuation": "  <li>One</li>\n      <li>Oranges</li>\n    </ul>\n  </main>\n</body>\n</html>\n\n",
      "balanced_tags": false,
      "unclosed_tags": [],
      "errors": [
        "Unexpected closing tag main",
        "Unexpected closing tag body",
        "Unexpected closing tag html"
      ]
    }
  ]
}