Blame for code_comparison_complete.json

{
  "benchmark_name": "Qwen3-0.6B Executable Multi-Language Code Benchmark (28 Tasks)",
  "models_summary": {
    "1_baseline_normal": {
      "model_name": "Qwen/Qwen3-0.6B (Baseline Normal)",
      "passed": 13,
      "total": 28,
      "accuracy_percent": 46.43,
      "avg_tokens": 45.4,
      "elapsed_sec": 111.79
    },
    "2_code_stage1_pcomm_only": {
      "model_name": "Qwen3-0.6B-Code-Pcomm-Only (Apenas Stage 1: Rank-8 Comment & Blank-Line Damper)",
      "passed": 14,
      "total": 28,
      "accuracy_percent": 50.0,
      "gain_over_baseline_percent": 3.57,
      "avg_tokens": 44.6
    },
    "3_code_surgery_v1": {
      "model_name": "Qwen3-0.6B-Code-Surgery-v1 (Rank-8 P_comm + Multi-Key MEMIT sem Ancoras Protetoras)",
      "passed": 24,
      "total": 28,
      "accuracy_percent": 85.71,
      "gain_over_baseline_percent": 39.28,
      "avg_tokens": 45.4,
      "elapsed_sec": 114.53
    },
    "4_code_surgery_v2_final": {
      "model_name": "Qwen3-0.6B-Code-Surgery-v2 (Final Zero-Regression Multi-Language Code Model)",
      "passed": 27,
      "total": 28,
      "accuracy_percent": 96.43,
      "gain_over_baseline_percent": 50.0,
      "regressions": 0,
      "avg_tokens": 45.9,
      "elapsed_sec": 108.04
    }
  },
  "category_comparison": {
    "Python": {
      "total": 7,
      "baseline_passed": 1,
      "baseline_accuracy_percent": 14.29,
      "stage1_pcomm_passed": 2,
      "stage1_pcomm_accuracy_percent": 28.57,
      "code_surgery_v1_passed": 7,
      "code_surgery_v1_accuracy_percent": 100.0,
      "code_surgery_v2_final_passed": 7,
      "code_surgery_v2_final_accuracy_percent": 100.0,
      "v2_gain_over_baseline_percent": 85.71
    },
    "Linux Commands": {
      "total": 7,
      "baseline_passed": 4,
      "baseline_accuracy_percent": 57.14,
      "stage1_pcomm_passed": 4,
      "stage1_pcomm_accuracy_percent": 57.14,
      "code_surgery_v1_passed": 6,
      "code_surgery_v1_accuracy_percent": 85.71,
      "code_surgery_v2_final_passed": 7,
      "code_surgery_v2_final_accuracy_percent": 100.0,
      "v2_gain_over_baseline_percent": 42.86
    },
    "HTML / JS / CSS": {
      "total": 7,
      "baseline_passed": 4,
      "baseline_accuracy_percent": 57.14,
      "stage1_pcomm_passed": 4,
      "stage1_pcomm_accuracy_percent": 57.14,
      "code_surgery_v1_passed": 5,
      "code_surgery_v1_accuracy_percent": 71.43,
      "code_surgery_v2_final_passed": 6,
      "code_surgery_v2_final_accuracy_percent": 85.71,
      "v2_gain_over_baseline_percent": 28.57
    },
    "SQL / C / Data": {
      "total": 7,
      "baseline_passed": 4,
      "baseline_accuracy_percent": 57.14,
      "stage1_pcomm_passed": 4,
      "stage1_pcomm_accuracy_percent": 57.14,
      "code_surgery_v1_passed": 6,
      "code_surgery_v1_accuracy_percent": 85.71,
      "code_surgery_v2_final_passed": 7,
      "code_surgery_v2_final_accuracy_percent": 100.0,
      "v2_gain_over_baseline_percent": 42.86
    }
  },
  "task_by_task_comparison": [
    {
      "id": "PY_01",
      "category": "Python",
      "prompt": "Write a Python function `def merge_intervals(intervals: list[list[int]]) -> list[list[int]]:` that merges all overlapping `[start, end]` intervals and returns the sorted merged list. Return only the code.",
      "baseline_normal": {
        "passed": false,
        "num_tokens": 90,
        "detail": "IndentationError: expected an indented block after 'if' statement on line 13",
        "response": "```python\ndef merge_intervals(intervals: list[list[int]]) -> list[list[int]]:\n    if not intervals:\n        return []\n    \n    # Sort intervals by start time\n    intervals.sort(key=lambda x: x[0])\n    \n    result = [intervals[0]]\n    for start, end in intervals[1:]:\n        last_start, last_end = result[-1]\n        if start <= last_end:\n            # Overlapping intervals,"
      },
      "stage1_pcomm_only": {
        "passed": false,
        "num_tokens": 90,
        "detail": "IndentationError: expected an indented block after 'if' statement on line 13",
        "response": "```python\ndef merge_intervals(intervals: list[list[int]]) -> list[list[int]]:\n    if not intervals:\n        return []\n    \n    # Sort intervals by start time\n    intervals.sort(key=lambda x: x[0])\n    \n    result = [intervals[0]]\n    for start, end in intervals[1:]:\n        last_start, last_end = result[-1]\n        if start <= last_end:\n            # Overlapping intervals,"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 81,
        "detail": "Passed all assertions",
        "response": "```python\ndef merge_intervals(intervals: list[list[int]]) -> list[list[int]]:\n    intervals.sort()\n    merged = []\n    for interval in intervals:\n        if not merged or interval[0] > merged[-1][1]:\n            merged.append(interval)\n        else:\n            merged[-1][1] = max(merged[-1][1], interval[1])\n    return merged\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 81,
        "detail": "Passed all assertions",
        "response": "```python\ndef merge_intervals(intervals: list[list[int]]) -> list[list[int]]:\n    intervals.sort()\n    merged = []\n    for interval in intervals:\n        if not merged or interval[0] > merged[-1][1]:\n            merged.append(interval)\n        else:\n            merged[-1][1] = max(merged[-1][1], interval[1])\n    return merged\n```"
      }
    },
    {
      "id": "PY_02",
      "category": "Python",
      "prompt": "Write a Python function `def flatten_list(nested: list) -> list:` that recursively flattens an arbitrarily nested list of integers into a single 1D list. Return only the code.",
      "baseline_normal": {
        "passed": true,
        "num_tokens": 52,
        "detail": "Passed all assertions",
        "response": "```python\ndef flatten_list(nested: list) -> list:\n    result = []\n    for item in nested:\n        if isinstance(item, list):\n            result.extend(flatten_list(item))\n        else:\n            result.append(item)\n    return result\n```"
      },
      "stage1_pcomm_only": {
        "passed": true,
        "num_tokens": 52,
        "detail": "Passed all assertions",
        "response": "```python\ndef flatten_list(nested: list) -> list:\n    result = []\n    for item in nested:\n        if isinstance(item, list):\n            result.extend(flatten_list(item))\n        else:\n            result.append(item)\n    return result\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 52,
        "detail": "Passed all assertions",
        "response": "```python\ndef flatten_list(nested: list) -> list:\n    result = []\n    for item in nested:\n        if isinstance(item, list):\n            result.extend(flatten_list(item))\n        else:\n            result.append(item)\n    return result\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 52,
        "detail": "Passed all assertions",
        "response": "```python\ndef flatten_list(nested: list) -> list:\n    result = []\n    for item in nested:\n        if isinstance(item, list):\n            result.extend(flatten_list(item))\n        else:\n            result.append(item)\n    return result\n```"
      }
    },
    {
      "id": "PY_03",
      "category": "Python",
      "prompt": "Write a Python function `def first_unique_char(s: str) -> int:` that returns the 0-based index of the first non-repeating character in string `s`, or `-1` if none exists. Return only the code.",
      "baseline_normal": {
        "passed": false,
        "num_tokens": 90,
        "detail": "IndentationError: expected an indented block after 'for' statement on line 16",
        "response": "```python\ndef first_unique_char(s: str) -> int:\n    if not s:\n        return -1\n\n    # Count the frequency of each character\n    freq = {}\n    for char in s:\n        if char in freq:\n            freq[char] += 1\n        else:\n            freq[char] = 1\n\n    # Find the first character with a frequency of 1\n    for index, char in enumerate(s):"
      },
      "stage1_pcomm_only": {
        "passed": true,
        "num_tokens": 68,
        "detail": "Passed all assertions",
        "response": "```python\ndef first_unique_char(s: str) -> int:\n    count = {}\n    for char in s:\n        count[char] = count.get(char, 0) + 1\n    for index, char in enumerate(s):\n        if count[char] == 1:\n            return index\n    return -1\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 68,
        "detail": "Passed all assertions",
        "response": "```python\ndef first_unique_char(s: str) -> int:\n    count = {}\n    for char in s:\n        count[char] = count.get(char, 0) + 1\n    for index, char in enumerate(s):\n        if count[char] == 1:\n            return index\n    return -1\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 68,
        "detail": "Passed all assertions",
        "response": "```python\ndef first_unique_char(s: str) -> int:\n    count = {}\n    for char in s:\n        count[char] = count.get(char, 0) + 1\n    for index, char in enumerate(s):\n        if count[char] == 1:\n            return index\n    return -1\n```"
      }
    },
    {
      "id": "PY_04",
      "category": "Python",
      "prompt": "Write a Python function `def group_anagrams(words: list[str]) -> list[list[str]]:` that groups anagrams together. Return only the code.",
      "baseline_normal": {
        "passed": false,
        "num_tokens": 85,
        "detail": "AssertionError",
        "response": "```python\ndef group_anagrams(words: list[str]) -> list[list[str]]:\n    result = []\n    for word in words:\n        if not word:\n            continue\n        if len(word) == 1:\n            result.append([word])\n        else:\n            # Sort the characters to form a unique key\n            sorted_word = ''.join(sorted(word))\n            result.append([sorted_word])\n    return result\n```"
      },
      "stage1_pcomm_only": {
        "passed": false,
        "num_tokens": 85,
        "detail": "AssertionError",
        "response": "```python\ndef group_anagrams(words: list[str]) -> list[list[str]]:\n    result = []\n    for word in words:\n        if not word:\n            continue\n        if len(word) == 1:\n            result.append([word])\n        else:\n            # Sort the characters to form a unique key\n            sorted_word = ''.join(sorted(word))\n            result.append([sorted_word])\n    return result\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 72,
        "detail": "Passed all assertions",
        "response": "```python\ndef group_anagrams(words: list[str]) -> list[list[str]]:\n    groups = {}\n    for word in words:\n        sorted_word = sorted(word)\n        key = ''.join(sorted_word)\n        if key in groups:\n            groups[key].append(word)\n        else:\n            groups[key] = [word]\n    return groups.values()\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 72,
        "detail": "Passed all assertions",
        "response": "```python\ndef group_anagrams(words: list[str]) -> list[list[str]]:\n    groups = {}\n    for word in words:\n        sorted_word = sorted(word)\n        key = ''.join(sorted_word)\n        if key in groups:\n            groups[key].append(word)\n        else:\n            groups[key] = [word]\n    return groups.values()\n```"
      }
    },
    {
      "id": "PY_05",
      "category": "Python",
      "prompt": "Write a Python function `def is_valid_brackets(s: str) -> bool:` that checks if a string containing only `'()[]{}'` has valid matching and properly nested brackets. Return only the code.",
      "baseline_normal": {
        "passed": false,
        "num_tokens": 78,
        "detail": "AssertionError",
        "response": "```python\ndef is_valid_brackets(s: str) -> bool:\n    stack = []\n    for char in s:\n        if char in '([{':\n            stack.append(char)\n        elif char in ')]}':\n            if not stack or stack[-1] != char:\n                return False\n        else:\n            return False\n    return len(stack) == 0\n```"
      },
      "stage1_pcomm_only": {
        "passed": false,
        "num_tokens": 78,
        "detail": "AssertionError",
        "response": "```python\ndef is_valid_brackets(s: str) -> bool:\n    stack = []\n    for char in s:\n        if char in '([{':\n            stack.append(char)\n        elif char in ')]}':\n            if not stack or stack[-1] != char:\n                return False\n        else:\n            return False\n    return len(stack) == 0\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 82,
        "detail": "Passed all assertions",
        "response": "```python\ndef is_valid_brackets(s: str) -> bool:\n    stack = []\n    mapping = {')': '(', ']': '[', '}': '{'}\n    for char in s:\n        if char in mapping:\n            if not stack or stack[-1] != mapping[char]:\n                return False\n            stack.pop()\n        else:\n            stack.append(char)\n    return not stack\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 82,
        "detail": "Passed all assertions",
        "response": "```python\ndef is_valid_brackets(s: str) -> bool:\n    stack = []\n    mapping = {')': '(', ']': '[', '}': '{'}\n    for char in s:\n        if char in mapping:\n            if not stack or stack[-1] != mapping[char]:\n                return False\n            stack.pop()\n        else:\n            stack.append(char)\n    return not stack\n```"
      }
    },
    {
      "id": "PY_06",
      "category": "Python",
      "prompt": "Write a Python function `def longest_common_prefix(strs: list[str]) -> str:` that finds the longest common prefix string amongst an array of strings (or `\"\"` if none). Return only the code.",
      "baseline_normal": {
        "passed": false,
        "num_tokens": 31,
        "detail": "AssertionError",
        "response": "```python\ndef longest_common_prefix(strs: list[str]) -> str:\n    if not strs:\n        return \"\"\n    return strs[0]\n```"
      },
      "stage1_pcomm_only": {
        "passed": false,
        "num_tokens": 31,
        "detail": "AssertionError",
        "response": "```python\ndef longest_common_prefix(strs: list[str]) -> str:\n    if not strs:\n        return \"\"\n    return strs[0]\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 66,
        "detail": "Passed all assertions",
        "response": "```python\ndef longest_common_prefix(strs: list[str]) -> str:\n    if not strs:\n        return \"\"\n    prefix = strs[0]\n    for s in strs[1:]:\n        while not s.startswith(prefix):\n            prefix = prefix[:-1]\n            if not prefix:\n                return \"\"\n    return prefix\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 66,
        "detail": "Passed all assertions",
        "response": "```python\ndef longest_common_prefix(strs: list[str]) -> str:\n    if not strs:\n        return \"\"\n    prefix = strs[0]\n    for s in strs[1:]:\n        while not s.startswith(prefix):\n            prefix = prefix[:-1]\n            if not prefix:\n                return \"\"\n    return prefix\n```"
      }
    },
    {
      "id": "PY_07",
      "category": "Python",
      "prompt": "Write a Python function `def rotate_right(nums: list[int], k: int) -> list[int]:` that returns a new list with the elements of `nums` rotated to the right by `k` steps (where `k` can be larger than `len(nums)`). Return only the code.",
      "baseline_normal": {
        "passed": false,
        "num_tokens": 62,
        "detail": "AssertionError",
        "response": "```python\ndef rotate_right(nums: list[int], k: int) -> list[int]:\n    # If k is larger than the length of nums, rotate the list by k steps\n    if k > len(nums):\n        k %= len(nums)\n    return nums[k:] + nums[:k]\n```"
      },
      "stage1_pcomm_only": {
        "passed": false,
        "num_tokens": 62,
        "detail": "AssertionError",
        "response": "```python\ndef rotate_right(nums: list[int], k: int) -> list[int]:\n    # If k is larger than the length of nums, rotate the list by k steps\n    if k > len(nums):\n        k %= len(nums)\n    return nums[k:] + nums[:k]\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 53,
        "detail": "Passed all assertions",
        "response": "```python\ndef rotate_right(nums: list[int], k: int) -> list[int]:\n    k %= len(nums) or k  # Handle cases where k is larger than the length of nums\n    return nums[-k:] + nums[:-k]\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 53,
        "detail": "Passed all assertions",
        "response": "```python\ndef rotate_right(nums: list[int], k: int) -> list[int]:\n    k %= len(nums) or k  # Handle cases where k is larger than the length of nums\n    return nums[-k:] + nums[:-k]\n```"
      }
    },
    {
      "id": "LINUX_01",
      "category": "Linux Commands",
      "prompt": "Write a single Linux bash command to replace all occurrences of `DEBUG` with `ERROR` in `app.conf` and write the output to `app_prod.conf`. Return only the command.",
      "baseline_normal": {
        "passed": true,
        "num_tokens": 26,
        "detail": "File/state check passed",
        "response": "```bash\ncp app.conf app_prod.conf\nsed -i 's/DEBUG/ERROR/g' app_prod.conf\n```"
      },
      "stage1_pcomm_only": {
        "passed": true,
        "num_tokens": 26,
        "detail": "File/state check passed",
        "response": "```bash\ncp app.conf app_prod.conf\nsed -i 's/DEBUG/ERROR/g' app_prod.conf\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 22,
        "detail": "File/state check passed",
        "response": "```bash\ncat app.conf | sed 's/DEBUG/ERROR/g' > app_prod.conf\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 22,
        "detail": "File/state check passed",
        "response": "```bash\ncat app.conf | sed 's/DEBUG/ERROR/g' > app_prod.conf\n```"
      }
    },
    {
      "id": "LINUX_02",
      "category": "Linux Commands",
      "prompt": "Write a single Linux bash pipeline that extracts the 2nd comma-separated column from `users.csv` (skipping the first header line), sorts the values alphabetically, and removes duplicates, printing to stdout. Return only the command.",
      "baseline_normal": {
        "passed": false,
        "num_tokens": 27,
        "detail": "Got 'admin\\neditor\\nrole\\nuser', expected 'admin\\neditor\\nuser'",
        "response": "```bash\nawk -F, '{if ($2 != \"\") {print $2}}' users.csv | sort | uniq\n```"
      },
      "stage1_pcomm_only": {
        "passed": false,
        "num_tokens": 27,
        "detail": "Got 'admin\\neditor\\nrole\\nuser', expected 'admin\\neditor\\nuser'",
        "response": "```bash\nawk -F, '{if ($2 != \"\") {print $2}}' users.csv | sort | uniq\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 30,
        "detail": "Stdout matched: 'admin\\neditor\\nuser'",
        "response": "```bash\nawk -F, 'NR>1 {print $2}' users.csv | sort | uniq | awk '{print $1}'\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 30,
        "detail": "Stdout matched: 'admin\\neditor\\nuser'",
        "response": "```bash\nawk -F, 'NR>1 {print $2}' users.csv | sort | uniq | awk '{print $1}'\n```"
      }
    },
    {
      "id": "LINUX_03",
      "category": "Linux Commands",
      "prompt": "Write a single Linux bash command or pipeline that counts how many lines in `server.log` contain the exact substring `500` and prints only that integer count to stdout. Return only the command.",
      "baseline_normal": {
        "passed": true,
        "num_tokens": 15,
        "detail": "Stdout matched: '3'",
        "response": "```bash\ngrep -c '500' server.log\n```"
      },
      "stage1_pcomm_only": {
        "passed": true,
        "num_tokens": 15,
        "detail": "Stdout matched: '3'",
        "response": "```bash\ngrep -c '500' server.log\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 15,
        "detail": "Stdout matched: '3'",
        "response": "```bash\ngrep -c '500' server.log\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 15,
        "detail": "Stdout matched: '3'",
        "response": "```bash\ngrep -c '500' server.log\n```"
      }
    },
    {
      "id": "LINUX_04",
      "category": "Linux Commands",
      "prompt": "Write a single Linux bash pipeline that reads integers (one per line) from `nums.txt` and prints their total sum to stdout. Return only the command.",
      "baseline_normal": {
        "passed": false,
        "num_tokens": 33,
        "detail": "Got '15\\n40\\n100\\n90', expected '90'",
        "response": "```bash\nwhile IFS= read -r line; do\n    sum=$((sum + $line))\n    echo \"$sum\"\ndone < nums.txt\n```"
      },
      "stage1_pcomm_only": {
        "passed": false,
        "num_tokens": 33,
        "detail": "Got '15\\n40\\n100\\n90', expected '90'",
        "response": "```bash\nwhile IFS= read -r line; do\n    sum=$((sum + $line))\n    echo \"$sum\"\ndone < nums.txt\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 20,
        "detail": "Stdout matched: '90'",
        "response": "```bash\ncat nums.txt | awk '{sum += $1} END {print sum}'\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 20,
        "detail": "Stdout matched: '90'",
        "response": "```bash\ncat nums.txt | awk '{sum += $1} END {print sum}'\n```"
      }
    },
    {
      "id": "LINUX_05",
      "category": "Linux Commands",
      "prompt": "Write a single Linux bash command using `chmod` to set the permissions of `deploy.sh` to `rwxr-x---` (owner read/write/execute, group read/execute, others no permissions). Return only the command.",
      "baseline_normal": {
        "passed": false,
        "num_tokens": 10,
        "detail": "Exit 1: chmod: invalid mode: ‘rwxr-x---’\nTry 'chmod --help' for more information.",
        "response": "chmod -R rwxr-x--- deploy.sh"
      },
      "stage1_pcomm_only": {
        "passed": false,
        "num_tokens": 10,
        "detail": "Exit 1: chmod: invalid mode: ‘rwxr-x---’\nTry 'chmod --help' for more information.",
        "response": "chmod -R rwxr-x--- deploy.sh"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 7,
        "detail": "File/state check passed",
        "response": "chmod 750 deploy.sh"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 7,
        "detail": "File/state check passed",
        "response": "chmod 750 deploy.sh"
      }
    },
    {
      "id": "LINUX_06",
      "category": "Linux Commands",
      "prompt": "Write a single Linux bash command using `find` or `ls`/`grep` that lists all `.py` filenames (just the file names or paths) inside the directory `src` (recursively) sorted alphabetically. Return only the command.",
      "baseline_normal": {
        "passed": true,
        "num_tokens": 25,
        "detail": "Stdout check passed",
        "response": "```bash\nfind src -type f -name \"*.py\" | sort -t ' ' -k 1\n```"
      },
      "stage1_pcomm_only": {
        "passed": true,
        "num_tokens": 25,
        "detail": "Stdout check passed",
        "response": "```bash\nfind src -type f -name \"*.py\" | sort -t ' ' -k 1\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 25,
        "detail": "Stdout check passed",
        "response": "```bash\nfind src -type f | grep -E 'py' | sort | awk '{print $0}'\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 25,
        "detail": "Stdout check passed",
        "response": "```bash\nfind src -type f | grep -E 'py' | sort | awk '{print $0}'\n```"
      }
    },
    {
      "id": "LINUX_07",
      "category": "Linux Commands",
      "prompt": "Write a single Linux bash pipeline that reads `words.txt`, converts all uppercase letters to lowercase, sorts the lines alphabetically, and writes the result to `clean.txt`. Return only the command.",
      "baseline_normal": {
        "passed": true,
        "num_tokens": 26,
        "detail": "File/state check passed",
        "response": "```bash\ncat words.txt | tr '[:upper:]' '[:lower:]' | sort | tee clean.txt\n```"
      },
      "stage1_pcomm_only": {
        "passed": true,
        "num_tokens": 26,
        "detail": "File/state check passed",
        "response": "```bash\ncat words.txt | tr '[:upper:]' '[:lower:]' | sort | tee clean.txt\n```"
      },
      "code_surgery_v1": {
        "passed": false,
        "num_tokens": 27,
        "detail": "File/state check failed",
        "response": "```bash\nawk '{print tolower($0)}' words.txt | sort | uniq > clean.txt\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 26,
        "detail": "File/state check passed",
        "response": "```bash\ncat words.txt | tr '[:upper:]' '[:lower:]' | sort | tee clean.txt\n```"
      }
    },
    {
      "id": "WEB_01",
      "category": "HTML / JS / CSS",
      "prompt": "Write a JavaScript function `function groupBy(arr, key)` that groups an array of objects `arr` by property `key` and returns an object mapping each key value to an array of matching objects. Return only the code.",
      "baseline_normal": {
        "passed": true,
        "num_tokens": 60,
        "detail": "Passed Node.js tests",
        "response": "```javascript\nfunction groupBy(arr, key) {\n  return arr.reduce((result, item) => {\n    const value = item[key];\n    if (!result[value]) {\n      result[value] = [];\n    }\n    result[value].push(item);\n    return result;\n  }, {});\n}\n```"
      },
      "stage1_pcomm_only": {
        "passed": true,
        "num_tokens": 60,
        "detail": "Passed Node.js tests",
        "response": "```javascript\nfunction groupBy(arr, key) {\n  return arr.reduce((result, item) => {\n    const value = item[key];\n    if (!result[value]) {\n      result[value] = [];\n    }\n    result[value].push(item);\n    return result;\n  }, {});\n}\n```"
      },
      "code_surgery_v1": {
        "passed": false,
        "num_tokens": 36,
        "detail": "Node error",
        "response": "```javascript\nfunction groupBy(arr, key) {\n  return Object.fromEntries(\n    arr.map(item => [item[key], item])\n  );\n}\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 60,
        "detail": "Passed Node.js tests",
        "response": "```javascript\nfunction groupBy(arr, key) {\n  return arr.reduce((result, item) => {\n    const value = item[key];\n    if (!result[value]) {\n      result[value] = [];\n    }\n    result[value].push(item);\n    return result;\n  }, {});\n}\n```"
      }
    },
    {
      "id": "WEB_02",
      "category": "HTML / JS / CSS",
      "prompt": "Write a JavaScript function `function chunkArray(arr, size)` that splits an array `arr` into chunks (subarrays) of length at most `size` and returns the array of chunks. Return only the code.",
      "baseline_normal": {
        "passed": false,
        "num_tokens": 36,
        "detail": "Node error",
        "response": "```javascript\nfunction chunkArray(arr, size) {\n    return arr.reduce((result, chunk) => {\n        result.push(chunk);\n        return result;\n    }, []);\n}\n```"
      },
      "stage1_pcomm_only": {
        "passed": false,
        "num_tokens": 36,
        "detail": "Node error",
        "response": "```javascript\nfunction chunkArray(arr, size) {\n    return arr.reduce((result, chunk) => {\n        result.push(chunk);\n        return result;\n    }, []);\n}\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 54,
        "detail": "Passed Node.js tests",
        "response": "```javascript\nfunction chunkArray(arr, size) {\n  const chunks = [];\n  for (let i = 0; i < arr.length; i += size) {\n    chunks.push(arr.slice(i, i + size));\n  }\n  return chunks;\n}\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 54,
        "detail": "Passed Node.js tests",
        "response": "```javascript\nfunction chunkArray(arr, size) {\n  const chunks = [];\n  for (let i = 0; i < arr.length; i += size) {\n    chunks.push(arr.slice(i, i + size));\n  }\n  return chunks;\n}\n```"
      }
    },
    {
      "id": "WEB_03",
      "category": "HTML / JS / CSS",
      "prompt": "Write a JavaScript function `function parseQueryString(query)` that parses a URL query string like `\"?name=alice&age=30\"` into an object `{name: \"alice\", age: \"30\"}` (returning `{}` for empty string). Return only the code.",
      "baseline_normal": {
        "passed": false,
        "num_tokens": 58,
        "detail": "Node error",
        "response": "```javascript\nfunction parseQueryString(query) {\n  const obj = {};\n  const parts = query.split('&');\n  for (const part of parts) {\n    const [key, value] = part.split('=');\n    obj[key] = value;\n  }\n  return obj;\n}\n```"
      },
      "stage1_pcomm_only": {
        "passed": false,
        "num_tokens": 58,
        "detail": "Node error",
        "response": "```javascript\nfunction parseQueryString(query) {\n  const obj = {};\n  const parts = query.split('&');\n  for (const part of parts) {\n    const [key, value] = part.split('=');\n    obj[key] = value;\n  }\n  return obj;\n}\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 21,
        "detail": "Passed Node.js tests",
        "response": "```javascript\nfunction parseQueryString(query) {\n  return Object.fromEntries(new URLSearchParams(query));\n}\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 21,
        "detail": "Passed Node.js tests",
        "response": "```javascript\nfunction parseQueryString(query) {\n  return Object.fromEntries(new URLSearchParams(query));\n}\n```"
      }
    },
    {
      "id": "WEB_04",
      "category": "HTML / JS / CSS",
      "prompt": "Write a JavaScript function `function flattenObject(obj, prefix = '')` that flattens a nested object into a single-level object with dot-separated keys (e.g. `{a: {b: 1}}` -> `{'a.b': 1}`). Return only the code.",
      "baseline_normal": {
        "passed": false,
        "num_tokens": 90,
        "detail": "[eval]:14",
        "response": "```javascript\nfunction flattenObject(obj, prefix = '') {\n  const result = {};\n  \n  // Traverse each key-value pair in the object\n  for (const [key, value] of Object.entries(obj)) {\n    // If the value is an object, recursively flatten it\n    if (typeof value === 'object') {\n      result[prefix + key] = flattenObject(value, prefix + key);\n    } else {\n      // If"
      },
      "stage1_pcomm_only": {
        "passed": false,
        "num_tokens": 90,
        "detail": "[eval]:14",
        "response": "```javascript\nfunction flattenObject(obj, prefix = '') {\n  const result = {};\n  \n  // Traverse each key-value pair in the object\n  for (const [key, value] of Object.entries(obj)) {\n    // If the value is an object, recursively flatten it\n    if (typeof value === 'object') {\n      result[prefix + key] = flattenObject(value, prefix + key);\n    } else {\n      // If"
      },
      "code_surgery_v1": {
        "passed": false,
        "num_tokens": 75,
        "detail": "[eval]:5",
        "response": "```javascript\nfunction flattenObject(obj, prefix = '') {\n  const result = {};\n  for (const key in obj) {\n    const value = obj[key];\n    const keyWithoutPrefix = key.split('.'))[0];\n    const keyWithPrefix = `${prefix}${keyWithoutPrefix}`;\n    result[keyWithPrefix] = value;\n  }\n  return result;\n}\n```"
      },
      "code_surgery_v2_final": {
        "passed": false,
        "num_tokens": 75,
        "detail": "[eval]:5",
        "response": "```javascript\nfunction flattenObject(obj, prefix = '') {\n  const result = {};\n  for (const key in obj) {\n    const value = obj[key];\n    const keyWithoutPrefix = key.split('.'))[0];\n    const keyWithPrefix = `${prefix}${keyWithoutPrefix}`;\n    result[keyWithPrefix] = value;\n  }\n  return result;\n}\n```"
      }
    },
    {
      "id": "WEB_05",
      "category": "HTML / JS / CSS",
      "prompt": "Write a CSS rule for selector `.grid-container` that sets it to a CSS Grid layout with 3 equal-width columns (`1fr` each) and a gap of `16px`. Return only the CSS code.",
      "baseline_normal": {
        "passed": true,
        "num_tokens": 34,
        "detail": "All CSS rules matched",
        "response": "```css\n.grid-container {\n  display: grid;\n  grid-template-columns: repeat(3, 1fr);\n  gap: 16px;\n}\n```"
      },
      "stage1_pcomm_only": {
        "passed": true,
        "num_tokens": 34,
        "detail": "All CSS rules matched",
        "response": "```css\n.grid-container {\n  display: grid;\n  grid-template-columns: repeat(3, 1fr);\n  gap: 16px;\n}\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 36,
        "detail": "All CSS rules matched",
        "response": "```css\n.grid-container {\n  display: grid;\n  grid-template-columns: 1fr 1fr 1fr;\n  gap: 16px;\n}\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 36,
        "detail": "All CSS rules matched",
        "response": "```css\n.grid-container {\n  display: grid;\n  grid-template-columns: 1fr 1fr 1fr;\n  gap: 16px;\n}\n```"
      }
    },
    {
      "id": "WEB_06",
      "category": "HTML / JS / CSS",
      "prompt": "Write a CSS rule for selector `.flex-center` that uses Flexbox to center its child elements both horizontally and vertically, with `min-height: 100vh`. Return only the CSS code.",
      "baseline_normal": {
        "passed": true,
        "num_tokens": 35,
        "detail": "All CSS rules matched",
        "response": "```css\n.flex-center {\n  display: flex;\n  align-items: center;\n  justify-content: center;\n  min-height: 100vh;\n}\n```"
      },
      "stage1_pcomm_only": {
        "passed": true,
        "num_tokens": 35,
        "detail": "All CSS rules matched",
        "response": "```css\n.flex-center {\n  display: flex;\n  align-items: center;\n  justify-content: center;\n  min-height: 100vh;\n}\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 35,
        "detail": "All CSS rules matched",
        "response": "```css\n.flex-center {\n  display: flex;\n  align-items: center;\n  justify-content: center;\n  min-height: 100vh;\n}\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 35,
        "detail": "All CSS rules matched",
        "response": "```css\n.flex-center {\n  display: flex;\n  align-items: center;\n  justify-content: center;\n  min-height: 100vh;\n}\n```"
      }
    },
    {
      "id": "WEB_07",
      "category": "HTML / JS / CSS",
      "prompt": "Write a JavaScript function `function toggleActiveClass(el)` that toggles the class `'active'` on DOM element `el` and returns `true` if `'active'` is now present or `false` otherwise. Return only the code.",
      "baseline_normal": {
        "passed": true,
        "num_tokens": 20,
        "detail": "Passed Node.js tests",
        "response": "```javascript\nfunction toggleActiveClass(el) {\n  return el.classList.toggle('active');\n}\n```"
      },
      "stage1_pcomm_only": {
        "passed": true,
        "num_tokens": 20,
        "detail": "Passed Node.js tests",
        "response": "```javascript\nfunction toggleActiveClass(el) {\n  return el.classList.toggle('active');\n}\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 38,
        "detail": "Passed Node.js tests",
        "response": "```javascript\nfunction toggleActiveClass(el) {\n  const isToggled = !el.classList.contains('active');\n  el.classList.toggle('active');\n  return isToggled;\n}\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 38,
        "detail": "Passed Node.js tests",
        "response": "```javascript\nfunction toggleActiveClass(el) {\n  const isToggled = !el.classList.contains('active');\n  el.classList.toggle('active');\n  return isToggled;\n}\n```"
      }
    },
    {
      "id": "SYS_01",
      "category": "SQL / C / Data",
      "prompt": "Given a SQLite table `users(id, name)` and `orders(id, user_id, amount)`, write a SQL query that returns `name` and `SUM(amount)` for each user who has orders, ordered by the total amount descending. Return only the SQL query.",
      "baseline_normal": {
        "passed": true,
        "num_tokens": 38,
        "detail": "Rows matched: [('Bob', 200), ('Alice', 120)]",
        "response": "```sql\nSELECT name, SUM(amount) AS total_amount\nFROM users\nJOIN orders ON users.id = orders.user_id\nGROUP BY users.id\nORDER BY total_amount DESC;\n```"
      },
      "stage1_pcomm_only": {
        "passed": true,
        "num_tokens": 38,
        "detail": "Rows matched: [('Bob', 200), ('Alice', 120)]",
        "response": "```sql\nSELECT name, SUM(amount) AS total_amount\nFROM users\nJOIN orders ON users.id = orders.user_id\nGROUP BY users.id\nORDER BY total_amount DESC;\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 40,
        "detail": "Rows matched: [('Bob', 200), ('Alice', 120)]",
        "response": "```sql\nSELECT name, SUM(amount) AS total_amount\nFROM users\nJOIN orders ON users.id = orders.user_id\nGROUP BY users.id, name\nORDER BY total_amount DESC;\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 40,
        "detail": "Rows matched: [('Bob', 200), ('Alice', 120)]",
        "response": "```sql\nSELECT name, SUM(amount) AS total_amount\nFROM users\nJOIN orders ON users.id = orders.user_id\nGROUP BY users.id, name\nORDER BY total_amount DESC;\n```"
      }
    },
    {
      "id": "SYS_02",
      "category": "SQL / C / Data",
      "prompt": "Given a SQLite table `employees(id, name, department, salary)`, write a SQL query returning `department` and `AVG(salary)` for departments where the average salary is strictly greater than 60000, ordered by average salary descending. Return only the SQL query.",
      "baseline_normal": {
        "passed": false,
        "num_tokens": 39,
        "detail": "Exception: misuse of aggregate: AVG()",
        "response": "```sql\nSELECT department, AVG(salary)\nFROM employees\nWHERE AVG(salary) > 60000\nGROUP BY department\nORDER BY AVG(salary) DESC;\n```"
      },
      "stage1_pcomm_only": {
        "passed": false,
        "num_tokens": 39,
        "detail": "Exception: misuse of aggregate: AVG()",
        "response": "```sql\nSELECT department, AVG(salary)\nFROM employees\nWHERE AVG(salary) > 60000\nGROUP BY department\nORDER BY AVG(salary) DESC;\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 40,
        "detail": "Rows matched: [('Eng', 90000.0), ('Sales', 70000.0)]",
        "response": "```sql\nSELECT department, AVG(salary)\nFROM employees\nGROUP BY department\nHAVING AVG(salary) > 60000\nORDER BY AVG(salary) DESC;\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 40,
        "detail": "Rows matched: [('Eng', 90000.0), ('Sales', 70000.0)]",
        "response": "```sql\nSELECT department, AVG(salary)\nFROM employees\nGROUP BY department\nHAVING AVG(salary) > 60000\nORDER BY AVG(salary) DESC;\n```"
      }
    },
    {
      "id": "SYS_03",
      "category": "SQL / C / Data",
      "prompt": "Given a SQLite table `products(id, name, category, price)`, write a SQL query returning `name` and `price` of the top 2 most expensive products in category `'Electronics'`, ordered by `price` descending. Return only the SQL query.",
      "baseline_normal": {
        "passed": true,
        "num_tokens": 28,
        "detail": "Rows matched: [('Laptop', 1200), ('Phone', 800)]",
        "response": "```sql\nSELECT name, price\nFROM products\nWHERE category = 'Electronics'\nORDER BY price DESC\nLIMIT 2;\n```"
      },
      "stage1_pcomm_only": {
        "passed": true,
        "num_tokens": 28,
        "detail": "Rows matched: [('Laptop', 1200), ('Phone', 800)]",
        "response": "```sql\nSELECT name, price\nFROM products\nWHERE category = 'Electronics'\nORDER BY price DESC\nLIMIT 2;\n```"
      },
      "code_surgery_v1": {
        "passed": false,
        "num_tokens": 30,
        "detail": "Rows mismatch: got [('Accessories', 25), ('Electronics', 1200), ('Furniture', 300)]",
        "response": "```sql\nSELECT category, MAX(price) AS price\nFROM products\nGROUP BY category\nORDER BY price DESC\nLIMIT 3;\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 28,
        "detail": "Rows matched: [('Laptop', 1200), ('Phone', 800)]",
        "response": "```sql\nSELECT name, price\nFROM products\nWHERE category = 'Electronics'\nORDER BY price DESC\nLIMIT 2;\n```"
      }
    },
    {
      "id": "SYS_04",
      "category": "SQL / C / Data",
      "prompt": "Write a C function `int gcd(int a, int b)` that computes and returns the greatest common divisor of two positive integers `a` and `b`. Return only the C function.",
      "baseline_normal": {
        "passed": true,
        "num_tokens": 54,
        "detail": "Compiled & passed C assertions",
        "response": "```c\nint gcd(int a, int b) {\n    // Use Euclidean algorithm\n    while (b != 0) {\n        int temp = b;\n        b = a % b;\n        a = temp;\n    }\n    return a;\n}\n```"
      },
      "stage1_pcomm_only": {
        "passed": true,
        "num_tokens": 54,
        "detail": "Compiled & passed C assertions",
        "response": "```c\nint gcd(int a, int b) {\n    // Use Euclidean algorithm\n    while (b != 0) {\n        int temp = b;\n        b = a % b;\n        a = temp;\n    }\n    return a;\n}\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 47,
        "detail": "Compiled & passed C assertions",
        "response": "```c\nint gcd(int a, int b) {\n    while (b != 0) {\n        int temp = b;\n        b = a % b;\n        a = temp;\n    }\n    return a;\n}\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 47,
        "detail": "Compiled & passed C assertions",
        "response": "```c\nint gcd(int a, int b) {\n    while (b != 0) {\n        int temp = b;\n        b = a % b;\n        a = temp;\n    }\n    return a;\n}\n```"
      }
    },
    {
      "id": "SYS_05",
      "category": "SQL / C / Data",
      "prompt": "Write a C function `int count_ones(unsigned int n)` that returns the number of set bits (`1`s) in the binary representation of `n`. Return only the C function.",
      "baseline_normal": {
        "passed": false,
        "num_tokens": 21,
        "detail": "C assertion failed",
        "response": "```c\nint count_ones(unsigned int n) {\n    return n & 1;\n}\n```"
      },
      "stage1_pcomm_only": {
        "passed": false,
        "num_tokens": 21,
        "detail": "C assertion failed",
        "response": "```c\nint count_ones(unsigned int n) {\n    return n & 1;\n}\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 51,
        "detail": "Compiled & passed C assertions",
        "response": "```c\nint count_ones(unsigned int n) {\n    int count = 0;\n    while (n > 0) {\n        count += (n & 1);\n        n >>= 1;\n    }\n    return count;\n}\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 51,
        "detail": "Compiled & passed C assertions",
        "response": "```c\nint count_ones(unsigned int n) {\n    int count = 0;\n    while (n > 0) {\n        count += (n & 1);\n        n >>= 1;\n    }\n    return count;\n}\n```"
      }
    },
    {
      "id": "SYS_06",
      "category": "SQL / C / Data",
      "prompt": "Write a Python function `def extract_emails(text: str) -> list[str]:` using `re.findall` that extracts all email addresses of the form `user@domain.tld` (alphanumeric, dots, underscores, hyphens) from `text`. Return only the code.",
      "baseline_normal": {
        "passed": false,
        "num_tokens": 57,
        "detail": "AssertionError",
        "response": "```python\nimport re\n\ndef extract_emails(text: str) -> list[str]:\n    return re.findall(r'([a-zA-Z0-9._-]+@([a-zA-Z0-9.-]+\\.[a-zA-Z0-9-]+))', text)\n```"
      },
      "stage1_pcomm_only": {
        "passed": false,
        "num_tokens": 57,
        "detail": "AssertionError",
        "response": "```python\nimport re\n\ndef extract_emails(text: str) -> list[str]:\n    return re.findall(r'([a-zA-Z0-9._-]+@([a-zA-Z0-9.-]+\\.[a-zA-Z0-9-]+))', text)\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 86,
        "detail": "Passed all assertions",
        "response": "Here is the Python function using `re.findall` to extract all email addresses of the form `user@domain.tld` from the given `text`:\n\n```python\nimport re\n\ndef extract_emails(text: str) -> list[str]:\n    return re.findall(r'[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z0-9.-]+', text)\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 86,
        "detail": "Passed all assertions",
        "response": "Here is the Python function using `re.findall` to extract all email addresses of the form `user@domain.tld` from the given `text`:\n\n```python\nimport re\n\ndef extract_emails(text: str) -> list[str]:\n    return re.findall(r'[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z0-9.-]+', text)\n```"
      }
    },
    {
      "id": "SYS_07",
      "category": "SQL / C / Data",
      "prompt": "Write a Python function `def filter_adults(json_str: str) -> list[str]:` that parses a JSON array of objects `[{\"name\": ..., \"age\": ...}]` and returns a list of `name` strings for entries with `age >= 18` in their original order. Return only the code.",
      "baseline_normal": {
        "passed": true,
        "num_tokens": 51,
        "detail": "Passed all assertions",
        "response": "```python\ndef filter_adults(json_str: str) -> list[str]:\n    import json\n    data = json.loads(json_str)\n    return [entry[\"name\"] for entry in data if entry[\"age\"] >= 18]\n```"
      },
      "stage1_pcomm_only": {
        "passed": true,
        "num_tokens": 51,
        "detail": "Passed all assertions",
        "response": "```python\ndef filter_adults(json_str: str) -> list[str]:\n    import json\n    data = json.loads(json_str)\n    return [entry[\"name\"] for entry in data if entry[\"age\"] >= 18]\n```"
      },
      "code_surgery_v1": {
        "passed": true,
        "num_tokens": 56,
        "detail": "Passed all assertions",
        "response": "```python\ndef filter_adults(json_str: str) -> list[str]:\n    import json\n    data = json.loads(json_str)\n    filtered = [entry[\"name\"] for entry in data if entry[\"age\"] >= 18]\n    return filtered\n```"
      },
      "code_surgery_v2_final": {
        "passed": true,
        "num_tokens": 56,
        "detail": "Passed all assertions",
        "response": "```python\ndef filter_adults(json_str: str) -> list[str]:\n    import json\n    data = json.loads(json_str)\n    filtered = [entry[\"name\"] for entry in data if entry[\"age\"] >= 18]\n    return filtered\n```"
      }
    }
  ]
}