Hugging Face's logo

zerolimites
/
Qwen3-0.6B-Code-Surgery-v2

Text Generation
Safetensors
GGUF
Rust
English
Portuguese
qwen3
code
python
bash
weight-surgery
memit
conversational
Model card Files
xet
 Community
Qwen3-0.6B-Code-Surgery-v2 / code_comparison_complete.json
zerolimites's picture
zerolimites
Upload Qwen3-0.6B-Code-Surgery-v2 (96.43% Pass@1 - Safetensors + GGUF Q8_0 & F16 + JSONs)
f10f49d verified
Raw History Blame Contribute Delete
48.2 kB
{
"benchmark_name": "Qwen3-0.6B Executable Multi-Language Code Benchmark (28 Tasks)",
"models_summary": {
"1_baseline_normal": {
"model_name": "Qwen/Qwen3-0.6B (Baseline Normal)",
"passed": 13,
"total": 28,
"accuracy_percent": 46.43,
"avg_tokens": 45.4,
"elapsed_sec": 111.79
},
"2_code_stage1_pcomm_only": {
"model_name": "Qwen3-0.6B-Code-Pcomm-Only (Apenas Stage 1: Rank-8 Comment & Blank-Line Damper)",
"passed": 14,
"total": 28,
"accuracy_percent": 50.0,
"gain_over_baseline_percent": 3.57,
"avg_tokens": 44.6
},
"3_code_surgery_v1": {
"model_name": "Qwen3-0.6B-Code-Surgery-v1 (Rank-8 P_comm + Multi-Key MEMIT sem Ancoras Protetoras)",
"passed": 24,
"total": 28,
"accuracy_percent": 85.71,
"gain_over_baseline_percent": 39.28,
"avg_tokens": 45.4,
"elapsed_sec": 114.53
},
"4_code_surgery_v2_final": {
"model_name": "Qwen3-0.6B-Code-Surgery-v2 (Final Zero-Regression Multi-Language Code Model)",
"passed": 27,
"total": 28,
"accuracy_percent": 96.43,
"gain_over_baseline_percent": 50.0,
"regressions": 0,
"avg_tokens": 45.9,
"elapsed_sec": 108.04
}
},
"category_comparison": {
"Python": {
"total": 7,
"baseline_passed": 1,
"baseline_accuracy_percent": 14.29,
"stage1_pcomm_passed": 2,
"stage1_pcomm_accuracy_percent": 28.57,
"code_surgery_v1_passed": 7,
"code_surgery_v1_accuracy_percent": 100.0,
"code_surgery_v2_final_passed": 7,
"code_surgery_v2_final_accuracy_percent": 100.0,
"v2_gain_over_baseline_percent": 85.71
},
"Linux Commands": {
"total": 7,
"baseline_passed": 4,
"baseline_accuracy_percent": 57.14,
"stage1_pcomm_passed": 4,
"stage1_pcomm_accuracy_percent": 57.14,
"code_surgery_v1_passed": 6,
"code_surgery_v1_accuracy_percent": 85.71,
"code_surgery_v2_final_passed": 7,
"code_surgery_v2_final_accuracy_percent": 100.0,
"v2_gain_over_baseline_percent": 42.86
},
"HTML / JS / CSS": {
"total": 7,
"baseline_passed": 4,
"baseline_accuracy_percent": 57.14,
"stage1_pcomm_passed": 4,
"stage1_pcomm_accuracy_percent": 57.14,
"code_surgery_v1_passed": 5,
"code_surgery_v1_accuracy_percent": 71.43,
"code_surgery_v2_final_passed": 6,
"code_surgery_v2_final_accuracy_percent": 85.71,
"v2_gain_over_baseline_percent": 28.57
},
"SQL / C / Data": {
"total": 7,
"baseline_passed": 4,
"baseline_accuracy_percent": 57.14,
"stage1_pcomm_passed": 4,
"stage1_pcomm_accuracy_percent": 57.14,
"code_surgery_v1_passed": 6,
"code_surgery_v1_accuracy_percent": 85.71,
"code_surgery_v2_final_passed": 7,
"code_surgery_v2_final_accuracy_percent": 100.0,
"v2_gain_over_baseline_percent": 42.86
}
},
"task_by_task_comparison": [
{
"id": "PY_01",
"category": "Python",
"prompt": "Write a Python function `def merge_intervals(intervals: list[list[int]]) -> list[list[int]]:` that merges all overlapping `[start, end]` intervals and returns the sorted merged list. Return only the code.",
"baseline_normal": {
"passed": false,
"num_tokens": 90,
"detail": "IndentationError: expected an indented block after 'if' statement on line 13",
"response": "```python\ndef merge_intervals(intervals: list[list[int]]) -> list[list[int]]:\n if not intervals:\n return []\n \n # Sort intervals by start time\n intervals.sort(key=lambda x: x[0])\n \n result = [intervals[0]]\n for start, end in intervals[1:]:\n last_start, last_end = result[-1]\n if start <= last_end:\n # Overlapping intervals,"
},
"stage1_pcomm_only": {
"passed": false,
"num_tokens": 90,
"detail": "IndentationError: expected an indented block after 'if' statement on line 13",
"response": "```python\ndef merge_intervals(intervals: list[list[int]]) -> list[list[int]]:\n if not intervals:\n return []\n \n # Sort intervals by start time\n intervals.sort(key=lambda x: x[0])\n \n result = [intervals[0]]\n for start, end in intervals[1:]:\n last_start, last_end = result[-1]\n if start <= last_end:\n # Overlapping intervals,"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 81,
"detail": "Passed all assertions",
"response": "```python\ndef merge_intervals(intervals: list[list[int]]) -> list[list[int]]:\n intervals.sort()\n merged = []\n for interval in intervals:\n if not merged or interval[0] > merged[-1][1]:\n merged.append(interval)\n else:\n merged[-1][1] = max(merged[-1][1], interval[1])\n return merged\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 81,
"detail": "Passed all assertions",
"response": "```python\ndef merge_intervals(intervals: list[list[int]]) -> list[list[int]]:\n intervals.sort()\n merged = []\n for interval in intervals:\n if not merged or interval[0] > merged[-1][1]:\n merged.append(interval)\n else:\n merged[-1][1] = max(merged[-1][1], interval[1])\n return merged\n```"
}
},
{
"id": "PY_02",
"category": "Python",
"prompt": "Write a Python function `def flatten_list(nested: list) -> list:` that recursively flattens an arbitrarily nested list of integers into a single 1D list. Return only the code.",
"baseline_normal": {
"passed": true,
"num_tokens": 52,
"detail": "Passed all assertions",
"response": "```python\ndef flatten_list(nested: list) -> list:\n result = []\n for item in nested:\n if isinstance(item, list):\n result.extend(flatten_list(item))\n else:\n result.append(item)\n return result\n```"
},
"stage1_pcomm_only": {
"passed": true,
"num_tokens": 52,
"detail": "Passed all assertions",
"response": "```python\ndef flatten_list(nested: list) -> list:\n result = []\n for item in nested:\n if isinstance(item, list):\n result.extend(flatten_list(item))\n else:\n result.append(item)\n return result\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 52,
"detail": "Passed all assertions",
"response": "```python\ndef flatten_list(nested: list) -> list:\n result = []\n for item in nested:\n if isinstance(item, list):\n result.extend(flatten_list(item))\n else:\n result.append(item)\n return result\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 52,
"detail": "Passed all assertions",
"response": "```python\ndef flatten_list(nested: list) -> list:\n result = []\n for item in nested:\n if isinstance(item, list):\n result.extend(flatten_list(item))\n else:\n result.append(item)\n return result\n```"
}
},
{
"id": "PY_03",
"category": "Python",
"prompt": "Write a Python function `def first_unique_char(s: str) -> int:` that returns the 0-based index of the first non-repeating character in string `s`, or `-1` if none exists. Return only the code.",
"baseline_normal": {
"passed": false,
"num_tokens": 90,
"detail": "IndentationError: expected an indented block after 'for' statement on line 16",
"response": "```python\ndef first_unique_char(s: str) -> int:\n if not s:\n return -1\n\n # Count the frequency of each character\n freq = {}\n for char in s:\n if char in freq:\n freq[char] += 1\n else:\n freq[char] = 1\n\n # Find the first character with a frequency of 1\n for index, char in enumerate(s):"
},
"stage1_pcomm_only": {
"passed": true,
"num_tokens": 68,
"detail": "Passed all assertions",
"response": "```python\ndef first_unique_char(s: str) -> int:\n count = {}\n for char in s:\n count[char] = count.get(char, 0) + 1\n for index, char in enumerate(s):\n if count[char] == 1:\n return index\n return -1\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 68,
"detail": "Passed all assertions",
"response": "```python\ndef first_unique_char(s: str) -> int:\n count = {}\n for char in s:\n count[char] = count.get(char, 0) + 1\n for index, char in enumerate(s):\n if count[char] == 1:\n return index\n return -1\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 68,
"detail": "Passed all assertions",
"response": "```python\ndef first_unique_char(s: str) -> int:\n count = {}\n for char in s:\n count[char] = count.get(char, 0) + 1\n for index, char in enumerate(s):\n if count[char] == 1:\n return index\n return -1\n```"
}
},
{
"id": "PY_04",
"category": "Python",
"prompt": "Write a Python function `def group_anagrams(words: list[str]) -> list[list[str]]:` that groups anagrams together. Return only the code.",
"baseline_normal": {
"passed": false,
"num_tokens": 85,
"detail": "AssertionError",
"response": "```python\ndef group_anagrams(words: list[str]) -> list[list[str]]:\n result = []\n for word in words:\n if not word:\n continue\n if len(word) == 1:\n result.append([word])\n else:\n # Sort the characters to form a unique key\n sorted_word = ''.join(sorted(word))\n result.append([sorted_word])\n return result\n```"
},
"stage1_pcomm_only": {
"passed": false,
"num_tokens": 85,
"detail": "AssertionError",
"response": "```python\ndef group_anagrams(words: list[str]) -> list[list[str]]:\n result = []\n for word in words:\n if not word:\n continue\n if len(word) == 1:\n result.append([word])\n else:\n # Sort the characters to form a unique key\n sorted_word = ''.join(sorted(word))\n result.append([sorted_word])\n return result\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 72,
"detail": "Passed all assertions",
"response": "```python\ndef group_anagrams(words: list[str]) -> list[list[str]]:\n groups = {}\n for word in words:\n sorted_word = sorted(word)\n key = ''.join(sorted_word)\n if key in groups:\n groups[key].append(word)\n else:\n groups[key] = [word]\n return groups.values()\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 72,
"detail": "Passed all assertions",
"response": "```python\ndef group_anagrams(words: list[str]) -> list[list[str]]:\n groups = {}\n for word in words:\n sorted_word = sorted(word)\n key = ''.join(sorted_word)\n if key in groups:\n groups[key].append(word)\n else:\n groups[key] = [word]\n return groups.values()\n```"
}
},
{
"id": "PY_05",
"category": "Python",
"prompt": "Write a Python function `def is_valid_brackets(s: str) -> bool:` that checks if a string containing only `'()[]{}'` has valid matching and properly nested brackets. Return only the code.",
"baseline_normal": {
"passed": false,
"num_tokens": 78,
"detail": "AssertionError",
"response": "```python\ndef is_valid_brackets(s: str) -> bool:\n stack = []\n for char in s:\n if char in '([{':\n stack.append(char)\n elif char in ')]}':\n if not stack or stack[-1] != char:\n return False\n else:\n return False\n return len(stack) == 0\n```"
},
"stage1_pcomm_only": {
"passed": false,
"num_tokens": 78,
"detail": "AssertionError",
"response": "```python\ndef is_valid_brackets(s: str) -> bool:\n stack = []\n for char in s:\n if char in '([{':\n stack.append(char)\n elif char in ')]}':\n if not stack or stack[-1] != char:\n return False\n else:\n return False\n return len(stack) == 0\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 82,
"detail": "Passed all assertions",
"response": "```python\ndef is_valid_brackets(s: str) -> bool:\n stack = []\n mapping = {')': '(', ']': '[', '}': '{'}\n for char in s:\n if char in mapping:\n if not stack or stack[-1] != mapping[char]:\n return False\n stack.pop()\n else:\n stack.append(char)\n return not stack\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 82,
"detail": "Passed all assertions",
"response": "```python\ndef is_valid_brackets(s: str) -> bool:\n stack = []\n mapping = {')': '(', ']': '[', '}': '{'}\n for char in s:\n if char in mapping:\n if not stack or stack[-1] != mapping[char]:\n return False\n stack.pop()\n else:\n stack.append(char)\n return not stack\n```"
}
},
{
"id": "PY_06",
"category": "Python",
"prompt": "Write a Python function `def longest_common_prefix(strs: list[str]) -> str:` that finds the longest common prefix string amongst an array of strings (or `\"\"` if none). Return only the code.",
"baseline_normal": {
"passed": false,
"num_tokens": 31,
"detail": "AssertionError",
"response": "```python\ndef longest_common_prefix(strs: list[str]) -> str:\n if not strs:\n return \"\"\n return strs[0]\n```"
},
"stage1_pcomm_only": {
"passed": false,
"num_tokens": 31,
"detail": "AssertionError",
"response": "```python\ndef longest_common_prefix(strs: list[str]) -> str:\n if not strs:\n return \"\"\n return strs[0]\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 66,
"detail": "Passed all assertions",
"response": "```python\ndef longest_common_prefix(strs: list[str]) -> str:\n if not strs:\n return \"\"\n prefix = strs[0]\n for s in strs[1:]:\n while not s.startswith(prefix):\n prefix = prefix[:-1]\n if not prefix:\n return \"\"\n return prefix\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 66,
"detail": "Passed all assertions",
"response": "```python\ndef longest_common_prefix(strs: list[str]) -> str:\n if not strs:\n return \"\"\n prefix = strs[0]\n for s in strs[1:]:\n while not s.startswith(prefix):\n prefix = prefix[:-1]\n if not prefix:\n return \"\"\n return prefix\n```"
}
},
{
"id": "PY_07",
"category": "Python",
"prompt": "Write a Python function `def rotate_right(nums: list[int], k: int) -> list[int]:` that returns a new list with the elements of `nums` rotated to the right by `k` steps (where `k` can be larger than `len(nums)`). Return only the code.",
"baseline_normal": {
"passed": false,
"num_tokens": 62,
"detail": "AssertionError",
"response": "```python\ndef rotate_right(nums: list[int], k: int) -> list[int]:\n # If k is larger than the length of nums, rotate the list by k steps\n if k > len(nums):\n k %= len(nums)\n return nums[k:] + nums[:k]\n```"
},
"stage1_pcomm_only": {
"passed": false,
"num_tokens": 62,
"detail": "AssertionError",
"response": "```python\ndef rotate_right(nums: list[int], k: int) -> list[int]:\n # If k is larger than the length of nums, rotate the list by k steps\n if k > len(nums):\n k %= len(nums)\n return nums[k:] + nums[:k]\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 53,
"detail": "Passed all assertions",
"response": "```python\ndef rotate_right(nums: list[int], k: int) -> list[int]:\n k %= len(nums) or k # Handle cases where k is larger than the length of nums\n return nums[-k:] + nums[:-k]\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 53,
"detail": "Passed all assertions",
"response": "```python\ndef rotate_right(nums: list[int], k: int) -> list[int]:\n k %= len(nums) or k # Handle cases where k is larger than the length of nums\n return nums[-k:] + nums[:-k]\n```"
}
},
{
"id": "LINUX_01",
"category": "Linux Commands",
"prompt": "Write a single Linux bash command to replace all occurrences of `DEBUG` with `ERROR` in `app.conf` and write the output to `app_prod.conf`. Return only the command.",
"baseline_normal": {
"passed": true,
"num_tokens": 26,
"detail": "File/state check passed",
"response": "```bash\ncp app.conf app_prod.conf\nsed -i 's/DEBUG/ERROR/g' app_prod.conf\n```"
},
"stage1_pcomm_only": {
"passed": true,
"num_tokens": 26,
"detail": "File/state check passed",
"response": "```bash\ncp app.conf app_prod.conf\nsed -i 's/DEBUG/ERROR/g' app_prod.conf\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 22,
"detail": "File/state check passed",
"response": "```bash\ncat app.conf | sed 's/DEBUG/ERROR/g' > app_prod.conf\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 22,
"detail": "File/state check passed",
"response": "```bash\ncat app.conf | sed 's/DEBUG/ERROR/g' > app_prod.conf\n```"
}
},
{
"id": "LINUX_02",
"category": "Linux Commands",
"prompt": "Write a single Linux bash pipeline that extracts the 2nd comma-separated column from `users.csv` (skipping the first header line), sorts the values alphabetically, and removes duplicates, printing to stdout. Return only the command.",
"baseline_normal": {
"passed": false,
"num_tokens": 27,
"detail": "Got 'admin\\neditor\\nrole\\nuser', expected 'admin\\neditor\\nuser'",
"response": "```bash\nawk -F, '{if ($2 != \"\") {print $2}}' users.csv | sort | uniq\n```"
},
"stage1_pcomm_only": {
"passed": false,
"num_tokens": 27,
"detail": "Got 'admin\\neditor\\nrole\\nuser', expected 'admin\\neditor\\nuser'",
"response": "```bash\nawk -F, '{if ($2 != \"\") {print $2}}' users.csv | sort | uniq\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 30,
"detail": "Stdout matched: 'admin\\neditor\\nuser'",
"response": "```bash\nawk -F, 'NR>1 {print $2}' users.csv | sort | uniq | awk '{print $1}'\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 30,
"detail": "Stdout matched: 'admin\\neditor\\nuser'",
"response": "```bash\nawk -F, 'NR>1 {print $2}' users.csv | sort | uniq | awk '{print $1}'\n```"
}
},
{
"id": "LINUX_03",
"category": "Linux Commands",
"prompt": "Write a single Linux bash command or pipeline that counts how many lines in `server.log` contain the exact substring `500` and prints only that integer count to stdout. Return only the command.",
"baseline_normal": {
"passed": true,
"num_tokens": 15,
"detail": "Stdout matched: '3'",
"response": "```bash\ngrep -c '500' server.log\n```"
},
"stage1_pcomm_only": {
"passed": true,
"num_tokens": 15,
"detail": "Stdout matched: '3'",
"response": "```bash\ngrep -c '500' server.log\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 15,
"detail": "Stdout matched: '3'",
"response": "```bash\ngrep -c '500' server.log\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 15,
"detail": "Stdout matched: '3'",
"response": "```bash\ngrep -c '500' server.log\n```"
}
},
{
"id": "LINUX_04",
"category": "Linux Commands",
"prompt": "Write a single Linux bash pipeline that reads integers (one per line) from `nums.txt` and prints their total sum to stdout. Return only the command.",
"baseline_normal": {
"passed": false,
"num_tokens": 33,
"detail": "Got '15\\n40\\n100\\n90', expected '90'",
"response": "```bash\nwhile IFS= read -r line; do\n sum=$((sum + $line))\n echo \"$sum\"\ndone < nums.txt\n```"
},
"stage1_pcomm_only": {
"passed": false,
"num_tokens": 33,
"detail": "Got '15\\n40\\n100\\n90', expected '90'",
"response": "```bash\nwhile IFS= read -r line; do\n sum=$((sum + $line))\n echo \"$sum\"\ndone < nums.txt\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 20,
"detail": "Stdout matched: '90'",
"response": "```bash\ncat nums.txt | awk '{sum += $1} END {print sum}'\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 20,
"detail": "Stdout matched: '90'",
"response": "```bash\ncat nums.txt | awk '{sum += $1} END {print sum}'\n```"
}
},
{
"id": "LINUX_05",
"category": "Linux Commands",
"prompt": "Write a single Linux bash command using `chmod` to set the permissions of `deploy.sh` to `rwxr-x---` (owner read/write/execute, group read/execute, others no permissions). Return only the command.",
"baseline_normal": {
"passed": false,
"num_tokens": 10,
"detail": "Exit 1: chmod: invalid mode: ‘rwxr-x---’\nTry 'chmod --help' for more information.",
"response": "chmod -R rwxr-x--- deploy.sh"
},
"stage1_pcomm_only": {
"passed": false,
"num_tokens": 10,
"detail": "Exit 1: chmod: invalid mode: ‘rwxr-x---’\nTry 'chmod --help' for more information.",
"response": "chmod -R rwxr-x--- deploy.sh"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 7,
"detail": "File/state check passed",
"response": "chmod 750 deploy.sh"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 7,
"detail": "File/state check passed",
"response": "chmod 750 deploy.sh"
}
},
{
"id": "LINUX_06",
"category": "Linux Commands",
"prompt": "Write a single Linux bash command using `find` or `ls`/`grep` that lists all `.py` filenames (just the file names or paths) inside the directory `src` (recursively) sorted alphabetically. Return only the command.",
"baseline_normal": {
"passed": true,
"num_tokens": 25,
"detail": "Stdout check passed",
"response": "```bash\nfind src -type f -name \"*.py\" | sort -t ' ' -k 1\n```"
},
"stage1_pcomm_only": {
"passed": true,
"num_tokens": 25,
"detail": "Stdout check passed",
"response": "```bash\nfind src -type f -name \"*.py\" | sort -t ' ' -k 1\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 25,
"detail": "Stdout check passed",
"response": "```bash\nfind src -type f | grep -E 'py' | sort | awk '{print $0}'\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 25,
"detail": "Stdout check passed",
"response": "```bash\nfind src -type f | grep -E 'py' | sort | awk '{print $0}'\n```"
}
},
{
"id": "LINUX_07",
"category": "Linux Commands",
"prompt": "Write a single Linux bash pipeline that reads `words.txt`, converts all uppercase letters to lowercase, sorts the lines alphabetically, and writes the result to `clean.txt`. Return only the command.",
"baseline_normal": {
"passed": true,
"num_tokens": 26,
"detail": "File/state check passed",
"response": "```bash\ncat words.txt | tr '[:upper:]' '[:lower:]' | sort | tee clean.txt\n```"
},
"stage1_pcomm_only": {
"passed": true,
"num_tokens": 26,
"detail": "File/state check passed",
"response": "```bash\ncat words.txt | tr '[:upper:]' '[:lower:]' | sort | tee clean.txt\n```"
},
"code_surgery_v1": {
"passed": false,
"num_tokens": 27,
"detail": "File/state check failed",
"response": "```bash\nawk '{print tolower($0)}' words.txt | sort | uniq > clean.txt\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 26,
"detail": "File/state check passed",
"response": "```bash\ncat words.txt | tr '[:upper:]' '[:lower:]' | sort | tee clean.txt\n```"
}
},
{
"id": "WEB_01",
"category": "HTML / JS / CSS",
"prompt": "Write a JavaScript function `function groupBy(arr, key)` that groups an array of objects `arr` by property `key` and returns an object mapping each key value to an array of matching objects. Return only the code.",
"baseline_normal": {
"passed": true,
"num_tokens": 60,
"detail": "Passed Node.js tests",
"response": "```javascript\nfunction groupBy(arr, key) {\n return arr.reduce((result, item) => {\n const value = item[key];\n if (!result[value]) {\n result[value] = [];\n }\n result[value].push(item);\n return result;\n }, {});\n}\n```"
},
"stage1_pcomm_only": {
"passed": true,
"num_tokens": 60,
"detail": "Passed Node.js tests",
"response": "```javascript\nfunction groupBy(arr, key) {\n return arr.reduce((result, item) => {\n const value = item[key];\n if (!result[value]) {\n result[value] = [];\n }\n result[value].push(item);\n return result;\n }, {});\n}\n```"
},
"code_surgery_v1": {
"passed": false,
"num_tokens": 36,
"detail": "Node error",
"response": "```javascript\nfunction groupBy(arr, key) {\n return Object.fromEntries(\n arr.map(item => [item[key], item])\n );\n}\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 60,
"detail": "Passed Node.js tests",
"response": "```javascript\nfunction groupBy(arr, key) {\n return arr.reduce((result, item) => {\n const value = item[key];\n if (!result[value]) {\n result[value] = [];\n }\n result[value].push(item);\n return result;\n }, {});\n}\n```"
}
},
{
"id": "WEB_02",
"category": "HTML / JS / CSS",
"prompt": "Write a JavaScript function `function chunkArray(arr, size)` that splits an array `arr` into chunks (subarrays) of length at most `size` and returns the array of chunks. Return only the code.",
"baseline_normal": {
"passed": false,
"num_tokens": 36,
"detail": "Node error",
"response": "```javascript\nfunction chunkArray(arr, size) {\n return arr.reduce((result, chunk) => {\n result.push(chunk);\n return result;\n }, []);\n}\n```"
},
"stage1_pcomm_only": {
"passed": false,
"num_tokens": 36,
"detail": "Node error",
"response": "```javascript\nfunction chunkArray(arr, size) {\n return arr.reduce((result, chunk) => {\n result.push(chunk);\n return result;\n }, []);\n}\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 54,
"detail": "Passed Node.js tests",
"response": "```javascript\nfunction chunkArray(arr, size) {\n const chunks = [];\n for (let i = 0; i < arr.length; i += size) {\n chunks.push(arr.slice(i, i + size));\n }\n return chunks;\n}\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 54,
"detail": "Passed Node.js tests",
"response": "```javascript\nfunction chunkArray(arr, size) {\n const chunks = [];\n for (let i = 0; i < arr.length; i += size) {\n chunks.push(arr.slice(i, i + size));\n }\n return chunks;\n}\n```"
}
},
{
"id": "WEB_03",
"category": "HTML / JS / CSS",
"prompt": "Write a JavaScript function `function parseQueryString(query)` that parses a URL query string like `\"?name=alice&age=30\"` into an object `{name: \"alice\", age: \"30\"}` (returning `{}` for empty string). Return only the code.",
"baseline_normal": {
"passed": false,
"num_tokens": 58,
"detail": "Node error",
"response": "```javascript\nfunction parseQueryString(query) {\n const obj = {};\n const parts = query.split('&');\n for (const part of parts) {\n const [key, value] = part.split('=');\n obj[key] = value;\n }\n return obj;\n}\n```"
},
"stage1_pcomm_only": {
"passed": false,
"num_tokens": 58,
"detail": "Node error",
"response": "```javascript\nfunction parseQueryString(query) {\n const obj = {};\n const parts = query.split('&');\n for (const part of parts) {\n const [key, value] = part.split('=');\n obj[key] = value;\n }\n return obj;\n}\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 21,
"detail": "Passed Node.js tests",
"response": "```javascript\nfunction parseQueryString(query) {\n return Object.fromEntries(new URLSearchParams(query));\n}\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 21,
"detail": "Passed Node.js tests",
"response": "```javascript\nfunction parseQueryString(query) {\n return Object.fromEntries(new URLSearchParams(query));\n}\n```"
}
},
{
"id": "WEB_04",
"category": "HTML / JS / CSS",
"prompt": "Write a JavaScript function `function flattenObject(obj, prefix = '')` that flattens a nested object into a single-level object with dot-separated keys (e.g. `{a: {b: 1}}` -> `{'a.b': 1}`). Return only the code.",
"baseline_normal": {
"passed": false,
"num_tokens": 90,
"detail": "[eval]:14",
"response": "```javascript\nfunction flattenObject(obj, prefix = '') {\n const result = {};\n \n // Traverse each key-value pair in the object\n for (const [key, value] of Object.entries(obj)) {\n // If the value is an object, recursively flatten it\n if (typeof value === 'object') {\n result[prefix + key] = flattenObject(value, prefix + key);\n } else {\n // If"
},
"stage1_pcomm_only": {
"passed": false,
"num_tokens": 90,
"detail": "[eval]:14",
"response": "```javascript\nfunction flattenObject(obj, prefix = '') {\n const result = {};\n \n // Traverse each key-value pair in the object\n for (const [key, value] of Object.entries(obj)) {\n // If the value is an object, recursively flatten it\n if (typeof value === 'object') {\n result[prefix + key] = flattenObject(value, prefix + key);\n } else {\n // If"
},
"code_surgery_v1": {
"passed": false,
"num_tokens": 75,
"detail": "[eval]:5",
"response": "```javascript\nfunction flattenObject(obj, prefix = '') {\n const result = {};\n for (const key in obj) {\n const value = obj[key];\n const keyWithoutPrefix = key.split('.'))[0];\n const keyWithPrefix = `${prefix}${keyWithoutPrefix}`;\n result[keyWithPrefix] = value;\n }\n return result;\n}\n```"
},
"code_surgery_v2_final": {
"passed": false,
"num_tokens": 75,
"detail": "[eval]:5",
"response": "```javascript\nfunction flattenObject(obj, prefix = '') {\n const result = {};\n for (const key in obj) {\n const value = obj[key];\n const keyWithoutPrefix = key.split('.'))[0];\n const keyWithPrefix = `${prefix}${keyWithoutPrefix}`;\n result[keyWithPrefix] = value;\n }\n return result;\n}\n```"
}
},
{
"id": "WEB_05",
"category": "HTML / JS / CSS",
"prompt": "Write a CSS rule for selector `.grid-container` that sets it to a CSS Grid layout with 3 equal-width columns (`1fr` each) and a gap of `16px`. Return only the CSS code.",
"baseline_normal": {
"passed": true,
"num_tokens": 34,
"detail": "All CSS rules matched",
"response": "```css\n.grid-container {\n display: grid;\n grid-template-columns: repeat(3, 1fr);\n gap: 16px;\n}\n```"
},
"stage1_pcomm_only": {
"passed": true,
"num_tokens": 34,
"detail": "All CSS rules matched",
"response": "```css\n.grid-container {\n display: grid;\n grid-template-columns: repeat(3, 1fr);\n gap: 16px;\n}\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 36,
"detail": "All CSS rules matched",
"response": "```css\n.grid-container {\n display: grid;\n grid-template-columns: 1fr 1fr 1fr;\n gap: 16px;\n}\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 36,
"detail": "All CSS rules matched",
"response": "```css\n.grid-container {\n display: grid;\n grid-template-columns: 1fr 1fr 1fr;\n gap: 16px;\n}\n```"
}
},
{
"id": "WEB_06",
"category": "HTML / JS / CSS",
"prompt": "Write a CSS rule for selector `.flex-center` that uses Flexbox to center its child elements both horizontally and vertically, with `min-height: 100vh`. Return only the CSS code.",
"baseline_normal": {
"passed": true,
"num_tokens": 35,
"detail": "All CSS rules matched",
"response": "```css\n.flex-center {\n display: flex;\n align-items: center;\n justify-content: center;\n min-height: 100vh;\n}\n```"
},
"stage1_pcomm_only": {
"passed": true,
"num_tokens": 35,
"detail": "All CSS rules matched",
"response": "```css\n.flex-center {\n display: flex;\n align-items: center;\n justify-content: center;\n min-height: 100vh;\n}\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 35,
"detail": "All CSS rules matched",
"response": "```css\n.flex-center {\n display: flex;\n align-items: center;\n justify-content: center;\n min-height: 100vh;\n}\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 35,
"detail": "All CSS rules matched",
"response": "```css\n.flex-center {\n display: flex;\n align-items: center;\n justify-content: center;\n min-height: 100vh;\n}\n```"
}
},
{
"id": "WEB_07",
"category": "HTML / JS / CSS",
"prompt": "Write a JavaScript function `function toggleActiveClass(el)` that toggles the class `'active'` on DOM element `el` and returns `true` if `'active'` is now present or `false` otherwise. Return only the code.",
"baseline_normal": {
"passed": true,
"num_tokens": 20,
"detail": "Passed Node.js tests",
"response": "```javascript\nfunction toggleActiveClass(el) {\n return el.classList.toggle('active');\n}\n```"
},
"stage1_pcomm_only": {
"passed": true,
"num_tokens": 20,
"detail": "Passed Node.js tests",
"response": "```javascript\nfunction toggleActiveClass(el) {\n return el.classList.toggle('active');\n}\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 38,
"detail": "Passed Node.js tests",
"response": "```javascript\nfunction toggleActiveClass(el) {\n const isToggled = !el.classList.contains('active');\n el.classList.toggle('active');\n return isToggled;\n}\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 38,
"detail": "Passed Node.js tests",
"response": "```javascript\nfunction toggleActiveClass(el) {\n const isToggled = !el.classList.contains('active');\n el.classList.toggle('active');\n return isToggled;\n}\n```"
}
},
{
"id": "SYS_01",
"category": "SQL / C / Data",
"prompt": "Given a SQLite table `users(id, name)` and `orders(id, user_id, amount)`, write a SQL query that returns `name` and `SUM(amount)` for each user who has orders, ordered by the total amount descending. Return only the SQL query.",
"baseline_normal": {
"passed": true,
"num_tokens": 38,
"detail": "Rows matched: [('Bob', 200), ('Alice', 120)]",
"response": "```sql\nSELECT name, SUM(amount) AS total_amount\nFROM users\nJOIN orders ON users.id = orders.user_id\nGROUP BY users.id\nORDER BY total_amount DESC;\n```"
},
"stage1_pcomm_only": {
"passed": true,
"num_tokens": 38,
"detail": "Rows matched: [('Bob', 200), ('Alice', 120)]",
"response": "```sql\nSELECT name, SUM(amount) AS total_amount\nFROM users\nJOIN orders ON users.id = orders.user_id\nGROUP BY users.id\nORDER BY total_amount DESC;\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 40,
"detail": "Rows matched: [('Bob', 200), ('Alice', 120)]",
"response": "```sql\nSELECT name, SUM(amount) AS total_amount\nFROM users\nJOIN orders ON users.id = orders.user_id\nGROUP BY users.id, name\nORDER BY total_amount DESC;\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 40,
"detail": "Rows matched: [('Bob', 200), ('Alice', 120)]",
"response": "```sql\nSELECT name, SUM(amount) AS total_amount\nFROM users\nJOIN orders ON users.id = orders.user_id\nGROUP BY users.id, name\nORDER BY total_amount DESC;\n```"
}
},
{
"id": "SYS_02",
"category": "SQL / C / Data",
"prompt": "Given a SQLite table `employees(id, name, department, salary)`, write a SQL query returning `department` and `AVG(salary)` for departments where the average salary is strictly greater than 60000, ordered by average salary descending. Return only the SQL query.",
"baseline_normal": {
"passed": false,
"num_tokens": 39,
"detail": "Exception: misuse of aggregate: AVG()",
"response": "```sql\nSELECT department, AVG(salary)\nFROM employees\nWHERE AVG(salary) > 60000\nGROUP BY department\nORDER BY AVG(salary) DESC;\n```"
},
"stage1_pcomm_only": {
"passed": false,
"num_tokens": 39,
"detail": "Exception: misuse of aggregate: AVG()",
"response": "```sql\nSELECT department, AVG(salary)\nFROM employees\nWHERE AVG(salary) > 60000\nGROUP BY department\nORDER BY AVG(salary) DESC;\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 40,
"detail": "Rows matched: [('Eng', 90000.0), ('Sales', 70000.0)]",
"response": "```sql\nSELECT department, AVG(salary)\nFROM employees\nGROUP BY department\nHAVING AVG(salary) > 60000\nORDER BY AVG(salary) DESC;\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 40,
"detail": "Rows matched: [('Eng', 90000.0), ('Sales', 70000.0)]",
"response": "```sql\nSELECT department, AVG(salary)\nFROM employees\nGROUP BY department\nHAVING AVG(salary) > 60000\nORDER BY AVG(salary) DESC;\n```"
}
},
{
"id": "SYS_03",
"category": "SQL / C / Data",
"prompt": "Given a SQLite table `products(id, name, category, price)`, write a SQL query returning `name` and `price` of the top 2 most expensive products in category `'Electronics'`, ordered by `price` descending. Return only the SQL query.",
"baseline_normal": {
"passed": true,
"num_tokens": 28,
"detail": "Rows matched: [('Laptop', 1200), ('Phone', 800)]",
"response": "```sql\nSELECT name, price\nFROM products\nWHERE category = 'Electronics'\nORDER BY price DESC\nLIMIT 2;\n```"
},
"stage1_pcomm_only": {
"passed": true,
"num_tokens": 28,
"detail": "Rows matched: [('Laptop', 1200), ('Phone', 800)]",
"response": "```sql\nSELECT name, price\nFROM products\nWHERE category = 'Electronics'\nORDER BY price DESC\nLIMIT 2;\n```"
},
"code_surgery_v1": {
"passed": false,
"num_tokens": 30,
"detail": "Rows mismatch: got [('Accessories', 25), ('Electronics', 1200), ('Furniture', 300)]",
"response": "```sql\nSELECT category, MAX(price) AS price\nFROM products\nGROUP BY category\nORDER BY price DESC\nLIMIT 3;\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 28,
"detail": "Rows matched: [('Laptop', 1200), ('Phone', 800)]",
"response": "```sql\nSELECT name, price\nFROM products\nWHERE category = 'Electronics'\nORDER BY price DESC\nLIMIT 2;\n```"
}
},
{
"id": "SYS_04",
"category": "SQL / C / Data",
"prompt": "Write a C function `int gcd(int a, int b)` that computes and returns the greatest common divisor of two positive integers `a` and `b`. Return only the C function.",
"baseline_normal": {
"passed": true,
"num_tokens": 54,
"detail": "Compiled & passed C assertions",
"response": "```c\nint gcd(int a, int b) {\n // Use Euclidean algorithm\n while (b != 0) {\n int temp = b;\n b = a % b;\n a = temp;\n }\n return a;\n}\n```"
},
"stage1_pcomm_only": {
"passed": true,
"num_tokens": 54,
"detail": "Compiled & passed C assertions",
"response": "```c\nint gcd(int a, int b) {\n // Use Euclidean algorithm\n while (b != 0) {\n int temp = b;\n b = a % b;\n a = temp;\n }\n return a;\n}\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 47,
"detail": "Compiled & passed C assertions",
"response": "```c\nint gcd(int a, int b) {\n while (b != 0) {\n int temp = b;\n b = a % b;\n a = temp;\n }\n return a;\n}\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 47,
"detail": "Compiled & passed C assertions",
"response": "```c\nint gcd(int a, int b) {\n while (b != 0) {\n int temp = b;\n b = a % b;\n a = temp;\n }\n return a;\n}\n```"
}
},
{
"id": "SYS_05",
"category": "SQL / C / Data",
"prompt": "Write a C function `int count_ones(unsigned int n)` that returns the number of set bits (`1`s) in the binary representation of `n`. Return only the C function.",
"baseline_normal": {
"passed": false,
"num_tokens": 21,
"detail": "C assertion failed",
"response": "```c\nint count_ones(unsigned int n) {\n return n & 1;\n}\n```"
},
"stage1_pcomm_only": {
"passed": false,
"num_tokens": 21,
"detail": "C assertion failed",
"response": "```c\nint count_ones(unsigned int n) {\n return n & 1;\n}\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 51,
"detail": "Compiled & passed C assertions",
"response": "```c\nint count_ones(unsigned int n) {\n int count = 0;\n while (n > 0) {\n count += (n & 1);\n n >>= 1;\n }\n return count;\n}\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 51,
"detail": "Compiled & passed C assertions",
"response": "```c\nint count_ones(unsigned int n) {\n int count = 0;\n while (n > 0) {\n count += (n & 1);\n n >>= 1;\n }\n return count;\n}\n```"
}
},
{
"id": "SYS_06",
"category": "SQL / C / Data",
"prompt": "Write a Python function `def extract_emails(text: str) -> list[str]:` using `re.findall` that extracts all email addresses of the form `user@domain.tld` (alphanumeric, dots, underscores, hyphens) from `text`. Return only the code.",
"baseline_normal": {
"passed": false,
"num_tokens": 57,
"detail": "AssertionError",
"response": "```python\nimport re\n\ndef extract_emails(text: str) -> list[str]:\n return re.findall(r'([a-zA-Z0-9._-]+@([a-zA-Z0-9.-]+\\.[a-zA-Z0-9-]+))', text)\n```"
},
"stage1_pcomm_only": {
"passed": false,
"num_tokens": 57,
"detail": "AssertionError",
"response": "```python\nimport re\n\ndef extract_emails(text: str) -> list[str]:\n return re.findall(r'([a-zA-Z0-9._-]+@([a-zA-Z0-9.-]+\\.[a-zA-Z0-9-]+))', text)\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 86,
"detail": "Passed all assertions",
"response": "Here is the Python function using `re.findall` to extract all email addresses of the form `user@domain.tld` from the given `text`:\n\n```python\nimport re\n\ndef extract_emails(text: str) -> list[str]:\n return re.findall(r'[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z0-9.-]+', text)\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 86,
"detail": "Passed all assertions",
"response": "Here is the Python function using `re.findall` to extract all email addresses of the form `user@domain.tld` from the given `text`:\n\n```python\nimport re\n\ndef extract_emails(text: str) -> list[str]:\n return re.findall(r'[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z0-9.-]+', text)\n```"
}
},
{
"id": "SYS_07",
"category": "SQL / C / Data",
"prompt": "Write a Python function `def filter_adults(json_str: str) -> list[str]:` that parses a JSON array of objects `[{\"name\": ..., \"age\": ...}]` and returns a list of `name` strings for entries with `age >= 18` in their original order. Return only the code.",
"baseline_normal": {
"passed": true,
"num_tokens": 51,
"detail": "Passed all assertions",
"response": "```python\ndef filter_adults(json_str: str) -> list[str]:\n import json\n data = json.loads(json_str)\n return [entry[\"name\"] for entry in data if entry[\"age\"] >= 18]\n```"
},
"stage1_pcomm_only": {
"passed": true,
"num_tokens": 51,
"detail": "Passed all assertions",
"response": "```python\ndef filter_adults(json_str: str) -> list[str]:\n import json\n data = json.loads(json_str)\n return [entry[\"name\"] for entry in data if entry[\"age\"] >= 18]\n```"
},
"code_surgery_v1": {
"passed": true,
"num_tokens": 56,
"detail": "Passed all assertions",
"response": "```python\ndef filter_adults(json_str: str) -> list[str]:\n import json\n data = json.loads(json_str)\n filtered = [entry[\"name\"] for entry in data if entry[\"age\"] >= 18]\n return filtered\n```"
},
"code_surgery_v2_final": {
"passed": true,
"num_tokens": 56,
"detail": "Passed all assertions",
"response": "```python\ndef filter_adults(json_str: str) -> list[str]:\n import json\n data = json.loads(json_str)\n filtered = [entry[\"name\"] for entry in data if entry[\"age\"] >= 18]\n return filtered\n```"
}
}
]
}