| { |
| "total": 28, |
| "passed": 13, |
| "accuracy": 46.43, |
| "avg_tokens": 45.4, |
| "elapsed_sec": 111.79, |
| "results": [ |
| { |
| "id": "PY_01", |
| "category": "Python", |
| "prompt": "Write a Python function `def merge_intervals(intervals: list[list[int]]) -> list[list[int]]:` that merges all overlapping `[start, end]` intervals and returns the sorted merged list. Return only the code.", |
| "passed": false, |
| "detail": "IndentationError: expected an indented block after 'if' statement on line 13", |
| "num_tokens": 90, |
| "response": "```python\ndef merge_intervals(intervals: list[list[int]]) -> list[list[int]]:\n if not intervals:\n return []\n \n # Sort intervals by start time\n intervals.sort(key=lambda x: x[0])\n \n result = [intervals[0]]\n for start, end in intervals[1:]:\n last_start, last_end = result[-1]\n if start <= last_end:\n # Overlapping intervals," |
| }, |
| { |
| "id": "PY_02", |
| "category": "Python", |
| "prompt": "Write a Python function `def flatten_list(nested: list) -> list:` that recursively flattens an arbitrarily nested list of integers into a single 1D list. Return only the code.", |
| "passed": true, |
| "detail": "Passed all assertions", |
| "num_tokens": 52, |
| "response": "```python\ndef flatten_list(nested: list) -> list:\n result = []\n for item in nested:\n if isinstance(item, list):\n result.extend(flatten_list(item))\n else:\n result.append(item)\n return result\n```" |
| }, |
| { |
| "id": "PY_03", |
| "category": "Python", |
| "prompt": "Write a Python function `def first_unique_char(s: str) -> int:` that returns the 0-based index of the first non-repeating character in string `s`, or `-1` if none exists. Return only the code.", |
| "passed": false, |
| "detail": "IndentationError: expected an indented block after 'for' statement on line 16", |
| "num_tokens": 90, |
| "response": "```python\ndef first_unique_char(s: str) -> int:\n if not s:\n return -1\n\n # Count the frequency of each character\n freq = {}\n for char in s:\n if char in freq:\n freq[char] += 1\n else:\n freq[char] = 1\n\n # Find the first character with a frequency of 1\n for index, char in enumerate(s):" |
| }, |
| { |
| "id": "PY_04", |
| "category": "Python", |
| "prompt": "Write a Python function `def group_anagrams(words: list[str]) -> list[list[str]]:` that groups anagrams together. Return only the code.", |
| "passed": false, |
| "detail": "AssertionError", |
| "num_tokens": 85, |
| "response": "```python\ndef group_anagrams(words: list[str]) -> list[list[str]]:\n result = []\n for word in words:\n if not word:\n continue\n if len(word) == 1:\n result.append([word])\n else:\n # Sort the characters to form a unique key\n sorted_word = ''.join(sorted(word))\n result.append([sorted_word])\n return result\n```" |
| }, |
| { |
| "id": "PY_05", |
| "category": "Python", |
| "prompt": "Write a Python function `def is_valid_brackets(s: str) -> bool:` that checks if a string containing only `'()[]{}'` has valid matching and properly nested brackets. Return only the code.", |
| "passed": false, |
| "detail": "AssertionError", |
| "num_tokens": 78, |
| "response": "```python\ndef is_valid_brackets(s: str) -> bool:\n stack = []\n for char in s:\n if char in '([{':\n stack.append(char)\n elif char in ')]}':\n if not stack or stack[-1] != char:\n return False\n else:\n return False\n return len(stack) == 0\n```" |
| }, |
| { |
| "id": "PY_06", |
| "category": "Python", |
| "prompt": "Write a Python function `def longest_common_prefix(strs: list[str]) -> str:` that finds the longest common prefix string amongst an array of strings (or `\"\"` if none). Return only the code.", |
| "passed": false, |
| "detail": "AssertionError", |
| "num_tokens": 31, |
| "response": "```python\ndef longest_common_prefix(strs: list[str]) -> str:\n if not strs:\n return \"\"\n return strs[0]\n```" |
| }, |
| { |
| "id": "PY_07", |
| "category": "Python", |
| "prompt": "Write a Python function `def rotate_right(nums: list[int], k: int) -> list[int]:` that returns a new list with the elements of `nums` rotated to the right by `k` steps (where `k` can be larger than `len(nums)`). Return only the code.", |
| "passed": false, |
| "detail": "AssertionError", |
| "num_tokens": 62, |
| "response": "```python\ndef rotate_right(nums: list[int], k: int) -> list[int]:\n # If k is larger than the length of nums, rotate the list by k steps\n if k > len(nums):\n k %= len(nums)\n return nums[k:] + nums[:k]\n```" |
| }, |
| { |
| "id": "LINUX_01", |
| "category": "Linux Commands", |
| "prompt": "Write a single Linux bash command to replace all occurrences of `DEBUG` with `ERROR` in `app.conf` and write the output to `app_prod.conf`. Return only the command.", |
| "passed": true, |
| "detail": "File/state check passed", |
| "num_tokens": 26, |
| "response": "```bash\ncp app.conf app_prod.conf\nsed -i 's/DEBUG/ERROR/g' app_prod.conf\n```" |
| }, |
| { |
| "id": "LINUX_02", |
| "category": "Linux Commands", |
| "prompt": "Write a single Linux bash pipeline that extracts the 2nd comma-separated column from `users.csv` (skipping the first header line), sorts the values alphabetically, and removes duplicates, printing to stdout. Return only the command.", |
| "passed": false, |
| "detail": "Got 'admin\\neditor\\nrole\\nuser', expected 'admin\\neditor\\nuser'", |
| "num_tokens": 27, |
| "response": "```bash\nawk -F, '{if ($2 != \"\") {print $2}}' users.csv | sort | uniq\n```" |
| }, |
| { |
| "id": "LINUX_03", |
| "category": "Linux Commands", |
| "prompt": "Write a single Linux bash command or pipeline that counts how many lines in `server.log` contain the exact substring `500` and prints only that integer count to stdout. Return only the command.", |
| "passed": true, |
| "detail": "Stdout matched: '3'", |
| "num_tokens": 15, |
| "response": "```bash\ngrep -c '500' server.log\n```" |
| }, |
| { |
| "id": "LINUX_04", |
| "category": "Linux Commands", |
| "prompt": "Write a single Linux bash pipeline that reads integers (one per line) from `nums.txt` and prints their total sum to stdout. Return only the command.", |
| "passed": false, |
| "detail": "Got '15\\n40\\n100\\n90', expected '90'", |
| "num_tokens": 33, |
| "response": "```bash\nwhile IFS= read -r line; do\n sum=$((sum + $line))\n echo \"$sum\"\ndone < nums.txt\n```" |
| }, |
| { |
| "id": "LINUX_05", |
| "category": "Linux Commands", |
| "prompt": "Write a single Linux bash command using `chmod` to set the permissions of `deploy.sh` to `rwxr-x---` (owner read/write/execute, group read/execute, others no permissions). Return only the command.", |
| "passed": false, |
| "detail": "Exit 1: chmod: invalid mode: ‘rwxr-x---’\nTry 'chmod --help' for more information.", |
| "num_tokens": 10, |
| "response": "chmod -R rwxr-x--- deploy.sh" |
| }, |
| { |
| "id": "LINUX_06", |
| "category": "Linux Commands", |
| "prompt": "Write a single Linux bash command using `find` or `ls`/`grep` that lists all `.py` filenames (just the file names or paths) inside the directory `src` (recursively) sorted alphabetically. Return only the command.", |
| "passed": true, |
| "detail": "Stdout check passed", |
| "num_tokens": 25, |
| "response": "```bash\nfind src -type f -name \"*.py\" | sort -t ' ' -k 1\n```" |
| }, |
| { |
| "id": "LINUX_07", |
| "category": "Linux Commands", |
| "prompt": "Write a single Linux bash pipeline that reads `words.txt`, converts all uppercase letters to lowercase, sorts the lines alphabetically, and writes the result to `clean.txt`. Return only the command.", |
| "passed": true, |
| "detail": "File/state check passed", |
| "num_tokens": 26, |
| "response": "```bash\ncat words.txt | tr '[:upper:]' '[:lower:]' | sort | tee clean.txt\n```" |
| }, |
| { |
| "id": "WEB_01", |
| "category": "HTML / JS / CSS", |
| "prompt": "Write a JavaScript function `function groupBy(arr, key)` that groups an array of objects `arr` by property `key` and returns an object mapping each key value to an array of matching objects. Return only the code.", |
| "passed": true, |
| "detail": "Passed Node.js tests", |
| "num_tokens": 60, |
| "response": "```javascript\nfunction groupBy(arr, key) {\n return arr.reduce((result, item) => {\n const value = item[key];\n if (!result[value]) {\n result[value] = [];\n }\n result[value].push(item);\n return result;\n }, {});\n}\n```" |
| }, |
| { |
| "id": "WEB_02", |
| "category": "HTML / JS / CSS", |
| "prompt": "Write a JavaScript function `function chunkArray(arr, size)` that splits an array `arr` into chunks (subarrays) of length at most `size` and returns the array of chunks. Return only the code.", |
| "passed": false, |
| "detail": "Node error", |
| "num_tokens": 36, |
| "response": "```javascript\nfunction chunkArray(arr, size) {\n return arr.reduce((result, chunk) => {\n result.push(chunk);\n return result;\n }, []);\n}\n```" |
| }, |
| { |
| "id": "WEB_03", |
| "category": "HTML / JS / CSS", |
| "prompt": "Write a JavaScript function `function parseQueryString(query)` that parses a URL query string like `\"?name=alice&age=30\"` into an object `{name: \"alice\", age: \"30\"}` (returning `{}` for empty string). Return only the code.", |
| "passed": false, |
| "detail": "Node error", |
| "num_tokens": 58, |
| "response": "```javascript\nfunction parseQueryString(query) {\n const obj = {};\n const parts = query.split('&');\n for (const part of parts) {\n const [key, value] = part.split('=');\n obj[key] = value;\n }\n return obj;\n}\n```" |
| }, |
| { |
| "id": "WEB_04", |
| "category": "HTML / JS / CSS", |
| "prompt": "Write a JavaScript function `function flattenObject(obj, prefix = '')` that flattens a nested object into a single-level object with dot-separated keys (e.g. `{a: {b: 1}}` -> `{'a.b': 1}`). Return only the code.", |
| "passed": false, |
| "detail": "[eval]:14", |
| "num_tokens": 90, |
| "response": "```javascript\nfunction flattenObject(obj, prefix = '') {\n const result = {};\n \n // Traverse each key-value pair in the object\n for (const [key, value] of Object.entries(obj)) {\n // If the value is an object, recursively flatten it\n if (typeof value === 'object') {\n result[prefix + key] = flattenObject(value, prefix + key);\n } else {\n // If" |
| }, |
| { |
| "id": "WEB_05", |
| "category": "HTML / JS / CSS", |
| "prompt": "Write a CSS rule for selector `.grid-container` that sets it to a CSS Grid layout with 3 equal-width columns (`1fr` each) and a gap of `16px`. Return only the CSS code.", |
| "passed": true, |
| "detail": "All CSS rules matched", |
| "num_tokens": 34, |
| "response": "```css\n.grid-container {\n display: grid;\n grid-template-columns: repeat(3, 1fr);\n gap: 16px;\n}\n```" |
| }, |
| { |
| "id": "WEB_06", |
| "category": "HTML / JS / CSS", |
| "prompt": "Write a CSS rule for selector `.flex-center` that uses Flexbox to center its child elements both horizontally and vertically, with `min-height: 100vh`. Return only the CSS code.", |
| "passed": true, |
| "detail": "All CSS rules matched", |
| "num_tokens": 35, |
| "response": "```css\n.flex-center {\n display: flex;\n align-items: center;\n justify-content: center;\n min-height: 100vh;\n}\n```" |
| }, |
| { |
| "id": "WEB_07", |
| "category": "HTML / JS / CSS", |
| "prompt": "Write a JavaScript function `function toggleActiveClass(el)` that toggles the class `'active'` on DOM element `el` and returns `true` if `'active'` is now present or `false` otherwise. Return only the code.", |
| "passed": true, |
| "detail": "Passed Node.js tests", |
| "num_tokens": 20, |
| "response": "```javascript\nfunction toggleActiveClass(el) {\n return el.classList.toggle('active');\n}\n```" |
| }, |
| { |
| "id": "SYS_01", |
| "category": "SQL / C / Data", |
| "prompt": "Given a SQLite table `users(id, name)` and `orders(id, user_id, amount)`, write a SQL query that returns `name` and `SUM(amount)` for each user who has orders, ordered by the total amount descending. Return only the SQL query.", |
| "passed": true, |
| "detail": "Rows matched: [('Bob', 200), ('Alice', 120)]", |
| "num_tokens": 38, |
| "response": "```sql\nSELECT name, SUM(amount) AS total_amount\nFROM users\nJOIN orders ON users.id = orders.user_id\nGROUP BY users.id\nORDER BY total_amount DESC;\n```" |
| }, |
| { |
| "id": "SYS_02", |
| "category": "SQL / C / Data", |
| "prompt": "Given a SQLite table `employees(id, name, department, salary)`, write a SQL query returning `department` and `AVG(salary)` for departments where the average salary is strictly greater than 60000, ordered by average salary descending. Return only the SQL query.", |
| "passed": false, |
| "detail": "Exception: misuse of aggregate: AVG()", |
| "num_tokens": 39, |
| "response": "```sql\nSELECT department, AVG(salary)\nFROM employees\nWHERE AVG(salary) > 60000\nGROUP BY department\nORDER BY AVG(salary) DESC;\n```" |
| }, |
| { |
| "id": "SYS_03", |
| "category": "SQL / C / Data", |
| "prompt": "Given a SQLite table `products(id, name, category, price)`, write a SQL query returning `name` and `price` of the top 2 most expensive products in category `'Electronics'`, ordered by `price` descending. Return only the SQL query.", |
| "passed": true, |
| "detail": "Rows matched: [('Laptop', 1200), ('Phone', 800)]", |
| "num_tokens": 28, |
| "response": "```sql\nSELECT name, price\nFROM products\nWHERE category = 'Electronics'\nORDER BY price DESC\nLIMIT 2;\n```" |
| }, |
| { |
| "id": "SYS_04", |
| "category": "SQL / C / Data", |
| "prompt": "Write a C function `int gcd(int a, int b)` that computes and returns the greatest common divisor of two positive integers `a` and `b`. Return only the C function.", |
| "passed": true, |
| "detail": "Compiled & passed C assertions", |
| "num_tokens": 54, |
| "response": "```c\nint gcd(int a, int b) {\n // Use Euclidean algorithm\n while (b != 0) {\n int temp = b;\n b = a % b;\n a = temp;\n }\n return a;\n}\n```" |
| }, |
| { |
| "id": "SYS_05", |
| "category": "SQL / C / Data", |
| "prompt": "Write a C function `int count_ones(unsigned int n)` that returns the number of set bits (`1`s) in the binary representation of `n`. Return only the C function.", |
| "passed": false, |
| "detail": "C assertion failed", |
| "num_tokens": 21, |
| "response": "```c\nint count_ones(unsigned int n) {\n return n & 1;\n}\n```" |
| }, |
| { |
| "id": "SYS_06", |
| "category": "SQL / C / Data", |
| "prompt": "Write a Python function `def extract_emails(text: str) -> list[str]:` using `re.findall` that extracts all email addresses of the form `user@domain.tld` (alphanumeric, dots, underscores, hyphens) from `text`. Return only the code.", |
| "passed": false, |
| "detail": "AssertionError", |
| "num_tokens": 57, |
| "response": "```python\nimport re\n\ndef extract_emails(text: str) -> list[str]:\n return re.findall(r'([a-zA-Z0-9._-]+@([a-zA-Z0-9.-]+\\.[a-zA-Z0-9-]+))', text)\n```" |
| }, |
| { |
| "id": "SYS_07", |
| "category": "SQL / C / Data", |
| "prompt": "Write a Python function `def filter_adults(json_str: str) -> list[str]:` that parses a JSON array of objects `[{\"name\": ..., \"age\": ...}]` and returns a list of `name` strings for entries with `age >= 18` in their original order. Return only the code.", |
| "passed": true, |
| "detail": "Passed all assertions", |
| "num_tokens": 51, |
| "response": "```python\ndef filter_adults(json_str: str) -> list[str]:\n import json\n data = json.loads(json_str)\n return [entry[\"name\"] for entry in data if entry[\"age\"] >= 18]\n```" |
| } |
| ], |
| "model": "Qwen/Qwen3-0.6B (Baseline Normal)", |
| "category_breakdown": { |
| "Python": { |
| "passed": 1, |
| "total": 7, |
| "accuracy_percent": 14.29, |
| "avg_tokens": 69.7 |
| }, |
| "Linux Commands": { |
| "passed": 4, |
| "total": 7, |
| "accuracy_percent": 57.14, |
| "avg_tokens": 23.1 |
| }, |
| "HTML / JS / CSS": { |
| "passed": 4, |
| "total": 7, |
| "accuracy_percent": 57.14, |
| "avg_tokens": 47.6 |
| }, |
| "SQL / C / Data": { |
| "passed": 4, |
| "total": 7, |
| "accuracy_percent": 57.14, |
| "avg_tokens": 41.1 |
| } |
| } |
| } |
