import os import re import json import sqlite3 import tempfile import subprocess # 28 Problemas Executáveis do Benchmark de Código para Qwen3-0.6B # Categorias: # 1. Python (7 tarefas: PY_01 .. PY_07) # 2. Linux Commands / Bash (7 tarefas: LINUX_01 .. LINUX_07) # 3. HTML / JS / CSS (7 tarefas: WEB_01 .. WEB_07) # 4. SQL / C / Regex / JSON (7 tarefas: SYS_01 .. SYS_07) CODE_BENCHMARK_PROBLEMS = [ # --- 1. PYTHON (7 Questões) --- { "id": "PY_01", "category": "Python", "prompt": "Write a Python function `def merge_intervals(intervals: list[list[int]]) -> list[list[int]]:` that merges all overlapping `[start, end]` intervals and returns the sorted merged list. Return only the code.", "test_type": "python", "test_code": """ assert merge_intervals([[1,3],[2,6],[8,10],[15,18]]) == [[1,6],[8,10],[15,18]] assert merge_intervals([[1,4],[4,5]]) == [[1,5]] assert merge_intervals([[5,7],[1,3]]) == [[1,3],[5,7]] """, }, { "id": "PY_02", "category": "Python", "prompt": "Write a Python function `def flatten_list(nested: list) -> list:` that recursively flattens an arbitrarily nested list of integers into a single 1D list. Return only the code.", "test_type": "python", "test_code": """ assert flatten_list([1, [2, [3, 4], 5], [], [6]]) == [1, 2, 3, 4, 5, 6] assert flatten_list([[1, 2], [3]]) == [1, 2, 3] assert flatten_list([]) == [] """, }, { "id": "PY_03", "category": "Python", "prompt": "Write a Python function `def first_unique_char(s: str) -> int:` that returns the 0-based index of the first non-repeating character in string `s`, or `-1` if none exists. Return only the code.", "test_type": "python", "test_code": """ assert first_unique_char("leetcode") == 0 assert first_unique_char("loveleetcode") == 2 assert first_unique_char("aabb") == -1 """, }, { "id": "PY_04", "category": "Python", "prompt": "Write a Python function `def group_anagrams(words: list[str]) -> list[list[str]]:` that groups anagrams together. Return only the code.", "test_type": "python", "test_code": """ res = group_anagrams(["eat","tea","tan","ate","nat","bat"]) norm = sorted([sorted(g) for g in res]) assert norm == [["ate","eat","tea"], ["bat"], ["nat","tan"]] """, }, { "id": "PY_05", "category": "Python", "prompt": "Write a Python function `def is_valid_brackets(s: str) -> bool:` that checks if a string containing only `'()[]{}'` has valid matching and properly nested brackets. Return only the code.", "test_type": "python", "test_code": """ assert is_valid_brackets("()[]{}") is True assert is_valid_brackets("([{}])") is True assert is_valid_brackets("(]") is False assert is_valid_brackets("([)]") is False assert is_valid_brackets("(") is False """, }, { "id": "PY_06", "category": "Python", "prompt": "Write a Python function `def longest_common_prefix(strs: list[str]) -> str:` that finds the longest common prefix string amongst an array of strings (or `\"\"` if none). Return only the code.", "test_type": "python", "test_code": """ assert longest_common_prefix(["flower","flow","flight"]) == "fl" assert longest_common_prefix(["dog","racecar","car"]) == "" assert longest_common_prefix(["interstellar","internet","internal"]) == "inter" """, }, { "id": "PY_07", "category": "Python", "prompt": "Write a Python function `def rotate_right(nums: list[int], k: int) -> list[int]:` that returns a new list with the elements of `nums` rotated to the right by `k` steps (where `k` can be larger than `len(nums)`). Return only the code.", "test_type": "python", "test_code": """ assert rotate_right([1,2,3,4,5,6,7], 3) == [5,6,7,1,2,3,4] assert rotate_right([1,2], 5) == [2,1] assert rotate_right([], 3) == [] """, }, # --- 2. LINUX COMMANDS / BASH (7 Questões) --- { "id": "LINUX_01", "category": "Linux Commands", "prompt": "Write a single Linux bash command to replace all occurrences of `DEBUG` with `ERROR` in `app.conf` and write the output to `app_prod.conf`. Return only the command.", "test_type": "bash", "setup_script": "printf 'mode=DEBUG\\nlog=INFO\\ndebug=DEBUG\\n' > app.conf", "check_script": "[ -f app_prod.conf ] && [ \"$(cat app_prod.conf)\" = \"$(printf 'mode=ERROR\\nlog=INFO\\ndebug=ERROR\\n')\" ]", }, { "id": "LINUX_02", "category": "Linux Commands", "prompt": "Write a single Linux bash pipeline that extracts the 2nd comma-separated column from `users.csv` (skipping the first header line), sorts the values alphabetically, and removes duplicates, printing to stdout. Return only the command.", "test_type": "bash", "setup_script": "printf 'id,role,name\\n1,admin,alice\\n2,user,bob\\n3,admin,charlie\\n4,editor,dave\\n' > users.csv", "expected_stdout": "admin\neditor\nuser", }, { "id": "LINUX_03", "category": "Linux Commands", "prompt": "Write a single Linux bash command or pipeline that counts how many lines in `server.log` contain the exact substring `500` and prints only that integer count to stdout. Return only the command.", "test_type": "bash", "setup_script": "printf '200 OK\\n500 ERR\\n404 NOT FOUND\\n500 FAIL\\n500 CRASH\\n' > server.log", "expected_stdout": "3", }, { "id": "LINUX_04", "category": "Linux Commands", "prompt": "Write a single Linux bash pipeline that reads integers (one per line) from `nums.txt` and prints their total sum to stdout. Return only the command.", "test_type": "bash", "setup_script": "printf '15\\n25\\n60\\n-10\\n' > nums.txt", "expected_stdout": "90", }, { "id": "LINUX_05", "category": "Linux Commands", "prompt": "Write a single Linux bash command using `chmod` to set the permissions of `deploy.sh` to `rwxr-x---` (owner read/write/execute, group read/execute, others no permissions). Return only the command.", "test_type": "bash", "setup_script": "touch deploy.sh && chmod 644 deploy.sh", "check_script": "[ \"$(stat -c '%a' deploy.sh)\" = \"750\" ]", }, { "id": "LINUX_06", "category": "Linux Commands", "prompt": "Write a single Linux bash command using `find` or `ls`/`grep` that lists all `.py` filenames (just the file names or paths) inside the directory `src` (recursively) sorted alphabetically. Return only the command.", "test_type": "bash", "setup_script": "mkdir -p src/sub && touch src/b.py src/a.txt src/sub/a.py", "check_stdout_contains": ["a.py", "b.py"], "check_stdout_not_contains": ["a.txt"], }, { "id": "LINUX_07", "category": "Linux Commands", "prompt": "Write a single Linux bash pipeline that reads `words.txt`, converts all uppercase letters to lowercase, sorts the lines alphabetically, and writes the result to `clean.txt`. Return only the command.", "test_type": "bash", "setup_script": "printf 'Banana\\nApple\\ncherry\\n' > words.txt", "check_script": "[ -f clean.txt ] && [ \"$(cat clean.txt)\" = \"$(printf 'apple\\nbanana\\ncherry\\n')\" ]", }, # --- 3. HTML / JS / CSS (7 Questões) --- { "id": "WEB_01", "category": "HTML / JS / CSS", "prompt": "Write a JavaScript function `function groupBy(arr, key)` that groups an array of objects `arr` by property `key` and returns an object mapping each key value to an array of matching objects. Return only the code.", "test_type": "node", "test_code": """ const data = [{t: 'a', v: 1}, {t: 'b', v: 2}, {t: 'a', v: 3}]; const res = groupBy(data, 't'); if (JSON.stringify(res) !== JSON.stringify({a: [{t: 'a', v: 1}, {t: 'a', v: 3}], b: [{t: 'b', v: 2}]})) process.exit(1); """, }, { "id": "WEB_02", "category": "HTML / JS / CSS", "prompt": "Write a JavaScript function `function chunkArray(arr, size)` that splits an array `arr` into chunks (subarrays) of length at most `size` and returns the array of chunks. Return only the code.", "test_type": "node", "test_code": """ const res = chunkArray([1, 2, 3, 4, 5], 2); if (JSON.stringify(res) !== JSON.stringify([[1, 2], [3, 4], [5]])) process.exit(1); if (JSON.stringify(chunkArray([], 3)) !== JSON.stringify([])) process.exit(1); """, }, { "id": "WEB_03", "category": "HTML / JS / CSS", "prompt": "Write a JavaScript function `function parseQueryString(query)` that parses a URL query string like `\"?name=alice&age=30\"` into an object `{name: \"alice\", age: \"30\"}` (returning `{}` for empty string). Return only the code.", "test_type": "node", "test_code": """ const r1 = parseQueryString("?name=alice&age=30"); if (r1.name !== "alice" || r1.age !== "30") process.exit(1); const r2 = parseQueryString(""); if (Object.keys(r2).length !== 0) process.exit(1); """, }, { "id": "WEB_04", "category": "HTML / JS / CSS", "prompt": "Write a JavaScript function `function flattenObject(obj, prefix = '')` that flattens a nested object into a single-level object with dot-separated keys (e.g. `{a: {b: 1}}` -> `{'a.b': 1}`). Return only the code.", "test_type": "node", "test_code": """ const res = flattenObject({a: {b: 1, c: {d: 2}}, e: 3}); if (res['a.b'] !== 1 || res['a.c.d'] !== 2 || res['e'] !== 3) process.exit(1); """, }, { "id": "WEB_05", "category": "HTML / JS / CSS", "prompt": "Write a CSS rule for selector `.grid-container` that sets it to a CSS Grid layout with 3 equal-width columns (`1fr` each) and a gap of `16px`. Return only the CSS code.", "test_type": "css", "required_patterns": [ r"\.grid-container\s*\{", r"display\s*:\s*grid", r"grid-template-columns\s*:\s*(repeat\(\s*3\s*,\s*1fr\s*\)|1fr\s+1fr\s+1fr)", r"(?:grid-)?gap\s*:\s*16px", ], }, { "id": "WEB_06", "category": "HTML / JS / CSS", "prompt": "Write a CSS rule for selector `.flex-center` that uses Flexbox to center its child elements both horizontally and vertically, with `min-height: 100vh`. Return only the CSS code.", "test_type": "css", "required_patterns": [ r"\.flex-center\s*\{", r"display\s*:\s*flex", r"justify-content\s*:\s*center", r"align-items\s*:\s*center", r"min-height\s*:\s*100vh", ], }, { "id": "WEB_07", "category": "HTML / JS / CSS", "prompt": "Write a JavaScript function `function toggleActiveClass(el)` that toggles the class `'active'` on DOM element `el` and returns `true` if `'active'` is now present or `false` otherwise. Return only the code.", "test_type": "node", "test_code": """ const set = new Set(); const mockEl = { classList: { toggle(c) { if (set.has(c)) { set.delete(c); return false; } set.add(c); return true; }, contains(c) { return set.has(c); }, add(c) { set.add(c); }, remove(c) { set.delete(c); } } }; const r1 = toggleActiveClass(mockEl); if (r1 !== true || !set.has('active')) process.exit(1); const r2 = toggleActiveClass(mockEl); if (r2 !== false || set.has('active')) process.exit(1); """, }, # --- 4. SQL / C / REGEX / JSON (7 Questões) --- { "id": "SYS_01", "category": "SQL / C / Data", "prompt": "Given a SQLite table `users(id, name)` and `orders(id, user_id, amount)`, write a SQL query that returns `name` and `SUM(amount)` for each user who has orders, ordered by the total amount descending. Return only the SQL query.", "test_type": "sql", "setup_sql": """ CREATE TABLE users (id INT, name TEXT); CREATE TABLE orders (id INT, user_id INT, amount INT); INSERT INTO users VALUES (1, 'Alice'), (2, 'Bob'), (3, 'Charlie'); INSERT INTO orders VALUES (10, 1, 50), (11, 1, 70), (12, 2, 200); """, "expected_rows": [("Bob", 200), ("Alice", 120)], }, { "id": "SYS_02", "category": "SQL / C / Data", "prompt": "Given a SQLite table `employees(id, name, department, salary)`, write a SQL query returning `department` and `AVG(salary)` for departments where the average salary is strictly greater than 60000, ordered by average salary descending. Return only the SQL query.", "test_type": "sql", "setup_sql": """ CREATE TABLE employees (id INT, name TEXT, department TEXT, salary REAL); INSERT INTO employees VALUES (1, 'A', 'Eng', 80000), (2, 'B', 'Eng', 100000), (3, 'C', 'HR', 50000), (4, 'D', 'Sales', 70000); """, "expected_rows": [("Eng", 90000.0), ("Sales", 70000.0)], }, { "id": "SYS_03", "category": "SQL / C / Data", "prompt": "Given a SQLite table `products(id, name, category, price)`, write a SQL query returning `name` and `price` of the top 2 most expensive products in category `'Electronics'`, ordered by `price` descending. Return only the SQL query.", "test_type": "sql", "setup_sql": """ CREATE TABLE products (id INT, name TEXT, category TEXT, price INT); INSERT INTO products VALUES (1, 'Laptop', 'Electronics', 1200), (2, 'Mouse', 'Electronics', 40), (3, 'Phone', 'Electronics', 800), (4, 'Desk', 'Furniture', 1500); """, "expected_rows": [("Laptop", 1200), ("Phone", 800)], }, { "id": "SYS_04", "category": "SQL / C / Data", "prompt": "Write a C function `int gcd(int a, int b)` that computes and returns the greatest common divisor of two positive integers `a` and `b`. Return only the C function.", "test_type": "c", "test_main": """ #include int main() { assert(gcd(48, 18) == 6); assert(gcd(100, 25) == 25); assert(gcd(17, 13) == 1); return 0; } """, }, { "id": "SYS_05", "category": "SQL / C / Data", "prompt": "Write a C function `int count_ones(unsigned int n)` that returns the number of set bits (`1`s) in the binary representation of `n`. Return only the C function.", "test_type": "c", "test_main": """ #include int main() { assert(count_ones(0) == 0); assert(count_ones(7) == 3); assert(count_ones(255) == 8); assert(count_ones(1024) == 1); return 0; } """, }, { "id": "SYS_06", "category": "SQL / C / Data", "prompt": "Write a Python function `def extract_emails(text: str) -> list[str]:` using `re.findall` that extracts all email addresses of the form `user@domain.tld` (alphanumeric, dots, underscores, hyphens) from `text`. Return only the code.", "test_type": "python", "test_code": """ s = "Contact support@qwen.ai or dev_team-1@sub.example.org for help!" assert extract_emails(s) == ["support@qwen.ai", "dev_team-1@sub.example.org"] """, }, { "id": "SYS_07", "category": "SQL / C / Data", "prompt": "Write a Python function `def filter_adults(json_str: str) -> list[str]:` that parses a JSON array of objects `[{\"name\": ..., \"age\": ...}]` and returns a list of `name` strings for entries with `age >= 18` in their original order. Return only the code.", "test_type": "python", "test_code": """ data = '[{"name": "Ana", "age": 22}, {"name": "Leo", "age": 15}, {"name": "Max", "age": 18}]' assert filter_adults(data) == ["Ana", "Max"] """, }, ] def extract_code_block(text: str) -> str: # 1. Closed markdown code block ```lang ... ``` m = re.findall(r"```(?:[a-zA-Z0-9_+-]*)\n(.*?)```", text, flags=re.DOTALL) if m: return m[0].strip() # 2. Unclosed markdown code block ```lang ... (if truncated before closing ```) m_open = re.search(r"```(?:[a-zA-Z0-9_+-]*)\n(.*)$", text, flags=re.DOTALL) if m_open: return m_open.group(1).strip() return text.strip() def verify_code_solution(problem: dict, response_text: str) -> tuple[bool, str]: code = extract_code_block(response_text) if not code: return False, "Empty code" ttype = problem["test_type"] try: if ttype == "python": full_py = "import re, json, math, collections\nfrom typing import *\n" + code + "\n" + problem["test_code"] res = subprocess.run( ["python3", "-c", full_py], capture_output=True, text=True, timeout=4, ) if res.returncode == 0: return True, "Passed all assertions" err = (res.stderr or res.stdout).strip().splitlines() return False, err[-1] if err else "Python error" elif ttype == "bash": with tempfile.TemporaryDirectory() as tmpdir: subprocess.run(["bash", "-c", problem["setup_script"]], cwd=tmpdir, check=True, timeout=4) res = subprocess.run( ["bash", "-c", code], cwd=tmpdir, capture_output=True, text=True, timeout=4, ) if res.returncode != 0: return False, f"Exit {res.returncode}: {res.stderr.strip()[:80]}" if "expected_stdout" in problem: out = res.stdout.strip() if out == problem["expected_stdout"].strip(): return True, f"Stdout matched: {out!r}" return False, f"Got {out!r}, expected {problem['expected_stdout']!r}" if "check_stdout_contains" in problem: out = res.stdout.strip() for req in problem["check_stdout_contains"]: if req not in out: return False, f"Missing {req!r} in {out!r}" for forbid in problem.get("check_stdout_not_contains", []): if forbid in out: return False, f"Forbidden {forbid!r} in {out!r}" return True, "Stdout check passed" if "check_script" in problem: chk = subprocess.run(["bash", "-c", problem["check_script"]], cwd=tmpdir, timeout=4) if chk.returncode == 0: return True, "File/state check passed" return False, "File/state check failed" elif ttype == "node": full_js = code + "\n" + problem["test_code"] res = subprocess.run( ["node", "-e", full_js], capture_output=True, text=True, timeout=4, ) if res.returncode == 0: return True, "Passed Node.js tests" err = (res.stderr or res.stdout).strip().splitlines() return False, err[0] if err else "Node error" elif ttype == "css": for pat in problem["required_patterns"]: if not re.search(pat, code, flags=re.IGNORECASE | re.DOTALL): return False, f"Missing CSS pattern: {pat}" return True, "All CSS rules matched" elif ttype == "sql": conn = sqlite3.connect(":memory:") cur = conn.cursor() cur.executescript(problem["setup_sql"]) # Take first statement if multiple stmt = code.strip().split(";")[0].strip() cur.execute(stmt) rows = cur.fetchall() conn.close() if rows == problem["expected_rows"]: return True, f"Rows matched: {rows}" return False, f"Got {rows}, expected {problem['expected_rows']}" elif ttype == "c": with tempfile.TemporaryDirectory() as tmpdir: c_file = os.path.join(tmpdir, "sol.c") bin_file = os.path.join(tmpdir, "sol") with open(c_file, "w") as f: f.write("#include \n#include \n" + code + "\n" + problem["test_main"]) comp = subprocess.run(["gcc", "-O1", c_file, "-o", bin_file], capture_output=True, text=True, timeout=5) if comp.returncode != 0: return False, "Compile error: " + comp.stderr.strip().splitlines()[-1][:80] run = subprocess.run([bin_file], capture_output=True, text=True, timeout=4) if run.returncode == 0: return True, "Compiled & passed C assertions" return False, "C assertion failed" except Exception as e: return False, f"Exception: {str(e)[:80]}" return False, "Unknown test type"