Blame for code_benchmark_problems.py

import os
import re
import json
import sqlite3
import tempfile
import subprocess

# 28 Problemas Executáveis do Benchmark de Código para Qwen3-0.6B
# Categorias:
# 1. Python (7 tarefas: PY_01 .. PY_07)
# 2. Linux Commands / Bash (7 tarefas: LINUX_01 .. LINUX_07)
# 3. HTML / JS / CSS (7 tarefas: WEB_01 .. WEB_07)
# 4. SQL / C / Regex / JSON (7 tarefas: SYS_01 .. SYS_07)

CODE_BENCHMARK_PROBLEMS = [
    # --- 1. PYTHON (7 Questões) ---
    {
        "id": "PY_01",
        "category": "Python",
        "prompt": "Write a Python function `def merge_intervals(intervals: list[list[int]]) -> list[list[int]]:` that merges all overlapping `[start, end]` intervals and returns the sorted merged list. Return only the code.",
        "test_type": "python",
        "test_code": """
assert merge_intervals([[1,3],[2,6],[8,10],[15,18]]) == [[1,6],[8,10],[15,18]]
assert merge_intervals([[1,4],[4,5]]) == [[1,5]]
assert merge_intervals([[5,7],[1,3]]) == [[1,3],[5,7]]
""",
    },
    {
        "id": "PY_02",
        "category": "Python",
        "prompt": "Write a Python function `def flatten_list(nested: list) -> list:` that recursively flattens an arbitrarily nested list of integers into a single 1D list. Return only the code.",
        "test_type": "python",
        "test_code": """
assert flatten_list([1, [2, [3, 4], 5], [], [6]]) == [1, 2, 3, 4, 5, 6]
assert flatten_list([[1, 2], [3]]) == [1, 2, 3]
assert flatten_list([]) == []
""",
    },
    {
        "id": "PY_03",
        "category": "Python",
        "prompt": "Write a Python function `def first_unique_char(s: str) -> int:` that returns the 0-based index of the first non-repeating character in string `s`, or `-1` if none exists. Return only the code.",
        "test_type": "python",
        "test_code": """
assert first_unique_char("leetcode") == 0
assert first_unique_char("loveleetcode") == 2
assert first_unique_char("aabb") == -1
""",
    },
    {
        "id": "PY_04",
        "category": "Python",
        "prompt": "Write a Python function `def group_anagrams(words: list[str]) -> list[list[str]]:` that groups anagrams together. Return only the code.",
        "test_type": "python",
        "test_code": """
res = group_anagrams(["eat","tea","tan","ate","nat","bat"])
norm = sorted([sorted(g) for g in res])
assert norm == [["ate","eat","tea"], ["bat"], ["nat","tan"]]
""",
    },
    {
        "id": "PY_05",
        "category": "Python",
        "prompt": "Write a Python function `def is_valid_brackets(s: str) -> bool:` that checks if a string containing only `'()[]{}'` has valid matching and properly nested brackets. Return only the code.",
        "test_type": "python",
        "test_code": """
assert is_valid_brackets("()[]{}") is True
assert is_valid_brackets("([{}])") is True
assert is_valid_brackets("(]") is False
assert is_valid_brackets("([)]") is False
assert is_valid_brackets("(") is False
""",
    },
    {
        "id": "PY_06",
        "category": "Python",
        "prompt": "Write a Python function `def longest_common_prefix(strs: list[str]) -> str:` that finds the longest common prefix string amongst an array of strings (or `\"\"` if none). Return only the code.",
        "test_type": "python",
        "test_code": """
assert longest_common_prefix(["flower","flow","flight"]) == "fl"
assert longest_common_prefix(["dog","racecar","car"]) == ""
assert longest_common_prefix(["interstellar","internet","internal"]) == "inter"
""",
    },
    {
        "id": "PY_07",
        "category": "Python",
        "prompt": "Write a Python function `def rotate_right(nums: list[int], k: int) -> list[int]:` that returns a new list with the elements of `nums` rotated to the right by `k` steps (where `k` can be larger than `len(nums)`). Return only the code.",
        "test_type": "python",
        "test_code": """
assert rotate_right([1,2,3,4,5,6,7], 3) == [5,6,7,1,2,3,4]
assert rotate_right([1,2], 5) == [2,1]
assert rotate_right([], 3) == []
""",
    },

    # --- 2. LINUX COMMANDS / BASH (7 Questões) ---
    {
        "id": "LINUX_01",
        "category": "Linux Commands",
        "prompt": "Write a single Linux bash command to replace all occurrences of `DEBUG` with `ERROR` in `app.conf` and write the output to `app_prod.conf`. Return only the command.",
        "test_type": "bash",
        "setup_script": "printf 'mode=DEBUG\\nlog=INFO\\ndebug=DEBUG\\n' > app.conf",
        "check_script": "[ -f app_prod.conf ] && [ \"$(cat app_prod.conf)\" = \"$(printf 'mode=ERROR\\nlog=INFO\\ndebug=ERROR\\n')\" ]",
    },
    {
        "id": "LINUX_02",
        "category": "Linux Commands",
        "prompt": "Write a single Linux bash pipeline that extracts the 2nd comma-separated column from `users.csv` (skipping the first header line), sorts the values alphabetically, and removes duplicates, printing to stdout. Return only the command.",
        "test_type": "bash",
        "setup_script": "printf 'id,role,name\\n1,admin,alice\\n2,user,bob\\n3,admin,charlie\\n4,editor,dave\\n' > users.csv",
        "expected_stdout": "admin\neditor\nuser",
    },
    {
        "id": "LINUX_03",
        "category": "Linux Commands",
        "prompt": "Write a single Linux bash command or pipeline that counts how many lines in `server.log` contain the exact substring `500` and prints only that integer count to stdout. Return only the command.",
        "test_type": "bash",
        "setup_script": "printf '200 OK\\n500 ERR\\n404 NOT FOUND\\n500 FAIL\\n500 CRASH\\n' > server.log",
        "expected_stdout": "3",
    },
    {
        "id": "LINUX_04",
        "category": "Linux Commands",
        "prompt": "Write a single Linux bash pipeline that reads integers (one per line) from `nums.txt` and prints their total sum to stdout. Return only the command.",
        "test_type": "bash",
        "setup_script": "printf '15\\n25\\n60\\n-10\\n' > nums.txt",
        "expected_stdout": "90",
    },
    {
        "id": "LINUX_05",
        "category": "Linux Commands",
        "prompt": "Write a single Linux bash command using `chmod` to set the permissions of `deploy.sh` to `rwxr-x---` (owner read/write/execute, group read/execute, others no permissions). Return only the command.",
        "test_type": "bash",
        "setup_script": "touch deploy.sh && chmod 644 deploy.sh",
        "check_script": "[ \"$(stat -c '%a' deploy.sh)\" = \"750\" ]",
    },
    {
        "id": "LINUX_06",
        "category": "Linux Commands",
        "prompt": "Write a single Linux bash command using `find` or `ls`/`grep` that lists all `.py` filenames (just the file names or paths) inside the directory `src` (recursively) sorted alphabetically. Return only the command.",
        "test_type": "bash",
        "setup_script": "mkdir -p src/sub && touch src/b.py src/a.txt src/sub/a.py",
        "check_stdout_contains": ["a.py", "b.py"],
        "check_stdout_not_contains": ["a.txt"],
    },
    {
        "id": "LINUX_07",
        "category": "Linux Commands",
        "prompt": "Write a single Linux bash pipeline that reads `words.txt`, converts all uppercase letters to lowercase, sorts the lines alphabetically, and writes the result to `clean.txt`. Return only the command.",
        "test_type": "bash",
        "setup_script": "printf 'Banana\\nApple\\ncherry\\n' > words.txt",
        "check_script": "[ -f clean.txt ] && [ \"$(cat clean.txt)\" = \"$(printf 'apple\\nbanana\\ncherry\\n')\" ]",
    },

    # --- 3. HTML / JS / CSS (7 Questões) ---
    {
        "id": "WEB_01",
        "category": "HTML / JS / CSS",
        "prompt": "Write a JavaScript function `function groupBy(arr, key)` that groups an array of objects `arr` by property `key` and returns an object mapping each key value to an array of matching objects. Return only the code.",
        "test_type": "node",
        "test_code": """
const data = [{t: 'a', v: 1}, {t: 'b', v: 2}, {t: 'a', v: 3}];
const res = groupBy(data, 't');
if (JSON.stringify(res) !== JSON.stringify({a: [{t: 'a', v: 1}, {t: 'a', v: 3}], b: [{t: 'b', v: 2}]})) process.exit(1);
""",
    },
    {
        "id": "WEB_02",
        "category": "HTML / JS / CSS",
        "prompt": "Write a JavaScript function `function chunkArray(arr, size)` that splits an array `arr` into chunks (subarrays) of length at most `size` and returns the array of chunks. Return only the code.",
        "test_type": "node",
        "test_code": """
const res = chunkArray([1, 2, 3, 4, 5], 2);
if (JSON.stringify(res) !== JSON.stringify([[1, 2], [3, 4], [5]])) process.exit(1);
if (JSON.stringify(chunkArray([], 3)) !== JSON.stringify([])) process.exit(1);
""",
    },
    {
        "id": "WEB_03",
        "category": "HTML / JS / CSS",
        "prompt": "Write a JavaScript function `function parseQueryString(query)` that parses a URL query string like `\"?name=alice&age=30\"` into an object `{name: \"alice\", age: \"30\"}` (returning `{}` for empty string). Return only the code.",
        "test_type": "node",
        "test_code": """
const r1 = parseQueryString("?name=alice&age=30");
if (r1.name !== "alice" || r1.age !== "30") process.exit(1);
const r2 = parseQueryString("");
if (Object.keys(r2).length !== 0) process.exit(1);
""",
    },
    {
        "id": "WEB_04",
        "category": "HTML / JS / CSS",
        "prompt": "Write a JavaScript function `function flattenObject(obj, prefix = '')` that flattens a nested object into a single-level object with dot-separated keys (e.g. `{a: {b: 1}}` -> `{'a.b': 1}`). Return only the code.",
        "test_type": "node",
        "test_code": """
const res = flattenObject({a: {b: 1, c: {d: 2}}, e: 3});
if (res['a.b'] !== 1 || res['a.c.d'] !== 2 || res['e'] !== 3) process.exit(1);
""",
    },
    {
        "id": "WEB_05",
        "category": "HTML / JS / CSS",
        "prompt": "Write a CSS rule for selector `.grid-container` that sets it to a CSS Grid layout with 3 equal-width columns (`1fr` each) and a gap of `16px`. Return only the CSS code.",
        "test_type": "css",
        "required_patterns": [
            r"\.grid-container\s*\{",
            r"display\s*:\s*grid",
            r"grid-template-columns\s*:\s*(repeat\(\s*3\s*,\s*1fr\s*\)|1fr\s+1fr\s+1fr)",
            r"(?:grid-)?gap\s*:\s*16px",
        ],
    },
    {
        "id": "WEB_06",
        "category": "HTML / JS / CSS",
        "prompt": "Write a CSS rule for selector `.flex-center` that uses Flexbox to center its child elements both horizontally and vertically, with `min-height: 100vh`. Return only the CSS code.",
        "test_type": "css",
        "required_patterns": [
            r"\.flex-center\s*\{",
            r"display\s*:\s*flex",
            r"justify-content\s*:\s*center",
            r"align-items\s*:\s*center",
            r"min-height\s*:\s*100vh",
        ],
    },
    {
        "id": "WEB_07",
        "category": "HTML / JS / CSS",
        "prompt": "Write a JavaScript function `function toggleActiveClass(el)` that toggles the class `'active'` on DOM element `el` and returns `true` if `'active'` is now present or `false` otherwise. Return only the code.",
        "test_type": "node",
        "test_code": """
const set = new Set();
const mockEl = {
  classList: {
    toggle(c) { if (set.has(c)) { set.delete(c); return false; } set.add(c); return true; },
    contains(c) { return set.has(c); },
    add(c) { set.add(c); },
    remove(c) { set.delete(c); }
  }
};
const r1 = toggleActiveClass(mockEl);
if (r1 !== true || !set.has('active')) process.exit(1);
const r2 = toggleActiveClass(mockEl);
if (r2 !== false || set.has('active')) process.exit(1);
""",
    },

    # --- 4. SQL / C / REGEX / JSON (7 Questões) ---
    {
        "id": "SYS_01",
        "category": "SQL / C / Data",
        "prompt": "Given a SQLite table `users(id, name)` and `orders(id, user_id, amount)`, write a SQL query that returns `name` and `SUM(amount)` for each user who has orders, ordered by the total amount descending. Return only the SQL query.",
        "test_type": "sql",
        "setup_sql": """
CREATE TABLE users (id INT, name TEXT);
CREATE TABLE orders (id INT, user_id INT, amount INT);
INSERT INTO users VALUES (1, 'Alice'), (2, 'Bob'), (3, 'Charlie');
INSERT INTO orders VALUES (10, 1, 50), (11, 1, 70), (12, 2, 200);
""",
        "expected_rows": [("Bob", 200), ("Alice", 120)],
    },
    {
        "id": "SYS_02",
        "category": "SQL / C / Data",
        "prompt": "Given a SQLite table `employees(id, name, department, salary)`, write a SQL query returning `department` and `AVG(salary)` for departments where the average salary is strictly greater than 60000, ordered by average salary descending. Return only the SQL query.",
        "test_type": "sql",
        "setup_sql": """
CREATE TABLE employees (id INT, name TEXT, department TEXT, salary REAL);
INSERT INTO employees VALUES (1, 'A', 'Eng', 80000), (2, 'B', 'Eng', 100000), (3, 'C', 'HR', 50000), (4, 'D', 'Sales', 70000);
""",
        "expected_rows": [("Eng", 90000.0), ("Sales", 70000.0)],
    },
    {
        "id": "SYS_03",
        "category": "SQL / C / Data",
        "prompt": "Given a SQLite table `products(id, name, category, price)`, write a SQL query returning `name` and `price` of the top 2 most expensive products in category `'Electronics'`, ordered by `price` descending. Return only the SQL query.",
        "test_type": "sql",
        "setup_sql": """
CREATE TABLE products (id INT, name TEXT, category TEXT, price INT);
INSERT INTO products VALUES (1, 'Laptop', 'Electronics', 1200), (2, 'Mouse', 'Electronics', 40), (3, 'Phone', 'Electronics', 800), (4, 'Desk', 'Furniture', 1500);
""",
        "expected_rows": [("Laptop", 1200), ("Phone", 800)],
    },
    {
        "id": "SYS_04",
        "category": "SQL / C / Data",
        "prompt": "Write a C function `int gcd(int a, int b)` that computes and returns the greatest common divisor of two positive integers `a` and `b`. Return only the C function.",
        "test_type": "c",
        "test_main": """
#include <assert.h>
int main() {
    assert(gcd(48, 18) == 6);
    assert(gcd(100, 25) == 25);
    assert(gcd(17, 13) == 1);
    return 0;
}
""",
    },
    {
        "id": "SYS_05",
        "category": "SQL / C / Data",
        "prompt": "Write a C function `int count_ones(unsigned int n)` that returns the number of set bits (`1`s) in the binary representation of `n`. Return only the C function.",
        "test_type": "c",
        "test_main": """
#include <assert.h>
int main() {
    assert(count_ones(0) == 0);
    assert(count_ones(7) == 3);
    assert(count_ones(255) == 8);
    assert(count_ones(1024) == 1);
    return 0;
}
""",
    },
    {
        "id": "SYS_06",
        "category": "SQL / C / Data",
        "prompt": "Write a Python function `def extract_emails(text: str) -> list[str]:` using `re.findall` that extracts all email addresses of the form `user@domain.tld` (alphanumeric, dots, underscores, hyphens) from `text`. Return only the code.",
        "test_type": "python",
        "test_code": """
s = "Contact support@qwen.ai or dev_team-1@sub.example.org for help!"
assert extract_emails(s) == ["support@qwen.ai", "dev_team-1@sub.example.org"]
""",
    },
    {
        "id": "SYS_07",
        "category": "SQL / C / Data",
        "prompt": "Write a Python function `def filter_adults(json_str: str) -> list[str]:` that parses a JSON array of objects `[{\"name\": ..., \"age\": ...}]` and returns a list of `name` strings for entries with `age >= 18` in their original order. Return only the code.",
        "test_type": "python",
        "test_code": """
data = '[{"name": "Ana", "age": 22}, {"name": "Leo", "age": 15}, {"name": "Max", "age": 18}]'
assert filter_adults(data) == ["Ana", "Max"]
""",
    },
]


def extract_code_block(text: str) -> str:
    # 1. Closed markdown code block ```lang ... ```
    m = re.findall(r"```(?:[a-zA-Z0-9_+-]*)\n(.*?)```", text, flags=re.DOTALL)
    if m:
        return m[0].strip()
    # 2. Unclosed markdown code block ```lang ... (if truncated before closing ```)
    m_open = re.search(r"```(?:[a-zA-Z0-9_+-]*)\n(.*)$", text, flags=re.DOTALL)
    if m_open:
        return m_open.group(1).strip()
    return text.strip()


def verify_code_solution(problem: dict, response_text: str) -> tuple[bool, str]:
    code = extract_code_block(response_text)
    if not code:
        return False, "Empty code"

    ttype = problem["test_type"]
    try:
        if ttype == "python":
            full_py = "import re, json, math, collections\nfrom typing import *\n" + code + "\n" + problem["test_code"]
            res = subprocess.run(
                ["python3", "-c", full_py],
                capture_output=True,
                text=True,
                timeout=4,
            )
            if res.returncode == 0:
                return True, "Passed all assertions"
            err = (res.stderr or res.stdout).strip().splitlines()
            return False, err[-1] if err else "Python error"

        elif ttype == "bash":
            with tempfile.TemporaryDirectory() as tmpdir:
                subprocess.run(["bash", "-c", problem["setup_script"]], cwd=tmpdir, check=True, timeout=4)
                res = subprocess.run(
                    ["bash", "-c", code],
                    cwd=tmpdir,
                    capture_output=True,
                    text=True,
                    timeout=4,
                )
                if res.returncode != 0:
                    return False, f"Exit {res.returncode}: {res.stderr.strip()[:80]}"
                if "expected_stdout" in problem:
                    out = res.stdout.strip()
                    if out == problem["expected_stdout"].strip():
                        return True, f"Stdout matched: {out!r}"
                    return False, f"Got {out!r}, expected {problem['expected_stdout']!r}"
                if "check_stdout_contains" in problem:
                    out = res.stdout.strip()
                    for req in problem["check_stdout_contains"]:
                        if req not in out:
                            return False, f"Missing {req!r} in {out!r}"
                    for forbid in problem.get("check_stdout_not_contains", []):
                        if forbid in out:
                            return False, f"Forbidden {forbid!r} in {out!r}"
                    return True, "Stdout check passed"
                if "check_script" in problem:
                    chk = subprocess.run(["bash", "-c", problem["check_script"]], cwd=tmpdir, timeout=4)
                    if chk.returncode == 0:
                        return True, "File/state check passed"
                    return False, "File/state check failed"

        elif ttype == "node":
            full_js = code + "\n" + problem["test_code"]
            res = subprocess.run(
                ["node", "-e", full_js],
                capture_output=True,
                text=True,
                timeout=4,
            )
            if res.returncode == 0:
                return True, "Passed Node.js tests"
            err = (res.stderr or res.stdout).strip().splitlines()
            return False, err[0] if err else "Node error"

        elif ttype == "css":
            for pat in problem["required_patterns"]:
                if not re.search(pat, code, flags=re.IGNORECASE | re.DOTALL):
                    return False, f"Missing CSS pattern: {pat}"
            return True, "All CSS rules matched"

        elif ttype == "sql":
            conn = sqlite3.connect(":memory:")
            cur = conn.cursor()
            cur.executescript(problem["setup_sql"])
            # Take first statement if multiple
            stmt = code.strip().split(";")[0].strip()
            cur.execute(stmt)
            rows = cur.fetchall()
            conn.close()
            if rows == problem["expected_rows"]:
                return True, f"Rows matched: {rows}"
            return False, f"Got {rows}, expected {problem['expected_rows']}"

        elif ttype == "c":
            with tempfile.TemporaryDirectory() as tmpdir:
                c_file = os.path.join(tmpdir, "sol.c")
                bin_file = os.path.join(tmpdir, "sol")
                with open(c_file, "w") as f:
                    f.write("#include <stdio.h>\n#include <stdlib.h>\n" + code + "\n" + problem["test_main"])
                comp = subprocess.run(["gcc", "-O1", c_file, "-o", bin_file], capture_output=True, text=True, timeout=5)
                if comp.returncode != 0:
                    return False, "Compile error: " + comp.stderr.strip().splitlines()[-1][:80]
                run = subprocess.run([bin_file], capture_output=True, text=True, timeout=4)
                if run.returncode == 0:
                    return True, "Compiled & passed C assertions"
                return False, "C assertion failed"

    except Exception as e:
        return False, f"Exception: {str(e)[:80]}"

    return False, "Unknown test type"