Download eval/taskkit.py from sebika123123/Qwen3.8-27B-Fast-Coder: direct link, hf CLI and curl.
- Browser
- Download file 1.76 kB
-
https://huggingface.co/sebika123123/Qwen3.8-27B-Fast-Coder/resolve/main/eval/taskkit.py
- Command line
-
hf download hf://sebika123123/Qwen3.8-27B-Fast-Coder/eval/taskkit.py
-
curl -L -o taskkit.py https://huggingface.co/sebika123123/Qwen3.8-27B-Fast-Coder/resolve/main/eval/taskkit.py
1.76 kB
| """What a task looks like, plus the helpers its check uses.""" | |
| import re | |
| TASKS = [] | |
| def add(id, title, difficulty, kind, prompt, files, check, reference, max_turns=20, answers=None, alternatives=None): | |
| """files: dict of path -> text, or a function returning one. check(sandbox, answer) -> (ok, detail). | |
| reference: the steps of a correct solution, used by selftest. Each step is a list of (tool, args) calls, | |
| or ("final", answer) for the last message. answers: for tasks judged on the final answer, (text, should_pass) | |
| pairs that selftest feeds to the check. alternatives: other correct solutions (step lists like reference) that | |
| selftest must also see pass, for tasks with more than one valid way to finish.""" | |
| TASKS.append(dict(id=id, title=title, difficulty=difficulty, kind=kind, prompt=prompt, files=files, check=check, | |
| reference=reference, max_turns=max_turns, answers=answers or [], alternatives=alternatives or [])) | |
| def node(sb, script, *args): | |
| return sb.exec("node", [script, *args], timeout=60) | |
| def py(sb, script, *args): | |
| return sb.exec("python", [script, *args], timeout=60) | |
| def why(rc, out): | |
| """A short reason from a failed script's output (its error line), or nothing when the script succeeded.""" | |
| if rc == 0: | |
| return "" | |
| lines = [l.strip() for l in out.splitlines() if l.strip()] | |
| hit = next((l for l in lines if re.match(r"\w*(Error|Exception)\b", l)), | |
| next((l for l in lines if re.search(r"error:|FAIL|failed", l)), lines[-1] if lines else "no output")) | |
| return f" ({hit[:140]})" | |
| def unchanged(sb, *names): | |
| """True when the named files are still exactly as the task gave them.""" | |
| return all(sb.get(n) == sb.files[n] for n in names) | |