File size: 17,628 Bytes
d19188c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
"""Conversation scenarios for self-generated calibration / evaluation chats and agent fixtures."""

import json


def fn(name, description, properties, required):
    return {
        "type": "function",
        "function": {
            "name": name,
            "description": description,
            "parameters": {"type": "object", "properties": properties, "required": required},
        },
    }


TOOLS = {
    "get_weather": fn(
        "get_weather", "Get the current weather for a city.",
        {"city": {"type": "string"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}}, ["city"],
    ),
    "terminal": fn(
        "terminal", "Run a shell command in the project sandbox and return stdout, stderr and the exit code.",
        {"command": {"type": "string", "description": "Command to run"}, "timeout": {"type": "integer"}}, ["command"],
    ),
    "read_file": fn(
        "read_file", "Read a text file and return its contents with line numbers.",
        {"path": {"type": "string"}, "offset": {"type": "integer"}, "limit": {"type": "integer"}}, ["path"],
    ),
    "write_file": fn(
        "write_file", "Create or overwrite a text file.",
        {"path": {"type": "string"}, "content": {"type": "string"}}, ["path", "content"],
    ),
    "web_search": fn(
        "web_search", "Search the web and return the top results.",
        {"query": {"type": "string"}, "max_results": {"type": "integer"}}, ["query"],
    ),
    "calculator": fn(
        "calculator", "Evaluate an arithmetic expression.", {"expression": {"type": "string"}}, ["expression"]
    ),
    "search_orders": fn(
        "search_orders", "Look up customer orders.",
        {
            "customer_id": {"type": "string"},
            "status": {"type": "string", "enum": ["open", "shipped", "returned"]},
            "filters": {
                "type": "object",
                "properties": {"min_total": {"type": "number"}, "tags": {"type": "array", "items": {"type": "string"}}},
            },
        },
        ["customer_id"],
    ),
}

FILES = {
    "app/config.py": 'import os\n\nDEBUG = os.environ.get("DEBUG", "0") == "1"\nTIMEOUT = 30\nRETRIES = 3\nDATABASE_URL = os.environ["DATABASE_URL"]\n',
    "README.md": "# Tidepool\n\nA small job queue.\n\n## Run\n\n    make dev\n\n## Test\n\n    pytest -q\n",
    "src/queue.py": (
        "import heapq\n\n\nclass JobQueue:\n    def __init__(self):\n        self._heap = []\n        self._count = 0\n\n"
        "    def push(self, job, priority=0):\n        heapq.heappush(self._heap, (priority, self._count, job))\n        self._count += 1\n\n"
        "    def pop(self):\n        return heapq.heappop(self._heap)[1]\n\n    def __len__(self):\n        return len(self._heap)\n"
    ),
}


def _terminal(cmd):
    """A tiny scripted shell over FILES: enough for an agent to explore, run tests and commit."""
    import re

    out, err, code = [], [], 0
    for part in re.split(r"\s*(?:&&|;|\|\|)\s*", cmd):
        part = part.strip()
        if not part:
            continue
        head = part.split("|")[0].strip()
        words = head.split()
        prog = words[0] if words else ""
        if "pytest" in part:
            fixed = "return heapq.heappop(self._heap)[2]" in FILES["src/queue.py"]
            if fixed:
                out.append("..                                                                       [100%]\n2 passed in 0.02s")
            else:
                code = 1
                out.append("F.                                                                       [100%]\n=== FAILURES ===\n___ test_pop_returns_job ___\n    def test_pop_returns_job():\n        q = JobQueue(); q.push('a')\n>       assert q.pop() == 'a'\nE       AssertionError: assert 0 == 'a'\ntests/test_queue.py:7: AssertionError\n1 failed, 1 passed in 0.03s")
        elif prog == "ls":
            target = next((w for w in words[1:] if not w.startswith("-")), ".").rstrip("/")
            names = sorted({f.split("/")[0] if target in (".", "") else f[len(target) + 1:].split("/")[0]
                            for f in list(FILES) + ["tests/test_queue.py", "Makefile"]
                            if target in (".", "") or f.startswith(target + "/")})
            out.append("\n".join(names) if names else "") if names else err.append(f"ls: {target}: No such file or directory")
        elif prog in ("cat", "head", "tail", "sed", "nl"):
            paths = [w for w in words[1:]
                     if not w.startswith("-") and not w.isdigit() and ">" not in w and "," not in w and w != "p"]
            for path in paths:
                path = path.lstrip("./")
                if path in FILES:
                    out.append(FILES[path].rstrip("\n"))
                elif path == "Makefile":
                    out.append("dev:\n\tpython -m app\n\ntest:\n\tpytest -q")
                elif path == "tests/test_queue.py":
                    out.append("from src.queue import JobQueue\n\n\ndef test_len():\n    q = JobQueue(); q.push('a'); assert len(q) == 1\n\n\ndef test_pop_returns_job():\n    q = JobQueue(); q.push('a')\n    assert q.pop() == 'a'")
                else:
                    code = 1; err.append(f"{prog}: {path}: No such file or directory")
        elif prog == "pwd":
            out.append("/workspace/tidepool")
        elif prog == "echo":
            out.append(" ".join(words[1:]).strip("'\""))
        elif prog == "find":
            out.append("\n".join("./" + f for f in sorted(list(FILES) + ["tests/test_queue.py", "Makefile"])))
        elif prog == "grep":
            pat = next((w.strip("'\"") for w in words[1:] if not w.startswith("-")), "")
            hits = [f"{f}:{i + 1}:{line}" for f, body in FILES.items() for i, line in enumerate(body.splitlines()) if pat and pat in line]
            out.append("\n".join(hits)) if hits else None
            code = 0 if hits else 1
        elif prog == "git":
            if "status" in part:
                out.append("On branch main\nnothing to commit, working tree clean")
            elif "diff" in part:
                out.append("")
            elif "log" in part:
                out.append("a1b2c3d Add priority queue\n9f8e7d6 Initial commit")
            else:
                out.append("")
        elif prog in ("python", "python3", "make"):
            out.append("")
        elif prog == "wc":
            out.append("      17 src/queue.py")
        else:
            code = 127; err.append(f"sh: {prog}: command not found")
    return {"exit_code": code, "stdout": "\n".join(x for x in out if x), "stderr": "\n".join(err)}


def run_tool(name, args):
    """Deterministic canned tool results."""
    if name == "get_weather":
        table = {"lisbon": (19, "clear"), "oslo": (3, "snow"), "tokyo": (14, "rain"), "cairo": (28, "sunny")}
        t, sky = table.get(str(args.get("city", "")).lower(), (12, "cloudy"))
        if args.get("unit") == "fahrenheit":
            return json.dumps({"city": args.get("city"), "temp_f": round(t * 9 / 5 + 32), "sky": sky})
        return json.dumps({"city": args.get("city"), "temp_c": t, "sky": sky})
    if name == "terminal":
        return json.dumps(_terminal(str(args.get("command", ""))))
    if name == "read_file":
        body = FILES.get(str(args.get("path", "")))
        if body is None:
            return json.dumps({"error": f"No such file: {args.get('path')}"})
        return "\n".join(f"{i + 1}\t{line}" for i, line in enumerate(body.splitlines()))
    if name == "write_file":
        path = str(args.get("path", "")).lstrip("./")
        FILES[path] = str(args.get("content", ""))
        return json.dumps({"ok": True, "bytes": len(FILES[path])})
    if name == "web_search":
        return json.dumps({"results": [
            {"title": "MLX documentation", "url": "https://ml-explore.github.io/mlx/", "snippet": "MLX is an array framework for machine learning on Apple silicon."},
            {"title": "Unified memory on Apple silicon", "url": "https://example.org/unified-memory", "snippet": "CPU and GPU share one memory pool, so arrays need no copies."},
        ]})
    if name == "calculator":
        try:
            return json.dumps({"result": eval(str(args.get("expression", "0")), {"__builtins__": {}}, {})})
        except Exception as exc:  # noqa: BLE001
            return json.dumps({"error": str(exc)})
    if name == "search_orders":
        return json.dumps({"orders": [
            {"id": "A-1042", "status": "shipped", "total": 129.5, "tags": ["gift"]},
            {"id": "A-1077", "status": "open", "total": 48.0, "tags": []},
        ]})
    return json.dumps({"error": f"unknown tool {name}"})


AGENT_SYSTEM = (
    "You are a coding agent working in a sandboxed repository. Use the tools to inspect and change files. "
    "Keep answers short and report what you actually did."
)

ARTICLE = (
    "The municipal archive reopened on Monday after a two-year renovation. The work replaced the roof, added a "
    "climate-controlled vault for the oldest parish registers, and moved the reading room to the ground floor so "
    "that visitors no longer need the narrow staircase. Opening hours are now 9:00 to 17:00 on weekdays and 10:00 "
    "to 14:00 on Saturdays. Researchers must book a desk at least one day ahead; the archive holds 24 desks. "
    "Digitisation continues: about 40 percent of the pre-1900 registers can be read online, and the team expects "
    "to reach 60 percent by the end of next year. The renovation cost 3.2 million euros, of which the region paid half."
)

# (id, split, category, kwargs)
S = []


def add(sid, split, category, user, tools=None, system=None, effort="low", max_tokens=700, followups=()):
    S.append(dict(id=sid, split=split, category=category, user=user, tools=tools or [], system=system,
                  effort=effort, max_tokens=max_tokens, followups=list(followups)))


# --- tool use (agent fixtures and calibration) ---
add("tool_weather_pair", "calib", "tools", "Is it warmer in Lisbon or in Oslo right now? Use the tool.", ["get_weather"])
add("tool_weather_single", "eval", "tools", "What's the weather in Tokyo in fahrenheit?", ["get_weather", "calculator"])
add("tool_none_needed", "calib", "tools", "Thanks! No lookup needed, just tell me a fun fact about lighthouses.", ["get_weather", "web_search"])
add("agent_fix_test", "calib", "agent", "The test suite is failing. Find out why and fix it.", ["terminal", "read_file", "write_file"], AGENT_SYSTEM, "medium", 900)
add("agent_explore", "eval", "agent", "What does this project do and how do I run its tests?", ["terminal", "read_file"], AGENT_SYSTEM, "low", 700)
add("agent_edit_config", "calib", "agent", "Change the request timeout to 60 seconds in the config and show me the diff in words.", ["terminal", "read_file", "write_file"], AGENT_SYSTEM, "low", 800)
add("agent_missing_file", "eval", "agent", "Open docs/ARCHITECTURE.md and summarise it.", ["read_file", "terminal"], AGENT_SYSTEM, "low", 500)
add("tool_nested_args", "calib", "tools", "Find customer C-88's shipped orders over 100 dollars that are tagged gift.", ["search_orders"], None, "low", 500)
add("tool_nested_args_eval", "eval", "tools", "List open orders for customer C-19 with a total of at least 40.", ["search_orders", "calculator"], None, "low", 500)
add("tool_search_then_answer", "calib", "tools", "Search for what MLX is and answer in two sentences with the source URL.", ["web_search"], None, "low", 500)
add("tool_calc_chain", "eval", "tools", "Use the calculator: what is (1234 * 57) - 999, and what is that divided by 7?", ["calculator"], None, "low", 500)
add("tool_multiturn", "calib", "tools", "What's the weather in Cairo?", ["get_weather"], None, "none", 400,
    followups=["And in Oslo? Which one needs a coat?"])
# --- coding ---
add("code_lru", "calib", "code", "Write a small LRU cache class in Python with get and put, O(1) each, plus three assert-based tests.", max_tokens=800)
add("code_js_debounce", "calib", "code", "Write a debounce function in TypeScript with a cancel method and explain when leading-edge firing is useful.", max_tokens=700)
add("code_sql", "eval", "code", "Given tables orders(id, customer_id, total, created_at) and customers(id, country), write SQL for monthly revenue per country for 2025, and say which index helps.", max_tokens=700)
add("code_bash", "calib", "code", "Write a bash script that finds the ten largest files under a directory given as $1, safely handling spaces in names.", max_tokens=600)
add("code_review", "eval", "code", "Review this function and point out bugs:\n\n```python\ndef median(xs):\n    xs.sort()\n    n = len(xs)\n    return xs[n/2] if n % 2 else (xs[n/2] + xs[n/2+1]) / 2\n```", max_tokens=700)
add("code_rust", "calib", "code", "Explain Rust's ownership rules with one short example that fails to compile and its fix.", max_tokens=700)
add("code_regex", "eval", "code", "Give a regular expression for ISO 8601 dates (YYYY-MM-DD) that rejects month 13 and day 32, with a brief explanation of each part.", max_tokens=600)
# --- reasoning / math ---
add("math_trains", "calib", "reasoning", "Two trains start 300 km apart and move toward each other at 70 km/h and 80 km/h. When do they meet, and how far has each travelled?", effort="medium")
add("math_prob", "eval", "reasoning", "I roll two fair dice. What is the probability that the product is even? Show the reasoning briefly.", effort="medium")
add("logic_puzzle", "calib", "reasoning", "Ana, Ben and Cy each own one pet: a cat, a dog, a fish. Ana does not own the dog. Ben owns neither the dog nor the fish. Who owns what?", effort="high", max_tokens=900)
add("estimate", "eval", "reasoning", "Estimate how many piano tuners work in a city of two million people. State your assumptions.", effort="medium")
add("plan_trip", "calib", "reasoning", "Plan a two-day itinerary for Lisbon for someone who likes architecture and dislikes crowds. Keep it under 200 words.", effort="low")
# --- summarisation / extraction / structured output ---
add("summ_article", "calib", "text", "Summarise in three bullet points:\n\n" + ARTICLE, effort="none")
add("extract_json", "calib", "structured", "Extract the facts from this text as JSON with keys opening_hours (object), desks (integer), cost_eur (number) and digitised_percent (integer). Output only the JSON.\n\n" + ARTICLE, effort="none", max_tokens=400)
add("extract_json_eval", "eval", "structured", "Return a JSON array of objects {\"name\", \"role\", \"year\"} for: Ada Lovelace wrote notes on the Analytical Engine in 1843; Grace Hopper built the A-0 compiler in 1952; Margaret Hamilton led Apollo flight software in 1969. Output only the JSON.", effort="none", max_tokens=400)
add("rewrite_tone", "eval", "text", "Rewrite this to be polite and concise: 'You sent the wrong file again. Fix it now, I am tired of waiting.'", effort="none", max_tokens=300)
add("classify", "calib", "structured", "Classify each review as positive, negative or mixed and answer as a markdown table:\n1. 'Battery lasts forever but the screen scratches easily.'\n2. 'Arrived broken.'\n3. 'Exactly what I needed.'", effort="none", max_tokens=400)
# --- knowledge / explanation ---
add("explain_tcp", "calib", "knowledge", "Explain the TCP three-way handshake to a new engineer in about 120 words.", effort="none", max_tokens=400)
add("explain_photosynth", "eval", "knowledge", "Explain photosynthesis at two levels: for a ten-year-old, then for a biology undergraduate.", effort="low", max_tokens=600)
add("compare", "calib", "knowledge", "Compare B-trees and LSM-trees for a write-heavy workload. Give a recommendation.", effort="low", max_tokens=600)
add("history", "eval", "knowledge", "Why did the Hanseatic League decline? Give the three main causes.", effort="low", max_tokens=500)
# --- multilingual ---
add("fr_email", "calib", "multilingual", "Rédige un court e-mail professionnel pour reporter une réunion à jeudi 15 h, en t'excusant du changement.", effort="none", max_tokens=350)
add("de_explain", "eval", "multilingual", "Erkläre in einfachem Deutsch, was eine Rekursion ist, mit einem Beispiel aus dem Alltag.", effort="none", max_tokens=350)
add("es_recipe", "calib", "multilingual", "Dame una receta sencilla de tortilla de patatas para dos personas, con cantidades.", effort="none", max_tokens=400)
add("ja_translate", "calib", "multilingual", "次の文を英語と中国語に翻訳してください:「明日の会議は午後三時からオンラインで行います。」", effort="none", max_tokens=300)
add("zh_summary", "eval", "multilingual", "用三句话解释什么是机器学习中的过拟合,并给出一个避免的方法。", effort="none", max_tokens=350)
add("pt_story", "calib", "multilingual", "Escreve uma história muito curta (80 palavras) sobre um farol e uma gaivota.", effort="none", max_tokens=350)
# --- system-prompted persona / format constraints ---
add("sys_terse", "calib", "system", "How do I undo my last git commit but keep the changes?", system="Answer in at most two sentences. Never use markdown.", effort="none", max_tokens=200)
add("sys_json_only", "eval", "system", "Book a table for four at 19:30 tomorrow under the name Okafor.", system='You are an intent parser. Reply with exactly one JSON object: {"intent": string, "slots": object}. No prose.', effort="none", max_tokens=250)
add("sys_refuse_scope", "calib", "system", "What's a good pasta recipe?", system="You are the support assistant for a tax-filing product. Only answer questions about the product; politely decline anything else.", effort="none", max_tokens=250)

SCENARIOS = S