"""Conversation scenarios for self-generated calibration / evaluation chats and agent fixtures.""" import json def fn(name, description, properties, required): return { "type": "function", "function": { "name": name, "description": description, "parameters": {"type": "object", "properties": properties, "required": required}, }, } TOOLS = { "get_weather": fn( "get_weather", "Get the current weather for a city.", {"city": {"type": "string"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}}, ["city"], ), "terminal": fn( "terminal", "Run a shell command in the project sandbox and return stdout, stderr and the exit code.", {"command": {"type": "string", "description": "Command to run"}, "timeout": {"type": "integer"}}, ["command"], ), "read_file": fn( "read_file", "Read a text file and return its contents with line numbers.", {"path": {"type": "string"}, "offset": {"type": "integer"}, "limit": {"type": "integer"}}, ["path"], ), "write_file": fn( "write_file", "Create or overwrite a text file.", {"path": {"type": "string"}, "content": {"type": "string"}}, ["path", "content"], ), "web_search": fn( "web_search", "Search the web and return the top results.", {"query": {"type": "string"}, "max_results": {"type": "integer"}}, ["query"], ), "calculator": fn( "calculator", "Evaluate an arithmetic expression.", {"expression": {"type": "string"}}, ["expression"] ), "search_orders": fn( "search_orders", "Look up customer orders.", { "customer_id": {"type": "string"}, "status": {"type": "string", "enum": ["open", "shipped", "returned"]}, "filters": { "type": "object", "properties": {"min_total": {"type": "number"}, "tags": {"type": "array", "items": {"type": "string"}}}, }, }, ["customer_id"], ), } FILES = { "app/config.py": 'import os\n\nDEBUG = os.environ.get("DEBUG", "0") == "1"\nTIMEOUT = 30\nRETRIES = 3\nDATABASE_URL = os.environ["DATABASE_URL"]\n', "README.md": "# Tidepool\n\nA small job queue.\n\n## Run\n\n make dev\n\n## Test\n\n pytest -q\n", "src/queue.py": ( "import heapq\n\n\nclass JobQueue:\n def __init__(self):\n self._heap = []\n self._count = 0\n\n" " def push(self, job, priority=0):\n heapq.heappush(self._heap, (priority, self._count, job))\n self._count += 1\n\n" " def pop(self):\n return heapq.heappop(self._heap)[1]\n\n def __len__(self):\n return len(self._heap)\n" ), } def _terminal(cmd): """A tiny scripted shell over FILES: enough for an agent to explore, run tests and commit.""" import re out, err, code = [], [], 0 for part in re.split(r"\s*(?:&&|;|\|\|)\s*", cmd): part = part.strip() if not part: continue head = part.split("|")[0].strip() words = head.split() prog = words[0] if words else "" if "pytest" in part: fixed = "return heapq.heappop(self._heap)[2]" in FILES["src/queue.py"] if fixed: out.append(".. [100%]\n2 passed in 0.02s") else: code = 1 out.append("F. [100%]\n=== FAILURES ===\n___ test_pop_returns_job ___\n def test_pop_returns_job():\n q = JobQueue(); q.push('a')\n> assert q.pop() == 'a'\nE AssertionError: assert 0 == 'a'\ntests/test_queue.py:7: AssertionError\n1 failed, 1 passed in 0.03s") elif prog == "ls": target = next((w for w in words[1:] if not w.startswith("-")), ".").rstrip("/") names = sorted({f.split("/")[0] if target in (".", "") else f[len(target) + 1:].split("/")[0] for f in list(FILES) + ["tests/test_queue.py", "Makefile"] if target in (".", "") or f.startswith(target + "/")}) out.append("\n".join(names) if names else "") if names else err.append(f"ls: {target}: No such file or directory") elif prog in ("cat", "head", "tail", "sed", "nl"): paths = [w for w in words[1:] if not w.startswith("-") and not w.isdigit() and ">" not in w and "," not in w and w != "p"] for path in paths: path = path.lstrip("./") if path in FILES: out.append(FILES[path].rstrip("\n")) elif path == "Makefile": out.append("dev:\n\tpython -m app\n\ntest:\n\tpytest -q") elif path == "tests/test_queue.py": out.append("from src.queue import JobQueue\n\n\ndef test_len():\n q = JobQueue(); q.push('a'); assert len(q) == 1\n\n\ndef test_pop_returns_job():\n q = JobQueue(); q.push('a')\n assert q.pop() == 'a'") else: code = 1; err.append(f"{prog}: {path}: No such file or directory") elif prog == "pwd": out.append("/workspace/tidepool") elif prog == "echo": out.append(" ".join(words[1:]).strip("'\"")) elif prog == "find": out.append("\n".join("./" + f for f in sorted(list(FILES) + ["tests/test_queue.py", "Makefile"]))) elif prog == "grep": pat = next((w.strip("'\"") for w in words[1:] if not w.startswith("-")), "") hits = [f"{f}:{i + 1}:{line}" for f, body in FILES.items() for i, line in enumerate(body.splitlines()) if pat and pat in line] out.append("\n".join(hits)) if hits else None code = 0 if hits else 1 elif prog == "git": if "status" in part: out.append("On branch main\nnothing to commit, working tree clean") elif "diff" in part: out.append("") elif "log" in part: out.append("a1b2c3d Add priority queue\n9f8e7d6 Initial commit") else: out.append("") elif prog in ("python", "python3", "make"): out.append("") elif prog == "wc": out.append(" 17 src/queue.py") else: code = 127; err.append(f"sh: {prog}: command not found") return {"exit_code": code, "stdout": "\n".join(x for x in out if x), "stderr": "\n".join(err)} def run_tool(name, args): """Deterministic canned tool results.""" if name == "get_weather": table = {"lisbon": (19, "clear"), "oslo": (3, "snow"), "tokyo": (14, "rain"), "cairo": (28, "sunny")} t, sky = table.get(str(args.get("city", "")).lower(), (12, "cloudy")) if args.get("unit") == "fahrenheit": return json.dumps({"city": args.get("city"), "temp_f": round(t * 9 / 5 + 32), "sky": sky}) return json.dumps({"city": args.get("city"), "temp_c": t, "sky": sky}) if name == "terminal": return json.dumps(_terminal(str(args.get("command", "")))) if name == "read_file": body = FILES.get(str(args.get("path", ""))) if body is None: return json.dumps({"error": f"No such file: {args.get('path')}"}) return "\n".join(f"{i + 1}\t{line}" for i, line in enumerate(body.splitlines())) if name == "write_file": path = str(args.get("path", "")).lstrip("./") FILES[path] = str(args.get("content", "")) return json.dumps({"ok": True, "bytes": len(FILES[path])}) if name == "web_search": return json.dumps({"results": [ {"title": "MLX documentation", "url": "https://ml-explore.github.io/mlx/", "snippet": "MLX is an array framework for machine learning on Apple silicon."}, {"title": "Unified memory on Apple silicon", "url": "https://example.org/unified-memory", "snippet": "CPU and GPU share one memory pool, so arrays need no copies."}, ]}) if name == "calculator": try: return json.dumps({"result": eval(str(args.get("expression", "0")), {"__builtins__": {}}, {})}) except Exception as exc: # noqa: BLE001 return json.dumps({"error": str(exc)}) if name == "search_orders": return json.dumps({"orders": [ {"id": "A-1042", "status": "shipped", "total": 129.5, "tags": ["gift"]}, {"id": "A-1077", "status": "open", "total": 48.0, "tags": []}, ]}) return json.dumps({"error": f"unknown tool {name}"}) AGENT_SYSTEM = ( "You are a coding agent working in a sandboxed repository. Use the tools to inspect and change files. " "Keep answers short and report what you actually did." ) ARTICLE = ( "The municipal archive reopened on Monday after a two-year renovation. The work replaced the roof, added a " "climate-controlled vault for the oldest parish registers, and moved the reading room to the ground floor so " "that visitors no longer need the narrow staircase. Opening hours are now 9:00 to 17:00 on weekdays and 10:00 " "to 14:00 on Saturdays. Researchers must book a desk at least one day ahead; the archive holds 24 desks. " "Digitisation continues: about 40 percent of the pre-1900 registers can be read online, and the team expects " "to reach 60 percent by the end of next year. The renovation cost 3.2 million euros, of which the region paid half." ) # (id, split, category, kwargs) S = [] def add(sid, split, category, user, tools=None, system=None, effort="low", max_tokens=700, followups=()): S.append(dict(id=sid, split=split, category=category, user=user, tools=tools or [], system=system, effort=effort, max_tokens=max_tokens, followups=list(followups))) # --- tool use (agent fixtures and calibration) --- add("tool_weather_pair", "calib", "tools", "Is it warmer in Lisbon or in Oslo right now? Use the tool.", ["get_weather"]) add("tool_weather_single", "eval", "tools", "What's the weather in Tokyo in fahrenheit?", ["get_weather", "calculator"]) add("tool_none_needed", "calib", "tools", "Thanks! No lookup needed, just tell me a fun fact about lighthouses.", ["get_weather", "web_search"]) add("agent_fix_test", "calib", "agent", "The test suite is failing. Find out why and fix it.", ["terminal", "read_file", "write_file"], AGENT_SYSTEM, "medium", 900) add("agent_explore", "eval", "agent", "What does this project do and how do I run its tests?", ["terminal", "read_file"], AGENT_SYSTEM, "low", 700) add("agent_edit_config", "calib", "agent", "Change the request timeout to 60 seconds in the config and show me the diff in words.", ["terminal", "read_file", "write_file"], AGENT_SYSTEM, "low", 800) add("agent_missing_file", "eval", "agent", "Open docs/ARCHITECTURE.md and summarise it.", ["read_file", "terminal"], AGENT_SYSTEM, "low", 500) add("tool_nested_args", "calib", "tools", "Find customer C-88's shipped orders over 100 dollars that are tagged gift.", ["search_orders"], None, "low", 500) add("tool_nested_args_eval", "eval", "tools", "List open orders for customer C-19 with a total of at least 40.", ["search_orders", "calculator"], None, "low", 500) add("tool_search_then_answer", "calib", "tools", "Search for what MLX is and answer in two sentences with the source URL.", ["web_search"], None, "low", 500) add("tool_calc_chain", "eval", "tools", "Use the calculator: what is (1234 * 57) - 999, and what is that divided by 7?", ["calculator"], None, "low", 500) add("tool_multiturn", "calib", "tools", "What's the weather in Cairo?", ["get_weather"], None, "none", 400, followups=["And in Oslo? Which one needs a coat?"]) # --- coding --- add("code_lru", "calib", "code", "Write a small LRU cache class in Python with get and put, O(1) each, plus three assert-based tests.", max_tokens=800) add("code_js_debounce", "calib", "code", "Write a debounce function in TypeScript with a cancel method and explain when leading-edge firing is useful.", max_tokens=700) add("code_sql", "eval", "code", "Given tables orders(id, customer_id, total, created_at) and customers(id, country), write SQL for monthly revenue per country for 2025, and say which index helps.", max_tokens=700) add("code_bash", "calib", "code", "Write a bash script that finds the ten largest files under a directory given as $1, safely handling spaces in names.", max_tokens=600) add("code_review", "eval", "code", "Review this function and point out bugs:\n\n```python\ndef median(xs):\n xs.sort()\n n = len(xs)\n return xs[n/2] if n % 2 else (xs[n/2] + xs[n/2+1]) / 2\n```", max_tokens=700) add("code_rust", "calib", "code", "Explain Rust's ownership rules with one short example that fails to compile and its fix.", max_tokens=700) add("code_regex", "eval", "code", "Give a regular expression for ISO 8601 dates (YYYY-MM-DD) that rejects month 13 and day 32, with a brief explanation of each part.", max_tokens=600) # --- reasoning / math --- add("math_trains", "calib", "reasoning", "Two trains start 300 km apart and move toward each other at 70 km/h and 80 km/h. When do they meet, and how far has each travelled?", effort="medium") add("math_prob", "eval", "reasoning", "I roll two fair dice. What is the probability that the product is even? Show the reasoning briefly.", effort="medium") add("logic_puzzle", "calib", "reasoning", "Ana, Ben and Cy each own one pet: a cat, a dog, a fish. Ana does not own the dog. Ben owns neither the dog nor the fish. Who owns what?", effort="high", max_tokens=900) add("estimate", "eval", "reasoning", "Estimate how many piano tuners work in a city of two million people. State your assumptions.", effort="medium") add("plan_trip", "calib", "reasoning", "Plan a two-day itinerary for Lisbon for someone who likes architecture and dislikes crowds. Keep it under 200 words.", effort="low") # --- summarisation / extraction / structured output --- add("summ_article", "calib", "text", "Summarise in three bullet points:\n\n" + ARTICLE, effort="none") add("extract_json", "calib", "structured", "Extract the facts from this text as JSON with keys opening_hours (object), desks (integer), cost_eur (number) and digitised_percent (integer). Output only the JSON.\n\n" + ARTICLE, effort="none", max_tokens=400) add("extract_json_eval", "eval", "structured", "Return a JSON array of objects {\"name\", \"role\", \"year\"} for: Ada Lovelace wrote notes on the Analytical Engine in 1843; Grace Hopper built the A-0 compiler in 1952; Margaret Hamilton led Apollo flight software in 1969. Output only the JSON.", effort="none", max_tokens=400) add("rewrite_tone", "eval", "text", "Rewrite this to be polite and concise: 'You sent the wrong file again. Fix it now, I am tired of waiting.'", effort="none", max_tokens=300) add("classify", "calib", "structured", "Classify each review as positive, negative or mixed and answer as a markdown table:\n1. 'Battery lasts forever but the screen scratches easily.'\n2. 'Arrived broken.'\n3. 'Exactly what I needed.'", effort="none", max_tokens=400) # --- knowledge / explanation --- add("explain_tcp", "calib", "knowledge", "Explain the TCP three-way handshake to a new engineer in about 120 words.", effort="none", max_tokens=400) add("explain_photosynth", "eval", "knowledge", "Explain photosynthesis at two levels: for a ten-year-old, then for a biology undergraduate.", effort="low", max_tokens=600) add("compare", "calib", "knowledge", "Compare B-trees and LSM-trees for a write-heavy workload. Give a recommendation.", effort="low", max_tokens=600) add("history", "eval", "knowledge", "Why did the Hanseatic League decline? Give the three main causes.", effort="low", max_tokens=500) # --- multilingual --- add("fr_email", "calib", "multilingual", "Rédige un court e-mail professionnel pour reporter une réunion à jeudi 15 h, en t'excusant du changement.", effort="none", max_tokens=350) add("de_explain", "eval", "multilingual", "Erkläre in einfachem Deutsch, was eine Rekursion ist, mit einem Beispiel aus dem Alltag.", effort="none", max_tokens=350) add("es_recipe", "calib", "multilingual", "Dame una receta sencilla de tortilla de patatas para dos personas, con cantidades.", effort="none", max_tokens=400) add("ja_translate", "calib", "multilingual", "次の文を英語と中国語に翻訳してください:「明日の会議は午後三時からオンラインで行います。」", effort="none", max_tokens=300) add("zh_summary", "eval", "multilingual", "用三句话解释什么是机器学习中的过拟合,并给出一个避免的方法。", effort="none", max_tokens=350) add("pt_story", "calib", "multilingual", "Escreve uma história muito curta (80 palavras) sobre um farol e uma gaivota.", effort="none", max_tokens=350) # --- system-prompted persona / format constraints --- add("sys_terse", "calib", "system", "How do I undo my last git commit but keep the changes?", system="Answer in at most two sentences. Never use markdown.", effort="none", max_tokens=200) add("sys_json_only", "eval", "system", "Book a table for four at 19:30 tomorrow under the name Okafor.", system='You are an intent parser. Reply with exactly one JSON object: {"intent": string, "slots": object}. No prose.', effort="none", max_tokens=250) add("sys_refuse_scope", "calib", "system", "What's a good pasta recipe?", system="You are the support assistant for a tax-filing product. Only answer questions about the product; politely decline anything else.", effort="none", max_tokens=250) SCENARIOS = S