VisualInference's picture
Inkling 2 Mini for MLX, 4bit: weights, card, validation results
d19188c verified
Raw History Blame Contribute Delete
17.6 kB
"""Conversation scenarios for self-generated calibration / evaluation chats and agent fixtures."""
import json
def fn(name, description, properties, required):
return {
"type": "function",
"function": {
"name": name,
"description": description,
"parameters": {"type": "object", "properties": properties, "required": required},
},
}
TOOLS = {
"get_weather": fn(
"get_weather", "Get the current weather for a city.",
{"city": {"type": "string"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}}, ["city"],
),
"terminal": fn(
"terminal", "Run a shell command in the project sandbox and return stdout, stderr and the exit code.",
{"command": {"type": "string", "description": "Command to run"}, "timeout": {"type": "integer"}}, ["command"],
),
"read_file": fn(
"read_file", "Read a text file and return its contents with line numbers.",
{"path": {"type": "string"}, "offset": {"type": "integer"}, "limit": {"type": "integer"}}, ["path"],
),
"write_file": fn(
"write_file", "Create or overwrite a text file.",
{"path": {"type": "string"}, "content": {"type": "string"}}, ["path", "content"],
),
"web_search": fn(
"web_search", "Search the web and return the top results.",
{"query": {"type": "string"}, "max_results": {"type": "integer"}}, ["query"],
),
"calculator": fn(
"calculator", "Evaluate an arithmetic expression.", {"expression": {"type": "string"}}, ["expression"]
),
"search_orders": fn(
"search_orders", "Look up customer orders.",
{
"customer_id": {"type": "string"},
"status": {"type": "string", "enum": ["open", "shipped", "returned"]},
"filters": {
"type": "object",
"properties": {"min_total": {"type": "number"}, "tags": {"type": "array", "items": {"type": "string"}}},
},
},
["customer_id"],
),
}
FILES = {
"app/config.py": 'import os\n\nDEBUG = os.environ.get("DEBUG", "0") == "1"\nTIMEOUT = 30\nRETRIES = 3\nDATABASE_URL = os.environ["DATABASE_URL"]\n',
"README.md": "# Tidepool\n\nA small job queue.\n\n## Run\n\n make dev\n\n## Test\n\n pytest -q\n",
"src/queue.py": (
"import heapq\n\n\nclass JobQueue:\n def __init__(self):\n self._heap = []\n self._count = 0\n\n"
" def push(self, job, priority=0):\n heapq.heappush(self._heap, (priority, self._count, job))\n self._count += 1\n\n"
" def pop(self):\n return heapq.heappop(self._heap)[1]\n\n def __len__(self):\n return len(self._heap)\n"
),
}
def _terminal(cmd):
"""A tiny scripted shell over FILES: enough for an agent to explore, run tests and commit."""
import re
out, err, code = [], [], 0
for part in re.split(r"\s*(?:&&|;|\|\|)\s*", cmd):
part = part.strip()
if not part:
continue
head = part.split("|")[0].strip()
words = head.split()
prog = words[0] if words else ""
if "pytest" in part:
fixed = "return heapq.heappop(self._heap)[2]" in FILES["src/queue.py"]
if fixed:
out.append(".. [100%]\n2 passed in 0.02s")
else:
code = 1
out.append("F. [100%]\n=== FAILURES ===\n___ test_pop_returns_job ___\n def test_pop_returns_job():\n q = JobQueue(); q.push('a')\n> assert q.pop() == 'a'\nE AssertionError: assert 0 == 'a'\ntests/test_queue.py:7: AssertionError\n1 failed, 1 passed in 0.03s")
elif prog == "ls":
target = next((w for w in words[1:] if not w.startswith("-")), ".").rstrip("/")
names = sorted({f.split("/")[0] if target in (".", "") else f[len(target) + 1:].split("/")[0]
for f in list(FILES) + ["tests/test_queue.py", "Makefile"]
if target in (".", "") or f.startswith(target + "/")})
out.append("\n".join(names) if names else "") if names else err.append(f"ls: {target}: No such file or directory")
elif prog in ("cat", "head", "tail", "sed", "nl"):
paths = [w for w in words[1:]
if not w.startswith("-") and not w.isdigit() and ">" not in w and "," not in w and w != "p"]
for path in paths:
path = path.lstrip("./")
if path in FILES:
out.append(FILES[path].rstrip("\n"))
elif path == "Makefile":
out.append("dev:\n\tpython -m app\n\ntest:\n\tpytest -q")
elif path == "tests/test_queue.py":
out.append("from src.queue import JobQueue\n\n\ndef test_len():\n q = JobQueue(); q.push('a'); assert len(q) == 1\n\n\ndef test_pop_returns_job():\n q = JobQueue(); q.push('a')\n assert q.pop() == 'a'")
else:
code = 1; err.append(f"{prog}: {path}: No such file or directory")
elif prog == "pwd":
out.append("/workspace/tidepool")
elif prog == "echo":
out.append(" ".join(words[1:]).strip("'\""))
elif prog == "find":
out.append("\n".join("./" + f for f in sorted(list(FILES) + ["tests/test_queue.py", "Makefile"])))
elif prog == "grep":
pat = next((w.strip("'\"") for w in words[1:] if not w.startswith("-")), "")
hits = [f"{f}:{i + 1}:{line}" for f, body in FILES.items() for i, line in enumerate(body.splitlines()) if pat and pat in line]
out.append("\n".join(hits)) if hits else None
code = 0 if hits else 1
elif prog == "git":
if "status" in part:
out.append("On branch main\nnothing to commit, working tree clean")
elif "diff" in part:
out.append("")
elif "log" in part:
out.append("a1b2c3d Add priority queue\n9f8e7d6 Initial commit")
else:
out.append("")
elif prog in ("python", "python3", "make"):
out.append("")
elif prog == "wc":
out.append(" 17 src/queue.py")
else:
code = 127; err.append(f"sh: {prog}: command not found")
return {"exit_code": code, "stdout": "\n".join(x for x in out if x), "stderr": "\n".join(err)}
def run_tool(name, args):
"""Deterministic canned tool results."""
if name == "get_weather":
table = {"lisbon": (19, "clear"), "oslo": (3, "snow"), "tokyo": (14, "rain"), "cairo": (28, "sunny")}
t, sky = table.get(str(args.get("city", "")).lower(), (12, "cloudy"))
if args.get("unit") == "fahrenheit":
return json.dumps({"city": args.get("city"), "temp_f": round(t * 9 / 5 + 32), "sky": sky})
return json.dumps({"city": args.get("city"), "temp_c": t, "sky": sky})
if name == "terminal":
return json.dumps(_terminal(str(args.get("command", ""))))
if name == "read_file":
body = FILES.get(str(args.get("path", "")))
if body is None:
return json.dumps({"error": f"No such file: {args.get('path')}"})
return "\n".join(f"{i + 1}\t{line}" for i, line in enumerate(body.splitlines()))
if name == "write_file":
path = str(args.get("path", "")).lstrip("./")
FILES[path] = str(args.get("content", ""))
return json.dumps({"ok": True, "bytes": len(FILES[path])})
if name == "web_search":
return json.dumps({"results": [
{"title": "MLX documentation", "url": "https://ml-explore.github.io/mlx/", "snippet": "MLX is an array framework for machine learning on Apple silicon."},
{"title": "Unified memory on Apple silicon", "url": "https://example.org/unified-memory", "snippet": "CPU and GPU share one memory pool, so arrays need no copies."},
]})
if name == "calculator":
try:
return json.dumps({"result": eval(str(args.get("expression", "0")), {"__builtins__": {}}, {})})
except Exception as exc: # noqa: BLE001
return json.dumps({"error": str(exc)})
if name == "search_orders":
return json.dumps({"orders": [
{"id": "A-1042", "status": "shipped", "total": 129.5, "tags": ["gift"]},
{"id": "A-1077", "status": "open", "total": 48.0, "tags": []},
]})
return json.dumps({"error": f"unknown tool {name}"})
AGENT_SYSTEM = (
"You are a coding agent working in a sandboxed repository. Use the tools to inspect and change files. "
"Keep answers short and report what you actually did."
)
ARTICLE = (
"The municipal archive reopened on Monday after a two-year renovation. The work replaced the roof, added a "
"climate-controlled vault for the oldest parish registers, and moved the reading room to the ground floor so "
"that visitors no longer need the narrow staircase. Opening hours are now 9:00 to 17:00 on weekdays and 10:00 "
"to 14:00 on Saturdays. Researchers must book a desk at least one day ahead; the archive holds 24 desks. "
"Digitisation continues: about 40 percent of the pre-1900 registers can be read online, and the team expects "
"to reach 60 percent by the end of next year. The renovation cost 3.2 million euros, of which the region paid half."
)
# (id, split, category, kwargs)
S = []
def add(sid, split, category, user, tools=None, system=None, effort="low", max_tokens=700, followups=()):
S.append(dict(id=sid, split=split, category=category, user=user, tools=tools or [], system=system,
effort=effort, max_tokens=max_tokens, followups=list(followups)))
# --- tool use (agent fixtures and calibration) ---
add("tool_weather_pair", "calib", "tools", "Is it warmer in Lisbon or in Oslo right now? Use the tool.", ["get_weather"])
add("tool_weather_single", "eval", "tools", "What's the weather in Tokyo in fahrenheit?", ["get_weather", "calculator"])
add("tool_none_needed", "calib", "tools", "Thanks! No lookup needed, just tell me a fun fact about lighthouses.", ["get_weather", "web_search"])
add("agent_fix_test", "calib", "agent", "The test suite is failing. Find out why and fix it.", ["terminal", "read_file", "write_file"], AGENT_SYSTEM, "medium", 900)
add("agent_explore", "eval", "agent", "What does this project do and how do I run its tests?", ["terminal", "read_file"], AGENT_SYSTEM, "low", 700)
add("agent_edit_config", "calib", "agent", "Change the request timeout to 60 seconds in the config and show me the diff in words.", ["terminal", "read_file", "write_file"], AGENT_SYSTEM, "low", 800)
add("agent_missing_file", "eval", "agent", "Open docs/ARCHITECTURE.md and summarise it.", ["read_file", "terminal"], AGENT_SYSTEM, "low", 500)
add("tool_nested_args", "calib", "tools", "Find customer C-88's shipped orders over 100 dollars that are tagged gift.", ["search_orders"], None, "low", 500)
add("tool_nested_args_eval", "eval", "tools", "List open orders for customer C-19 with a total of at least 40.", ["search_orders", "calculator"], None, "low", 500)
add("tool_search_then_answer", "calib", "tools", "Search for what MLX is and answer in two sentences with the source URL.", ["web_search"], None, "low", 500)
add("tool_calc_chain", "eval", "tools", "Use the calculator: what is (1234 * 57) - 999, and what is that divided by 7?", ["calculator"], None, "low", 500)
add("tool_multiturn", "calib", "tools", "What's the weather in Cairo?", ["get_weather"], None, "none", 400,
followups=["And in Oslo? Which one needs a coat?"])
# --- coding ---
add("code_lru", "calib", "code", "Write a small LRU cache class in Python with get and put, O(1) each, plus three assert-based tests.", max_tokens=800)
add("code_js_debounce", "calib", "code", "Write a debounce function in TypeScript with a cancel method and explain when leading-edge firing is useful.", max_tokens=700)
add("code_sql", "eval", "code", "Given tables orders(id, customer_id, total, created_at) and customers(id, country), write SQL for monthly revenue per country for 2025, and say which index helps.", max_tokens=700)
add("code_bash", "calib", "code", "Write a bash script that finds the ten largest files under a directory given as $1, safely handling spaces in names.", max_tokens=600)
add("code_review", "eval", "code", "Review this function and point out bugs:\n\n```python\ndef median(xs):\n xs.sort()\n n = len(xs)\n return xs[n/2] if n % 2 else (xs[n/2] + xs[n/2+1]) / 2\n```", max_tokens=700)
add("code_rust", "calib", "code", "Explain Rust's ownership rules with one short example that fails to compile and its fix.", max_tokens=700)
add("code_regex", "eval", "code", "Give a regular expression for ISO 8601 dates (YYYY-MM-DD) that rejects month 13 and day 32, with a brief explanation of each part.", max_tokens=600)
# --- reasoning / math ---
add("math_trains", "calib", "reasoning", "Two trains start 300 km apart and move toward each other at 70 km/h and 80 km/h. When do they meet, and how far has each travelled?", effort="medium")
add("math_prob", "eval", "reasoning", "I roll two fair dice. What is the probability that the product is even? Show the reasoning briefly.", effort="medium")
add("logic_puzzle", "calib", "reasoning", "Ana, Ben and Cy each own one pet: a cat, a dog, a fish. Ana does not own the dog. Ben owns neither the dog nor the fish. Who owns what?", effort="high", max_tokens=900)
add("estimate", "eval", "reasoning", "Estimate how many piano tuners work in a city of two million people. State your assumptions.", effort="medium")
add("plan_trip", "calib", "reasoning", "Plan a two-day itinerary for Lisbon for someone who likes architecture and dislikes crowds. Keep it under 200 words.", effort="low")
# --- summarisation / extraction / structured output ---
add("summ_article", "calib", "text", "Summarise in three bullet points:\n\n" + ARTICLE, effort="none")
add("extract_json", "calib", "structured", "Extract the facts from this text as JSON with keys opening_hours (object), desks (integer), cost_eur (number) and digitised_percent (integer). Output only the JSON.\n\n" + ARTICLE, effort="none", max_tokens=400)
add("extract_json_eval", "eval", "structured", "Return a JSON array of objects {\"name\", \"role\", \"year\"} for: Ada Lovelace wrote notes on the Analytical Engine in 1843; Grace Hopper built the A-0 compiler in 1952; Margaret Hamilton led Apollo flight software in 1969. Output only the JSON.", effort="none", max_tokens=400)
add("rewrite_tone", "eval", "text", "Rewrite this to be polite and concise: 'You sent the wrong file again. Fix it now, I am tired of waiting.'", effort="none", max_tokens=300)
add("classify", "calib", "structured", "Classify each review as positive, negative or mixed and answer as a markdown table:\n1. 'Battery lasts forever but the screen scratches easily.'\n2. 'Arrived broken.'\n3. 'Exactly what I needed.'", effort="none", max_tokens=400)
# --- knowledge / explanation ---
add("explain_tcp", "calib", "knowledge", "Explain the TCP three-way handshake to a new engineer in about 120 words.", effort="none", max_tokens=400)
add("explain_photosynth", "eval", "knowledge", "Explain photosynthesis at two levels: for a ten-year-old, then for a biology undergraduate.", effort="low", max_tokens=600)
add("compare", "calib", "knowledge", "Compare B-trees and LSM-trees for a write-heavy workload. Give a recommendation.", effort="low", max_tokens=600)
add("history", "eval", "knowledge", "Why did the Hanseatic League decline? Give the three main causes.", effort="low", max_tokens=500)
# --- multilingual ---
add("fr_email", "calib", "multilingual", "Rédige un court e-mail professionnel pour reporter une réunion à jeudi 15 h, en t'excusant du changement.", effort="none", max_tokens=350)
add("de_explain", "eval", "multilingual", "Erkläre in einfachem Deutsch, was eine Rekursion ist, mit einem Beispiel aus dem Alltag.", effort="none", max_tokens=350)
add("es_recipe", "calib", "multilingual", "Dame una receta sencilla de tortilla de patatas para dos personas, con cantidades.", effort="none", max_tokens=400)
add("ja_translate", "calib", "multilingual", "次の文を英語と中国語に翻訳してください:「明日の会議は午後三時からオンラインで行います。」", effort="none", max_tokens=300)
add("zh_summary", "eval", "multilingual", "用三句话解释什么是机器学习中的过拟合,并给出一个避免的方法。", effort="none", max_tokens=350)
add("pt_story", "calib", "multilingual", "Escreve uma história muito curta (80 palavras) sobre um farol e uma gaivota.", effort="none", max_tokens=350)
# --- system-prompted persona / format constraints ---
add("sys_terse", "calib", "system", "How do I undo my last git commit but keep the changes?", system="Answer in at most two sentences. Never use markdown.", effort="none", max_tokens=200)
add("sys_json_only", "eval", "system", "Book a table for four at 19:30 tomorrow under the name Okafor.", system='You are an intent parser. Reply with exactly one JSON object: {"intent": string, "slots": object}. No prose.', effort="none", max_tokens=250)
add("sys_refuse_scope", "calib", "system", "What's a good pasta recipe?", system="You are the support assistant for a tax-filing product. Only answer questions about the product; politely decline anything else.", effort="none", max_tokens=250)
SCENARIOS = S