Ratel / scripts /bake_llm.py
Berca Akbayır
Claude Opus 4.8 (1M context)
Step-by-step gateway traces, 100-tool catalog, Run button & per-panel metrics (#3)
3812402 unverified
Raw History Blame Contribute Delete
11.9 kB
#!/usr/bin/env python3
"""
Offline LLM bake — record, once, what a real model does with vs without Ratel,
across all three retrieval methods, so the methods can be compared.
Same idea as the internal kestral-ratel-demo, adapted for a static Space:
kestral ran the model + Ratel live per turn; we can't (no server), so we run
each curated prompt ONCE here and the Space replays the result.
Retrieval uses Ratel's DEFAULT embedding model. If the Ratel Python SDK is
importable, this script registers the 100-tool catalog into a hybrid
`ToolCatalog` (no embed-model override → Ratel's default) and searches it live,
per method. If the SDK isn't installed, it falls back to the top-5 that bake.py
already baked into data/examples.json (also produced with Ratel's default
embedder), so the script still runs with only `openai` available.
For EACH retriever method (Sparse=bm25, Dense=semantic, Hybrid) we run the same
prompt on the model twice — with and without Ratel — so every method has its own
with/without pair (3 methods × 2 conditions per prompt):
without Ratel -> all 100 tool schemas handed to the model
with Ratel -> only that method's top-5
Per run we save exactly what the model produced:
• the tool(s) it selected + arguments • its natural-language answer
• real token spend (prompt/completion/total) • gold-set coverage
plus the retrieval (top-5 ids + scores) that fed the with-Ratel run. A per-method
comparison summary is written alongside.
Run:
pip install openai # + `ratel-ai` for live retrieval (optional)
python scripts/bake_llm.py # reads .env for LLM_BASE_URL / LLM_MODEL / LLM_API_KEY
Reads: data/tools.json, data/examples.json, data/meta.json (run bake.py first)
Writes: data/llm.json
"""
from __future__ import annotations
import json
import os
import re
import sys
from pathlib import Path
from openai import OpenAI
ROOT = Path(__file__).resolve().parent.parent
DATA = ROOT / "data"
def load_dotenv(path: Path) -> None:
"""Populate os.environ from a .env file (KEY=VALUE lines). No dependency;
real environment variables always win over the file."""
if not path.exists():
return
for line in path.read_text().splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, _, value = line.partition("=")
os.environ.setdefault(key.strip(), value.strip().strip("'\""))
load_dotenv(ROOT / ".env")
BASE_URL = os.environ.get("LLM_BASE_URL", "http://localhost:11434/v1")
MODEL = os.environ.get("LLM_MODEL", "qwen3:4b")
API_KEY = os.environ.get("LLM_API_KEY", "not-needed") # local servers ignore it
# Sparse / Dense / Hybrid — the three retrieval methods we produce answers for.
METHODS = ["bm25", "semantic", "hybrid"]
METHOD_LABEL = {"bm25": "sparse", "semantic": "dense", "hybrid": "hybrid"}
TOP_K = 3
SYSTEM = (
"You are a task-management assistant for a project workspace. Fulfil the user's "
"request by calling the most appropriate tool(s). Invent plausible values for any "
"required arguments the request does not specify. Reads are always safe; only write "
"(create/update/comment/link) when the request clearly asks for it. After the tools "
"run, reply in one or two short sentences describing what you did."
)
_THINK = re.compile(r"<think>.*?</think>", re.DOTALL)
def norm(s: str) -> str:
"""Must match norm() in app.js."""
return re.sub(r"\s+", " ", s.strip().lower())
def openai_tool(t: dict) -> dict:
return {
"type": "function",
"function": {"name": t["name"], "description": t["description"], "parameters": t["inputSchema"]},
}
def clean(text: str | None) -> str:
return _THINK.sub("", text or "").strip()
# ── Ratel retrieval (default embedding model) ─────────────────────────────────
def build_ratel_catalog(tools: list[dict]):
"""Register the catalog into a hybrid ToolCatalog using Ratel's DEFAULT
embedding model (no override). Returns None if the SDK isn't installed."""
try:
from ratel_ai import ExecutableTool, ToolCatalog
except Exception: # noqa: BLE001 — SDK optional; caller falls back to examples.json
return None
cat = ToolCatalog(method="hybrid") # default embedder — same as scripts/bake.py
for t in tools:
cat.register(
ExecutableTool(
id=t["id"],
name=t["name"],
description=t["description"],
input_schema=t["inputSchema"],
output_schema={"type": "object", "properties": {}},
execute=lambda args: {},
)
)
return cat
def retrieve(catalog, example: dict, method: str) -> list[dict]:
"""Top-5 for one method: live Ratel search when the SDK is present, else the
top-5 bake.py already stored in examples.json (also Ratel-default retrieval)."""
if catalog is not None:
hits = catalog.search(example["query"], TOP_K, method=method)
return [{"id": h.tool_id, "score": round(h.score, 6)} for h in hits]
return [{"id": h["id"], "score": h["score"]} for h in example["results"][method][:TOP_K]]
# ── one model turn ────────────────────────────────────────────────────────────
def run_turn(client: OpenAI, query: str, tools: list[dict], gold: list[str]) -> dict:
"""Model picks tool(s); we hand back a neutral mock result; model summarizes.
contextTokens = prompt tokens of the tool-loaded first call — the number
Ratel actually moves (70 schemas vs 5).
"""
messages = [{"role": "system", "content": SYSTEM}, {"role": "user", "content": query}]
try:
first = client.chat.completions.create(
model=MODEL, messages=messages, tools=[openai_tool(t) for t in tools],
tool_choice="auto", temperature=0,
)
except Exception as e: # noqa: BLE001 — record the failure, keep baking the rest
return {"error": str(e)[:200]}
msg = first.choices[0].message
context_tokens = getattr(first.usage, "prompt_tokens", 0) or 0
prompt_tokens = context_tokens
completion_tokens = getattr(first.usage, "completion_tokens", 0) or 0
calls = msg.tool_calls or []
selected, inputs = [], []
for c in calls:
selected.append(c.function.name)
try:
inputs.append({"tool": c.function.name, "args": json.loads(c.function.arguments or "{}")})
except json.JSONDecodeError:
inputs.append({"tool": c.function.name, "args": c.function.arguments})
answer = clean(msg.content)
if calls:
# Neutral mock result so the model produces a closing sentence
# (kestral mocked tool executes the same way — symmetric across conditions).
messages.append(msg.model_dump())
for c in calls:
messages.append({"role": "tool", "tool_call_id": c.id, "content": json.dumps({"ok": True})})
try:
second = client.chat.completions.create(model=MODEL, messages=messages, temperature=0)
answer = clean(second.choices[0].message.content) or answer
prompt_tokens += getattr(second.usage, "prompt_tokens", 0) or 0
completion_tokens += getattr(second.usage, "completion_tokens", 0) or 0
except Exception: # noqa: BLE001 — summary is optional; keep the selection + tokens
pass
gold_set = set(gold)
return {
"tools": selected,
"inputs": inputs,
"answer": answer[:400],
"correct": gold_set.issubset(set(selected)) if gold_set else False,
"goldHits": len(gold_set & set(selected)),
"contextTokens": context_tokens,
"promptTokens": prompt_tokens,
"completionTokens": completion_tokens,
"totalTokens": prompt_tokens + completion_tokens,
}
def summarize(res: dict) -> str:
if res.get("error"):
return f"error: {res['error']}"
tools = ", ".join(res["tools"]) or "(no tool call)"
return f"{tools} · {'gold' if res['correct'] else 'miss'} · {res['contextTokens']} ctx tokens"
def avg(values: list[int]) -> int:
vals = [v for v in values if v is not None]
return round(sum(vals) / len(vals)) if vals else 0
def main() -> int:
tools = json.loads((DATA / "tools.json").read_text())
examples = json.loads((DATA / "examples.json").read_text())
meta = json.loads((DATA / "meta.json").read_text()) if (DATA / "meta.json").exists() else {}
by_id = {t["id"]: t for t in tools}
catalog = build_ratel_catalog(tools)
source = "live Ratel SDK (default embedding model)" if catalog is not None else "examples.json (bake.py, default embedder)"
client = OpenAI(base_url=BASE_URL, api_key=API_KEY)
print(f"Model {MODEL} @ {BASE_URL}", flush=True)
print(f"Retrieval: {source} · {len(examples)} prompts × {len(METHODS)} methods × (without + with)\n", flush=True)
out = {
"model": MODEL,
# Deliberately NOT the raw BASE_URL — this file ships in a public Space;
# don't leak a private inference-gateway host.
"endpoint": "self-hosted (OpenAI-compatible)",
"topK": TOP_K,
"embedModel": meta.get("embedModel", "ratel-default"),
"retrievalSource": "live" if catalog is not None else "baked",
"methods": METHODS,
"examples": {},
}
for i, ex in enumerate(examples, 1):
query, gold = ex["query"], ex["goldTop5"]
print(f"[{i}/{len(examples)}] {query!r}", flush=True)
methods_out = {}
for method in METHODS:
hits = retrieve(catalog, ex, method)
subset = [by_id[h["id"]] for h in hits if h["id"] in by_id]
# Same prompt on the model, both conditions, for THIS retriever method.
without = run_turn(client, query, tools, gold) # all 100 tool schemas
with_ = run_turn(client, query, subset, gold) # this method's top-5
methods_out[method] = {"retrieval": hits, "without": without, "with": with_}
print(f" {METHOD_LABEL[method]:6s} ({method:8s}) — without: {summarize(without)}", flush=True)
print(f" {'':6s} {'':8s} with: {summarize(with_)}", flush=True)
out["examples"][norm(query)] = {"query": query, "gold": gold, "methods": methods_out}
# ── per-method comparison summary (with vs without, for each retriever) ──
exs = list(out["examples"].values())
ok = lambda res: 1 if (not res.get("error") and res.get("correct")) else 0
def agg(select):
picks = [select(e) for e in exs]
return {
"goldCovered": sum(ok(p) for p in picks),
"avgContextTokens": avg([p.get("contextTokens") for p in picks]),
"avgTotalTokens": avg([p.get("totalTokens") for p in picks]),
}
out["comparison"] = {"n": len(exs)}
for m in METHODS:
out["comparison"][m] = {
"without": agg(lambda e, m=m: e["methods"][m]["without"]),
"with": agg(lambda e, m=m: e["methods"][m]["with"]),
}
(DATA / "llm.json").write_text(json.dumps(out, ensure_ascii=False, indent=2))
n = len(exs)
print(f"\nMethod comparison over {n} prompts (gold-covered · avg context tokens):", flush=True)
for m in METHODS:
cm = out["comparison"][m]
print(f" {METHOD_LABEL[m]:6s} ({m:8s}) — without {cm['without']['goldCovered']}/{n} · {cm['without']['avgContextTokens']}"
f" | with {cm['with']['goldCovered']}/{n} · {cm['with']['avgContextTokens']}", flush=True)
print(f"\nWrote {DATA / 'llm.json'}", flush=True)
return 0
if __name__ == "__main__":
sys.exit(main())