#!/usr/bin/env python3 """ Offline LLM bake — record, once, what a real model does with vs without Ratel, across all three retrieval methods, so the methods can be compared. Same idea as the internal kestral-ratel-demo, adapted for a static Space: kestral ran the model + Ratel live per turn; we can't (no server), so we run each curated prompt ONCE here and the Space replays the result. Retrieval uses Ratel's DEFAULT embedding model. If the Ratel Python SDK is importable, this script registers the 100-tool catalog into a hybrid `ToolCatalog` (no embed-model override → Ratel's default) and searches it live, per method. If the SDK isn't installed, it falls back to the top-5 that bake.py already baked into data/examples.json (also produced with Ratel's default embedder), so the script still runs with only `openai` available. For EACH retriever method (Sparse=bm25, Dense=semantic, Hybrid) we run the same prompt on the model twice — with and without Ratel — so every method has its own with/without pair (3 methods × 2 conditions per prompt): without Ratel -> all 100 tool schemas handed to the model with Ratel -> only that method's top-5 Per run we save exactly what the model produced: • the tool(s) it selected + arguments • its natural-language answer • real token spend (prompt/completion/total) • gold-set coverage plus the retrieval (top-5 ids + scores) that fed the with-Ratel run. A per-method comparison summary is written alongside. Run: pip install openai # + `ratel-ai` for live retrieval (optional) python scripts/bake_llm.py # reads .env for LLM_BASE_URL / LLM_MODEL / LLM_API_KEY Reads: data/tools.json, data/examples.json, data/meta.json (run bake.py first) Writes: data/llm.json """ from __future__ import annotations import json import os import re import sys from pathlib import Path from openai import OpenAI ROOT = Path(__file__).resolve().parent.parent DATA = ROOT / "data" def load_dotenv(path: Path) -> None: """Populate os.environ from a .env file (KEY=VALUE lines). No dependency; real environment variables always win over the file.""" if not path.exists(): return for line in path.read_text().splitlines(): line = line.strip() if not line or line.startswith("#") or "=" not in line: continue key, _, value = line.partition("=") os.environ.setdefault(key.strip(), value.strip().strip("'\"")) load_dotenv(ROOT / ".env") BASE_URL = os.environ.get("LLM_BASE_URL", "http://localhost:11434/v1") MODEL = os.environ.get("LLM_MODEL", "qwen3:4b") API_KEY = os.environ.get("LLM_API_KEY", "not-needed") # local servers ignore it # Sparse / Dense / Hybrid — the three retrieval methods we produce answers for. METHODS = ["bm25", "semantic", "hybrid"] METHOD_LABEL = {"bm25": "sparse", "semantic": "dense", "hybrid": "hybrid"} TOP_K = 3 SYSTEM = ( "You are a task-management assistant for a project workspace. Fulfil the user's " "request by calling the most appropriate tool(s). Invent plausible values for any " "required arguments the request does not specify. Reads are always safe; only write " "(create/update/comment/link) when the request clearly asks for it. After the tools " "run, reply in one or two short sentences describing what you did." ) _THINK = re.compile(r".*?", re.DOTALL) def norm(s: str) -> str: """Must match norm() in app.js.""" return re.sub(r"\s+", " ", s.strip().lower()) def openai_tool(t: dict) -> dict: return { "type": "function", "function": {"name": t["name"], "description": t["description"], "parameters": t["inputSchema"]}, } def clean(text: str | None) -> str: return _THINK.sub("", text or "").strip() # ── Ratel retrieval (default embedding model) ───────────────────────────────── def build_ratel_catalog(tools: list[dict]): """Register the catalog into a hybrid ToolCatalog using Ratel's DEFAULT embedding model (no override). Returns None if the SDK isn't installed.""" try: from ratel_ai import ExecutableTool, ToolCatalog except Exception: # noqa: BLE001 — SDK optional; caller falls back to examples.json return None cat = ToolCatalog(method="hybrid") # default embedder — same as scripts/bake.py for t in tools: cat.register( ExecutableTool( id=t["id"], name=t["name"], description=t["description"], input_schema=t["inputSchema"], output_schema={"type": "object", "properties": {}}, execute=lambda args: {}, ) ) return cat def retrieve(catalog, example: dict, method: str) -> list[dict]: """Top-5 for one method: live Ratel search when the SDK is present, else the top-5 bake.py already stored in examples.json (also Ratel-default retrieval).""" if catalog is not None: hits = catalog.search(example["query"], TOP_K, method=method) return [{"id": h.tool_id, "score": round(h.score, 6)} for h in hits] return [{"id": h["id"], "score": h["score"]} for h in example["results"][method][:TOP_K]] # ── one model turn ──────────────────────────────────────────────────────────── def run_turn(client: OpenAI, query: str, tools: list[dict], gold: list[str]) -> dict: """Model picks tool(s); we hand back a neutral mock result; model summarizes. contextTokens = prompt tokens of the tool-loaded first call — the number Ratel actually moves (70 schemas vs 5). """ messages = [{"role": "system", "content": SYSTEM}, {"role": "user", "content": query}] try: first = client.chat.completions.create( model=MODEL, messages=messages, tools=[openai_tool(t) for t in tools], tool_choice="auto", temperature=0, ) except Exception as e: # noqa: BLE001 — record the failure, keep baking the rest return {"error": str(e)[:200]} msg = first.choices[0].message context_tokens = getattr(first.usage, "prompt_tokens", 0) or 0 prompt_tokens = context_tokens completion_tokens = getattr(first.usage, "completion_tokens", 0) or 0 calls = msg.tool_calls or [] selected, inputs = [], [] for c in calls: selected.append(c.function.name) try: inputs.append({"tool": c.function.name, "args": json.loads(c.function.arguments or "{}")}) except json.JSONDecodeError: inputs.append({"tool": c.function.name, "args": c.function.arguments}) answer = clean(msg.content) if calls: # Neutral mock result so the model produces a closing sentence # (kestral mocked tool executes the same way — symmetric across conditions). messages.append(msg.model_dump()) for c in calls: messages.append({"role": "tool", "tool_call_id": c.id, "content": json.dumps({"ok": True})}) try: second = client.chat.completions.create(model=MODEL, messages=messages, temperature=0) answer = clean(second.choices[0].message.content) or answer prompt_tokens += getattr(second.usage, "prompt_tokens", 0) or 0 completion_tokens += getattr(second.usage, "completion_tokens", 0) or 0 except Exception: # noqa: BLE001 — summary is optional; keep the selection + tokens pass gold_set = set(gold) return { "tools": selected, "inputs": inputs, "answer": answer[:400], "correct": gold_set.issubset(set(selected)) if gold_set else False, "goldHits": len(gold_set & set(selected)), "contextTokens": context_tokens, "promptTokens": prompt_tokens, "completionTokens": completion_tokens, "totalTokens": prompt_tokens + completion_tokens, } def summarize(res: dict) -> str: if res.get("error"): return f"error: {res['error']}" tools = ", ".join(res["tools"]) or "(no tool call)" return f"{tools} · {'gold' if res['correct'] else 'miss'} · {res['contextTokens']} ctx tokens" def avg(values: list[int]) -> int: vals = [v for v in values if v is not None] return round(sum(vals) / len(vals)) if vals else 0 def main() -> int: tools = json.loads((DATA / "tools.json").read_text()) examples = json.loads((DATA / "examples.json").read_text()) meta = json.loads((DATA / "meta.json").read_text()) if (DATA / "meta.json").exists() else {} by_id = {t["id"]: t for t in tools} catalog = build_ratel_catalog(tools) source = "live Ratel SDK (default embedding model)" if catalog is not None else "examples.json (bake.py, default embedder)" client = OpenAI(base_url=BASE_URL, api_key=API_KEY) print(f"Model {MODEL} @ {BASE_URL}", flush=True) print(f"Retrieval: {source} · {len(examples)} prompts × {len(METHODS)} methods × (without + with)\n", flush=True) out = { "model": MODEL, # Deliberately NOT the raw BASE_URL — this file ships in a public Space; # don't leak a private inference-gateway host. "endpoint": "self-hosted (OpenAI-compatible)", "topK": TOP_K, "embedModel": meta.get("embedModel", "ratel-default"), "retrievalSource": "live" if catalog is not None else "baked", "methods": METHODS, "examples": {}, } for i, ex in enumerate(examples, 1): query, gold = ex["query"], ex["goldTop5"] print(f"[{i}/{len(examples)}] {query!r}", flush=True) methods_out = {} for method in METHODS: hits = retrieve(catalog, ex, method) subset = [by_id[h["id"]] for h in hits if h["id"] in by_id] # Same prompt on the model, both conditions, for THIS retriever method. without = run_turn(client, query, tools, gold) # all 100 tool schemas with_ = run_turn(client, query, subset, gold) # this method's top-5 methods_out[method] = {"retrieval": hits, "without": without, "with": with_} print(f" {METHOD_LABEL[method]:6s} ({method:8s}) — without: {summarize(without)}", flush=True) print(f" {'':6s} {'':8s} with: {summarize(with_)}", flush=True) out["examples"][norm(query)] = {"query": query, "gold": gold, "methods": methods_out} # ── per-method comparison summary (with vs without, for each retriever) ── exs = list(out["examples"].values()) ok = lambda res: 1 if (not res.get("error") and res.get("correct")) else 0 def agg(select): picks = [select(e) for e in exs] return { "goldCovered": sum(ok(p) for p in picks), "avgContextTokens": avg([p.get("contextTokens") for p in picks]), "avgTotalTokens": avg([p.get("totalTokens") for p in picks]), } out["comparison"] = {"n": len(exs)} for m in METHODS: out["comparison"][m] = { "without": agg(lambda e, m=m: e["methods"][m]["without"]), "with": agg(lambda e, m=m: e["methods"][m]["with"]), } (DATA / "llm.json").write_text(json.dumps(out, ensure_ascii=False, indent=2)) n = len(exs) print(f"\nMethod comparison over {n} prompts (gold-covered · avg context tokens):", flush=True) for m in METHODS: cm = out["comparison"][m] print(f" {METHOD_LABEL[m]:6s} ({m:8s}) — without {cm['without']['goldCovered']}/{n} · {cm['without']['avgContextTokens']}" f" | with {cm['with']['goldCovered']}/{n} · {cm['with']['avgContextTokens']}", flush=True) print(f"\nWrote {DATA / 'llm.json'}", flush=True) return 0 if __name__ == "__main__": sys.exit(main())