Ratel / scripts /bake.py
Berca Akbayır
Claude Opus 4.8 (1M context)
Step-by-step gateway traces, 100-tool catalog, Run button & per-panel metrics (#3)
3812402 unverified
Raw History Blame Contribute Delete
6.05 kB
#!/usr/bin/env python3
"""
Offline bake step for the Ratel "with vs without" Hugging Face demo.
The real Ratel SDK is a compiled native binary and cannot run in a static
browser Space. So we run the REAL SDK here, once, on a normal machine, and
export its output to static JSON the browser loads directly.
For every curated example prompt we run Ratel's real `bm25` / `semantic` /
`hybrid` retrieval over the fixed 100-tool catalog and store the ranked hits.
We also count each tool schema's tokens with a real tokenizer (o200k_base) so
the demo can show the context-size delta between "load all 100 tools" and
"retrieve the top-k".
Run: pip install -r scripts/requirements.txt && python scripts/bake.py
Reads: scripts/corpus.json (tools + gold set + skills, extracted from the corpus)
Writes: data/tools.json, data/examples.json, data/meta.json
The Space itself never imports ratel_ai — it only reads data/*.json.
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
from ratel_ai import ToolCatalog, ExecutableTool
import tiktoken
ROOT = Path(__file__).resolve().parent.parent
CORPUS = ROOT / "scripts" / "corpus.json"
DATA = ROOT / "data"
METHODS = ["bm25", "semantic", "hybrid"]
TOP_K_BAKED = 10 # rank list length stored per prompt/method
TOP_K_ACCURACY = 3 # gold hits must land in top-3 (demo top-k)
EMBED_MODEL = "BAAI/bge-small-en-v1.5"
TOKENIZER = "o200k_base"
def tool_schema_for_llm(tool: dict) -> dict:
"""The shape an LLM tool definition serializes to — what actually costs tokens."""
return {
"name": tool["name"],
"description": tool["description"],
"input_schema": tool["inputSchema"],
}
def count_tokens(enc, tool: dict) -> int:
blob = json.dumps(tool_schema_for_llm(tool), separators=(",", ":"), ensure_ascii=False)
return len(enc.encode(blob))
def main() -> int:
corpus = json.loads(CORPUS.read_text())
tools = corpus["tools"]
gold = corpus["gold"]
enc = tiktoken.get_encoding(TOKENIZER)
# ── 1. Register the full catalog into a hybrid catalog (builds bge-small embeddings) ──
print(f"Registering {len(tools)} tools into a hybrid ToolCatalog…", flush=True)
cat = ToolCatalog(method="hybrid")
for t in tools:
cat.register(
ExecutableTool(
id=t["id"],
name=t["name"],
description=t["description"],
input_schema=t["inputSchema"],
output_schema={"type": "object", "properties": {}},
execute=lambda args: {},
)
)
# ── 2. Per-tool token counts ──
tool_tokens = {t["id"]: count_tokens(enc, t) for t in tools}
total_tokens = sum(tool_tokens.values())
print(f"Catalog token cost (all {len(tools)} tools, {TOKENIZER}): {total_tokens} tokens", flush=True)
tools_out = [
{
"id": t["id"],
"name": t["name"],
"description": t["description"],
"category": t["category"],
"group": t["group"],
"inputSchema": t["inputSchema"],
"isWorkflowSkill": t.get("isWorkflowSkill", False),
"tokens": tool_tokens[t["id"]],
}
for t in tools
]
# ── 3. Bake real retrieval per (example prompt × method) ──
print(f"Baking retrieval for {len(gold)} example prompts × {len(METHODS)} methods…", flush=True)
examples_out = []
accuracy_report = []
for case in gold:
query = case["query"]
gold_top5 = case["expectTop5"]
results = {}
for method in METHODS:
hits = cat.search(query, TOP_K_BAKED, method=method)
ranked = [{"id": h.tool_id, "score": round(h.score, 6)} for h in hits]
results[method] = ranked
top5_ids = [h["id"] for h in ranked[:TOP_K_ACCURACY]]
hit_count = sum(1 for g in gold_top5 if g in top5_ids)
accuracy_report.append((method, query, hit_count, len(gold_top5)))
retrieved_tokens = {
method: sum(tool_tokens.get(h["id"], 0) for h in results[method][:TOP_K_ACCURACY])
for method in METHODS
}
examples_out.append(
{
"query": query,
"goldTop5": gold_top5,
"source": case.get("source", ""),
"results": results,
"retrievedTokens": retrieved_tokens,
}
)
# ── 4. Validation summary (mirrors scripts/probe-gold.ts) ──
print("\nGold-set validation (hits in top-5 / expected):", flush=True)
per_method_pass = {m: 0 for m in METHODS}
per_method_total = {m: 0 for m in METHODS}
for method, query, hit, expected in accuracy_report:
per_method_total[method] += 1
if hit == expected:
per_method_pass[method] += 1
for m in METHODS:
print(f" {m:9s}: {per_method_pass[m]}/{per_method_total[m]} prompts fully covered", flush=True)
# ── 5. Write ──
DATA.mkdir(exist_ok=True)
(DATA / "tools.json").write_text(json.dumps(tools_out, ensure_ascii=False, indent=2))
(DATA / "examples.json").write_text(json.dumps(examples_out, ensure_ascii=False, indent=2))
meta = {
"toolCount": len(tools),
"totalCatalogTokens": total_tokens,
"topKAccuracy": TOP_K_ACCURACY,
"topKBaked": TOP_K_BAKED,
"methods": METHODS,
"embedModel": EMBED_MODEL,
"tokenizer": TOKENIZER,
"sdk": "ratel-ai 0.4.2 (Rust core 0.4.0)",
"note": "Retrieval baked offline from the real Ratel SDK; the Space runs BM25 live for custom prompts.",
"goldPassByMethod": {m: f"{per_method_pass[m]}/{per_method_total[m]}" for m in METHODS},
}
(DATA / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2))
print(f"\nWrote:\n {DATA/'tools.json'}\n {DATA/'examples.json'}\n {DATA/'meta.json'}", flush=True)
return 0
if __name__ == "__main__":
sys.exit(main())