#!/usr/bin/env python3 """ Offline bake step for the Ratel "with vs without" Hugging Face demo. The real Ratel SDK is a compiled native binary and cannot run in a static browser Space. So we run the REAL SDK here, once, on a normal machine, and export its output to static JSON the browser loads directly. For every curated example prompt we run Ratel's real `bm25` / `semantic` / `hybrid` retrieval over the fixed 100-tool catalog and store the ranked hits. We also count each tool schema's tokens with a real tokenizer (o200k_base) so the demo can show the context-size delta between "load all 100 tools" and "retrieve the top-k". Run: pip install -r scripts/requirements.txt && python scripts/bake.py Reads: scripts/corpus.json (tools + gold set + skills, extracted from the corpus) Writes: data/tools.json, data/examples.json, data/meta.json The Space itself never imports ratel_ai — it only reads data/*.json. """ from __future__ import annotations import json import sys from pathlib import Path from ratel_ai import ToolCatalog, ExecutableTool import tiktoken ROOT = Path(__file__).resolve().parent.parent CORPUS = ROOT / "scripts" / "corpus.json" DATA = ROOT / "data" METHODS = ["bm25", "semantic", "hybrid"] TOP_K_BAKED = 10 # rank list length stored per prompt/method TOP_K_ACCURACY = 3 # gold hits must land in top-3 (demo top-k) EMBED_MODEL = "BAAI/bge-small-en-v1.5" TOKENIZER = "o200k_base" def tool_schema_for_llm(tool: dict) -> dict: """The shape an LLM tool definition serializes to — what actually costs tokens.""" return { "name": tool["name"], "description": tool["description"], "input_schema": tool["inputSchema"], } def count_tokens(enc, tool: dict) -> int: blob = json.dumps(tool_schema_for_llm(tool), separators=(",", ":"), ensure_ascii=False) return len(enc.encode(blob)) def main() -> int: corpus = json.loads(CORPUS.read_text()) tools = corpus["tools"] gold = corpus["gold"] enc = tiktoken.get_encoding(TOKENIZER) # ── 1. Register the full catalog into a hybrid catalog (builds bge-small embeddings) ── print(f"Registering {len(tools)} tools into a hybrid ToolCatalog…", flush=True) cat = ToolCatalog(method="hybrid") for t in tools: cat.register( ExecutableTool( id=t["id"], name=t["name"], description=t["description"], input_schema=t["inputSchema"], output_schema={"type": "object", "properties": {}}, execute=lambda args: {}, ) ) # ── 2. Per-tool token counts ── tool_tokens = {t["id"]: count_tokens(enc, t) for t in tools} total_tokens = sum(tool_tokens.values()) print(f"Catalog token cost (all {len(tools)} tools, {TOKENIZER}): {total_tokens} tokens", flush=True) tools_out = [ { "id": t["id"], "name": t["name"], "description": t["description"], "category": t["category"], "group": t["group"], "inputSchema": t["inputSchema"], "isWorkflowSkill": t.get("isWorkflowSkill", False), "tokens": tool_tokens[t["id"]], } for t in tools ] # ── 3. Bake real retrieval per (example prompt × method) ── print(f"Baking retrieval for {len(gold)} example prompts × {len(METHODS)} methods…", flush=True) examples_out = [] accuracy_report = [] for case in gold: query = case["query"] gold_top5 = case["expectTop5"] results = {} for method in METHODS: hits = cat.search(query, TOP_K_BAKED, method=method) ranked = [{"id": h.tool_id, "score": round(h.score, 6)} for h in hits] results[method] = ranked top5_ids = [h["id"] for h in ranked[:TOP_K_ACCURACY]] hit_count = sum(1 for g in gold_top5 if g in top5_ids) accuracy_report.append((method, query, hit_count, len(gold_top5))) retrieved_tokens = { method: sum(tool_tokens.get(h["id"], 0) for h in results[method][:TOP_K_ACCURACY]) for method in METHODS } examples_out.append( { "query": query, "goldTop5": gold_top5, "source": case.get("source", ""), "results": results, "retrievedTokens": retrieved_tokens, } ) # ── 4. Validation summary (mirrors scripts/probe-gold.ts) ── print("\nGold-set validation (hits in top-5 / expected):", flush=True) per_method_pass = {m: 0 for m in METHODS} per_method_total = {m: 0 for m in METHODS} for method, query, hit, expected in accuracy_report: per_method_total[method] += 1 if hit == expected: per_method_pass[method] += 1 for m in METHODS: print(f" {m:9s}: {per_method_pass[m]}/{per_method_total[m]} prompts fully covered", flush=True) # ── 5. Write ── DATA.mkdir(exist_ok=True) (DATA / "tools.json").write_text(json.dumps(tools_out, ensure_ascii=False, indent=2)) (DATA / "examples.json").write_text(json.dumps(examples_out, ensure_ascii=False, indent=2)) meta = { "toolCount": len(tools), "totalCatalogTokens": total_tokens, "topKAccuracy": TOP_K_ACCURACY, "topKBaked": TOP_K_BAKED, "methods": METHODS, "embedModel": EMBED_MODEL, "tokenizer": TOKENIZER, "sdk": "ratel-ai 0.4.2 (Rust core 0.4.0)", "note": "Retrieval baked offline from the real Ratel SDK; the Space runs BM25 live for custom prompts.", "goldPassByMethod": {m: f"{per_method_pass[m]}/{per_method_total[m]}" for m in METHODS}, } (DATA / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2)) print(f"\nWrote:\n {DATA/'tools.json'}\n {DATA/'examples.json'}\n {DATA/'meta.json'}", flush=True) return 0 if __name__ == "__main__": sys.exit(main())