File size: 6,054 Bytes
6e72953 3812402 6e72953 3812402 6e72953 e09b1e2 6e72953 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 | #!/usr/bin/env python3
"""
Offline bake step for the Ratel "with vs without" Hugging Face demo.
The real Ratel SDK is a compiled native binary and cannot run in a static
browser Space. So we run the REAL SDK here, once, on a normal machine, and
export its output to static JSON the browser loads directly.
For every curated example prompt we run Ratel's real `bm25` / `semantic` /
`hybrid` retrieval over the fixed 100-tool catalog and store the ranked hits.
We also count each tool schema's tokens with a real tokenizer (o200k_base) so
the demo can show the context-size delta between "load all 100 tools" and
"retrieve the top-k".
Run: pip install -r scripts/requirements.txt && python scripts/bake.py
Reads: scripts/corpus.json (tools + gold set + skills, extracted from the corpus)
Writes: data/tools.json, data/examples.json, data/meta.json
The Space itself never imports ratel_ai — it only reads data/*.json.
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
from ratel_ai import ToolCatalog, ExecutableTool
import tiktoken
ROOT = Path(__file__).resolve().parent.parent
CORPUS = ROOT / "scripts" / "corpus.json"
DATA = ROOT / "data"
METHODS = ["bm25", "semantic", "hybrid"]
TOP_K_BAKED = 10 # rank list length stored per prompt/method
TOP_K_ACCURACY = 3 # gold hits must land in top-3 (demo top-k)
EMBED_MODEL = "BAAI/bge-small-en-v1.5"
TOKENIZER = "o200k_base"
def tool_schema_for_llm(tool: dict) -> dict:
"""The shape an LLM tool definition serializes to — what actually costs tokens."""
return {
"name": tool["name"],
"description": tool["description"],
"input_schema": tool["inputSchema"],
}
def count_tokens(enc, tool: dict) -> int:
blob = json.dumps(tool_schema_for_llm(tool), separators=(",", ":"), ensure_ascii=False)
return len(enc.encode(blob))
def main() -> int:
corpus = json.loads(CORPUS.read_text())
tools = corpus["tools"]
gold = corpus["gold"]
enc = tiktoken.get_encoding(TOKENIZER)
# ── 1. Register the full catalog into a hybrid catalog (builds bge-small embeddings) ──
print(f"Registering {len(tools)} tools into a hybrid ToolCatalog…", flush=True)
cat = ToolCatalog(method="hybrid")
for t in tools:
cat.register(
ExecutableTool(
id=t["id"],
name=t["name"],
description=t["description"],
input_schema=t["inputSchema"],
output_schema={"type": "object", "properties": {}},
execute=lambda args: {},
)
)
# ── 2. Per-tool token counts ──
tool_tokens = {t["id"]: count_tokens(enc, t) for t in tools}
total_tokens = sum(tool_tokens.values())
print(f"Catalog token cost (all {len(tools)} tools, {TOKENIZER}): {total_tokens} tokens", flush=True)
tools_out = [
{
"id": t["id"],
"name": t["name"],
"description": t["description"],
"category": t["category"],
"group": t["group"],
"inputSchema": t["inputSchema"],
"isWorkflowSkill": t.get("isWorkflowSkill", False),
"tokens": tool_tokens[t["id"]],
}
for t in tools
]
# ── 3. Bake real retrieval per (example prompt × method) ──
print(f"Baking retrieval for {len(gold)} example prompts × {len(METHODS)} methods…", flush=True)
examples_out = []
accuracy_report = []
for case in gold:
query = case["query"]
gold_top5 = case["expectTop5"]
results = {}
for method in METHODS:
hits = cat.search(query, TOP_K_BAKED, method=method)
ranked = [{"id": h.tool_id, "score": round(h.score, 6)} for h in hits]
results[method] = ranked
top5_ids = [h["id"] for h in ranked[:TOP_K_ACCURACY]]
hit_count = sum(1 for g in gold_top5 if g in top5_ids)
accuracy_report.append((method, query, hit_count, len(gold_top5)))
retrieved_tokens = {
method: sum(tool_tokens.get(h["id"], 0) for h in results[method][:TOP_K_ACCURACY])
for method in METHODS
}
examples_out.append(
{
"query": query,
"goldTop5": gold_top5,
"source": case.get("source", ""),
"results": results,
"retrievedTokens": retrieved_tokens,
}
)
# ── 4. Validation summary (mirrors scripts/probe-gold.ts) ──
print("\nGold-set validation (hits in top-5 / expected):", flush=True)
per_method_pass = {m: 0 for m in METHODS}
per_method_total = {m: 0 for m in METHODS}
for method, query, hit, expected in accuracy_report:
per_method_total[method] += 1
if hit == expected:
per_method_pass[method] += 1
for m in METHODS:
print(f" {m:9s}: {per_method_pass[m]}/{per_method_total[m]} prompts fully covered", flush=True)
# ── 5. Write ──
DATA.mkdir(exist_ok=True)
(DATA / "tools.json").write_text(json.dumps(tools_out, ensure_ascii=False, indent=2))
(DATA / "examples.json").write_text(json.dumps(examples_out, ensure_ascii=False, indent=2))
meta = {
"toolCount": len(tools),
"totalCatalogTokens": total_tokens,
"topKAccuracy": TOP_K_ACCURACY,
"topKBaked": TOP_K_BAKED,
"methods": METHODS,
"embedModel": EMBED_MODEL,
"tokenizer": TOKENIZER,
"sdk": "ratel-ai 0.4.2 (Rust core 0.4.0)",
"note": "Retrieval baked offline from the real Ratel SDK; the Space runs BM25 live for custom prompts.",
"goldPassByMethod": {m: f"{per_method_pass[m]}/{per_method_total[m]}" for m in METHODS},
}
(DATA / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2))
print(f"\nWrote:\n {DATA/'tools.json'}\n {DATA/'examples.json'}\n {DATA/'meta.json'}", flush=True)
return 0
if __name__ == "__main__":
sys.exit(main())
|