Berca Akbayır
Claude Opus 4.8 (1M context)
Step-by-step gateway traces, 100-tool catalog, Run button & per-panel metrics (#3)
3812402 unverified Download scripts/bake.py from ratel-ai/Ratel: direct link, hf CLI and curl.
- Browser
- Download file 6.05 kB
-
https://huggingface.co/spaces/ratel-ai/Ratel/resolve/main/scripts/bake.py
- Command line
-
hf download hf://spaces/ratel-ai/Ratel/scripts/bake.py
-
curl -L -o bake.py https://huggingface.co/spaces/ratel-ai/Ratel/resolve/main/scripts/bake.py
6.05 kB
| #!/usr/bin/env python3 | |
| """ | |
| Offline bake step for the Ratel "with vs without" Hugging Face demo. | |
| The real Ratel SDK is a compiled native binary and cannot run in a static | |
| browser Space. So we run the REAL SDK here, once, on a normal machine, and | |
| export its output to static JSON the browser loads directly. | |
| For every curated example prompt we run Ratel's real `bm25` / `semantic` / | |
| `hybrid` retrieval over the fixed 100-tool catalog and store the ranked hits. | |
| We also count each tool schema's tokens with a real tokenizer (o200k_base) so | |
| the demo can show the context-size delta between "load all 100 tools" and | |
| "retrieve the top-k". | |
| Run: pip install -r scripts/requirements.txt && python scripts/bake.py | |
| Reads: scripts/corpus.json (tools + gold set + skills, extracted from the corpus) | |
| Writes: data/tools.json, data/examples.json, data/meta.json | |
| The Space itself never imports ratel_ai — it only reads data/*.json. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import sys | |
| from pathlib import Path | |
| from ratel_ai import ToolCatalog, ExecutableTool | |
| import tiktoken | |
| ROOT = Path(__file__).resolve().parent.parent | |
| CORPUS = ROOT / "scripts" / "corpus.json" | |
| DATA = ROOT / "data" | |
| METHODS = ["bm25", "semantic", "hybrid"] | |
| TOP_K_BAKED = 10 # rank list length stored per prompt/method | |
| TOP_K_ACCURACY = 3 # gold hits must land in top-3 (demo top-k) | |
| EMBED_MODEL = "BAAI/bge-small-en-v1.5" | |
| TOKENIZER = "o200k_base" | |
| def tool_schema_for_llm(tool: dict) -> dict: | |
| """The shape an LLM tool definition serializes to — what actually costs tokens.""" | |
| return { | |
| "name": tool["name"], | |
| "description": tool["description"], | |
| "input_schema": tool["inputSchema"], | |
| } | |
| def count_tokens(enc, tool: dict) -> int: | |
| blob = json.dumps(tool_schema_for_llm(tool), separators=(",", ":"), ensure_ascii=False) | |
| return len(enc.encode(blob)) | |
| def main() -> int: | |
| corpus = json.loads(CORPUS.read_text()) | |
| tools = corpus["tools"] | |
| gold = corpus["gold"] | |
| enc = tiktoken.get_encoding(TOKENIZER) | |
| # ── 1. Register the full catalog into a hybrid catalog (builds bge-small embeddings) ── | |
| print(f"Registering {len(tools)} tools into a hybrid ToolCatalog…", flush=True) | |
| cat = ToolCatalog(method="hybrid") | |
| for t in tools: | |
| cat.register( | |
| ExecutableTool( | |
| id=t["id"], | |
| name=t["name"], | |
| description=t["description"], | |
| input_schema=t["inputSchema"], | |
| output_schema={"type": "object", "properties": {}}, | |
| execute=lambda args: {}, | |
| ) | |
| ) | |
| # ── 2. Per-tool token counts ── | |
| tool_tokens = {t["id"]: count_tokens(enc, t) for t in tools} | |
| total_tokens = sum(tool_tokens.values()) | |
| print(f"Catalog token cost (all {len(tools)} tools, {TOKENIZER}): {total_tokens} tokens", flush=True) | |
| tools_out = [ | |
| { | |
| "id": t["id"], | |
| "name": t["name"], | |
| "description": t["description"], | |
| "category": t["category"], | |
| "group": t["group"], | |
| "inputSchema": t["inputSchema"], | |
| "isWorkflowSkill": t.get("isWorkflowSkill", False), | |
| "tokens": tool_tokens[t["id"]], | |
| } | |
| for t in tools | |
| ] | |
| # ── 3. Bake real retrieval per (example prompt × method) ── | |
| print(f"Baking retrieval for {len(gold)} example prompts × {len(METHODS)} methods…", flush=True) | |
| examples_out = [] | |
| accuracy_report = [] | |
| for case in gold: | |
| query = case["query"] | |
| gold_top5 = case["expectTop5"] | |
| results = {} | |
| for method in METHODS: | |
| hits = cat.search(query, TOP_K_BAKED, method=method) | |
| ranked = [{"id": h.tool_id, "score": round(h.score, 6)} for h in hits] | |
| results[method] = ranked | |
| top5_ids = [h["id"] for h in ranked[:TOP_K_ACCURACY]] | |
| hit_count = sum(1 for g in gold_top5 if g in top5_ids) | |
| accuracy_report.append((method, query, hit_count, len(gold_top5))) | |
| retrieved_tokens = { | |
| method: sum(tool_tokens.get(h["id"], 0) for h in results[method][:TOP_K_ACCURACY]) | |
| for method in METHODS | |
| } | |
| examples_out.append( | |
| { | |
| "query": query, | |
| "goldTop5": gold_top5, | |
| "source": case.get("source", ""), | |
| "results": results, | |
| "retrievedTokens": retrieved_tokens, | |
| } | |
| ) | |
| # ── 4. Validation summary (mirrors scripts/probe-gold.ts) ── | |
| print("\nGold-set validation (hits in top-5 / expected):", flush=True) | |
| per_method_pass = {m: 0 for m in METHODS} | |
| per_method_total = {m: 0 for m in METHODS} | |
| for method, query, hit, expected in accuracy_report: | |
| per_method_total[method] += 1 | |
| if hit == expected: | |
| per_method_pass[method] += 1 | |
| for m in METHODS: | |
| print(f" {m:9s}: {per_method_pass[m]}/{per_method_total[m]} prompts fully covered", flush=True) | |
| # ── 5. Write ── | |
| DATA.mkdir(exist_ok=True) | |
| (DATA / "tools.json").write_text(json.dumps(tools_out, ensure_ascii=False, indent=2)) | |
| (DATA / "examples.json").write_text(json.dumps(examples_out, ensure_ascii=False, indent=2)) | |
| meta = { | |
| "toolCount": len(tools), | |
| "totalCatalogTokens": total_tokens, | |
| "topKAccuracy": TOP_K_ACCURACY, | |
| "topKBaked": TOP_K_BAKED, | |
| "methods": METHODS, | |
| "embedModel": EMBED_MODEL, | |
| "tokenizer": TOKENIZER, | |
| "sdk": "ratel-ai 0.4.2 (Rust core 0.4.0)", | |
| "note": "Retrieval baked offline from the real Ratel SDK; the Space runs BM25 live for custom prompts.", | |
| "goldPassByMethod": {m: f"{per_method_pass[m]}/{per_method_total[m]}" for m in METHODS}, | |
| } | |
| (DATA / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2)) | |
| print(f"\nWrote:\n {DATA/'tools.json'}\n {DATA/'examples.json'}\n {DATA/'meta.json'}", flush=True) | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |