Ratel / scripts /bake.ts
Berca Akbayır
Claude Opus 4.8 (1M context)
Add Ratel "with vs without" tool-retrieval Hugging Face Space (#1)
6e72953 unverified
Raw History Blame Contribute Delete
3.61 kB
/**
* TypeScript parity of scripts/bake.py — bakes the demo's retrieval data using the
* Ratel TypeScript SDK (`@ratel-ai/sdk`) instead of the Python one. Both SDKs wrap the
* same Rust core, so the ranked results are identical; this file exists so the demo
* documents both surfaces. `bake.py` is the one the repo actually runs.
*
* Run: npm i @ratel-ai/sdk gpt-tokenizer && npx tsx scripts/bake.ts
* Reads: scripts/corpus.json Writes: data/{tools,examples,meta}.json
*/
import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
import { resolve, dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { ToolCatalog } from "@ratel-ai/sdk";
import { encode } from "gpt-tokenizer/model/gpt-4o"; // o200k_base, matches bake.py
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const METHODS = ["bm25", "semantic", "hybrid"] as const;
const TOP_K_BAKED = 10;
const TOP_K_ACCURACY = 5;
type Tool = {
id: string; name: string; description: string; category: string;
group: string; inputSchema: unknown; isWorkflowSkill?: boolean;
};
type Gold = { query: string; expectTop5: string[]; source: string };
const corpus = JSON.parse(readFileSync(resolve(ROOT, "scripts/corpus.json"), "utf8"));
const tools: Tool[] = corpus.tools;
const gold: Gold[] = corpus.gold;
const tokensFor = (t: Tool) =>
encode(JSON.stringify({ name: t.name, description: t.description, input_schema: t.inputSchema })).length;
async function main() {
const catalog = new ToolCatalog({ method: "hybrid" });
for (const t of tools) {
catalog.register({
id: t.id, name: t.name, description: t.description,
inputSchema: t.inputSchema, outputSchema: { type: "object", properties: {} },
execute: async () => ({}),
});
}
await catalog.buildEmbeddings(); // downloads bge-small on first run
const toolTokens = new Map(tools.map((t) => [t.id, tokensFor(t)]));
const totalTokens = [...toolTokens.values()].reduce((a, b) => a + b, 0);
const toolsOut = tools.map((t) => ({ ...t, tokens: toolTokens.get(t.id) }));
const examplesOut = [];
for (const c of gold) {
const results: Record<string, { id: string; score: number }[]> = {};
const retrievedTokens: Record<string, number> = {};
for (const method of METHODS) {
const hits = await catalog.search(c.query, TOP_K_BAKED, "direct", method);
results[method] = hits.map((h) => ({ id: h.toolId, score: Number(h.score.toFixed(6)) }));
retrievedTokens[method] = results[method]
.slice(0, TOP_K_ACCURACY)
.reduce((s, h) => s + (toolTokens.get(h.id) ?? 0), 0);
}
examplesOut.push({ query: c.query, goldTop5: c.expectTop5, source: c.source, results, retrievedTokens });
}
mkdirSync(resolve(ROOT, "data"), { recursive: true });
writeFileSync(resolve(ROOT, "data/tools.json"), JSON.stringify(toolsOut, null, 2));
writeFileSync(resolve(ROOT, "data/examples.json"), JSON.stringify(examplesOut, null, 2));
writeFileSync(
resolve(ROOT, "data/meta.json"),
JSON.stringify(
{
toolCount: tools.length, totalCatalogTokens: totalTokens,
topKAccuracy: TOP_K_ACCURACY, topKBaked: TOP_K_BAKED, methods: METHODS,
embedModel: "BAAI/bge-small-en-v1.5", tokenizer: "o200k_base",
sdk: "@ratel-ai/sdk 0.4.1 (Rust core 0.4.0)",
note: "Retrieval baked offline from the real Ratel SDK; the Space runs BM25 live for custom prompts.",
},
null,
2,
),
);
console.log(`Baked ${tools.length} tools · ${totalTokens} catalog tokens · ${gold.length} example prompts`);
}
main();