/** * TypeScript parity of scripts/bake.py — bakes the demo's retrieval data using the * Ratel TypeScript SDK (`@ratel-ai/sdk`) instead of the Python one. Both SDKs wrap the * same Rust core, so the ranked results are identical; this file exists so the demo * documents both surfaces. `bake.py` is the one the repo actually runs. * * Run: npm i @ratel-ai/sdk gpt-tokenizer && npx tsx scripts/bake.ts * Reads: scripts/corpus.json Writes: data/{tools,examples,meta}.json */ import { readFileSync, writeFileSync, mkdirSync } from "node:fs"; import { resolve, dirname } from "node:path"; import { fileURLToPath } from "node:url"; import { ToolCatalog } from "@ratel-ai/sdk"; import { encode } from "gpt-tokenizer/model/gpt-4o"; // o200k_base, matches bake.py const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const METHODS = ["bm25", "semantic", "hybrid"] as const; const TOP_K_BAKED = 10; const TOP_K_ACCURACY = 5; type Tool = { id: string; name: string; description: string; category: string; group: string; inputSchema: unknown; isWorkflowSkill?: boolean; }; type Gold = { query: string; expectTop5: string[]; source: string }; const corpus = JSON.parse(readFileSync(resolve(ROOT, "scripts/corpus.json"), "utf8")); const tools: Tool[] = corpus.tools; const gold: Gold[] = corpus.gold; const tokensFor = (t: Tool) => encode(JSON.stringify({ name: t.name, description: t.description, input_schema: t.inputSchema })).length; async function main() { const catalog = new ToolCatalog({ method: "hybrid" }); for (const t of tools) { catalog.register({ id: t.id, name: t.name, description: t.description, inputSchema: t.inputSchema, outputSchema: { type: "object", properties: {} }, execute: async () => ({}), }); } await catalog.buildEmbeddings(); // downloads bge-small on first run const toolTokens = new Map(tools.map((t) => [t.id, tokensFor(t)])); const totalTokens = [...toolTokens.values()].reduce((a, b) => a + b, 0); const toolsOut = tools.map((t) => ({ ...t, tokens: toolTokens.get(t.id) })); const examplesOut = []; for (const c of gold) { const results: Record = {}; const retrievedTokens: Record = {}; for (const method of METHODS) { const hits = await catalog.search(c.query, TOP_K_BAKED, "direct", method); results[method] = hits.map((h) => ({ id: h.toolId, score: Number(h.score.toFixed(6)) })); retrievedTokens[method] = results[method] .slice(0, TOP_K_ACCURACY) .reduce((s, h) => s + (toolTokens.get(h.id) ?? 0), 0); } examplesOut.push({ query: c.query, goldTop5: c.expectTop5, source: c.source, results, retrievedTokens }); } mkdirSync(resolve(ROOT, "data"), { recursive: true }); writeFileSync(resolve(ROOT, "data/tools.json"), JSON.stringify(toolsOut, null, 2)); writeFileSync(resolve(ROOT, "data/examples.json"), JSON.stringify(examplesOut, null, 2)); writeFileSync( resolve(ROOT, "data/meta.json"), JSON.stringify( { toolCount: tools.length, totalCatalogTokens: totalTokens, topKAccuracy: TOP_K_ACCURACY, topKBaked: TOP_K_BAKED, methods: METHODS, embedModel: "BAAI/bge-small-en-v1.5", tokenizer: "o200k_base", sdk: "@ratel-ai/sdk 0.4.1 (Rust core 0.4.0)", note: "Retrieval baked offline from the real Ratel SDK; the Space runs BM25 live for custom prompts.", }, null, 2, ), ); console.log(`Baked ${tools.length} tools · ${totalTokens} catalog tokens · ${gold.length} example prompts`); } main();