Berca Akbayır
Claude Opus 4.8 (1M context)
Step-by-step gateway traces, 100-tool catalog, Run button & per-panel metrics (#3)
3812402 unverified Download scripts/bake_llm.py from ratel-ai/Ratel: direct link, hf CLI and curl.
- Browser
- Download file 11.9 kB
-
https://huggingface.co/spaces/ratel-ai/Ratel/resolve/main/scripts/bake_llm.py
- Command line
-
hf download hf://spaces/ratel-ai/Ratel/scripts/bake_llm.py
-
curl -L -o bake_llm.py https://huggingface.co/spaces/ratel-ai/Ratel/resolve/main/scripts/bake_llm.py
11.9 kB
| #!/usr/bin/env python3 | |
| """ | |
| Offline LLM bake — record, once, what a real model does with vs without Ratel, | |
| across all three retrieval methods, so the methods can be compared. | |
| Same idea as the internal kestral-ratel-demo, adapted for a static Space: | |
| kestral ran the model + Ratel live per turn; we can't (no server), so we run | |
| each curated prompt ONCE here and the Space replays the result. | |
| Retrieval uses Ratel's DEFAULT embedding model. If the Ratel Python SDK is | |
| importable, this script registers the 100-tool catalog into a hybrid | |
| `ToolCatalog` (no embed-model override → Ratel's default) and searches it live, | |
| per method. If the SDK isn't installed, it falls back to the top-5 that bake.py | |
| already baked into data/examples.json (also produced with Ratel's default | |
| embedder), so the script still runs with only `openai` available. | |
| For EACH retriever method (Sparse=bm25, Dense=semantic, Hybrid) we run the same | |
| prompt on the model twice — with and without Ratel — so every method has its own | |
| with/without pair (3 methods × 2 conditions per prompt): | |
| without Ratel -> all 100 tool schemas handed to the model | |
| with Ratel -> only that method's top-5 | |
| Per run we save exactly what the model produced: | |
| • the tool(s) it selected + arguments • its natural-language answer | |
| • real token spend (prompt/completion/total) • gold-set coverage | |
| plus the retrieval (top-5 ids + scores) that fed the with-Ratel run. A per-method | |
| comparison summary is written alongside. | |
| Run: | |
| pip install openai # + `ratel-ai` for live retrieval (optional) | |
| python scripts/bake_llm.py # reads .env for LLM_BASE_URL / LLM_MODEL / LLM_API_KEY | |
| Reads: data/tools.json, data/examples.json, data/meta.json (run bake.py first) | |
| Writes: data/llm.json | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import re | |
| import sys | |
| from pathlib import Path | |
| from openai import OpenAI | |
| ROOT = Path(__file__).resolve().parent.parent | |
| DATA = ROOT / "data" | |
| def load_dotenv(path: Path) -> None: | |
| """Populate os.environ from a .env file (KEY=VALUE lines). No dependency; | |
| real environment variables always win over the file.""" | |
| if not path.exists(): | |
| return | |
| for line in path.read_text().splitlines(): | |
| line = line.strip() | |
| if not line or line.startswith("#") or "=" not in line: | |
| continue | |
| key, _, value = line.partition("=") | |
| os.environ.setdefault(key.strip(), value.strip().strip("'\"")) | |
| load_dotenv(ROOT / ".env") | |
| BASE_URL = os.environ.get("LLM_BASE_URL", "http://localhost:11434/v1") | |
| MODEL = os.environ.get("LLM_MODEL", "qwen3:4b") | |
| API_KEY = os.environ.get("LLM_API_KEY", "not-needed") # local servers ignore it | |
| # Sparse / Dense / Hybrid — the three retrieval methods we produce answers for. | |
| METHODS = ["bm25", "semantic", "hybrid"] | |
| METHOD_LABEL = {"bm25": "sparse", "semantic": "dense", "hybrid": "hybrid"} | |
| TOP_K = 3 | |
| SYSTEM = ( | |
| "You are a task-management assistant for a project workspace. Fulfil the user's " | |
| "request by calling the most appropriate tool(s). Invent plausible values for any " | |
| "required arguments the request does not specify. Reads are always safe; only write " | |
| "(create/update/comment/link) when the request clearly asks for it. After the tools " | |
| "run, reply in one or two short sentences describing what you did." | |
| ) | |
| _THINK = re.compile(r"<think>.*?</think>", re.DOTALL) | |
| def norm(s: str) -> str: | |
| """Must match norm() in app.js.""" | |
| return re.sub(r"\s+", " ", s.strip().lower()) | |
| def openai_tool(t: dict) -> dict: | |
| return { | |
| "type": "function", | |
| "function": {"name": t["name"], "description": t["description"], "parameters": t["inputSchema"]}, | |
| } | |
| def clean(text: str | None) -> str: | |
| return _THINK.sub("", text or "").strip() | |
| # ── Ratel retrieval (default embedding model) ───────────────────────────────── | |
| def build_ratel_catalog(tools: list[dict]): | |
| """Register the catalog into a hybrid ToolCatalog using Ratel's DEFAULT | |
| embedding model (no override). Returns None if the SDK isn't installed.""" | |
| try: | |
| from ratel_ai import ExecutableTool, ToolCatalog | |
| except Exception: # noqa: BLE001 — SDK optional; caller falls back to examples.json | |
| return None | |
| cat = ToolCatalog(method="hybrid") # default embedder — same as scripts/bake.py | |
| for t in tools: | |
| cat.register( | |
| ExecutableTool( | |
| id=t["id"], | |
| name=t["name"], | |
| description=t["description"], | |
| input_schema=t["inputSchema"], | |
| output_schema={"type": "object", "properties": {}}, | |
| execute=lambda args: {}, | |
| ) | |
| ) | |
| return cat | |
| def retrieve(catalog, example: dict, method: str) -> list[dict]: | |
| """Top-5 for one method: live Ratel search when the SDK is present, else the | |
| top-5 bake.py already stored in examples.json (also Ratel-default retrieval).""" | |
| if catalog is not None: | |
| hits = catalog.search(example["query"], TOP_K, method=method) | |
| return [{"id": h.tool_id, "score": round(h.score, 6)} for h in hits] | |
| return [{"id": h["id"], "score": h["score"]} for h in example["results"][method][:TOP_K]] | |
| # ── one model turn ──────────────────────────────────────────────────────────── | |
| def run_turn(client: OpenAI, query: str, tools: list[dict], gold: list[str]) -> dict: | |
| """Model picks tool(s); we hand back a neutral mock result; model summarizes. | |
| contextTokens = prompt tokens of the tool-loaded first call — the number | |
| Ratel actually moves (70 schemas vs 5). | |
| """ | |
| messages = [{"role": "system", "content": SYSTEM}, {"role": "user", "content": query}] | |
| try: | |
| first = client.chat.completions.create( | |
| model=MODEL, messages=messages, tools=[openai_tool(t) for t in tools], | |
| tool_choice="auto", temperature=0, | |
| ) | |
| except Exception as e: # noqa: BLE001 — record the failure, keep baking the rest | |
| return {"error": str(e)[:200]} | |
| msg = first.choices[0].message | |
| context_tokens = getattr(first.usage, "prompt_tokens", 0) or 0 | |
| prompt_tokens = context_tokens | |
| completion_tokens = getattr(first.usage, "completion_tokens", 0) or 0 | |
| calls = msg.tool_calls or [] | |
| selected, inputs = [], [] | |
| for c in calls: | |
| selected.append(c.function.name) | |
| try: | |
| inputs.append({"tool": c.function.name, "args": json.loads(c.function.arguments or "{}")}) | |
| except json.JSONDecodeError: | |
| inputs.append({"tool": c.function.name, "args": c.function.arguments}) | |
| answer = clean(msg.content) | |
| if calls: | |
| # Neutral mock result so the model produces a closing sentence | |
| # (kestral mocked tool executes the same way — symmetric across conditions). | |
| messages.append(msg.model_dump()) | |
| for c in calls: | |
| messages.append({"role": "tool", "tool_call_id": c.id, "content": json.dumps({"ok": True})}) | |
| try: | |
| second = client.chat.completions.create(model=MODEL, messages=messages, temperature=0) | |
| answer = clean(second.choices[0].message.content) or answer | |
| prompt_tokens += getattr(second.usage, "prompt_tokens", 0) or 0 | |
| completion_tokens += getattr(second.usage, "completion_tokens", 0) or 0 | |
| except Exception: # noqa: BLE001 — summary is optional; keep the selection + tokens | |
| pass | |
| gold_set = set(gold) | |
| return { | |
| "tools": selected, | |
| "inputs": inputs, | |
| "answer": answer[:400], | |
| "correct": gold_set.issubset(set(selected)) if gold_set else False, | |
| "goldHits": len(gold_set & set(selected)), | |
| "contextTokens": context_tokens, | |
| "promptTokens": prompt_tokens, | |
| "completionTokens": completion_tokens, | |
| "totalTokens": prompt_tokens + completion_tokens, | |
| } | |
| def summarize(res: dict) -> str: | |
| if res.get("error"): | |
| return f"error: {res['error']}" | |
| tools = ", ".join(res["tools"]) or "(no tool call)" | |
| return f"{tools} · {'gold' if res['correct'] else 'miss'} · {res['contextTokens']} ctx tokens" | |
| def avg(values: list[int]) -> int: | |
| vals = [v for v in values if v is not None] | |
| return round(sum(vals) / len(vals)) if vals else 0 | |
| def main() -> int: | |
| tools = json.loads((DATA / "tools.json").read_text()) | |
| examples = json.loads((DATA / "examples.json").read_text()) | |
| meta = json.loads((DATA / "meta.json").read_text()) if (DATA / "meta.json").exists() else {} | |
| by_id = {t["id"]: t for t in tools} | |
| catalog = build_ratel_catalog(tools) | |
| source = "live Ratel SDK (default embedding model)" if catalog is not None else "examples.json (bake.py, default embedder)" | |
| client = OpenAI(base_url=BASE_URL, api_key=API_KEY) | |
| print(f"Model {MODEL} @ {BASE_URL}", flush=True) | |
| print(f"Retrieval: {source} · {len(examples)} prompts × {len(METHODS)} methods × (without + with)\n", flush=True) | |
| out = { | |
| "model": MODEL, | |
| # Deliberately NOT the raw BASE_URL — this file ships in a public Space; | |
| # don't leak a private inference-gateway host. | |
| "endpoint": "self-hosted (OpenAI-compatible)", | |
| "topK": TOP_K, | |
| "embedModel": meta.get("embedModel", "ratel-default"), | |
| "retrievalSource": "live" if catalog is not None else "baked", | |
| "methods": METHODS, | |
| "examples": {}, | |
| } | |
| for i, ex in enumerate(examples, 1): | |
| query, gold = ex["query"], ex["goldTop5"] | |
| print(f"[{i}/{len(examples)}] {query!r}", flush=True) | |
| methods_out = {} | |
| for method in METHODS: | |
| hits = retrieve(catalog, ex, method) | |
| subset = [by_id[h["id"]] for h in hits if h["id"] in by_id] | |
| # Same prompt on the model, both conditions, for THIS retriever method. | |
| without = run_turn(client, query, tools, gold) # all 100 tool schemas | |
| with_ = run_turn(client, query, subset, gold) # this method's top-5 | |
| methods_out[method] = {"retrieval": hits, "without": without, "with": with_} | |
| print(f" {METHOD_LABEL[method]:6s} ({method:8s}) — without: {summarize(without)}", flush=True) | |
| print(f" {'':6s} {'':8s} with: {summarize(with_)}", flush=True) | |
| out["examples"][norm(query)] = {"query": query, "gold": gold, "methods": methods_out} | |
| # ── per-method comparison summary (with vs without, for each retriever) ── | |
| exs = list(out["examples"].values()) | |
| ok = lambda res: 1 if (not res.get("error") and res.get("correct")) else 0 | |
| def agg(select): | |
| picks = [select(e) for e in exs] | |
| return { | |
| "goldCovered": sum(ok(p) for p in picks), | |
| "avgContextTokens": avg([p.get("contextTokens") for p in picks]), | |
| "avgTotalTokens": avg([p.get("totalTokens") for p in picks]), | |
| } | |
| out["comparison"] = {"n": len(exs)} | |
| for m in METHODS: | |
| out["comparison"][m] = { | |
| "without": agg(lambda e, m=m: e["methods"][m]["without"]), | |
| "with": agg(lambda e, m=m: e["methods"][m]["with"]), | |
| } | |
| (DATA / "llm.json").write_text(json.dumps(out, ensure_ascii=False, indent=2)) | |
| n = len(exs) | |
| print(f"\nMethod comparison over {n} prompts (gold-covered · avg context tokens):", flush=True) | |
| for m in METHODS: | |
| cm = out["comparison"][m] | |
| print(f" {METHOD_LABEL[m]:6s} ({m:8s}) — without {cm['without']['goldCovered']}/{n} · {cm['without']['avgContextTokens']}" | |
| f" | with {cm['with']['goldCovered']}/{n} · {cm['with']['avgContextTokens']}", flush=True) | |
| print(f"\nWrote {DATA / 'llm.json'}", flush=True) | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |