| import json, re, sys, urllib.request |
|
|
| SYSTEM = json.loads(open(sys.argv[1]).readline())['messages'][0]['content'] |
| GOLD = [json.loads(l) for l in open(sys.argv[2])] |
| PORT = sys.argv[3] |
| TAG = sys.argv[4] |
|
|
| def ask(question): |
| payload = { |
| "model": "parser", |
| "messages": [ |
| {"role": "system", "content": SYSTEM}, |
| {"role": "user", "content": question}, |
| ], |
| "temperature": 0.0, |
| "max_tokens": 600, |
| } |
| req = urllib.request.Request( |
| f"http://localhost:{PORT}/v1/chat/completions", |
| data=json.dumps(payload).encode(), |
| headers={"Content-Type": "application/json"}, |
| ) |
| with urllib.request.urlopen(req, timeout=60) as r: |
| body = json.loads(r.read()) |
| return body["choices"][0]["message"]["content"] |
|
|
| route_correct = 0 |
| endpoint_correct = 0 |
| json_valid = 0 |
| n = len(GOLD) |
| rows = [] |
| for i, g in enumerate(GOLD): |
| |
| tagged_q = f"[ROUTE: {g['expected_route']}] {g['question']}" |
| try: |
| raw = ask(tagged_q) |
| m = re.search(r"\{.*\}", raw, re.DOTALL) |
| parsed = json.loads(m.group(0)) if m else None |
| json_valid += 1 if parsed else 0 |
| got_route = parsed.get("route") if parsed else None |
| got_endpoints = sorted(c.get("endpoint", "") for c in parsed.get("calls", [])) if parsed else [] |
| exp_endpoints = sorted(g.get("expected_endpoints", [])) |
| r_ok = got_route == g["expected_route"] |
| e_ok = got_endpoints == exp_endpoints |
| route_correct += r_ok |
| endpoint_correct += e_ok |
| rows.append({"q": g["question"], "expected_route": g["expected_route"], "got_route": got_route, "route_ok": r_ok, "endpoint_ok": e_ok, "got_endpoints": got_endpoints, "exp_endpoints": exp_endpoints}) |
| except Exception as ex: |
| rows.append({"q": g["question"], "error": str(ex)}) |
| if (i + 1) % 20 == 0: |
| print(f"[{TAG}] {i+1}/{n}", file=sys.stderr) |
|
|
| print(json.dumps({ |
| "tag": TAG, |
| "n": n, |
| "json_valid_pct": round(100 * json_valid / n, 1), |
| "route_accuracy_pct": round(100 * route_correct / n, 1), |
| "endpoint_exact_match_pct": round(100 * endpoint_correct / n, 1), |
| "rows": rows, |
| }, indent=2)) |
|
|
