File size: 2,255 Bytes
b7c9121 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | import json, re, sys, urllib.request
SYSTEM = json.loads(open(sys.argv[1]).readline())['messages'][0]['content']
GOLD = [json.loads(l) for l in open(sys.argv[2])]
PORT = sys.argv[3]
TAG = sys.argv[4]
def ask(question):
payload = {
"model": "parser",
"messages": [
{"role": "system", "content": SYSTEM},
{"role": "user", "content": question},
],
"temperature": 0.0,
"max_tokens": 600,
}
req = urllib.request.Request(
f"http://localhost:{PORT}/v1/chat/completions",
data=json.dumps(payload).encode(),
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(req, timeout=60) as r:
body = json.loads(r.read())
return body["choices"][0]["message"]["content"]
route_correct = 0
endpoint_correct = 0
json_valid = 0
n = len(GOLD)
rows = []
for i, g in enumerate(GOLD):
# Match production/training format: route is GIVEN, model builds the DAG.
tagged_q = f"[ROUTE: {g['expected_route']}] {g['question']}"
try:
raw = ask(tagged_q)
m = re.search(r"\{.*\}", raw, re.DOTALL)
parsed = json.loads(m.group(0)) if m else None
json_valid += 1 if parsed else 0
got_route = parsed.get("route") if parsed else None
got_endpoints = sorted(c.get("endpoint", "") for c in parsed.get("calls", [])) if parsed else []
exp_endpoints = sorted(g.get("expected_endpoints", []))
r_ok = got_route == g["expected_route"]
e_ok = got_endpoints == exp_endpoints
route_correct += r_ok
endpoint_correct += e_ok
rows.append({"q": g["question"], "expected_route": g["expected_route"], "got_route": got_route, "route_ok": r_ok, "endpoint_ok": e_ok, "got_endpoints": got_endpoints, "exp_endpoints": exp_endpoints})
except Exception as ex:
rows.append({"q": g["question"], "error": str(ex)})
if (i + 1) % 20 == 0:
print(f"[{TAG}] {i+1}/{n}", file=sys.stderr)
print(json.dumps({
"tag": TAG,
"n": n,
"json_valid_pct": round(100 * json_valid / n, 1),
"route_accuracy_pct": round(100 * route_correct / n, 1),
"endpoint_exact_match_pct": round(100 * endpoint_correct / n, 1),
"rows": rows,
}, indent=2))
|