import json, re, sys, urllib.request

SYSTEM = json.loads(open(sys.argv[1]).readline())['messages'][0]['content']
GOLD = [json.loads(l) for l in open(sys.argv[2])]
PORT = sys.argv[3]
TAG = sys.argv[4]

def ask(question):
    payload = {
        "model": "parser",
        "messages": [
            {"role": "system", "content": SYSTEM},
            {"role": "user", "content": question},
        ],
        "temperature": 0.0,
        "max_tokens": 600,
    }
    req = urllib.request.Request(
        f"http://localhost:{PORT}/v1/chat/completions",
        data=json.dumps(payload).encode(),
        headers={"Content-Type": "application/json"},
    )
    with urllib.request.urlopen(req, timeout=60) as r:
        body = json.loads(r.read())
    return body["choices"][0]["message"]["content"]

route_correct = 0
endpoint_correct = 0
json_valid = 0
n = len(GOLD)
rows = []
for i, g in enumerate(GOLD):
    # Match production/training format: route is GIVEN, model builds the DAG.
    tagged_q = f"[ROUTE: {g['expected_route']}] {g['question']}"
    try:
        raw = ask(tagged_q)
        m = re.search(r"\{.*\}", raw, re.DOTALL)
        parsed = json.loads(m.group(0)) if m else None
        json_valid += 1 if parsed else 0
        got_route = parsed.get("route") if parsed else None
        got_endpoints = sorted(c.get("endpoint", "") for c in parsed.get("calls", [])) if parsed else []
        exp_endpoints = sorted(g.get("expected_endpoints", []))
        r_ok = got_route == g["expected_route"]
        e_ok = got_endpoints == exp_endpoints
        route_correct += r_ok
        endpoint_correct += e_ok
        rows.append({"q": g["question"], "expected_route": g["expected_route"], "got_route": got_route, "route_ok": r_ok, "endpoint_ok": e_ok, "got_endpoints": got_endpoints, "exp_endpoints": exp_endpoints})
    except Exception as ex:
        rows.append({"q": g["question"], "error": str(ex)})
    if (i + 1) % 20 == 0:
        print(f"[{TAG}] {i+1}/{n}", file=sys.stderr)

print(json.dumps({
    "tag": TAG,
    "n": n,
    "json_valid_pct": round(100 * json_valid / n, 1),
    "route_accuracy_pct": round(100 * route_correct / n, 1),
    "endpoint_exact_match_pct": round(100 * endpoint_correct / n, 1),
    "rows": rows,
}, indent=2))