File size: 2,425 Bytes
32bed67 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | """End-to-end check of the served openjev classifier: POST the reference NLI pairs to /classify and
compare the returned probabilities with the CPU reference (ref/reference.pt).
python scripts/check_classify_api.py --url http://localhost:8000 [--model AlexWortega/openjev]
"""
import argparse
import json
import time
import urllib.request
import torch
TEMPLATE = "Premise: {premise}\nHypothesis: {hypothesis}"
LABELS = ["contradiction", "entailment", "neutral"]
def post(url, payload, timeout=600):
req = urllib.request.Request(
url, data=json.dumps(payload).encode(), headers={"Content-Type": "application/json"}, method="POST"
)
with urllib.request.urlopen(req, timeout=timeout) as r:
return json.loads(r.read())
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--url", default="http://localhost:8000")
ap.add_argument("--model", default="AlexWortega/openjev")
ap.add_argument("--ref", default="/home/ttuser/experiments/openjev/ref/reference.pt")
ap.add_argument("--batch", type=int, default=20, help="pairs per /classify request")
args = ap.parse_args()
ref = torch.load(args.ref)
texts = ref["texts"]
rprobs = ref["fp32"]["probs"]
got = []
t0 = time.time()
for i in range(0, len(texts), args.batch):
chunk = texts[i : i + args.batch]
t1 = time.time()
out = post(f"{args.url}/classify", {"model": args.model, "input": chunk})
dt = time.time() - t1
print(f"batch {i // args.batch}: {len(chunk)} inputs in {dt:.2f}s ({dt / len(chunk) * 1e3:.0f} ms/input)")
got.extend(out["data"])
total = time.time() - t0
probs = torch.tensor([d["probs"] for d in got], dtype=torch.float32)
labels = [d["label"] for d in got]
agree = (probs.argmax(-1) == rprobs.argmax(-1)).float().mean().item()
dprob = (probs - rprobs).abs()
print(f"\n{len(texts)} pairs in {total:.2f}s; label agreement vs CPU fp32 reference: {agree:.3f}; "
f"max|dprob| {dprob.max():.4f} mean|dprob| {dprob.mean():.4f}")
for i, (p, h) in enumerate(ref["pairs"]):
print(f" {i:2d} server={labels[i][:3].upper()} {[round(x, 4) for x in probs[i].tolist()]} "
f"ref={LABELS[int(rprobs[i].argmax())][:3].upper()} {[round(x, 4) for x in rprobs[i].tolist()]}")
assert agree == 1.0, "label mismatch vs reference"
print("OK")
if __name__ == "__main__":
main()
|