| |
| """Validate a running hybrid GLM-5.2 vLLM server. |
| |
| Checks: completion coherence (short prompt), decode throughput, per-GPU |
| memory versus the 96GB target ceiling, and reported KV-cache capacity. |
| |
| Usage: validate_serve.py [--port 8199] [--long N_TOKENS] |
| """ |
| import argparse |
| import json |
| import subprocess |
| import time |
| import urllib.request |
|
|
|
|
| def post(port, path, body): |
| req = urllib.request.Request( |
| f"http://localhost:{port}{path}", |
| data=json.dumps(body).encode(), |
| headers={"Content-Type": "application/json"}, |
| ) |
| with urllib.request.urlopen(req, timeout=1800) as r: |
| return json.load(r) |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--port", type=int, default=8199) |
| ap.add_argument("--long", type=int, default=0, |
| help="also run a long-context probe of this many tokens") |
| args = ap.parse_args() |
|
|
| models = json.load( |
| urllib.request.urlopen(f"http://localhost:{args.port}/v1/models") |
| ) |
| model = models["data"][0]["id"] |
| print("model:", model) |
|
|
| prompts = [ |
| "The capital of France is", |
| "def fibonacci(n):\n ", |
| "Water is composed of the elements", |
| ] |
| for p in prompts: |
| t0 = time.time() |
| out = post(args.port, "/v1/completions", { |
| "model": model, "prompt": p, "max_tokens": 48, "temperature": 0.0, |
| }) |
| text = out["choices"][0]["text"] |
| n = out["usage"]["completion_tokens"] |
| dt = time.time() - t0 |
| print(f"\n>>> {p!r}\n{text!r}\n[{n} tok in {dt:.1f}s = {n/dt:.1f} tok/s]") |
|
|
| if args.long: |
| filler = ("The quick brown fox jumps over the lazy dog. " * 8 + "\n") |
| target_words = int(args.long * 0.75) |
| doc = (filler * (target_words // len(filler.split()) + 1)) |
| needle = "\nIMPORTANT: the secret code word is BLUEBERRY42.\n" |
| prompt = (doc[: len(doc) // 2] + needle + doc[len(doc) // 2:] |
| + "\n\nWhat is the secret code word mentioned above? Answer:") |
| t0 = time.time() |
| out = post(args.port, "/v1/completions", { |
| "model": model, "prompt": prompt, "max_tokens": 16, |
| "temperature": 0.0, |
| }) |
| dt = time.time() - t0 |
| u = out["usage"] |
| print(f"\n=== long-context probe: {u['prompt_tokens']} prompt tokens, " |
| f"{dt:.0f}s ===") |
| print("answer:", out["choices"][0]["text"].strip()) |
|
|
| print("\n=== per-GPU memory ===") |
| smi = subprocess.run( |
| ["nvidia-smi", "--query-gpu=index,memory.used", |
| "--format=csv,noheader,nounits"], |
| capture_output=True, text=True, |
| ).stdout |
| worst = 0 |
| for line in smi.strip().splitlines(): |
| idx, used = [int(x) for x in line.split(",")] |
| worst = max(worst, used) |
| flag = "OK" if used <= 97887 else "OVER 96GB TARGET" |
| print(f"gpu{idx}: {used} MiB [{flag}]") |
| print(f"worst GPU: {worst} MiB vs 97887 MiB target " |
| f"({'FITS' if worst <= 97887 else 'DOES NOT FIT'})") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
