homerquan's picture
Add standard Q4_K_M text model and Q8_0 vision/audio companion
336c50f verified
Raw History Blame Contribute Delete
1.79 kB
#!/usr/bin/env python3
import json
import subprocess
import time
import urllib.request
from validation_common import ROOT, TEXTS, cosine, post
with (ROOT / "logs/bf16-server.log").open("w") as log:
command = [str(ROOT / "llama.cpp/build/bin/llama-server"),
"-m", str(ROOT / "work/embeddinggemma-2-BF16.gguf"),
"--embeddings", "--pooling", "mean", "-c", "2048", "-b", "2048", "-ub", "2048",
"--parallel", "1", "-ngl", "0", "-t", "10", "--host", "127.0.0.1", "--port", "18080"]
log.write(json.dumps(command)+"\n"); log.flush()
server = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT)
try:
for attempt in range(120):
if server.poll() is not None:
raise RuntimeError("BF16 server exited; inspect logs/bf16-server.log")
try:
with urllib.request.urlopen("http://127.0.0.1:18080/health", timeout=1) as response:
if response.status == 200:
break
except Exception:
time.sleep(0.5)
else:
raise TimeoutError("BF16 server startup timed out")
vectors, report = post("http://127.0.0.1:18080/v1/embeddings", {"input": TEXTS, "encoding_format": "float"}, "bf16-text")
report.update(related_cosine=cosine(vectors[0], vectors[1]), unrelated_cosine=cosine(vectors[0], vectors[2]))
assert report["related_cosine"] > report["unrelated_cosine"]
(ROOT / "logs/bf16-validation.json").write_text(json.dumps(report, indent=2)+"\n")
print(json.dumps(report, indent=2))
finally:
server.terminate()
try:
server.wait(timeout=10)
except subprocess.TimeoutExpired:
server.kill(); server.wait()