File size: 1,788 Bytes
336c50f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
#!/usr/bin/env python3
import json
import subprocess
import time
import urllib.request
from validation_common import ROOT, TEXTS, cosine, post

with (ROOT / "logs/bf16-server.log").open("w") as log:
    command = [str(ROOT / "llama.cpp/build/bin/llama-server"),
               "-m", str(ROOT / "work/embeddinggemma-2-BF16.gguf"),
               "--embeddings", "--pooling", "mean", "-c", "2048", "-b", "2048", "-ub", "2048",
               "--parallel", "1", "-ngl", "0", "-t", "10", "--host", "127.0.0.1", "--port", "18080"]
    log.write(json.dumps(command)+"\n"); log.flush()
    server = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT)
    try:
        for attempt in range(120):
            if server.poll() is not None:
                raise RuntimeError("BF16 server exited; inspect logs/bf16-server.log")
            try:
                with urllib.request.urlopen("http://127.0.0.1:18080/health", timeout=1) as response:
                    if response.status == 200:
                        break
            except Exception:
                time.sleep(0.5)
        else:
            raise TimeoutError("BF16 server startup timed out")
        vectors, report = post("http://127.0.0.1:18080/v1/embeddings", {"input": TEXTS, "encoding_format": "float"}, "bf16-text")
        report.update(related_cosine=cosine(vectors[0], vectors[1]), unrelated_cosine=cosine(vectors[0], vectors[2]))
        assert report["related_cosine"] > report["unrelated_cosine"]
        (ROOT / "logs/bf16-validation.json").write_text(json.dumps(report, indent=2)+"\n")
        print(json.dumps(report, indent=2))
    finally:
        server.terminate()
        try:
            server.wait(timeout=10)
        except subprocess.TimeoutExpired:
            server.kill(); server.wait()