homerquan's picture
Add standard Q4_K_M text model and Q8_0 vision/audio companion
336c50f verified
Raw History Blame Contribute Delete
1.93 kB
#!/usr/bin/env python3
import argparse
import json
from validation_common import ROOT, TEXTS, content_audio, content_image, cosine, fixtures, post
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://127.0.0.1:12435")
parser.add_argument("--model", default="mirrorneuronlab/embeddinggemma-2:q4_k_m")
args = parser.parse_args()
url = args.base_url.rstrip("/") + "/engines/llama.cpp/v1/embeddings"
def request(name, inputs):
return post(url, dict(model=args.model, input=inputs, encoding_format="float"), "dmr-"+name)
vectors, text = request("text", TEXTS)
text.update(related_cosine=cosine(vectors[0], vectors[1]), unrelated_cosine=cosine(vectors[0], vectors[2]))
assert text["related_cosine"] > text["unrelated_cosine"]
repeat, repeated = request("repeat", TEXTS)
repeated["max_abs_difference"] = max(abs(x-y) for a,b in zip(vectors, repeat) for x,y in zip(a,b))
assert repeated["max_abs_difference"] < 1e-5
reference = json.loads((ROOT / "logs/bf16-text.response.json").read_text())
reference = [item["embedding"] for item in sorted(reference["data"], key=lambda x: x["index"])]
bf16_cosines = [cosine(a, b) for a, b in zip(vectors, reference)]
assert min(bf16_cosines) > 0.90
image_path, audio_path = fixtures()
_, image = request("image", [content_image(image_path)])
_, audio = request("audio", [content_audio(audio_path)])
_, mixed = request("mixed-batch", [TEXTS[0], content_image(image_path), content_audio(audio_path)])
report = dict(passed=True, model=args.model, endpoint=url, text=text, repeat=repeated,
cosine_to_bf16_text=bf16_cosines, image=image, audio=audio, mixed_batch=mixed,
scope="Functional smoke checks on four texts, one solid-color image, and one synthetic audio tone. Not a retrieval or multimodal quality benchmark.")
(ROOT / "logs/dmr-validation.json").write_text(json.dumps(report, indent=2)+"\n")
print(json.dumps(report, indent=2))