#!/usr/bin/env python3 import argparse import json from validation_common import ROOT, TEXTS, content_audio, content_image, cosine, fixtures, post parser = argparse.ArgumentParser() parser.add_argument("--base-url", default="http://127.0.0.1:12435") parser.add_argument("--model", default="mirrorneuronlab/embeddinggemma-2:q4_k_m") args = parser.parse_args() url = args.base_url.rstrip("/") + "/engines/llama.cpp/v1/embeddings" def request(name, inputs): return post(url, dict(model=args.model, input=inputs, encoding_format="float"), "dmr-"+name) vectors, text = request("text", TEXTS) text.update(related_cosine=cosine(vectors[0], vectors[1]), unrelated_cosine=cosine(vectors[0], vectors[2])) assert text["related_cosine"] > text["unrelated_cosine"] repeat, repeated = request("repeat", TEXTS) repeated["max_abs_difference"] = max(abs(x-y) for a,b in zip(vectors, repeat) for x,y in zip(a,b)) assert repeated["max_abs_difference"] < 1e-5 reference = json.loads((ROOT / "logs/bf16-text.response.json").read_text()) reference = [item["embedding"] for item in sorted(reference["data"], key=lambda x: x["index"])] bf16_cosines = [cosine(a, b) for a, b in zip(vectors, reference)] assert min(bf16_cosines) > 0.90 image_path, audio_path = fixtures() _, image = request("image", [content_image(image_path)]) _, audio = request("audio", [content_audio(audio_path)]) _, mixed = request("mixed-batch", [TEXTS[0], content_image(image_path), content_audio(audio_path)]) report = dict(passed=True, model=args.model, endpoint=url, text=text, repeat=repeated, cosine_to_bf16_text=bf16_cosines, image=image, audio=audio, mixed_batch=mixed, scope="Functional smoke checks on four texts, one solid-color image, and one synthetic audio tone. Not a retrieval or multimodal quality benchmark.") (ROOT / "logs/dmr-validation.json").write_text(json.dumps(report, indent=2)+"\n") print(json.dumps(report, indent=2))