embeddinggemma-2-GGUF / reproduce /scripts /validation_common.py
homerquan's picture
Add standard Q4_K_M text model and Q8_0 vision/audio companion
336c50f verified
Raw History Blame Contribute Delete
3.25 kB
import base64
import json
import math
import pathlib
import struct
import time
import urllib.error
import urllib.request
import wave
import zlib
ROOT = pathlib.Path(__file__).resolve().parents[1]
TEXTS = [
"task: search result | query: What causes the northern lights?",
"title: none | text: The northern lights are caused by charged particles from the Sun interacting with Earth's atmosphere.",
"title: none | text: To bake a chocolate cake, mix flour, eggs, cocoa powder, and sugar, then bake in the oven.",
"task: sentence similarity | query: Bonjour, le soleil brille aujourd'hui.",
]
def post(url, payload, log_name):
(ROOT / f"logs/{log_name}.request.json").write_text(json.dumps(payload, indent=2) + "\n")
start = time.monotonic()
request = urllib.request.Request(url, data=json.dumps(payload).encode(), headers={"Content-Type": "application/json"})
try:
with urllib.request.urlopen(request, timeout=240) as response:
status = response.status
body = response.read()
except urllib.error.HTTPError as error:
(ROOT / f"logs/{log_name}.error.txt").write_bytes(error.read())
raise
(ROOT / f"logs/{log_name}.response.json").write_bytes(body)
result = json.loads(body)
vectors = [item["embedding"] for item in sorted(result["data"], key=lambda x: x["index"])]
assert len(vectors) == len(payload["input"]), result
for vector in vectors:
assert len(vector) == 768, len(vector)
assert all(math.isfinite(x) for x in vector)
assert abs(norm(vector) - 1.0) < 1e-4, norm(vector)
return vectors, dict(http_status=status, elapsed_seconds=time.monotonic()-start,
vectors=len(vectors), dimensions=768, finite=True,
l2_norms=[norm(v) for v in vectors])
def norm(vector):
return math.sqrt(sum(x*x for x in vector))
def cosine(a, b):
return sum(x*y for x, y in zip(a, b)) / (norm(a)*norm(b))
def fixtures():
folder = ROOT / "work/fixtures"
folder.mkdir(parents=True, exist_ok=True)
def chunk(kind, data):
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind+data) & 0xffffffff)
# A simple deterministic raster fixture, not an evaluation dataset.
png = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 224, 224, 8, 2, 0, 0, 0))
png += chunk(b"IDAT", zlib.compress((b"\x00" + bytes([230, 20, 20])*224)*224)) + chunk(b"IEND", b"")
image = folder / "red-square.png"
image.write_bytes(png)
audio = folder / "tone-440hz.wav"
samples = [int(8000*math.sin(2*math.pi*440*i/16000)) for i in range(32000)]
with wave.open(str(audio), "wb") as output:
output.setnchannels(1)
output.setsampwidth(2)
output.setframerate(16000)
output.writeframes(struct.pack("<"+"h"*len(samples), *samples))
return image, audio
def content_image(path):
return {"content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64," + base64.b64encode(path.read_bytes()).decode()}}]}
def content_audio(path):
return {"content": [{"type": "input_audio", "input_audio": {"data": base64.b64encode(path.read_bytes()).decode(), "format": "wav"}}]}