import base64 import json import math import pathlib import struct import time import urllib.error import urllib.request import wave import zlib ROOT = pathlib.Path(__file__).resolve().parents[1] TEXTS = [ "task: search result | query: What causes the northern lights?", "title: none | text: The northern lights are caused by charged particles from the Sun interacting with Earth's atmosphere.", "title: none | text: To bake a chocolate cake, mix flour, eggs, cocoa powder, and sugar, then bake in the oven.", "task: sentence similarity | query: Bonjour, le soleil brille aujourd'hui.", ] def post(url, payload, log_name): (ROOT / f"logs/{log_name}.request.json").write_text(json.dumps(payload, indent=2) + "\n") start = time.monotonic() request = urllib.request.Request(url, data=json.dumps(payload).encode(), headers={"Content-Type": "application/json"}) try: with urllib.request.urlopen(request, timeout=240) as response: status = response.status body = response.read() except urllib.error.HTTPError as error: (ROOT / f"logs/{log_name}.error.txt").write_bytes(error.read()) raise (ROOT / f"logs/{log_name}.response.json").write_bytes(body) result = json.loads(body) vectors = [item["embedding"] for item in sorted(result["data"], key=lambda x: x["index"])] assert len(vectors) == len(payload["input"]), result for vector in vectors: assert len(vector) == 768, len(vector) assert all(math.isfinite(x) for x in vector) assert abs(norm(vector) - 1.0) < 1e-4, norm(vector) return vectors, dict(http_status=status, elapsed_seconds=time.monotonic()-start, vectors=len(vectors), dimensions=768, finite=True, l2_norms=[norm(v) for v in vectors]) def norm(vector): return math.sqrt(sum(x*x for x in vector)) def cosine(a, b): return sum(x*y for x, y in zip(a, b)) / (norm(a)*norm(b)) def fixtures(): folder = ROOT / "work/fixtures" folder.mkdir(parents=True, exist_ok=True) def chunk(kind, data): return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind+data) & 0xffffffff) # A simple deterministic raster fixture, not an evaluation dataset. png = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 224, 224, 8, 2, 0, 0, 0)) png += chunk(b"IDAT", zlib.compress((b"\x00" + bytes([230, 20, 20])*224)*224)) + chunk(b"IEND", b"") image = folder / "red-square.png" image.write_bytes(png) audio = folder / "tone-440hz.wav" samples = [int(8000*math.sin(2*math.pi*440*i/16000)) for i in range(32000)] with wave.open(str(audio), "wb") as output: output.setnchannels(1) output.setsampwidth(2) output.setframerate(16000) output.writeframes(struct.pack("<"+"h"*len(samples), *samples)) return image, audio def content_image(path): return {"content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64," + base64.b64encode(path.read_bytes()).decode()}}]} def content_audio(path): return {"content": [{"type": "input_audio", "input_audio": {"data": base64.b64encode(path.read_bytes()).decode(), "format": "wav"}}]}