bdrgon's picture
Validate Gemma exports on full drum corpus and publish results
448ab1d
Raw History Blame Contribute Delete
7.71 kB
#!/usr/bin/env python
"""Export experimental EmbeddingGemma 2 tower ONNX variants only."""
from __future__ import annotations
import argparse
from pathlib import Path
from export_common import (
MODEL_ID, REVISION, ROOT, OPSET, EMBEDDING_DIM, EXTERNAL_DATA,
export_patches, load_reference, quantize_int8, quantize_int4,
print_artifact, sha256_file,
)
import numpy as np
import torch
import gc
from huggingface_hub import hf_hub_download
from safetensors import safe_open
from transformers.models.gemma4.modeling_gemma4 import Gemma4AudioModel
TOWER_OUT = ROOT / "audio_tower.onnx"
TOWER_TEMP = ROOT / "audio_tower.tmp.onnx"
TOWER_INT8_OUT = ROOT / "audio_tower_int8.onnx"
TOWER_INT8_TEMP = ROOT / "audio_tower_int8.tmp.onnx"
TOWER_INT4_OUT = ROOT / "audio_tower_int4.onnx"
TOWER_INT4_TEMP = ROOT / "audio_tower_int4.tmp.onnx"
class AudioTower(torch.nn.Module):
def __init__(self, model: Gemma4AudioModel):
super().__init__()
self.model = model
def forward(self, input_features, input_features_mask):
result = self.model(
input_features=input_features,
attention_mask=input_features_mask,
return_dict=True,
)
return result.last_hidden_state, result.attention_mask
def check_tower(
model_path: Path,
features: np.ndarray,
feature_mask: np.ndarray,
reference_states: np.ndarray,
reference_mask: np.ndarray,
*,
min_cosine: float,
max_abs_diff: float | None,
) -> dict[str, float]:
import onnx
import onnxruntime as ort
onnx.checker.check_model(str(model_path))
options = ort.SessionOptions()
options.log_severity_level = 3
options.intra_op_num_threads = 1
options.inter_op_num_threads = 1
session = ort.InferenceSession(str(model_path), options, providers=["CPUExecutionProvider"])
states, mask = session.run(
["audio_features", "audio_mask"],
{"input_features": features, "input_features_mask": feature_mask},
)
states = states.astype(np.float32)
if not np.isfinite(states).all():
raise RuntimeError(f"{model_path.name} produced non-finite audio features")
if not np.array_equal(mask, reference_mask):
raise RuntimeError(f"{model_path.name} changed the audio padding mask")
if states.shape != reference_states.shape:
raise RuntimeError(f"{model_path.name} output shape {states.shape} != PyTorch {reference_states.shape}")
ref_flat, out_flat = reference_states.reshape(-1), states.reshape(-1)
cosine = float(
np.dot(ref_flat, out_flat)
/ max(float(np.linalg.norm(ref_flat) * np.linalg.norm(out_flat)), 1e-12)
)
max_diff = float(np.max(np.abs(reference_states - states)))
if cosine < min_cosine or (max_abs_diff is not None and max_diff > max_abs_diff):
raise RuntimeError(f"{model_path.name} parity failed: cosine={cosine}, max_abs_diff={max_diff}")
return {"cosine": cosine, "max_abs_diff": max_diff}
def export_tower(config, processor, checkpoint: str) -> None:
if config.audio_config is None:
raise RuntimeError("Pinned checkpoint has no audio tower")
model = Gemma4AudioModel(config.audio_config)
expected_keys = set(model.state_dict())
audio_state = {}
with safe_open(checkpoint, framework="pt", device="cpu") as weights:
for key in weights.keys():
if key.startswith("audio_tower."):
key = key.removeprefix("audio_tower.")
if key in expected_keys:
audio_state[key] = weights.get_tensor(f"audio_tower.{key}")
if not audio_state:
raise RuntimeError("Pinned checkpoint contained no audio_tower weights")
incompatible = model.load_state_dict(audio_state, strict=True)
if incompatible.missing_keys or incompatible.unexpected_keys:
raise RuntimeError(f"Audio tower state mismatch: {incompatible}")
model.eval()
tower = AudioTower(model).eval()
sample_rate = int(processor.feature_extractor.sampling_rate)
audio3 = (0.4 * np.sin(2 * np.pi * 220 * np.arange(3 * sample_rate) / sample_rate)).astype(np.float32)
audio1 = (0.4 * np.sin(2 * np.pi * 220 * np.arange(sample_rate) / sample_rate)).astype(np.float32)
inputs3 = processor(audio=[audio3], sampling_rate=sample_rate, return_tensors="pt")
inputs1 = processor(audio=[audio1], sampling_rate=sample_rate, return_tensors="pt")
features3 = inputs3["input_features"].to(torch.float32)
mask3 = inputs3["input_features_mask"].to(torch.bool)
features1 = inputs1["input_features"].to(torch.float32)
mask1 = inputs1["input_features_mask"].to(torch.bool)
with torch.inference_mode():
ref3, ref_mask3 = tower(features3, mask3)
ref1, ref_mask1 = tower(features1, mask1)
x3, m3 = features3.cpu().numpy(), mask3.cpu().numpy()
x1, m1 = features1.cpu().numpy(), mask1.cpu().numpy()
y3, ym3 = ref3.to(torch.float32).cpu().numpy(), ref_mask3.cpu().numpy()
y1, ym1 = ref1.to(torch.float32).cpu().numpy(), ref_mask1.cpu().numpy()
dynamic_axes = {
"input_features": {1: "feature_frames"},
"input_features_mask": {1: "feature_frames"},
"audio_features": {1: "audio_tokens"},
"audio_mask": {1: "audio_tokens"},
}
with export_patches(include_text=False):
torch.onnx.export(
tower,
(features3, mask3),
str(TOWER_TEMP),
input_names=["input_features", "input_features_mask"],
output_names=["audio_features", "audio_mask"],
opset_version=OPSET,
dynamo=False,
do_constant_folding=True,
dynamic_axes=dynamic_axes,
)
raw3 = check_tower(TOWER_TEMP, x3, m3, y3, ym3, min_cosine=0.99999, max_abs_diff=5e-4)
TOWER_TEMP.replace(TOWER_OUT)
quantize_int8(TOWER_OUT, TOWER_INT8_TEMP)
int8_3 = check_tower(TOWER_INT8_TEMP, x3, m3, y3, ym3, min_cosine=0.97, max_abs_diff=None)
TOWER_INT8_TEMP.replace(TOWER_INT8_OUT)
quantize_int4(TOWER_OUT, TOWER_INT4_TEMP)
int4_3 = check_tower(TOWER_INT4_TEMP, x3, m3, y3, ym3, min_cosine=0.80, max_abs_diff=None)
TOWER_INT4_TEMP.replace(TOWER_INT4_OUT)
parity = {}
for path, threshold in (
(TOWER_OUT, 0.99999),
(TOWER_INT8_OUT, 0.97),
(TOWER_INT4_OUT, 0.80),
):
parity[path] = check_tower(
path,
x1,
m1,
y1,
ym1,
min_cosine=threshold,
max_abs_diff=5e-4 if path == TOWER_OUT else None,
)
print_artifact(TOWER_OUT, raw3, "Tower fp32; 3 s")
print_artifact(TOWER_INT8_OUT, int8_3, "Tower int8; 3 s")
print_artifact(TOWER_INT4_OUT, int4_3, "Tower int4; 3 s")
print(f"Tower short-audio cosine: fp32={parity[TOWER_OUT]['cosine']:.7f}, "
f"int8={parity[TOWER_INT8_OUT]['cosine']:.7f}, int4={parity[TOWER_INT4_OUT]['cosine']:.7f}")
del model, tower, audio_state, expected_keys
del features3, features1, mask3, mask1, ref3, ref1, ref_mask3, ref_mask1
del inputs3, inputs1, x3, x1, m3, m1, y3, y1, ym3, ym1, parity
gc.collect()
def main() -> None:
from validation import parser, run_validation
args = parser(__doc__).parse_args()
print("EXPERIMENTAL: all artifacts from this exporter require downstream qualification.", flush=True)
if not args.validate_only:
config, processor = load_reference()
checkpoint = hf_hub_download(MODEL_ID, "model.safetensors", revision=REVISION)
export_tower(config, processor, checkpoint)
if not args.skip_validation:
raise SystemExit(0 if run_validation("tower", args) else 1)
if __name__ == "__main__":
main()