#!/usr/bin/env python """Export experimental EmbeddingGemma 2 tower ONNX variants only.""" from __future__ import annotations import argparse from pathlib import Path from export_common import ( MODEL_ID, REVISION, ROOT, OPSET, EMBEDDING_DIM, EXTERNAL_DATA, export_patches, load_reference, quantize_int8, quantize_int4, print_artifact, sha256_file, ) import numpy as np import torch import gc from huggingface_hub import hf_hub_download from safetensors import safe_open from transformers.models.gemma4.modeling_gemma4 import Gemma4AudioModel TOWER_OUT = ROOT / "audio_tower.onnx" TOWER_TEMP = ROOT / "audio_tower.tmp.onnx" TOWER_INT8_OUT = ROOT / "audio_tower_int8.onnx" TOWER_INT8_TEMP = ROOT / "audio_tower_int8.tmp.onnx" TOWER_INT4_OUT = ROOT / "audio_tower_int4.onnx" TOWER_INT4_TEMP = ROOT / "audio_tower_int4.tmp.onnx" class AudioTower(torch.nn.Module): def __init__(self, model: Gemma4AudioModel): super().__init__() self.model = model def forward(self, input_features, input_features_mask): result = self.model( input_features=input_features, attention_mask=input_features_mask, return_dict=True, ) return result.last_hidden_state, result.attention_mask def check_tower( model_path: Path, features: np.ndarray, feature_mask: np.ndarray, reference_states: np.ndarray, reference_mask: np.ndarray, *, min_cosine: float, max_abs_diff: float | None, ) -> dict[str, float]: import onnx import onnxruntime as ort onnx.checker.check_model(str(model_path)) options = ort.SessionOptions() options.log_severity_level = 3 options.intra_op_num_threads = 1 options.inter_op_num_threads = 1 session = ort.InferenceSession(str(model_path), options, providers=["CPUExecutionProvider"]) states, mask = session.run( ["audio_features", "audio_mask"], {"input_features": features, "input_features_mask": feature_mask}, ) states = states.astype(np.float32) if not np.isfinite(states).all(): raise RuntimeError(f"{model_path.name} produced non-finite audio features") if not np.array_equal(mask, reference_mask): raise RuntimeError(f"{model_path.name} changed the audio padding mask") if states.shape != reference_states.shape: raise RuntimeError(f"{model_path.name} output shape {states.shape} != PyTorch {reference_states.shape}") ref_flat, out_flat = reference_states.reshape(-1), states.reshape(-1) cosine = float( np.dot(ref_flat, out_flat) / max(float(np.linalg.norm(ref_flat) * np.linalg.norm(out_flat)), 1e-12) ) max_diff = float(np.max(np.abs(reference_states - states))) if cosine < min_cosine or (max_abs_diff is not None and max_diff > max_abs_diff): raise RuntimeError(f"{model_path.name} parity failed: cosine={cosine}, max_abs_diff={max_diff}") return {"cosine": cosine, "max_abs_diff": max_diff} def export_tower(config, processor, checkpoint: str) -> None: if config.audio_config is None: raise RuntimeError("Pinned checkpoint has no audio tower") model = Gemma4AudioModel(config.audio_config) expected_keys = set(model.state_dict()) audio_state = {} with safe_open(checkpoint, framework="pt", device="cpu") as weights: for key in weights.keys(): if key.startswith("audio_tower."): key = key.removeprefix("audio_tower.") if key in expected_keys: audio_state[key] = weights.get_tensor(f"audio_tower.{key}") if not audio_state: raise RuntimeError("Pinned checkpoint contained no audio_tower weights") incompatible = model.load_state_dict(audio_state, strict=True) if incompatible.missing_keys or incompatible.unexpected_keys: raise RuntimeError(f"Audio tower state mismatch: {incompatible}") model.eval() tower = AudioTower(model).eval() sample_rate = int(processor.feature_extractor.sampling_rate) audio3 = (0.4 * np.sin(2 * np.pi * 220 * np.arange(3 * sample_rate) / sample_rate)).astype(np.float32) audio1 = (0.4 * np.sin(2 * np.pi * 220 * np.arange(sample_rate) / sample_rate)).astype(np.float32) inputs3 = processor(audio=[audio3], sampling_rate=sample_rate, return_tensors="pt") inputs1 = processor(audio=[audio1], sampling_rate=sample_rate, return_tensors="pt") features3 = inputs3["input_features"].to(torch.float32) mask3 = inputs3["input_features_mask"].to(torch.bool) features1 = inputs1["input_features"].to(torch.float32) mask1 = inputs1["input_features_mask"].to(torch.bool) with torch.inference_mode(): ref3, ref_mask3 = tower(features3, mask3) ref1, ref_mask1 = tower(features1, mask1) x3, m3 = features3.cpu().numpy(), mask3.cpu().numpy() x1, m1 = features1.cpu().numpy(), mask1.cpu().numpy() y3, ym3 = ref3.to(torch.float32).cpu().numpy(), ref_mask3.cpu().numpy() y1, ym1 = ref1.to(torch.float32).cpu().numpy(), ref_mask1.cpu().numpy() dynamic_axes = { "input_features": {1: "feature_frames"}, "input_features_mask": {1: "feature_frames"}, "audio_features": {1: "audio_tokens"}, "audio_mask": {1: "audio_tokens"}, } with export_patches(include_text=False): torch.onnx.export( tower, (features3, mask3), str(TOWER_TEMP), input_names=["input_features", "input_features_mask"], output_names=["audio_features", "audio_mask"], opset_version=OPSET, dynamo=False, do_constant_folding=True, dynamic_axes=dynamic_axes, ) raw3 = check_tower(TOWER_TEMP, x3, m3, y3, ym3, min_cosine=0.99999, max_abs_diff=5e-4) TOWER_TEMP.replace(TOWER_OUT) quantize_int8(TOWER_OUT, TOWER_INT8_TEMP) int8_3 = check_tower(TOWER_INT8_TEMP, x3, m3, y3, ym3, min_cosine=0.97, max_abs_diff=None) TOWER_INT8_TEMP.replace(TOWER_INT8_OUT) quantize_int4(TOWER_OUT, TOWER_INT4_TEMP) int4_3 = check_tower(TOWER_INT4_TEMP, x3, m3, y3, ym3, min_cosine=0.80, max_abs_diff=None) TOWER_INT4_TEMP.replace(TOWER_INT4_OUT) parity = {} for path, threshold in ( (TOWER_OUT, 0.99999), (TOWER_INT8_OUT, 0.97), (TOWER_INT4_OUT, 0.80), ): parity[path] = check_tower( path, x1, m1, y1, ym1, min_cosine=threshold, max_abs_diff=5e-4 if path == TOWER_OUT else None, ) print_artifact(TOWER_OUT, raw3, "Tower fp32; 3 s") print_artifact(TOWER_INT8_OUT, int8_3, "Tower int8; 3 s") print_artifact(TOWER_INT4_OUT, int4_3, "Tower int4; 3 s") print(f"Tower short-audio cosine: fp32={parity[TOWER_OUT]['cosine']:.7f}, " f"int8={parity[TOWER_INT8_OUT]['cosine']:.7f}, int4={parity[TOWER_INT4_OUT]['cosine']:.7f}") del model, tower, audio_state, expected_keys del features3, features1, mask3, mask1, ref3, ref1, ref_mask3, ref_mask1 del inputs3, inputs1, x3, x1, m3, m1, y3, y1, ym3, ym1, parity gc.collect() def main() -> None: from validation import parser, run_validation args = parser(__doc__).parse_args() print("EXPERIMENTAL: all artifacts from this exporter require downstream qualification.", flush=True) if not args.validate_only: config, processor = load_reference() checkpoint = hf_hub_download(MODEL_ID, "model.safetensors", revision=REVISION) export_tower(config, processor, checkpoint) if not args.skip_validation: raise SystemExit(0 if run_validation("tower", args) else 1) if __name__ == "__main__": main()