Download export_tower.py from bdrgon/embeddinggemma2-audio-tower-onnx: direct link, hf CLI and curl.
- Browser
- Download file 7.71 kB
-
https://huggingface.co/bdrgon/embeddinggemma2-audio-tower-onnx/resolve/main/export_tower.py
- Command line
-
hf download hf://bdrgon/embeddinggemma2-audio-tower-onnx/export_tower.py
-
curl -L -o export_tower.py https://huggingface.co/bdrgon/embeddinggemma2-audio-tower-onnx/resolve/main/export_tower.py
7.71 kB
| #!/usr/bin/env python | |
| """Export experimental EmbeddingGemma 2 tower ONNX variants only.""" | |
| from __future__ import annotations | |
| import argparse | |
| from pathlib import Path | |
| from export_common import ( | |
| MODEL_ID, REVISION, ROOT, OPSET, EMBEDDING_DIM, EXTERNAL_DATA, | |
| export_patches, load_reference, quantize_int8, quantize_int4, | |
| print_artifact, sha256_file, | |
| ) | |
| import numpy as np | |
| import torch | |
| import gc | |
| from huggingface_hub import hf_hub_download | |
| from safetensors import safe_open | |
| from transformers.models.gemma4.modeling_gemma4 import Gemma4AudioModel | |
| TOWER_OUT = ROOT / "audio_tower.onnx" | |
| TOWER_TEMP = ROOT / "audio_tower.tmp.onnx" | |
| TOWER_INT8_OUT = ROOT / "audio_tower_int8.onnx" | |
| TOWER_INT8_TEMP = ROOT / "audio_tower_int8.tmp.onnx" | |
| TOWER_INT4_OUT = ROOT / "audio_tower_int4.onnx" | |
| TOWER_INT4_TEMP = ROOT / "audio_tower_int4.tmp.onnx" | |
| class AudioTower(torch.nn.Module): | |
| def __init__(self, model: Gemma4AudioModel): | |
| super().__init__() | |
| self.model = model | |
| def forward(self, input_features, input_features_mask): | |
| result = self.model( | |
| input_features=input_features, | |
| attention_mask=input_features_mask, | |
| return_dict=True, | |
| ) | |
| return result.last_hidden_state, result.attention_mask | |
| def check_tower( | |
| model_path: Path, | |
| features: np.ndarray, | |
| feature_mask: np.ndarray, | |
| reference_states: np.ndarray, | |
| reference_mask: np.ndarray, | |
| *, | |
| min_cosine: float, | |
| max_abs_diff: float | None, | |
| ) -> dict[str, float]: | |
| import onnx | |
| import onnxruntime as ort | |
| onnx.checker.check_model(str(model_path)) | |
| options = ort.SessionOptions() | |
| options.log_severity_level = 3 | |
| options.intra_op_num_threads = 1 | |
| options.inter_op_num_threads = 1 | |
| session = ort.InferenceSession(str(model_path), options, providers=["CPUExecutionProvider"]) | |
| states, mask = session.run( | |
| ["audio_features", "audio_mask"], | |
| {"input_features": features, "input_features_mask": feature_mask}, | |
| ) | |
| states = states.astype(np.float32) | |
| if not np.isfinite(states).all(): | |
| raise RuntimeError(f"{model_path.name} produced non-finite audio features") | |
| if not np.array_equal(mask, reference_mask): | |
| raise RuntimeError(f"{model_path.name} changed the audio padding mask") | |
| if states.shape != reference_states.shape: | |
| raise RuntimeError(f"{model_path.name} output shape {states.shape} != PyTorch {reference_states.shape}") | |
| ref_flat, out_flat = reference_states.reshape(-1), states.reshape(-1) | |
| cosine = float( | |
| np.dot(ref_flat, out_flat) | |
| / max(float(np.linalg.norm(ref_flat) * np.linalg.norm(out_flat)), 1e-12) | |
| ) | |
| max_diff = float(np.max(np.abs(reference_states - states))) | |
| if cosine < min_cosine or (max_abs_diff is not None and max_diff > max_abs_diff): | |
| raise RuntimeError(f"{model_path.name} parity failed: cosine={cosine}, max_abs_diff={max_diff}") | |
| return {"cosine": cosine, "max_abs_diff": max_diff} | |
| def export_tower(config, processor, checkpoint: str) -> None: | |
| if config.audio_config is None: | |
| raise RuntimeError("Pinned checkpoint has no audio tower") | |
| model = Gemma4AudioModel(config.audio_config) | |
| expected_keys = set(model.state_dict()) | |
| audio_state = {} | |
| with safe_open(checkpoint, framework="pt", device="cpu") as weights: | |
| for key in weights.keys(): | |
| if key.startswith("audio_tower."): | |
| key = key.removeprefix("audio_tower.") | |
| if key in expected_keys: | |
| audio_state[key] = weights.get_tensor(f"audio_tower.{key}") | |
| if not audio_state: | |
| raise RuntimeError("Pinned checkpoint contained no audio_tower weights") | |
| incompatible = model.load_state_dict(audio_state, strict=True) | |
| if incompatible.missing_keys or incompatible.unexpected_keys: | |
| raise RuntimeError(f"Audio tower state mismatch: {incompatible}") | |
| model.eval() | |
| tower = AudioTower(model).eval() | |
| sample_rate = int(processor.feature_extractor.sampling_rate) | |
| audio3 = (0.4 * np.sin(2 * np.pi * 220 * np.arange(3 * sample_rate) / sample_rate)).astype(np.float32) | |
| audio1 = (0.4 * np.sin(2 * np.pi * 220 * np.arange(sample_rate) / sample_rate)).astype(np.float32) | |
| inputs3 = processor(audio=[audio3], sampling_rate=sample_rate, return_tensors="pt") | |
| inputs1 = processor(audio=[audio1], sampling_rate=sample_rate, return_tensors="pt") | |
| features3 = inputs3["input_features"].to(torch.float32) | |
| mask3 = inputs3["input_features_mask"].to(torch.bool) | |
| features1 = inputs1["input_features"].to(torch.float32) | |
| mask1 = inputs1["input_features_mask"].to(torch.bool) | |
| with torch.inference_mode(): | |
| ref3, ref_mask3 = tower(features3, mask3) | |
| ref1, ref_mask1 = tower(features1, mask1) | |
| x3, m3 = features3.cpu().numpy(), mask3.cpu().numpy() | |
| x1, m1 = features1.cpu().numpy(), mask1.cpu().numpy() | |
| y3, ym3 = ref3.to(torch.float32).cpu().numpy(), ref_mask3.cpu().numpy() | |
| y1, ym1 = ref1.to(torch.float32).cpu().numpy(), ref_mask1.cpu().numpy() | |
| dynamic_axes = { | |
| "input_features": {1: "feature_frames"}, | |
| "input_features_mask": {1: "feature_frames"}, | |
| "audio_features": {1: "audio_tokens"}, | |
| "audio_mask": {1: "audio_tokens"}, | |
| } | |
| with export_patches(include_text=False): | |
| torch.onnx.export( | |
| tower, | |
| (features3, mask3), | |
| str(TOWER_TEMP), | |
| input_names=["input_features", "input_features_mask"], | |
| output_names=["audio_features", "audio_mask"], | |
| opset_version=OPSET, | |
| dynamo=False, | |
| do_constant_folding=True, | |
| dynamic_axes=dynamic_axes, | |
| ) | |
| raw3 = check_tower(TOWER_TEMP, x3, m3, y3, ym3, min_cosine=0.99999, max_abs_diff=5e-4) | |
| TOWER_TEMP.replace(TOWER_OUT) | |
| quantize_int8(TOWER_OUT, TOWER_INT8_TEMP) | |
| int8_3 = check_tower(TOWER_INT8_TEMP, x3, m3, y3, ym3, min_cosine=0.97, max_abs_diff=None) | |
| TOWER_INT8_TEMP.replace(TOWER_INT8_OUT) | |
| quantize_int4(TOWER_OUT, TOWER_INT4_TEMP) | |
| int4_3 = check_tower(TOWER_INT4_TEMP, x3, m3, y3, ym3, min_cosine=0.80, max_abs_diff=None) | |
| TOWER_INT4_TEMP.replace(TOWER_INT4_OUT) | |
| parity = {} | |
| for path, threshold in ( | |
| (TOWER_OUT, 0.99999), | |
| (TOWER_INT8_OUT, 0.97), | |
| (TOWER_INT4_OUT, 0.80), | |
| ): | |
| parity[path] = check_tower( | |
| path, | |
| x1, | |
| m1, | |
| y1, | |
| ym1, | |
| min_cosine=threshold, | |
| max_abs_diff=5e-4 if path == TOWER_OUT else None, | |
| ) | |
| print_artifact(TOWER_OUT, raw3, "Tower fp32; 3 s") | |
| print_artifact(TOWER_INT8_OUT, int8_3, "Tower int8; 3 s") | |
| print_artifact(TOWER_INT4_OUT, int4_3, "Tower int4; 3 s") | |
| print(f"Tower short-audio cosine: fp32={parity[TOWER_OUT]['cosine']:.7f}, " | |
| f"int8={parity[TOWER_INT8_OUT]['cosine']:.7f}, int4={parity[TOWER_INT4_OUT]['cosine']:.7f}") | |
| del model, tower, audio_state, expected_keys | |
| del features3, features1, mask3, mask1, ref3, ref1, ref_mask3, ref_mask1 | |
| del inputs3, inputs1, x3, x1, m3, m1, y3, y1, ym3, ym1, parity | |
| gc.collect() | |
| def main() -> None: | |
| from validation import parser, run_validation | |
| args = parser(__doc__).parse_args() | |
| print("EXPERIMENTAL: all artifacts from this exporter require downstream qualification.", flush=True) | |
| if not args.validate_only: | |
| config, processor = load_reference() | |
| checkpoint = hf_hub_download(MODEL_ID, "model.safetensors", revision=REVISION) | |
| export_tower(config, processor, checkpoint) | |
| if not args.skip_validation: | |
| raise SystemExit(0 if run_validation("tower", args) else 1) | |
| if __name__ == "__main__": | |
| main() | |