Download export_common.py from bdrgon/embeddinggemma2-audio-tower-onnx: direct link, hf CLI and curl.
- Browser
- Download file 7.12 kB
-
https://huggingface.co/bdrgon/embeddinggemma2-audio-tower-onnx/resolve/main/export_common.py
- Command line
-
hf download hf://bdrgon/embeddinggemma2-audio-tower-onnx/export_common.py
-
curl -L -o export_common.py https://huggingface.co/bdrgon/embeddinggemma2-audio-tower-onnx/resolve/main/export_common.py
7.12 kB
| #!/usr/bin/env python | |
| """Shared primitives for the two independent experimental Gemma exporters.""" | |
| from __future__ import annotations | |
| import hashlib | |
| import importlib.metadata as metadata | |
| import os | |
| from contextlib import contextmanager, ExitStack | |
| from pathlib import Path | |
| from unittest.mock import patch | |
| os.environ.setdefault("OPENBLAS_NUM_THREADS", "1") | |
| os.environ.setdefault("OMP_NUM_THREADS", "1") | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| from transformers import AutoConfig, AutoProcessor | |
| from transformers.models.gemma4.modeling_gemma4 import Gemma4AudioAttention, Gemma4AudioModel | |
| MODEL_ID = "google/embeddinggemma-2" | |
| REVISION = "914f7f89142e33e77833254d9c9b90c3cef7303b" | |
| ROOT = Path(__file__).resolve().parent | |
| OPSET = 18 | |
| EMBEDDING_DIM = 512 | |
| EXTERNAL_DATA = "audio_embedding.data" | |
| def onnx_bidirectional_mask( | |
| config, inputs_embeds, attention_mask, and_mask_function=None, **kwargs | |
| ) -> torch.Tensor: | |
| """Build Gemma4's local audio mask with broadcast ops instead of vmap.""" | |
| seq_len = inputs_embeds.shape[1] | |
| positions = torch.arange(seq_len, device=inputs_embeds.device) | |
| query = positions.view(1, 1, seq_len, 1) | |
| key = positions.view(1, 1, 1, seq_len) | |
| distance = query - key | |
| left = (distance >= 0) & (distance < config.attention_context_left - 1) | |
| right = (distance < 0) & (-distance < config.attention_context_right) | |
| mask = left | right | |
| if attention_mask is not None: | |
| mask = mask & attention_mask[:, None, None, :].to(torch.bool) | |
| return mask.expand(inputs_embeds.shape[0], -1, -1, -1) | |
| def onnx_extract_block_context(self, hidden_states: torch.Tensor) -> torch.Tensor: | |
| """Replace Gemma4's unfold-based context extraction with ONNX-lowerable Gather.""" | |
| batch, seq_len, num_heads, head_dim = hidden_states.shape | |
| padded = F.pad( | |
| hidden_states, | |
| (0, 0, 0, 0, self.max_past_horizon, self.max_future_horizon + self.chunk_size - 1), | |
| ) | |
| num_blocks = (seq_len + self.chunk_size - 1) // self.chunk_size | |
| starts = torch.arange(num_blocks, device=hidden_states.device) * self.chunk_size | |
| offsets = torch.arange(self.context_size, device=hidden_states.device) | |
| indices = (starts[:, None] + offsets[None, :]).reshape(-1) | |
| contexts = torch.index_select(padded, 1, indices) | |
| return contexts.reshape(batch, num_blocks, self.context_size, num_heads, head_dim) | |
| def _mask_for_text_attention( | |
| mask: torch.Tensor, config, inputs_embeds: torch.Tensor | |
| ) -> torch.Tensor: | |
| if getattr(config, "_attn_implementation", "sdpa") == "eager": | |
| additive = torch.zeros(mask.shape, dtype=inputs_embeds.dtype, device=inputs_embeds.device) | |
| return additive.masked_fill(~mask, torch.finfo(inputs_embeds.dtype).min) | |
| return mask | |
| def onnx_text_bidirectional_mask( | |
| config, inputs_embeds, attention_mask, and_mask_function=None, **kwargs | |
| ) -> torch.Tensor: | |
| """Build EmbeddingGemma's full text mask with broadcast tensor operations.""" | |
| seq_len = inputs_embeds.shape[1] | |
| positions = torch.arange(seq_len, device=inputs_embeds.device) | |
| query = positions.view(1, 1, seq_len, 1) | |
| key = positions.view(1, 1, 1, seq_len) | |
| mask = torch.ones_like(query - key, dtype=torch.bool) | |
| if attention_mask is not None: | |
| mask = mask & attention_mask[:, None, None, :].to(torch.bool) | |
| mask = mask.expand(inputs_embeds.shape[0], -1, -1, -1) | |
| return _mask_for_text_attention(mask, config, inputs_embeds) | |
| def onnx_sliding_window_mask( | |
| config, inputs_embeds, attention_mask, and_mask_function=None, **kwargs | |
| ) -> torch.Tensor: | |
| """Build EmbeddingGemma's sliding-window mask with broadcast tensor operations.""" | |
| seq_len = inputs_embeds.shape[1] | |
| positions = torch.arange(seq_len, device=inputs_embeds.device) | |
| query = positions.view(1, 1, seq_len, 1) | |
| key = positions.view(1, 1, 1, seq_len) | |
| mask = torch.abs(query - key) <= config.sliding_window | |
| if attention_mask is not None: | |
| mask = mask & attention_mask[:, None, None, :].to(torch.bool) | |
| mask = mask.expand(inputs_embeds.shape[0], -1, -1, -1) | |
| return _mask_for_text_attention(mask, config, inputs_embeds) | |
| def quantize_int8(source: Path, destination: Path) -> None: | |
| from onnxruntime.quantization import QuantType, quantize_dynamic | |
| quantize_dynamic( | |
| str(source), | |
| str(destination), | |
| weight_type=QuantType.QUInt8, | |
| extra_options={"AddQDQPairToWeight": False}, | |
| ) | |
| def quantize_int4(source: Path, destination: Path) -> None: | |
| from onnxruntime.quantization.matmul_nbits_quantizer import ( | |
| DefaultWeightOnlyQuantConfig, | |
| MatMulNBitsQuantizer, | |
| ) | |
| config = DefaultWeightOnlyQuantConfig( | |
| bits=4, | |
| block_size=128, | |
| is_symmetric=True, | |
| accuracy_level=4, | |
| ) | |
| quantizer = MatMulNBitsQuantizer(str(source), algo_config=config) | |
| quantizer.process() | |
| quantizer.model.save_model_to_file(str(destination), use_external_data_format=False) | |
| def sha256_file(path: Path) -> str: | |
| digest = hashlib.sha256() | |
| with path.open("rb") as source: | |
| for chunk in iter(lambda: source.read(8 * 1024 * 1024), b""): | |
| digest.update(chunk) | |
| return digest.hexdigest() | |
| def print_artifact(path: Path, parity: dict[str, float], label: str) -> None: | |
| print( | |
| f"{label}: {path.name} | {path.stat().st_size:,} bytes | sha256={sha256_file(path)}" | |
| f" | cosine={parity['cosine']:.7f} | max_abs_diff={parity['max_abs_diff']:.7g}" | |
| ) | |
| def export_patches(*, include_text: bool = False): | |
| """Patch only tracing; always restore upstream code, including on failure.""" | |
| import transformers.models.gemma4.modeling_gemma4 as gemma4_modeling | |
| with ExitStack() as stack: | |
| stack.enter_context(patch.object(gemma4_modeling, "create_bidirectional_mask", onnx_bidirectional_mask)) | |
| stack.enter_context(patch.object(Gemma4AudioAttention, "_extract_block_context", onnx_extract_block_context)) | |
| if include_text: | |
| import transformers.models.embedding_gemma2.modeling_embedding_gemma2 as eg2_modeling | |
| stack.enter_context(patch.object(eg2_modeling, "create_bidirectional_mask", onnx_text_bidirectional_mask)) | |
| stack.enter_context(patch.object(eg2_modeling, "create_bidirectional_sliding_window_mask", onnx_sliding_window_mask)) | |
| yield | |
| def load_reference(): | |
| """Validate the reference stack and load pinned configuration/processor.""" | |
| if torch.__version__.split("+")[0] != "2.8.0": | |
| raise RuntimeError(f"Use pinned torch 2.8.0; found {torch.__version__}") | |
| if metadata.version("transformers") != "5.19.0": | |
| raise RuntimeError("Use pinned transformers 5.19.0") | |
| torch.set_num_threads(1) | |
| config = AutoConfig.from_pretrained(MODEL_ID, revision=REVISION) | |
| resolved_commit = getattr(config, "_commit_hash", None) | |
| if resolved_commit and resolved_commit != REVISION: | |
| raise RuntimeError(f"Hub resolved {REVISION} to unexpected commit {resolved_commit}") | |
| processor = AutoProcessor.from_pretrained(MODEL_ID, revision=REVISION) | |
| return config, processor | |