#!/usr/bin/env python """Shared primitives for the two independent experimental Gemma exporters.""" from __future__ import annotations import hashlib import importlib.metadata as metadata import os from contextlib import contextmanager, ExitStack from pathlib import Path from unittest.mock import patch os.environ.setdefault("OPENBLAS_NUM_THREADS", "1") os.environ.setdefault("OMP_NUM_THREADS", "1") import numpy as np import torch import torch.nn.functional as F from transformers import AutoConfig, AutoProcessor from transformers.models.gemma4.modeling_gemma4 import Gemma4AudioAttention, Gemma4AudioModel MODEL_ID = "google/embeddinggemma-2" REVISION = "914f7f89142e33e77833254d9c9b90c3cef7303b" ROOT = Path(__file__).resolve().parent OPSET = 18 EMBEDDING_DIM = 512 EXTERNAL_DATA = "audio_embedding.data" def onnx_bidirectional_mask( config, inputs_embeds, attention_mask, and_mask_function=None, **kwargs ) -> torch.Tensor: """Build Gemma4's local audio mask with broadcast ops instead of vmap.""" seq_len = inputs_embeds.shape[1] positions = torch.arange(seq_len, device=inputs_embeds.device) query = positions.view(1, 1, seq_len, 1) key = positions.view(1, 1, 1, seq_len) distance = query - key left = (distance >= 0) & (distance < config.attention_context_left - 1) right = (distance < 0) & (-distance < config.attention_context_right) mask = left | right if attention_mask is not None: mask = mask & attention_mask[:, None, None, :].to(torch.bool) return mask.expand(inputs_embeds.shape[0], -1, -1, -1) def onnx_extract_block_context(self, hidden_states: torch.Tensor) -> torch.Tensor: """Replace Gemma4's unfold-based context extraction with ONNX-lowerable Gather.""" batch, seq_len, num_heads, head_dim = hidden_states.shape padded = F.pad( hidden_states, (0, 0, 0, 0, self.max_past_horizon, self.max_future_horizon + self.chunk_size - 1), ) num_blocks = (seq_len + self.chunk_size - 1) // self.chunk_size starts = torch.arange(num_blocks, device=hidden_states.device) * self.chunk_size offsets = torch.arange(self.context_size, device=hidden_states.device) indices = (starts[:, None] + offsets[None, :]).reshape(-1) contexts = torch.index_select(padded, 1, indices) return contexts.reshape(batch, num_blocks, self.context_size, num_heads, head_dim) def _mask_for_text_attention( mask: torch.Tensor, config, inputs_embeds: torch.Tensor ) -> torch.Tensor: if getattr(config, "_attn_implementation", "sdpa") == "eager": additive = torch.zeros(mask.shape, dtype=inputs_embeds.dtype, device=inputs_embeds.device) return additive.masked_fill(~mask, torch.finfo(inputs_embeds.dtype).min) return mask def onnx_text_bidirectional_mask( config, inputs_embeds, attention_mask, and_mask_function=None, **kwargs ) -> torch.Tensor: """Build EmbeddingGemma's full text mask with broadcast tensor operations.""" seq_len = inputs_embeds.shape[1] positions = torch.arange(seq_len, device=inputs_embeds.device) query = positions.view(1, 1, seq_len, 1) key = positions.view(1, 1, 1, seq_len) mask = torch.ones_like(query - key, dtype=torch.bool) if attention_mask is not None: mask = mask & attention_mask[:, None, None, :].to(torch.bool) mask = mask.expand(inputs_embeds.shape[0], -1, -1, -1) return _mask_for_text_attention(mask, config, inputs_embeds) def onnx_sliding_window_mask( config, inputs_embeds, attention_mask, and_mask_function=None, **kwargs ) -> torch.Tensor: """Build EmbeddingGemma's sliding-window mask with broadcast tensor operations.""" seq_len = inputs_embeds.shape[1] positions = torch.arange(seq_len, device=inputs_embeds.device) query = positions.view(1, 1, seq_len, 1) key = positions.view(1, 1, 1, seq_len) mask = torch.abs(query - key) <= config.sliding_window if attention_mask is not None: mask = mask & attention_mask[:, None, None, :].to(torch.bool) mask = mask.expand(inputs_embeds.shape[0], -1, -1, -1) return _mask_for_text_attention(mask, config, inputs_embeds) def quantize_int8(source: Path, destination: Path) -> None: from onnxruntime.quantization import QuantType, quantize_dynamic quantize_dynamic( str(source), str(destination), weight_type=QuantType.QUInt8, extra_options={"AddQDQPairToWeight": False}, ) def quantize_int4(source: Path, destination: Path) -> None: from onnxruntime.quantization.matmul_nbits_quantizer import ( DefaultWeightOnlyQuantConfig, MatMulNBitsQuantizer, ) config = DefaultWeightOnlyQuantConfig( bits=4, block_size=128, is_symmetric=True, accuracy_level=4, ) quantizer = MatMulNBitsQuantizer(str(source), algo_config=config) quantizer.process() quantizer.model.save_model_to_file(str(destination), use_external_data_format=False) def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as source: for chunk in iter(lambda: source.read(8 * 1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def print_artifact(path: Path, parity: dict[str, float], label: str) -> None: print( f"{label}: {path.name} | {path.stat().st_size:,} bytes | sha256={sha256_file(path)}" f" | cosine={parity['cosine']:.7f} | max_abs_diff={parity['max_abs_diff']:.7g}" ) @contextmanager def export_patches(*, include_text: bool = False): """Patch only tracing; always restore upstream code, including on failure.""" import transformers.models.gemma4.modeling_gemma4 as gemma4_modeling with ExitStack() as stack: stack.enter_context(patch.object(gemma4_modeling, "create_bidirectional_mask", onnx_bidirectional_mask)) stack.enter_context(patch.object(Gemma4AudioAttention, "_extract_block_context", onnx_extract_block_context)) if include_text: import transformers.models.embedding_gemma2.modeling_embedding_gemma2 as eg2_modeling stack.enter_context(patch.object(eg2_modeling, "create_bidirectional_mask", onnx_text_bidirectional_mask)) stack.enter_context(patch.object(eg2_modeling, "create_bidirectional_sliding_window_mask", onnx_sliding_window_mask)) yield def load_reference(): """Validate the reference stack and load pinned configuration/processor.""" if torch.__version__.split("+")[0] != "2.8.0": raise RuntimeError(f"Use pinned torch 2.8.0; found {torch.__version__}") if metadata.version("transformers") != "5.19.0": raise RuntimeError("Use pinned transformers 5.19.0") torch.set_num_threads(1) config = AutoConfig.from_pretrained(MODEL_ID, revision=REVISION) resolved_commit = getattr(config, "_commit_hash", None) if resolved_commit and resolved_commit != REVISION: raise RuntimeError(f"Hub resolved {REVISION} to unexpected commit {resolved_commit}") processor = AutoProcessor.from_pretrained(MODEL_ID, revision=REVISION) return config, processor