bdrgon's picture
Separate Gemma exporters and mark all artifacts experimental
4ac5d79
Raw History Blame Contribute Delete
7.12 kB
#!/usr/bin/env python
"""Shared primitives for the two independent experimental Gemma exporters."""
from __future__ import annotations
import hashlib
import importlib.metadata as metadata
import os
from contextlib import contextmanager, ExitStack
from pathlib import Path
from unittest.mock import patch
os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
os.environ.setdefault("OMP_NUM_THREADS", "1")
import numpy as np
import torch
import torch.nn.functional as F
from transformers import AutoConfig, AutoProcessor
from transformers.models.gemma4.modeling_gemma4 import Gemma4AudioAttention, Gemma4AudioModel
MODEL_ID = "google/embeddinggemma-2"
REVISION = "914f7f89142e33e77833254d9c9b90c3cef7303b"
ROOT = Path(__file__).resolve().parent
OPSET = 18
EMBEDDING_DIM = 512
EXTERNAL_DATA = "audio_embedding.data"
def onnx_bidirectional_mask(
config, inputs_embeds, attention_mask, and_mask_function=None, **kwargs
) -> torch.Tensor:
"""Build Gemma4's local audio mask with broadcast ops instead of vmap."""
seq_len = inputs_embeds.shape[1]
positions = torch.arange(seq_len, device=inputs_embeds.device)
query = positions.view(1, 1, seq_len, 1)
key = positions.view(1, 1, 1, seq_len)
distance = query - key
left = (distance >= 0) & (distance < config.attention_context_left - 1)
right = (distance < 0) & (-distance < config.attention_context_right)
mask = left | right
if attention_mask is not None:
mask = mask & attention_mask[:, None, None, :].to(torch.bool)
return mask.expand(inputs_embeds.shape[0], -1, -1, -1)
def onnx_extract_block_context(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Replace Gemma4's unfold-based context extraction with ONNX-lowerable Gather."""
batch, seq_len, num_heads, head_dim = hidden_states.shape
padded = F.pad(
hidden_states,
(0, 0, 0, 0, self.max_past_horizon, self.max_future_horizon + self.chunk_size - 1),
)
num_blocks = (seq_len + self.chunk_size - 1) // self.chunk_size
starts = torch.arange(num_blocks, device=hidden_states.device) * self.chunk_size
offsets = torch.arange(self.context_size, device=hidden_states.device)
indices = (starts[:, None] + offsets[None, :]).reshape(-1)
contexts = torch.index_select(padded, 1, indices)
return contexts.reshape(batch, num_blocks, self.context_size, num_heads, head_dim)
def _mask_for_text_attention(
mask: torch.Tensor, config, inputs_embeds: torch.Tensor
) -> torch.Tensor:
if getattr(config, "_attn_implementation", "sdpa") == "eager":
additive = torch.zeros(mask.shape, dtype=inputs_embeds.dtype, device=inputs_embeds.device)
return additive.masked_fill(~mask, torch.finfo(inputs_embeds.dtype).min)
return mask
def onnx_text_bidirectional_mask(
config, inputs_embeds, attention_mask, and_mask_function=None, **kwargs
) -> torch.Tensor:
"""Build EmbeddingGemma's full text mask with broadcast tensor operations."""
seq_len = inputs_embeds.shape[1]
positions = torch.arange(seq_len, device=inputs_embeds.device)
query = positions.view(1, 1, seq_len, 1)
key = positions.view(1, 1, 1, seq_len)
mask = torch.ones_like(query - key, dtype=torch.bool)
if attention_mask is not None:
mask = mask & attention_mask[:, None, None, :].to(torch.bool)
mask = mask.expand(inputs_embeds.shape[0], -1, -1, -1)
return _mask_for_text_attention(mask, config, inputs_embeds)
def onnx_sliding_window_mask(
config, inputs_embeds, attention_mask, and_mask_function=None, **kwargs
) -> torch.Tensor:
"""Build EmbeddingGemma's sliding-window mask with broadcast tensor operations."""
seq_len = inputs_embeds.shape[1]
positions = torch.arange(seq_len, device=inputs_embeds.device)
query = positions.view(1, 1, seq_len, 1)
key = positions.view(1, 1, 1, seq_len)
mask = torch.abs(query - key) <= config.sliding_window
if attention_mask is not None:
mask = mask & attention_mask[:, None, None, :].to(torch.bool)
mask = mask.expand(inputs_embeds.shape[0], -1, -1, -1)
return _mask_for_text_attention(mask, config, inputs_embeds)
def quantize_int8(source: Path, destination: Path) -> None:
from onnxruntime.quantization import QuantType, quantize_dynamic
quantize_dynamic(
str(source),
str(destination),
weight_type=QuantType.QUInt8,
extra_options={"AddQDQPairToWeight": False},
)
def quantize_int4(source: Path, destination: Path) -> None:
from onnxruntime.quantization.matmul_nbits_quantizer import (
DefaultWeightOnlyQuantConfig,
MatMulNBitsQuantizer,
)
config = DefaultWeightOnlyQuantConfig(
bits=4,
block_size=128,
is_symmetric=True,
accuracy_level=4,
)
quantizer = MatMulNBitsQuantizer(str(source), algo_config=config)
quantizer.process()
quantizer.model.save_model_to_file(str(destination), use_external_data_format=False)
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as source:
for chunk in iter(lambda: source.read(8 * 1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def print_artifact(path: Path, parity: dict[str, float], label: str) -> None:
print(
f"{label}: {path.name} | {path.stat().st_size:,} bytes | sha256={sha256_file(path)}"
f" | cosine={parity['cosine']:.7f} | max_abs_diff={parity['max_abs_diff']:.7g}"
)
@contextmanager
def export_patches(*, include_text: bool = False):
"""Patch only tracing; always restore upstream code, including on failure."""
import transformers.models.gemma4.modeling_gemma4 as gemma4_modeling
with ExitStack() as stack:
stack.enter_context(patch.object(gemma4_modeling, "create_bidirectional_mask", onnx_bidirectional_mask))
stack.enter_context(patch.object(Gemma4AudioAttention, "_extract_block_context", onnx_extract_block_context))
if include_text:
import transformers.models.embedding_gemma2.modeling_embedding_gemma2 as eg2_modeling
stack.enter_context(patch.object(eg2_modeling, "create_bidirectional_mask", onnx_text_bidirectional_mask))
stack.enter_context(patch.object(eg2_modeling, "create_bidirectional_sliding_window_mask", onnx_sliding_window_mask))
yield
def load_reference():
"""Validate the reference stack and load pinned configuration/processor."""
if torch.__version__.split("+")[0] != "2.8.0":
raise RuntimeError(f"Use pinned torch 2.8.0; found {torch.__version__}")
if metadata.version("transformers") != "5.19.0":
raise RuntimeError("Use pinned transformers 5.19.0")
torch.set_num_threads(1)
config = AutoConfig.from_pretrained(MODEL_ID, revision=REVISION)
resolved_commit = getattr(config, "_commit_hash", None)
if resolved_commit and resolved_commit != REVISION:
raise RuntimeError(f"Hub resolved {REVISION} to unexpected commit {resolved_commit}")
processor = AutoProcessor.from_pretrained(MODEL_ID, revision=REVISION)
return config, processor