tarekziade's picture
tarekziade HF Staff
Publish MIC usage, slim fixture and reviewed Transformers expectations
ffb7204 verified
|
Raw History Blame Contribute Delete
4.97 kB

EmbeddingGemma: sentence embeddings with Transformers

This page is the Transformers contract of the EmbeddingGemma MIC and also a usage example. doc-builder renders the runnable blocks below as documentation, and its pytest plugin runs them as written against google/embeddinggemma-300m. The MIC runner runs the same blocks against the slim fixture and the checkpoint in integration.json. Lines marked # doc-builder: hide only matter there: they swap in the pinned checkpoint or variant and print the result record.

Embed queries and documents

import torch
from transformers import AutoModel, AutoTokenizer, set_seed
import json  # doc-builder: hide
import os  # doc-builder: hide

model_id = "google/embeddinggemma-300m"
model_id = os.environ.get("MIC_MODEL", model_id)  # doc-builder: hide
variant = json.loads(os.environ.get("MIC_MODEL_KWARGS", "{}"))  # doc-builder: hide
device = "cuda" if torch.cuda.is_available() else "cpu"
set_seed(0)  # seeds torch, NumPy, and Python: loading and dropout draw random numbers

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModel.from_pretrained(model_id).to(device)
if variant:  # doc-builder: hide
    del model  # doc-builder: hide
    model = AutoModel.from_pretrained(model_id, **variant).to(device)  # doc-builder: hide
model.eval()

query = "Which planet is known as the Red Planet?"
documents = [
    "Venus is often called Earth's twin because of its similar size and proximity.",
    "Mars, known for its reddish appearance, is often referred to as the Red Planet.",
    "Jupiter, the largest planet in our solar system, has a prominent red spot.",
    "Saturn, famous for its rings, is sometimes mistaken for the Red Planet.",
]

# EmbeddingGemma uses a prompt prefix for queries; documents are embedded as-is.
query_inputs = tokenizer(["task: search result | query: " + query], return_tensors="pt", padding=True).to(device)
doc_inputs = tokenizer(documents, return_tensors="pt", padding=True).to(device)

def mean_pool(outputs, attention_mask):
    hidden = outputs.last_hidden_state.float()
    mask = attention_mask.unsqueeze(-1).float()
    return (hidden * mask).sum(dim=1) / mask.sum(dim=1)

with torch.no_grad():
    query_embedding = mean_pool(model(**query_inputs), query_inputs["attention_mask"])
    document_embeddings = mean_pool(model(**doc_inputs), doc_inputs["attention_mask"])

query_embedding = torch.nn.functional.normalize(query_embedding, dim=-1)
document_embeddings = torch.nn.functional.normalize(document_embeddings, dim=-1)
similarities = query_embedding @ document_embeddings.T
print(query_embedding.shape, document_embeddings.shape)
print(similarities)

One training step

# A contrastive-style loss: pull the matching planet (index 1) towards the query.
# Recompute the embeddings with gradients: the inference ones ran under no_grad.
model.train()
query = torch.nn.functional.normalize(mean_pool(model(**query_inputs), query_inputs["attention_mask"]), dim=-1)
docs = torch.nn.functional.normalize(mean_pool(model(**doc_inputs), doc_inputs["attention_mask"]), dim=-1)
target = torch.tensor([1], device=device)
loss = torch.nn.functional.cross_entropy(query @ docs.T / 0.05, target)

optimizer = torch.optim.SGD(model.parameters(), lr=1e-4)
optimizer.zero_grad()
loss.backward()
before = {n: p.detach().cpu().clone() for n, p in model.named_parameters() if p.requires_grad}  # doc-builder: hide
optimizer.step()
print(f"loss: {loss.item():.4f}")

if os.environ.get("MIC_RUNNER"):  # doc-builder: hide
    named = dict(model.named_parameters())  # doc-builder: hide
    checked = ("layers.0.self_attn.q_proj.weight", "layers.0.mlp.up_proj.weight")  # doc-builder: hide
    result = {  # doc-builder: hide
        "runtime": {  # doc-builder: hide
            "device": model.device.type,  # doc-builder: hide
            "dtype": str(model.dtype).removeprefix("torch."),  # doc-builder: hide
            "attn_implementation": model.config._attn_implementation,  # doc-builder: hide
        },  # doc-builder: hide
        "query_embedding": query_embedding[0, :8].tolist(),  # doc-builder: hide
        "document_embeddings": document_embeddings[:, :8].tolist(),  # doc-builder: hide
        "similarities": similarities.tolist(),  # doc-builder: hide
        "training": {  # doc-builder: hide
            "loss": loss.item(),  # doc-builder: hide
            "gradients": {n: bool(named[n].grad is not None and torch.count_nonzero(named[n].grad) > 0 and torch.isfinite(named[n].grad).all()) for n in checked},  # doc-builder: hide
            "updated": any(not torch.equal(p.detach().cpu(), before[n]) for n, p in model.named_parameters() if n in before),  # doc-builder: hide
        },  # doc-builder: hide
    }  # doc-builder: hide
    print(json.dumps(result))  # doc-builder: hide