tarekziade's picture
tarekziade HF Staff
Publish MIC usage, slim fixture and reviewed Transformers expectations
ffb7204 verified
|
Raw History Blame Contribute Delete
4.97 kB
# EmbeddingGemma: sentence embeddings with Transformers
This page is the Transformers contract of the EmbeddingGemma MIC and also a usage example. [doc-builder](https://github.com/huggingface/doc-builder/blob/main/docs/runnable-code-blocks.md) renders the `runnable` blocks below as documentation, and its pytest plugin runs them as written against `google/embeddinggemma-300m`. The MIC runner runs the same blocks against the slim fixture and the checkpoint in [`integration.json`](../../integration.json). Lines marked `# doc-builder: hide` only matter there: they swap in the pinned checkpoint or variant and print the result record.
## Embed queries and documents
```py runnable:embed
import torch
from transformers import AutoModel, AutoTokenizer, set_seed
import json # doc-builder: hide
import os # doc-builder: hide
model_id = "google/embeddinggemma-300m"
model_id = os.environ.get("MIC_MODEL", model_id) # doc-builder: hide
variant = json.loads(os.environ.get("MIC_MODEL_KWARGS", "{}")) # doc-builder: hide
device = "cuda" if torch.cuda.is_available() else "cpu"
set_seed(0) # seeds torch, NumPy, and Python: loading and dropout draw random numbers
tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModel.from_pretrained(model_id).to(device)
if variant: # doc-builder: hide
del model # doc-builder: hide
model = AutoModel.from_pretrained(model_id, **variant).to(device) # doc-builder: hide
model.eval()
query = "Which planet is known as the Red Planet?"
documents = [
"Venus is often called Earth's twin because of its similar size and proximity.",
"Mars, known for its reddish appearance, is often referred to as the Red Planet.",
"Jupiter, the largest planet in our solar system, has a prominent red spot.",
"Saturn, famous for its rings, is sometimes mistaken for the Red Planet.",
]
# EmbeddingGemma uses a prompt prefix for queries; documents are embedded as-is.
query_inputs = tokenizer(["task: search result | query: " + query], return_tensors="pt", padding=True).to(device)
doc_inputs = tokenizer(documents, return_tensors="pt", padding=True).to(device)
def mean_pool(outputs, attention_mask):
hidden = outputs.last_hidden_state.float()
mask = attention_mask.unsqueeze(-1).float()
return (hidden * mask).sum(dim=1) / mask.sum(dim=1)
with torch.no_grad():
query_embedding = mean_pool(model(**query_inputs), query_inputs["attention_mask"])
document_embeddings = mean_pool(model(**doc_inputs), doc_inputs["attention_mask"])
query_embedding = torch.nn.functional.normalize(query_embedding, dim=-1)
document_embeddings = torch.nn.functional.normalize(document_embeddings, dim=-1)
similarities = query_embedding @ document_embeddings.T
print(query_embedding.shape, document_embeddings.shape)
print(similarities)
```
## One training step
```py runnable:embed:2
# A contrastive-style loss: pull the matching planet (index 1) towards the query.
# Recompute the embeddings with gradients: the inference ones ran under no_grad.
model.train()
query = torch.nn.functional.normalize(mean_pool(model(**query_inputs), query_inputs["attention_mask"]), dim=-1)
docs = torch.nn.functional.normalize(mean_pool(model(**doc_inputs), doc_inputs["attention_mask"]), dim=-1)
target = torch.tensor([1], device=device)
loss = torch.nn.functional.cross_entropy(query @ docs.T / 0.05, target)
optimizer = torch.optim.SGD(model.parameters(), lr=1e-4)
optimizer.zero_grad()
loss.backward()
before = {n: p.detach().cpu().clone() for n, p in model.named_parameters() if p.requires_grad} # doc-builder: hide
optimizer.step()
print(f"loss: {loss.item():.4f}")
if os.environ.get("MIC_RUNNER"): # doc-builder: hide
named = dict(model.named_parameters()) # doc-builder: hide
checked = ("layers.0.self_attn.q_proj.weight", "layers.0.mlp.up_proj.weight") # doc-builder: hide
result = { # doc-builder: hide
"runtime": { # doc-builder: hide
"device": model.device.type, # doc-builder: hide
"dtype": str(model.dtype).removeprefix("torch."), # doc-builder: hide
"attn_implementation": model.config._attn_implementation, # doc-builder: hide
}, # doc-builder: hide
"query_embedding": query_embedding[0, :8].tolist(), # doc-builder: hide
"document_embeddings": document_embeddings[:, :8].tolist(), # doc-builder: hide
"similarities": similarities.tolist(), # doc-builder: hide
"training": { # doc-builder: hide
"loss": loss.item(), # doc-builder: hide
"gradients": {n: bool(named[n].grad is not None and torch.count_nonzero(named[n].grad) > 0 and torch.isfinite(named[n].grad).all()) for n in checked}, # doc-builder: hide
"updated": any(not torch.equal(p.detach().cpu(), before[n]) for n, p in model.named_parameters() if n in before), # doc-builder: hide
}, # doc-builder: hide
} # doc-builder: hide
print(json.dumps(result)) # doc-builder: hide
```