tomaarsen's picture
tomaarsen HF Staff
Publish MIC usage, slim fixture and reviewed Transformers expectations
e2c0c3a verified
|
Raw History Blame Contribute Delete
2.59 kB
# SentenceTransformer: encoding and scoring
Run the public Sentence Transformers API with a small, fixed retrieval example.
```py runnable:retrieval
import torch
from sentence_transformers import SentenceTransformer
import json # doc-builder: hide
import os # doc-builder: hide
model_id = 'nomic-ai/modernbert-embed-base'
model_id = os.environ.get('MIC_MODEL', model_id) # doc-builder: hide
revision = os.environ.get('MIC_MODEL_REVISION') or None # doc-builder: hide
variant = json.loads(os.environ.get('MIC_MODEL_KWARGS', '{}')) # doc-builder: hide
torch.manual_seed(0)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
model = SentenceTransformer(
model_id,
device=device,
revision=revision, # doc-builder: hide
model_kwargs=variant, # doc-builder: hide
)
queries = ['Which planet is known as the Red Planet?', 'Where do penguins live?']
documents = ["Venus is often called Earth's twin.", 'Mars is known for its reddish appearance and is called the Red Planet.', 'Penguins live in the Southern Hemisphere, including Antarctica.', 'Jupiter is the largest planet in our solar system and has a prominent red spot.']
query_embeddings = model.encode_query(queries, show_progress_bar=False, convert_to_tensor=True)
document_embeddings = model.encode_document(documents, show_progress_bar=False, convert_to_tensor=True)
scores = model.similarity(query_embeddings, document_embeddings)
assert scores.shape == (2, 4)
assert torch.isfinite(scores).all()
assert torch.isfinite(query_embeddings).all() and torch.isfinite(document_embeddings).all()
assert (query_embeddings[0] - query_embeddings[1]).abs().max() > 0.001
individual = model.encode_query(queries, batch_size=1, show_progress_bar=False, convert_to_tensor=True)
batch_max_delta = float((query_embeddings - individual).abs().max())
if query_embeddings.dtype == torch.float32:
torch.testing.assert_close(query_embeddings, individual, atol=1e-4, rtol=1e-4)
if os.environ.get('MIC_RUNNER'): # doc-builder: hide
backbone = next(module for module in model.modules() if isinstance(module, __import__('transformers').PreTrainedModel))
result = {
'runtime': {'device': model.device.type, 'dtype': str(next(model.parameters()).dtype).removeprefix('torch.'), 'attn_implementation': backbone.config._attn_implementation},
'scores': scores.float().cpu().tolist(),
'batch_max_delta': batch_max_delta,
'query_embeddings': query_embeddings.float().cpu().tolist(),
'document_embeddings': document_embeddings.float().cpu().tolist(),
}
print(json.dumps(result))
```