tarekziade's picture
tarekziade HF Staff
Publish MIC usage, slim fixture and reviewed Transformers expectations
b17382a verified
|
Raw History Blame Contribute Delete
4.54 kB
# Florence-2: image captioning with Transformers
This page is the Transformers contract of the Florence-2 MIC and also a usage example. [doc-builder](https://github.com/huggingface/doc-builder/blob/main/docs/runnable-code-blocks.md) renders the `runnable` blocks below as documentation, and its pytest plugin runs them as written against `florence-community/Florence-2-base`. The MIC runner runs the same blocks against each slim fixture and each checkpoint in [`integration.json`](../../integration.json). Lines marked `# doc-builder: hide` only matter there: they swap in the pinned checkpoint or variant and print the result record.
## Inference: caption two images
```py runnable:florence2
import torch
import numpy as np
from PIL import Image
from transformers import AutoProcessor, Florence2ForConditionalGeneration, set_seed
import json # doc-builder: hide
import os # doc-builder: hide
model_id = "florence-community/Florence-2-base"
model_id = os.environ.get("MIC_MODEL", model_id) # doc-builder: hide
variant = json.loads(os.environ.get("MIC_MODEL_KWARGS", "{}")) # doc-builder: hide
device = "cuda" if torch.cuda.is_available() else "cpu"
set_seed(0)
processor = AutoProcessor.from_pretrained(model_id)
# The checkpoint is stored in float16; fine-tune in float32 (float16 training needs loss scaling)
model = Florence2ForConditionalGeneration.from_pretrained(model_id, dtype=torch.float32).to(device)
if variant: # doc-builder: hide
del model # doc-builder: hide
model = Florence2ForConditionalGeneration.from_pretrained(model_id, **{"dtype": torch.float32, **variant}).to(device) # doc-builder: hide
model.eval()
# Two clearly distinct synthetic images: random noise and a smooth color gradient
rng = np.random.default_rng(42)
img1 = Image.fromarray(rng.integers(0, 255, (224, 224, 3), dtype=np.uint8))
ramp = np.linspace(0, 255, 224, dtype=np.uint8)
img2 = Image.fromarray(np.stack([np.tile(ramp, (224, 1)), np.tile(ramp[:, None], (1, 224)), np.full((224, 224), 64, np.uint8)], axis=-1))
prompt = "<CAPTION>"
inputs = processor(text=[prompt, prompt], images=[img1, img2], return_tensors="pt")
# BatchFeature.to casts only floating-point tensors (pixel_values); token ids keep their dtype
inputs = inputs.to(device, dtype=model.dtype)
input_ids = inputs["input_ids"]
pixel_values = inputs["pixel_values"]
attention_mask = inputs.get("attention_mask")
with torch.no_grad():
outputs = model(input_ids=input_ids, pixel_values=pixel_values, attention_mask=attention_mask)
logits = outputs.logits
# Compact probe: first 4 positions, first 4 vocab entries per sample
probe = logits[:, :4, :4].float().tolist()
print(probe)
```
## One training step
```py runnable:florence2:2
# Decoder targets: a distinct short caption per image, padding ignored by the loss
targets = processor.tokenizer(["a red car", "two dogs on a beach"], add_special_tokens=False, padding=True, return_tensors="pt")
labels = targets.input_ids.masked_fill(targets.attention_mask == 0, -100).to(device)
model.train()
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
loss = model(input_ids=input_ids, pixel_values=pixel_values, attention_mask=attention_mask, labels=labels).loss
loss.backward()
before = {name: p.detach().clone() for name, p in model.named_parameters()} # doc-builder: hide
optimizer.step()
print(f"loss: {loss.item():.4f}")
if os.environ.get("MIC_RUNNER"): # doc-builder: hide
named = dict(model.named_parameters())
checked = (
"model.language_model.encoder.layers.0.self_attn.k_proj.weight",
"model.language_model.decoder.layers.0.self_attn.k_proj.weight",
"model.multi_modal_projector.image_projection.weight",
)
result = {
"runtime": {
"device": model.device.type,
"dtype": str(model.dtype).removeprefix("torch."),
"attn_implementation": model.config._attn_implementation,
"attn_implementations": {n: getattr(model.config, n)._attn_implementation for n in sorted(model.config.sub_configs) if getattr(getattr(model.config, n, None), '_attn_implementation', None)},
},
"inference_logits": probe,
"training": {
"loss": loss.item(),
"gradients": {
n: bool(named[n].grad is not None and torch.count_nonzero(named[n].grad) > 0 and torch.isfinite(named[n].grad).all())
for n in checked
},
"updated": any(not torch.equal(p, before[n]) for n, p in named.items()),
},
}
print(json.dumps(result))
```