tarekziade's picture
tarekziade HF Staff
Publish MIC usage, slim fixture and reviewed Transformers expectations
b17382a verified
|
Raw History Blame Contribute Delete
4.54 kB

Florence-2: image captioning with Transformers

This page is the Transformers contract of the Florence-2 MIC and also a usage example. doc-builder renders the runnable blocks below as documentation, and its pytest plugin runs them as written against florence-community/Florence-2-base. The MIC runner runs the same blocks against each slim fixture and each checkpoint in integration.json. Lines marked # doc-builder: hide only matter there: they swap in the pinned checkpoint or variant and print the result record.

Inference: caption two images

import torch
import numpy as np
from PIL import Image
from transformers import AutoProcessor, Florence2ForConditionalGeneration, set_seed
import json  # doc-builder: hide
import os  # doc-builder: hide

model_id = "florence-community/Florence-2-base"
model_id = os.environ.get("MIC_MODEL", model_id)  # doc-builder: hide
variant = json.loads(os.environ.get("MIC_MODEL_KWARGS", "{}"))  # doc-builder: hide
device = "cuda" if torch.cuda.is_available() else "cpu"
set_seed(0)

processor = AutoProcessor.from_pretrained(model_id)
# The checkpoint is stored in float16; fine-tune in float32 (float16 training needs loss scaling)
model = Florence2ForConditionalGeneration.from_pretrained(model_id, dtype=torch.float32).to(device)
if variant:  # doc-builder: hide
    del model  # doc-builder: hide
    model = Florence2ForConditionalGeneration.from_pretrained(model_id, **{"dtype": torch.float32, **variant}).to(device)  # doc-builder: hide
model.eval()

# Two clearly distinct synthetic images: random noise and a smooth color gradient
rng = np.random.default_rng(42)
img1 = Image.fromarray(rng.integers(0, 255, (224, 224, 3), dtype=np.uint8))
ramp = np.linspace(0, 255, 224, dtype=np.uint8)
img2 = Image.fromarray(np.stack([np.tile(ramp, (224, 1)), np.tile(ramp[:, None], (1, 224)), np.full((224, 224), 64, np.uint8)], axis=-1))

prompt = "<CAPTION>"
inputs = processor(text=[prompt, prompt], images=[img1, img2], return_tensors="pt")

# BatchFeature.to casts only floating-point tensors (pixel_values); token ids keep their dtype
inputs = inputs.to(device, dtype=model.dtype)
input_ids = inputs["input_ids"]
pixel_values = inputs["pixel_values"]
attention_mask = inputs.get("attention_mask")

with torch.no_grad():
    outputs = model(input_ids=input_ids, pixel_values=pixel_values, attention_mask=attention_mask)
logits = outputs.logits

# Compact probe: first 4 positions, first 4 vocab entries per sample
probe = logits[:, :4, :4].float().tolist()
print(probe)

One training step

# Decoder targets: a distinct short caption per image, padding ignored by the loss
targets = processor.tokenizer(["a red car", "two dogs on a beach"], add_special_tokens=False, padding=True, return_tensors="pt")
labels = targets.input_ids.masked_fill(targets.attention_mask == 0, -100).to(device)

model.train()
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
loss = model(input_ids=input_ids, pixel_values=pixel_values, attention_mask=attention_mask, labels=labels).loss
loss.backward()
before = {name: p.detach().clone() for name, p in model.named_parameters()}  # doc-builder: hide
optimizer.step()
print(f"loss: {loss.item():.4f}")

if os.environ.get("MIC_RUNNER"):  # doc-builder: hide
    named = dict(model.named_parameters())
    checked = (
        "model.language_model.encoder.layers.0.self_attn.k_proj.weight",
        "model.language_model.decoder.layers.0.self_attn.k_proj.weight",
        "model.multi_modal_projector.image_projection.weight",
    )
    result = {
        "runtime": {
            "device": model.device.type,
            "dtype": str(model.dtype).removeprefix("torch."),
            "attn_implementation": model.config._attn_implementation,
            "attn_implementations": {n: getattr(model.config, n)._attn_implementation for n in sorted(model.config.sub_configs) if getattr(getattr(model.config, n, None), '_attn_implementation', None)},
        },
        "inference_logits": probe,
        "training": {
            "loss": loss.item(),
            "gradients": {
                n: bool(named[n].grad is not None and torch.count_nonzero(named[n].grad) > 0 and torch.isfinite(named[n].grad).all())
                for n in checked
            },
            "updated": any(not torch.equal(p, before[n]) for n, p in named.items()),
        },
    }
    print(json.dumps(result))