tarekziade's picture
tarekziade HF Staff
Publish MIC usage, slim fixture and reviewed Transformers expectations
47866a2 verified
|
Raw History Blame Contribute Delete
5.34 kB

Janus-Pro-1B: multimodal understanding with Transformers

This page is the Transformers contract of the Janus-Pro-1B MIC and also a usage example. doc-builder renders the runnable blocks below as documentation, and its pytest plugin runs them as written against deepseek-community/Janus-Pro-1B. The MIC runner runs the same blocks against each slim fixture and each checkpoint in integration.json. Lines marked # doc-builder: hide only matter there: they swap in the pinned checkpoint or variant and print the result record.

Multimodal text understanding

import numpy as np
import torch
from PIL import Image
from transformers import JanusForConditionalGeneration, JanusProcessor, set_seed
import json  # doc-builder: hide
import os  # doc-builder: hide

model_id = "deepseek-community/Janus-Pro-1B"
model_id = os.environ.get("MIC_MODEL", model_id)  # doc-builder: hide
variant = json.loads(os.environ.get("MIC_MODEL_KWARGS", "{}"))  # doc-builder: hide
device = "cuda" if torch.cuda.is_available() else "cpu"
set_seed(0)  # seeds torch, NumPy, and Python: loading, dropout, and data augmentation draw random numbers

processor = JanusProcessor.from_pretrained(model_id)
model = JanusForConditionalGeneration.from_pretrained(model_id).to(device)
if variant:  # doc-builder: hide
    del model  # doc-builder: hide
    model = JanusForConditionalGeneration.from_pretrained(model_id, **variant).to(device)  # doc-builder: hide

# Two synthetic images of different sizes so batch perturbation is meaningful
def textured_image(height, width, colour, seed):
    # The base colour plus a gradient and seeded noise: every patch differs, so patch order, crops and resizing all change the pixels.
    rng = np.random.default_rng(seed)
    y, x = np.mgrid[0:height, 0:width]
    ramp = (48 * (x / width + y / height) - 48)[..., None]
    pixels = np.asarray(colour, dtype=np.float64) + ramp + rng.normal(0, 24, (height, width, 3))
    return Image.fromarray(np.clip(pixels, 0, 255).astype(np.uint8))

img1 = textured_image(64, 64, (120, 80, 200), seed=0)
img2 = textured_image(48, 96, (50, 180, 100), seed=1)

messages1 = [{"role": "user", "content": [{"type": "image", "image": img1}, {"type": "text", "text": "What color is this?"}]}]
messages2 = [{"role": "user", "content": [{"type": "image", "image": img2}, {"type": "text", "text": "Describe the image."}]}]

inputs1 = processor.apply_chat_template(
    messages1, add_generation_prompt=True, generation_mode="text",
    tokenize=True, return_dict=True, return_tensors="pt",
)
inputs2 = processor.apply_chat_template(
    messages2, add_generation_prompt=True, generation_mode="text",
    tokenize=True, return_dict=True, return_tensors="pt",
)

# BatchFeature.to casts only floating tensors (pixel_values) to model.dtype;
# integer tensors (input_ids, attention_mask) keep their original dtype
inputs1 = inputs1.to(device, dtype=model.dtype)
inputs2 = inputs2.to(device, dtype=model.dtype)

model.eval()
with torch.no_grad():
    out1 = model(**inputs1)
    out2 = model(**inputs2)

# Compact probe: 16 logits spread across the vocabulary at the last position of each sequence
vocab_ids = torch.linspace(0, out1.logits.shape[-1] - 1, 16).long()
s1 = inputs1["input_ids"].shape[1]
s2 = inputs2["input_ids"].shape[1]
logits_probe = torch.cat([
    out1.logits[0, s1 - 1, vocab_ids],
    out2.logits[0, s2 - 1, vocab_ids],
]).float()
print(f"logits: {logits_probe.tolist()}")

One training step

# Build labels: copy input_ids, mask padding and image-token positions to -100
labels = inputs1["input_ids"].clone()
if "attention_mask" in inputs1:
    labels[inputs1["attention_mask"] == 0] = -100
labels[labels == model.config.image_token_id] = -100

model.train()
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
loss = model(**inputs1, labels=labels).loss
loss.backward()
before = {n: p.detach().cpu().clone() for n, p in model.named_parameters() if p.requires_grad}  # doc-builder: hide
optimizer.step()
print(f"loss: {loss.item():.4f}")

if os.environ.get("MIC_RUNNER"):  # doc-builder: hide
    named = dict(model.named_parameters())
    checked = (
        "model.language_model.layers.0.self_attn.q_proj.weight",
        "model.language_model.layers.0.mlp.up_proj.weight",
    )
    result = {
        "runtime": {
            "device": model.device.type,
            "dtype": str(model.dtype).removeprefix("torch."),
            "attn_implementation": model.config._attn_implementation,
            "attn_implementations": {n: getattr(model.config, n)._attn_implementation for n in sorted(model.config.sub_configs) if getattr(getattr(model.config, n, None), '_attn_implementation', None)},
        },
        "logits_probe": logits_probe.tolist(),
        "training": {
            "loss": loss.item(),
            "gradients": {
                n: bool(named[n].grad is not None and torch.count_nonzero(named[n].grad) > 0 and torch.isfinite(named[n].grad).all())
                for n in checked
            },
            "updated": any(not torch.equal(p.detach().cpu(), before[n]) for n, p in model.named_parameters() if n in before),
        },
    }
    print(json.dumps(result))