tarekziade HF Staff
Publish MIC usage, slim fixture and reviewed Transformers expectations
47866a2 verified |
Download integrations/transformers/usage.md from huggingface-mic/Janus-Pro-1B-mic: direct link, hf CLI and curl.
- Browser
- Download file 5.34 kB
-
https://huggingface.co/huggingface-mic/Janus-Pro-1B-mic/resolve/main/integrations/transformers/usage.md
- Command line
-
hf download hf://huggingface-mic/Janus-Pro-1B-mic/integrations/transformers/usage.md
-
curl -L -o usage.md https://huggingface.co/huggingface-mic/Janus-Pro-1B-mic/resolve/main/integrations/transformers/usage.md
5.34 kB
Janus-Pro-1B: multimodal understanding with Transformers
This page is the Transformers contract of the Janus-Pro-1B MIC and also a usage example. doc-builder renders the runnable blocks below as documentation, and its pytest plugin runs them as written against deepseek-community/Janus-Pro-1B. The MIC runner runs the same blocks against each slim fixture and each checkpoint in integration.json. Lines marked # doc-builder: hide only matter there: they swap in the pinned checkpoint or variant and print the result record.
Multimodal text understanding
import numpy as np
import torch
from PIL import Image
from transformers import JanusForConditionalGeneration, JanusProcessor, set_seed
import json # doc-builder: hide
import os # doc-builder: hide
model_id = "deepseek-community/Janus-Pro-1B"
model_id = os.environ.get("MIC_MODEL", model_id) # doc-builder: hide
variant = json.loads(os.environ.get("MIC_MODEL_KWARGS", "{}")) # doc-builder: hide
device = "cuda" if torch.cuda.is_available() else "cpu"
set_seed(0) # seeds torch, NumPy, and Python: loading, dropout, and data augmentation draw random numbers
processor = JanusProcessor.from_pretrained(model_id)
model = JanusForConditionalGeneration.from_pretrained(model_id).to(device)
if variant: # doc-builder: hide
del model # doc-builder: hide
model = JanusForConditionalGeneration.from_pretrained(model_id, **variant).to(device) # doc-builder: hide
# Two synthetic images of different sizes so batch perturbation is meaningful
def textured_image(height, width, colour, seed):
# The base colour plus a gradient and seeded noise: every patch differs, so patch order, crops and resizing all change the pixels.
rng = np.random.default_rng(seed)
y, x = np.mgrid[0:height, 0:width]
ramp = (48 * (x / width + y / height) - 48)[..., None]
pixels = np.asarray(colour, dtype=np.float64) + ramp + rng.normal(0, 24, (height, width, 3))
return Image.fromarray(np.clip(pixels, 0, 255).astype(np.uint8))
img1 = textured_image(64, 64, (120, 80, 200), seed=0)
img2 = textured_image(48, 96, (50, 180, 100), seed=1)
messages1 = [{"role": "user", "content": [{"type": "image", "image": img1}, {"type": "text", "text": "What color is this?"}]}]
messages2 = [{"role": "user", "content": [{"type": "image", "image": img2}, {"type": "text", "text": "Describe the image."}]}]
inputs1 = processor.apply_chat_template(
messages1, add_generation_prompt=True, generation_mode="text",
tokenize=True, return_dict=True, return_tensors="pt",
)
inputs2 = processor.apply_chat_template(
messages2, add_generation_prompt=True, generation_mode="text",
tokenize=True, return_dict=True, return_tensors="pt",
)
# BatchFeature.to casts only floating tensors (pixel_values) to model.dtype;
# integer tensors (input_ids, attention_mask) keep their original dtype
inputs1 = inputs1.to(device, dtype=model.dtype)
inputs2 = inputs2.to(device, dtype=model.dtype)
model.eval()
with torch.no_grad():
out1 = model(**inputs1)
out2 = model(**inputs2)
# Compact probe: 16 logits spread across the vocabulary at the last position of each sequence
vocab_ids = torch.linspace(0, out1.logits.shape[-1] - 1, 16).long()
s1 = inputs1["input_ids"].shape[1]
s2 = inputs2["input_ids"].shape[1]
logits_probe = torch.cat([
out1.logits[0, s1 - 1, vocab_ids],
out2.logits[0, s2 - 1, vocab_ids],
]).float()
print(f"logits: {logits_probe.tolist()}")
One training step
# Build labels: copy input_ids, mask padding and image-token positions to -100
labels = inputs1["input_ids"].clone()
if "attention_mask" in inputs1:
labels[inputs1["attention_mask"] == 0] = -100
labels[labels == model.config.image_token_id] = -100
model.train()
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
loss = model(**inputs1, labels=labels).loss
loss.backward()
before = {n: p.detach().cpu().clone() for n, p in model.named_parameters() if p.requires_grad} # doc-builder: hide
optimizer.step()
print(f"loss: {loss.item():.4f}")
if os.environ.get("MIC_RUNNER"): # doc-builder: hide
named = dict(model.named_parameters())
checked = (
"model.language_model.layers.0.self_attn.q_proj.weight",
"model.language_model.layers.0.mlp.up_proj.weight",
)
result = {
"runtime": {
"device": model.device.type,
"dtype": str(model.dtype).removeprefix("torch."),
"attn_implementation": model.config._attn_implementation,
"attn_implementations": {n: getattr(model.config, n)._attn_implementation for n in sorted(model.config.sub_configs) if getattr(getattr(model.config, n, None), '_attn_implementation', None)},
},
"logits_probe": logits_probe.tolist(),
"training": {
"loss": loss.item(),
"gradients": {
n: bool(named[n].grad is not None and torch.count_nonzero(named[n].grad) > 0 and torch.isfinite(named[n].grad).all())
for n in checked
},
"updated": any(not torch.equal(p.detach().cpu(), before[n]) for n, p in model.named_parameters() if n in before),
},
}
print(json.dumps(result))