tarekziade's picture
tarekziade HF Staff
Publish MIC usage, slim fixture and reviewed Transformers expectations
802afb4 verified
|
Raw History Blame Contribute Delete
3.95 kB

PegasusX: summarization with Transformers

This page is the Transformers contract of the PegasusX MIC and also a usage example. doc-builder renders the runnable blocks below as documentation, and its pytest plugin runs them as written against google/pegasus-x-base. The MIC runner runs the same blocks against each slim fixture and each checkpoint in integration.json. Lines marked # doc-builder: hide only matter there: they swap in the pinned checkpoint or variant and print the result record.

Summarize an article

import torch
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, set_seed
import json  # doc-builder: hide
import os  # doc-builder: hide

model_id = "google/pegasus-x-base"
model_id = os.environ.get("MIC_MODEL", model_id)  # doc-builder: hide
variant = json.loads(os.environ.get("MIC_MODEL_KWARGS", "{}"))  # doc-builder: hide
device = "cuda" if torch.cuda.is_available() else "cpu"
set_seed(0)  # seeds torch, NumPy, and Python: loading, dropout, and data augmentation draw random numbers

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForSeq2SeqLM.from_pretrained(model_id).to(device)
if variant:  # doc-builder: hide
    del model  # doc-builder: hide
    model = AutoModelForSeq2SeqLM.from_pretrained(model_id, **variant).to(device)  # doc-builder: hide

articles = [
    "PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow.",
    "The company said it would cut 10% of its workforce and close several offices as part of a restructuring plan. The move is expected to save $200 million annually.",
]
inputs = tokenizer(articles, return_tensors="pt", padding=True, truncation=True, max_length=128).to(device)

with torch.no_grad():
    summary_ids = model.generate(**inputs, max_length=32, num_beams=2)
summary_text = tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)
print(summary_text)

One training step

labels = tokenizer(
    ["California utility cuts power to prevent wildfires.", "Company announces layoffs and office closures."],
    return_tensors="pt", padding=True, truncation=True, max_length=32
).to(device)
labels["input_ids"][labels["input_ids"] == tokenizer.pad_token_id] = -100

model.train()
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
loss = model(**inputs, labels=labels["input_ids"]).loss
loss.backward()
before = {n: p.detach().cpu().clone() for n, p in model.named_parameters() if p.requires_grad}  # doc-builder: hide
optimizer.step()
print(f"loss: {loss.item():.4f}")

if os.environ.get("MIC_RUNNER"):  # doc-builder: hide
    named = dict(model.named_parameters())
    checked = ("model.encoder.layers.0.self_attn.q_proj.weight", "model.decoder.layers.0.self_attn.q_proj.weight")
    result = {
        "runtime": {
            "device": model.device.type,
            "dtype": str(model.dtype).removeprefix("torch."),
            "attn_implementation": model.config._attn_implementation,
        },
        "summary_ids": summary_ids.tolist(),
        "summary_text": summary_text,
        "logits": {"probe": model(**inputs, decoder_input_ids=summary_ids[:, :1]).logits[:, 0, :5].float().tolist()},
        "training": {
            "loss": loss.item(),
            "gradients": {n: bool(named[n].grad is not None and torch.count_nonzero(named[n].grad) > 0 and torch.isfinite(named[n].grad).all()) for n in checked},
            "updated": any(not torch.equal(p.detach().cpu(), before[n]) for n, p in model.named_parameters() if n in before),
        },
    }
    print(json.dumps(result))