# PegasusX: summarization with Transformers This page is the Transformers contract of the PegasusX MIC and also a usage example. [doc-builder](https://github.com/huggingface/doc-builder/blob/main/docs/runnable-code-blocks.md) renders the `runnable` blocks below as documentation, and its pytest plugin runs them as written against `google/pegasus-x-base`. The MIC runner runs the same blocks against each slim fixture and each checkpoint in [`integration.json`](../../integration.json). Lines marked `# doc-builder: hide` only matter there: they swap in the pinned checkpoint or variant and print the result record. ## Summarize an article ```py runnable:summarize import torch from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, set_seed import json # doc-builder: hide import os # doc-builder: hide model_id = "google/pegasus-x-base" model_id = os.environ.get("MIC_MODEL", model_id) # doc-builder: hide variant = json.loads(os.environ.get("MIC_MODEL_KWARGS", "{}")) # doc-builder: hide device = "cuda" if torch.cuda.is_available() else "cpu" set_seed(0) # seeds torch, NumPy, and Python: loading, dropout, and data augmentation draw random numbers tokenizer = AutoTokenizer.from_pretrained(model_id) model = AutoModelForSeq2SeqLM.from_pretrained(model_id).to(device) if variant: # doc-builder: hide del model # doc-builder: hide model = AutoModelForSeq2SeqLM.from_pretrained(model_id, **variant).to(device) # doc-builder: hide articles = [ "PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow.", "The company said it would cut 10% of its workforce and close several offices as part of a restructuring plan. The move is expected to save $200 million annually.", ] inputs = tokenizer(articles, return_tensors="pt", padding=True, truncation=True, max_length=128).to(device) with torch.no_grad(): summary_ids = model.generate(**inputs, max_length=32, num_beams=2) summary_text = tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False) print(summary_text) ``` ## One training step ```py runnable:summarize:2 labels = tokenizer( ["California utility cuts power to prevent wildfires.", "Company announces layoffs and office closures."], return_tensors="pt", padding=True, truncation=True, max_length=32 ).to(device) labels["input_ids"][labels["input_ids"] == tokenizer.pad_token_id] = -100 model.train() optimizer = torch.optim.SGD(model.parameters(), lr=0.01) loss = model(**inputs, labels=labels["input_ids"]).loss loss.backward() before = {n: p.detach().cpu().clone() for n, p in model.named_parameters() if p.requires_grad} # doc-builder: hide optimizer.step() print(f"loss: {loss.item():.4f}") if os.environ.get("MIC_RUNNER"): # doc-builder: hide named = dict(model.named_parameters()) checked = ("model.encoder.layers.0.self_attn.q_proj.weight", "model.decoder.layers.0.self_attn.q_proj.weight") result = { "runtime": { "device": model.device.type, "dtype": str(model.dtype).removeprefix("torch."), "attn_implementation": model.config._attn_implementation, }, "summary_ids": summary_ids.tolist(), "summary_text": summary_text, "logits": {"probe": model(**inputs, decoder_input_ids=summary_ids[:, :1]).logits[:, 0, :5].float().tolist()}, "training": { "loss": loss.item(), "gradients": {n: bool(named[n].grad is not None and torch.count_nonzero(named[n].grad) > 0 and torch.isfinite(named[n].grad).all()) for n in checked}, "updated": any(not torch.equal(p.detach().cpu(), before[n]) for n, p in model.named_parameters() if n in before), }, } print(json.dumps(result)) ```