Download diag/diagnose_padding_trigger.py from codemaivanngu/simct: direct link, hf CLI and curl.
- Browser
- Download file 10.9 kB
-
https://huggingface.co/codemaivanngu/simct/resolve/main/diag/diagnose_padding_trigger.py
- Command line
-
hf download hf://codemaivanngu/simct/diag/diagnose_padding_trigger.py
-
curl -L -o diagnose_padding_trigger.py https://huggingface.co/codemaivanngu/simct/resolve/main/diag/diagnose_padding_trigger.py
10.9 kB
| """Find which property of the capture's own arrays makes a padded forward differ. | |
| Confirmed on `failure-zusi05xj`: a padded-batch eager forward reproduces the trainer's | |
| stored logprobs BIT-EXACTLY (max 0.0) while a per-row forward reproduces the engine's | |
| (max 0.33); the two differ by 1.7597 max on 150 positions. A synthetic sweep that copied | |
| the capture's geometry (lengths 506/224/4096/644, longest at index 2, pad id 0, train | |
| mode, position_ids = cumsum-1 with pads forced to 1) reports max|delta logit| = 0.00000 | |
| for every variant. So the trigger is a property of the capture's DATA, not its shape. | |
| This ablates the real arrays one input at a time and reports the content-independent | |
| statistic `max |delta logit|` between the padded forward and the per-row forward: | |
| base ids + attention_mask exactly as stored, position_ids = cumsum-1 | |
| base_nopos same, but no position_ids passed | |
| pad_zero padded region of every row replaced by token id 0 | |
| attn_prefix attention_mask rebuilt as a plain prefix-ones mask from its own sum | |
| attn_all_ones attention_mask replaced by all ones | |
| long_only_pad single-row batch of the long row, padded to the stored width | |
| long_only the long row alone at its own length (no padding at all) | |
| If `attn_prefix` or `pad_zero` collapses the delta, that names the trigger. | |
| Read-only and structure-only: it prints mask shape statistics and deltas, never prompt | |
| text or token ids. | |
| CUDA_VISIBLE_DEVICES=0 bash $SRC/experiments/runai/python-b200-host.sh \ | |
| /tmp/diagnose_padding_trigger.py "$CAPDIR" --out /tmp/pad_trigger.json | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| from typing import Any | |
| def max_logit_delta(a, b, chunk: int = 256) -> dict[str, Any]: | |
| """max |a - b| over the vocabulary, per position, in chunks.""" | |
| n = int(min(a.shape[0], b.shape[0])) | |
| worst, worst_pos = 0.0, -1 | |
| means: list[float] = [] | |
| for lo in range(0, n, chunk): | |
| hi = min(lo + chunk, n) | |
| d = (a[lo:hi].float() - b[lo:hi].float()).abs() | |
| means.append(float(d.mean())) | |
| m = float(d.max()) | |
| if m > worst: | |
| worst, worst_pos = m, lo + int(d.max(dim=1).values.argmax()) | |
| del d | |
| return {"positions": n, "max_abs_dlogit": worst, "worst_position": worst_pos, | |
| "mean_abs_dlogit": sum(means) / len(means) if means else 0.0} | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("capture", type=Path) | |
| ap.add_argument("--out", type=Path, default=None) | |
| ap.add_argument("--dtype", choices=("bfloat16", "float32"), default="bfloat16") | |
| ap.add_argument("--row", type=int, default=-1, help="row to compare; -1 = longest") | |
| args = ap.parse_args() | |
| import torch | |
| from transformers import AutoModelForCausalLM | |
| meta = json.loads((args.capture / "metadata.json").read_text()) | |
| batch = torch.load(args.capture / "batch.pt", map_location="cpu", weights_only=True) | |
| ids = batch["stu_input_ids"].long() | |
| attn = batch["stu_attn_mask"].long() | |
| lengths = [int(v) for v in attn.sum(-1).tolist()] | |
| width = int(ids.shape[1]) | |
| r = args.row if args.row >= 0 else max(range(len(lengths)), key=lambda i: lengths[i]) | |
| report: dict[str, Any] = { | |
| "capture": str(args.capture), "rows": int(ids.shape[0]), "width": width, | |
| "lengths": lengths, "row": r, | |
| "model_training": meta.get("model_training"), | |
| "attention_backend": meta.get("attention_backend"), | |
| } | |
| print("TRIGGER_META=" + json.dumps( | |
| {k: report[k] for k in ("rows", "width", "lengths", "row", "model_training")}, | |
| sort_keys=True), flush=True) | |
| # Structural description of the masks only: counts and whether the ones form a | |
| # plain prefix. No token ids and no text leave the machine. | |
| struct = [] | |
| for i in range(ids.shape[0]): | |
| a = attn[i] | |
| ones = int(a.sum()) | |
| first_zero = int((a == 0).nonzero()[0]) if bool((a == 0).any()) else -1 | |
| struct.append({"row": i, "ones": ones, "first_zero": first_zero, | |
| "is_prefix_ones": bool(int(a[:ones].sum()) == ones)}) | |
| report["mask_structure"] = struct | |
| print("TRIGGER_MASK_STRUCTURE=" + json.dumps(struct), flush=True) | |
| model = AutoModelForCausalLM.from_pretrained( | |
| args.capture / "student", dtype=getattr(torch, args.dtype), | |
| attn_implementation="eager", local_files_only=True, | |
| trust_remote_code=False).cuda() | |
| model.train(bool(meta.get("model_training"))) | |
| n_r = int(lengths[r]) | |
| lone_ids = ids[r:r + 1, :n_r].clone() | |
| def positions_for(a): | |
| p = a.long().cumsum(-1) - 1 | |
| p.masked_fill_(a == 0, 1) | |
| return p | |
| pos_stored = positions_for(attn) | |
| # --------------------------------------------------------------------- | |
| # Per-row delta for the stored configuration, for EVERY row. | |
| # | |
| # Discriminator between two mechanisms: if rows AFTER the first are affected | |
| # (row 1 sees row 0, row 3 sees rows 0-2) then the trainer's forward couples | |
| # rows -- a concatenation/packing-style defect. If only the LONGEST row is | |
| # affected it is a length/width effect. The two have different fixes. | |
| # --------------------------------------------------------------------- | |
| with torch.inference_mode(): | |
| padded_all = model(input_ids=ids.cuda(), attention_mask=attn.cuda(), | |
| position_ids=pos_stored.cuda(), use_cache=False).logits | |
| per_row: dict[str, Any] = {} | |
| for i in range(ids.shape[0]): | |
| n_i = int(lengths[i]) | |
| with torch.inference_mode(): | |
| ref_i = model(input_ids=ids[i:i + 1, :n_i].cuda(), use_cache=False).logits[0] | |
| per_row[str(i)] = max_logit_delta(padded_all[i, :n_i], ref_i) | |
| del ref_i | |
| torch.cuda.empty_cache() | |
| print("TRIGGER_PER_ROW=" + json.dumps( | |
| {"row": i, "length": n_i, **per_row[str(i)]}, sort_keys=True), flush=True) | |
| del padded_all | |
| torch.cuda.empty_cache() | |
| report["per_row_stored_config"] = per_row | |
| # Structure of the padded region only: is it a single constant pad value? No | |
| # token ids are printed, just counts. | |
| pad_region = [] | |
| for i in range(ids.shape[0]): | |
| L = int(lengths[i]) | |
| if L >= width: | |
| pad_region.append({"row": i, "padded_tokens": 0}) | |
| continue | |
| seg = ids[i, L:] | |
| uniq = int(torch.unique(seg).numel()) | |
| pad_region.append({"row": i, "padded_tokens": int(seg.numel()), | |
| "distinct_pad_values": uniq, "all_equal": bool(uniq == 1)}) | |
| report["pad_region_structure"] = pad_region | |
| print("TRIGGER_PAD_REGION=" + json.dumps(pad_region), flush=True) | |
| prefix = torch.zeros_like(attn) | |
| for i, L in enumerate(lengths): | |
| prefix[i, :L] = 1 | |
| pad_ids = torch.zeros((1, width), dtype=torch.long) | |
| pad_attn = torch.zeros((1, width), dtype=torch.long) | |
| pad_ids[0, :n_r] = ids[r, :n_r] | |
| pad_attn[0, :n_r] = 1 | |
| n_rows = int(ids.shape[0]) | |
| # One reference per row: that row forwarded alone, no mask and no position_ids. | |
| # This is the semantics SGLang serves, and it is what the fix must reproduce. | |
| refs: list[Any] = [] | |
| for i in range(n_rows): | |
| n_i = int(lengths[i]) | |
| with torch.inference_mode(): | |
| refs.append(model(input_ids=ids[i:i + 1, :n_i].cuda(), | |
| use_cache=False).logits[0]) | |
| torch.cuda.empty_cache() | |
| plans: list[tuple[str, Any, Any, Any, list[int]]] = [ | |
| ("base", ids, attn, pos_stored, list(range(n_rows))), | |
| ("base_nopos", ids, attn, None, list(range(n_rows))), | |
| ("pad_zero", torch.where(attn.bool(), ids, torch.zeros_like(ids)), attn, | |
| pos_stored, list(range(n_rows))), | |
| ("attn_prefix", ids, prefix, None, list(range(n_rows))), | |
| ("attn_all_ones", ids, torch.ones_like(attn), None, list(range(n_rows))), | |
| ("long_only_pad", pad_ids, pad_attn, None, [r]), | |
| ("long_only", lone_ids, torch.ones_like(lone_ids), None, [r]), | |
| ] | |
| variants: dict[str, dict[str, Any]] = {} | |
| per_variant_row: dict[str, dict[str, Any]] = {} | |
| for name, x, a, p, row_map in plans: | |
| with torch.inference_mode(): | |
| out = model(input_ids=x.cuda(), | |
| attention_mask=(a.cuda() if a is not None else None), | |
| position_ids=(p.cuda() if p is not None else None), | |
| use_cache=False).logits | |
| out3 = out if out.dim() == 3 else out.unsqueeze(0) | |
| del out | |
| torch.cuda.empty_cache() | |
| per_variant_row[name] = {} | |
| for j, capture_row in enumerate(row_map): | |
| n_i = int(lengths[capture_row]) | |
| d = max_logit_delta(out3[j, :n_i], refs[capture_row]) | |
| per_variant_row[name][str(capture_row)] = d | |
| print("TRIGGER_VARIANT_ROW=" + json.dumps( | |
| {"variant": name, "row": capture_row, "length": n_i, | |
| "max_abs_dlogit": d["max_abs_dlogit"], | |
| "mean_abs_dlogit": round(d["mean_abs_dlogit"], 6)}, sort_keys=True), | |
| flush=True) | |
| del out3 | |
| torch.cuda.empty_cache() | |
| # Headline = the longest row, which is the one the parity guard trips on. | |
| variants[name] = per_variant_row[name][str(r)] | |
| print("TRIGGER_VARIANT=" + json.dumps({"variant": name, **variants[name]}, | |
| sort_keys=True), flush=True) | |
| del refs | |
| torch.cuda.empty_cache() | |
| report["per_variant_row"] = per_variant_row | |
| # Does dropping position_ids reproduce the per-row reference for EVERY row? | |
| fixed = {k: round(v["max_abs_dlogit"], 6) | |
| for k, v in per_variant_row.get("base_nopos", {}).items()} | |
| broken = {k: round(v["max_abs_dlogit"], 6) | |
| for k, v in per_variant_row.get("base", {}).items()} | |
| report["fix_check"] = {"base_max_per_row": broken, "base_nopos_max_per_row": fixed, | |
| "fix_removes_the_effect_on_all_rows": | |
| all(v == 0.0 for v in fixed.values()) if fixed else None} | |
| print("TRIGGER_FIX_CHECK=" + json.dumps(report["fix_check"], sort_keys=True), flush=True) | |
| report["variants"] = variants | |
| ranked = sorted((v["max_abs_dlogit"], k) for k, v in variants.items()) | |
| report["smallest_delta_variant"] = ranked[0][1] if ranked else None | |
| print("TRIGGER_RANKED=" + json.dumps( | |
| {"smallest": ranked[0][1] if ranked else None, | |
| "deltas": {k: round(v["max_abs_dlogit"], 6) for k, v in variants.items()}}, | |
| sort_keys=True), flush=True) | |
| if args.out: | |
| args.out.write_text(json.dumps(report, indent=2, sort_keys=True, default=str) + "\n") | |
| print("TRIGGER_OUT=" + str(args.out), flush=True) | |
| print("TRIGGER_DONE=1", flush=True) | |
| if __name__ == "__main__": | |
| main() | |