File size: 10,857 Bytes
cdb8f01
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5960155
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
cdb8f01
 
 
 
 
 
 
 
593092d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
cdb8f01
 
593092d
 
 
 
 
 
 
 
 
cdb8f01
593092d
 
 
 
 
 
 
 
 
 
 
 
 
 
cdb8f01
 
593092d
 
 
 
 
 
 
 
 
 
 
 
cdb8f01
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
"""Find which property of the capture's own arrays makes a padded forward differ.

Confirmed on `failure-zusi05xj`: a padded-batch eager forward reproduces the trainer's
stored logprobs BIT-EXACTLY (max 0.0) while a per-row forward reproduces the engine's
(max 0.33); the two differ by 1.7597 max on 150 positions. A synthetic sweep that copied
the capture's geometry (lengths 506/224/4096/644, longest at index 2, pad id 0, train
mode, position_ids = cumsum-1 with pads forced to 1) reports max|delta logit| = 0.00000
for every variant. So the trigger is a property of the capture's DATA, not its shape.

This ablates the real arrays one input at a time and reports the content-independent
statistic `max |delta logit|` between the padded forward and the per-row forward:

  base            ids + attention_mask exactly as stored, position_ids = cumsum-1
  base_nopos      same, but no position_ids passed
  pad_zero        padded region of every row replaced by token id 0
  attn_prefix     attention_mask rebuilt as a plain prefix-ones mask from its own sum
  attn_all_ones   attention_mask replaced by all ones
  long_only_pad   single-row batch of the long row, padded to the stored width
  long_only       the long row alone at its own length (no padding at all)

If `attn_prefix` or `pad_zero` collapses the delta, that names the trigger.

Read-only and structure-only: it prints mask shape statistics and deltas, never prompt
text or token ids.

  CUDA_VISIBLE_DEVICES=0 bash $SRC/experiments/runai/python-b200-host.sh \
    /tmp/diagnose_padding_trigger.py "$CAPDIR" --out /tmp/pad_trigger.json
"""

from __future__ import annotations

import argparse
import json
from pathlib import Path
from typing import Any


def max_logit_delta(a, b, chunk: int = 256) -> dict[str, Any]:
    """max |a - b| over the vocabulary, per position, in chunks."""
    n = int(min(a.shape[0], b.shape[0]))
    worst, worst_pos = 0.0, -1
    means: list[float] = []
    for lo in range(0, n, chunk):
        hi = min(lo + chunk, n)
        d = (a[lo:hi].float() - b[lo:hi].float()).abs()
        means.append(float(d.mean()))
        m = float(d.max())
        if m > worst:
            worst, worst_pos = m, lo + int(d.max(dim=1).values.argmax())
        del d
    return {"positions": n, "max_abs_dlogit": worst, "worst_position": worst_pos,
            "mean_abs_dlogit": sum(means) / len(means) if means else 0.0}


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("capture", type=Path)
    ap.add_argument("--out", type=Path, default=None)
    ap.add_argument("--dtype", choices=("bfloat16", "float32"), default="bfloat16")
    ap.add_argument("--row", type=int, default=-1, help="row to compare; -1 = longest")
    args = ap.parse_args()

    import torch
    from transformers import AutoModelForCausalLM

    meta = json.loads((args.capture / "metadata.json").read_text())
    batch = torch.load(args.capture / "batch.pt", map_location="cpu", weights_only=True)
    ids = batch["stu_input_ids"].long()
    attn = batch["stu_attn_mask"].long()
    lengths = [int(v) for v in attn.sum(-1).tolist()]
    width = int(ids.shape[1])
    r = args.row if args.row >= 0 else max(range(len(lengths)), key=lambda i: lengths[i])

    report: dict[str, Any] = {
        "capture": str(args.capture), "rows": int(ids.shape[0]), "width": width,
        "lengths": lengths, "row": r,
        "model_training": meta.get("model_training"),
        "attention_backend": meta.get("attention_backend"),
    }
    print("TRIGGER_META=" + json.dumps(
        {k: report[k] for k in ("rows", "width", "lengths", "row", "model_training")},
        sort_keys=True), flush=True)

    # Structural description of the masks only: counts and whether the ones form a
    # plain prefix. No token ids and no text leave the machine.
    struct = []
    for i in range(ids.shape[0]):
        a = attn[i]
        ones = int(a.sum())
        first_zero = int((a == 0).nonzero()[0]) if bool((a == 0).any()) else -1
        struct.append({"row": i, "ones": ones, "first_zero": first_zero,
                       "is_prefix_ones": bool(int(a[:ones].sum()) == ones)})
    report["mask_structure"] = struct
    print("TRIGGER_MASK_STRUCTURE=" + json.dumps(struct), flush=True)

    model = AutoModelForCausalLM.from_pretrained(
        args.capture / "student", dtype=getattr(torch, args.dtype),
        attn_implementation="eager", local_files_only=True,
        trust_remote_code=False).cuda()
    model.train(bool(meta.get("model_training")))

    n_r = int(lengths[r])
    lone_ids = ids[r:r + 1, :n_r].clone()

    def positions_for(a):
        p = a.long().cumsum(-1) - 1
        p.masked_fill_(a == 0, 1)
        return p

    pos_stored = positions_for(attn)

    # ---------------------------------------------------------------------
    # Per-row delta for the stored configuration, for EVERY row.
    #
    # Discriminator between two mechanisms: if rows AFTER the first are affected
    # (row 1 sees row 0, row 3 sees rows 0-2) then the trainer's forward couples
    # rows -- a concatenation/packing-style defect. If only the LONGEST row is
    # affected it is a length/width effect. The two have different fixes.
    # ---------------------------------------------------------------------
    with torch.inference_mode():
        padded_all = model(input_ids=ids.cuda(), attention_mask=attn.cuda(),
                           position_ids=pos_stored.cuda(), use_cache=False).logits
    per_row: dict[str, Any] = {}
    for i in range(ids.shape[0]):
        n_i = int(lengths[i])
        with torch.inference_mode():
            ref_i = model(input_ids=ids[i:i + 1, :n_i].cuda(), use_cache=False).logits[0]
        per_row[str(i)] = max_logit_delta(padded_all[i, :n_i], ref_i)
        del ref_i
        torch.cuda.empty_cache()
        print("TRIGGER_PER_ROW=" + json.dumps(
            {"row": i, "length": n_i, **per_row[str(i)]}, sort_keys=True), flush=True)
    del padded_all
    torch.cuda.empty_cache()
    report["per_row_stored_config"] = per_row

    # Structure of the padded region only: is it a single constant pad value? No
    # token ids are printed, just counts.
    pad_region = []
    for i in range(ids.shape[0]):
        L = int(lengths[i])
        if L >= width:
            pad_region.append({"row": i, "padded_tokens": 0})
            continue
        seg = ids[i, L:]
        uniq = int(torch.unique(seg).numel())
        pad_region.append({"row": i, "padded_tokens": int(seg.numel()),
                           "distinct_pad_values": uniq, "all_equal": bool(uniq == 1)})
    report["pad_region_structure"] = pad_region
    print("TRIGGER_PAD_REGION=" + json.dumps(pad_region), flush=True)

    prefix = torch.zeros_like(attn)
    for i, L in enumerate(lengths):
        prefix[i, :L] = 1
    pad_ids = torch.zeros((1, width), dtype=torch.long)
    pad_attn = torch.zeros((1, width), dtype=torch.long)
    pad_ids[0, :n_r] = ids[r, :n_r]
    pad_attn[0, :n_r] = 1

    n_rows = int(ids.shape[0])
    # One reference per row: that row forwarded alone, no mask and no position_ids.
    # This is the semantics SGLang serves, and it is what the fix must reproduce.
    refs: list[Any] = []
    for i in range(n_rows):
        n_i = int(lengths[i])
        with torch.inference_mode():
            refs.append(model(input_ids=ids[i:i + 1, :n_i].cuda(),
                              use_cache=False).logits[0])
        torch.cuda.empty_cache()

    plans: list[tuple[str, Any, Any, Any, list[int]]] = [
        ("base", ids, attn, pos_stored, list(range(n_rows))),
        ("base_nopos", ids, attn, None, list(range(n_rows))),
        ("pad_zero", torch.where(attn.bool(), ids, torch.zeros_like(ids)), attn,
         pos_stored, list(range(n_rows))),
        ("attn_prefix", ids, prefix, None, list(range(n_rows))),
        ("attn_all_ones", ids, torch.ones_like(attn), None, list(range(n_rows))),
        ("long_only_pad", pad_ids, pad_attn, None, [r]),
        ("long_only", lone_ids, torch.ones_like(lone_ids), None, [r]),
    ]
    variants: dict[str, dict[str, Any]] = {}
    per_variant_row: dict[str, dict[str, Any]] = {}
    for name, x, a, p, row_map in plans:
        with torch.inference_mode():
            out = model(input_ids=x.cuda(),
                        attention_mask=(a.cuda() if a is not None else None),
                        position_ids=(p.cuda() if p is not None else None),
                        use_cache=False).logits
        out3 = out if out.dim() == 3 else out.unsqueeze(0)
        del out
        torch.cuda.empty_cache()
        per_variant_row[name] = {}
        for j, capture_row in enumerate(row_map):
            n_i = int(lengths[capture_row])
            d = max_logit_delta(out3[j, :n_i], refs[capture_row])
            per_variant_row[name][str(capture_row)] = d
            print("TRIGGER_VARIANT_ROW=" + json.dumps(
                {"variant": name, "row": capture_row, "length": n_i,
                 "max_abs_dlogit": d["max_abs_dlogit"],
                 "mean_abs_dlogit": round(d["mean_abs_dlogit"], 6)}, sort_keys=True),
                flush=True)
        del out3
        torch.cuda.empty_cache()
        # Headline = the longest row, which is the one the parity guard trips on.
        variants[name] = per_variant_row[name][str(r)]
        print("TRIGGER_VARIANT=" + json.dumps({"variant": name, **variants[name]},
                                             sort_keys=True), flush=True)
    del refs
    torch.cuda.empty_cache()
    report["per_variant_row"] = per_variant_row
    # Does dropping position_ids reproduce the per-row reference for EVERY row?
    fixed = {k: round(v["max_abs_dlogit"], 6)
             for k, v in per_variant_row.get("base_nopos", {}).items()}
    broken = {k: round(v["max_abs_dlogit"], 6)
              for k, v in per_variant_row.get("base", {}).items()}
    report["fix_check"] = {"base_max_per_row": broken, "base_nopos_max_per_row": fixed,
                           "fix_removes_the_effect_on_all_rows":
                               all(v == 0.0 for v in fixed.values()) if fixed else None}
    print("TRIGGER_FIX_CHECK=" + json.dumps(report["fix_check"], sort_keys=True), flush=True)

    report["variants"] = variants
    ranked = sorted((v["max_abs_dlogit"], k) for k, v in variants.items())
    report["smallest_delta_variant"] = ranked[0][1] if ranked else None
    print("TRIGGER_RANKED=" + json.dumps(
        {"smallest": ranked[0][1] if ranked else None,
         "deltas": {k: round(v["max_abs_dlogit"], 6) for k, v in variants.items()}},
        sort_keys=True), flush=True)
    if args.out:
        args.out.write_text(json.dumps(report, indent=2, sort_keys=True, default=str) + "\n")
        print("TRIGGER_OUT=" + str(args.out), flush=True)
    print("TRIGGER_DONE=1", flush=True)


if __name__ == "__main__":
    main()