gn-encoder / gpu_encode.py
infiniteviolets's picture
Fix batched seam: feed each row's own final token, and capture continuation logits after the forward pass
a8e4e2b verified
Raw History Blame Contribute Delete
19.8 kB
"""GPU-resident batched encoder, for ZeroGPU / CUDA Spaces.
The CPU path in sentence_encode.py regenerates a rejected sentence up to
`attempts` times *sequentially*. Measured on real runs, ~82% of sampled tokens
belong to attempts that are later discarded, so almost all of the wall clock is
spent on work that runs one row at a time.
Those attempts are independent, so on a GPU they are one batch:
cache.batch_repeat_interleave(N) fan a batch-1 prefix out to N rows
... sample N sentences in parallel, each with its own payload position ...
cache.batch_select_indices([w]) keep the winner's KV, drop the rest
cache.crop(-(steps - n_w)) trim the winner's surplus tokens
`batch_select_indices` also removes the rollback/replay logic entirely: the
winner's KV is already in place, so nothing has to be recomputed.
Everything per-step lives on the device -- roles, masking, penalties, anti-repeat
and nucleus sampling -- and only the chosen token ids come back. That matters
because the constraint is O(batch) work: at batch 32 the CPU version would cost
~64 ms/step against a ~7 ms GPU step, so batching without moving the constraint
would simply relocate the bottleneck.
Hardware (ZeroGPU, 2026): NVIDIA RTX Pro 6000 Blackwell, 96 GB GDDR7, 1.79 TB/s.
`large` (the default) is half a card -- 48 GB, 1x quota; `xlarge` is the full
card -- 96 GB, 2x quota. For a 3B model the KV cache is ~72 KiB/token, so batch
is never memory-bound; see `autosize_batch`.
"""
from __future__ import annotations
import numpy as np
import torch
import sentence_encode as se
# --------------------------------------------------------------------------- #
# batch sizing
# --------------------------------------------------------------------------- #
def model_geometry(model) -> dict:
c = model.config
heads = getattr(c, "num_attention_heads", 1)
kv = getattr(c, "num_key_value_heads", heads) or heads
hd = getattr(c, "head_dim", None) or (getattr(c, "hidden_size", 0) // max(heads, 1))
return {"layers": getattr(c, "num_hidden_layers", 0), "kv_heads": kv,
"head_dim": hd, "vocab": getattr(c, "vocab_size", 0)}
def autosize_batch(model, seq_budget=1024, reserve=0.30, cap=16, floor=4,
verbose=True) -> int:
"""Rows to run in parallel.
Decode is memory-*bandwidth* bound: the weights are read once per step no
matter how many rows ride along, so extra rows are nearly free until the
step turns compute-bound. For a 3B model on this hardware that crossover is
far above any batch we would want, and the KV cache is tiny, so VRAM is not
the binding constraint -- `cap` is. It exists because best-of-N stops paying
off long before the GPU saturates, and because every row still costs one
tokenizer decode and one bookkeeping pass on the CPU each step.
"""
g = model_geometry(model)
esz = 2 if model.dtype in (torch.float16, torch.bfloat16) else 4
kv_per_tok = 2 * g["layers"] * g["kv_heads"] * g["head_dim"] * esz
# per row: KV for the whole sequence, plus a logits row and the (V, max_c)
# role scratch the constraint allocates per row
per_row = kv_per_tok * seq_budget + g["vocab"] * 4 * 8
if torch.cuda.is_available():
free, total = torch.cuda.mem_get_info()
else:
free = total = 8 << 30
fits = int((free * (1.0 - reserve)) // max(per_row, 1))
n = max(floor, min(cap, fits))
if verbose:
print(f"[gpu] {free/2**30:.1f} GiB free of {total/2**30:.1f} GiB | "
f"KV {kv_per_tok/1024:.0f} KiB/token | {per_row/2**20:.1f} MiB/row | "
f"fits {fits} -> batch {n}", flush=True)
return n
# --------------------------------------------------------------------------- #
# device-resident constraint
# --------------------------------------------------------------------------- #
class GpuConstraint:
"""se.Encoder's constraint, evaluated for N rows at once on the device.
Each row carries its own payload position and character position, so the
keyed alphabet differs per row; the whole thing is still one set of batched
gathers. Semantics follow Encoder.constrain exactly -- wrong-bit and junk
tokens to -inf, skip characters priced at `skip_penalty`.
"""
def __init__(self, tok, bits, skip_penalty, key, device, max_pos=4096,
vocab_size=None):
base = se.Encoder(tok, bits, skip_penalty, key)
# must match the model's logits width, which can exceed len(tok)
base._build(int(vocab_size or len(tok)))
self.device, self.nbits = device, len(bits)
self.skip_penalty = float(skip_penalty)
self.max_c = base.max_c
self.junk_np = base.junk
d = device
self.clipped = torch.as_tensor(base._clipped.astype(np.int64)).to(d) # (V,C)
self.letter = torch.as_tensor(base._letter_at).to(d) # (V,C)
self.fixed = torch.as_tensor(base._fixed.astype(np.int64)).to(d) # (V,C)
self.junk = torch.as_tensor(base.junk).to(d) # (V,)
self.bits_pad = torch.as_tensor(
np.append(np.asarray(bits, dtype=np.int64), -1)).to(d) # (nbits+1,)
# position -> letter-role table, precomputed once. _letter_roles is a
# SHA-256 schedule on the CPU; hoisting it out of the step loop is what
# lets a step be pure tensor work.
self.max_pos = max_pos
table = np.stack([se._letter_roles(key, p) for p in range(max_pos)]) # (P,26)
self.pos_role = torch.as_tensor(table.astype(np.int64)).to(d)
self.V = self.clipped.shape[0]
def ensure_pos(self, need: int, key):
if need + self.max_c < self.max_pos:
return
grow = int(need + self.max_c + 1024)
table = np.stack([se._letter_roles(key, p) for p in range(grow)])
self.pos_role = torch.as_tensor(table.astype(np.int64)).to(self.device)
self.max_pos = grow
def __call__(self, consumed: torch.Tensor, char_pos: torch.Tensor,
logits: torch.Tensor):
"""consumed/char_pos: (N,) int64. logits: (N,V) float.
Returns (masked (N,V), bits_enc (N,V) int64)."""
N, V, C = logits.shape[0], self.V, self.max_c
done = consumed >= self.nbits
# (N, C, 26) keyed roles for each row's next C character positions
offs = torch.arange(C, device=self.device).unsqueeze(0) # (1,C)
rows = (char_pos.unsqueeze(1) + offs).clamp_(0, self.max_pos - 1) # (N,C)
rmap = self.pos_role[rows] # (N,C,26)
# letters -> their live role; everything else keeps its build-time role.
# Advanced indexing rather than gather-on-an-expanded-view: the latter
# would broadcast a (N,V,C,26) shape, 26x more elements than the answer.
n_ix = torch.arange(N, device=self.device).view(N, 1, 1)
c_ix = torch.arange(C, device=self.device).view(1, 1, C)
role = rmap[n_ix, c_ix, self.clipped.unsqueeze(0)] # (N,V,C)
role = torch.where(self.letter.unsqueeze(0), role, self.fixed.unsqueeze(0))
enc = (role == 0) | (role == 1)
cum = torch.cumsum(enc.long(), dim=2) - enc.long() # bit offset
bit_pos = consumed.view(N, 1, 1) + cum
within = enc & (bit_pos < self.nbits)
target = self.bits_pad[bit_pos.clamp(0, self.nbits)]
bad = (within & (role != target)).any(dim=2) | self.junk.unsqueeze(0)
out = logits - self.skip_penalty * (role == se.SKIP).sum(dim=2)
out = out.masked_fill(bad, float("-inf"))
bits_enc = within.sum(dim=2).masked_fill(bad, 0)
# rows whose payload is already spent are unconstrained but still may not
# emit junk -- matching Encoder.constrain's rem <= 0 branch
if done.any():
free = logits.masked_fill(self.junk.unsqueeze(0), float("-inf"))
out = torch.where(done.view(N, 1), free, out)
bits_enc = torch.where(done.view(N, 1), torch.zeros_like(bits_enc), bits_enc)
return out, bits_enc
# --------------------------------------------------------------------------- #
# sampling, on device
# --------------------------------------------------------------------------- #
def sample_rows(scores, temperature, top_p, gen):
"""Nucleus sample one token per row from (N,V) scores containing -inf."""
if temperature <= 1e-6:
return scores.argmax(dim=-1)
z = scores / temperature
probs = torch.softmax(z.float(), dim=-1)
probs = torch.nan_to_num(probs, nan=0.0)
if 0.0 < top_p < 1.0:
sp, si = torch.sort(probs, dim=-1, descending=True)
keep = (torch.cumsum(sp, dim=-1) - sp) < top_p
keep[..., 0] = True
probs = torch.zeros_like(probs).scatter_(-1, si, sp * keep)
tot = probs.sum(dim=-1, keepdim=True)
probs = torch.where(tot > 0, probs / tot.clamp_min(1e-30),
torch.zeros_like(probs))
dead = (probs.sum(dim=-1) <= 0)
if dead.any(): # fully cornered row: fall back to argmax
probs[dead] = 0.0
probs[dead, scores[dead].argmax(dim=-1)] = 1.0
return torch.multinomial(probs, 1, generator=gen).squeeze(-1)
def antirepeat_rows(scores, hist, freq_penalty, window, no_repeat_ngram):
"""Per-row frequency penalty and n-gram ban, applied in place on (N,V)."""
if freq_penalty > 0:
for i, h in enumerate(hist):
if not h:
continue
ids, counts = np.unique(np.asarray(h[-window:]), return_counts=True)
t_ids = torch.as_tensor(ids, device=scores.device, dtype=torch.long)
t_cnt = torch.as_tensor(counts, device=scores.device, dtype=scores.dtype)
row = scores[i]
finite = torch.isfinite(row[t_ids])
row[t_ids] = torch.where(finite, row[t_ids] - freq_penalty * t_cnt,
row[t_ids])
n = no_repeat_ngram
if n and n >= 1:
for i, h in enumerate(hist):
if len(h) < n - 1:
continue
prefix = tuple(h[-(n - 1):]) if n > 1 else ()
banned = [h[j + n - 1] for j in range(len(h) - n + 1)
if tuple(h[j:j + n - 1]) == prefix]
if banned:
scores[i, torch.as_tensor(sorted(set(banned)),
device=scores.device)] = float("-inf")
return scores
# --------------------------------------------------------------------------- #
# batched sentence generation
# --------------------------------------------------------------------------- #
class _Row:
__slots__ = ("sent", "toks", "consumed", "start", "ex", "done", "eos",
"complete", "logits")
def __init__(self, consumed):
self.sent, self.toks, self.consumed = "", [], consumed
self.start = consumed
self.ex, self.done, self.eos, self.complete = [], False, False, False
self.logits = None
def verdict(self, budget, vocab):
exa = np.array(self.ex) if self.ex else np.array([0.0])
coined = len(se.coined_words(self.sent, vocab))
caps = se.register_oddity(self.sent)
shocks = int((exa > se.SHOCK_NATS).sum())
worst = float(exa.max())
gained = self.consumed - self.start
# A sentence that carries no payload bit makes no progress, so it cannot
# be "acceptable" however clean it reads. Without this a row that emits a
# bare "." scores zero shocks, zero coined words and a low worst token,
# and therefore wins the round -- best-of-N finds that degenerate optimum
# immediately, where the CPU path hid it by stopping at the first
# acceptable candidate rather than the best one.
ok = ((self.complete or self.eos) and gained > 0
and budget.verdict(shocks, worst, coined, caps))
return {"shocks": shocks, "worst": worst, "coined": coined, "caps": caps,
"ok": ok, "n": len(self.toks), "gained": gained}
def generate_batched(model, tok, prompt, bits, *, key=None, batch=8,
temperature=0.9, top_p=0.95, bit_bonus=1.0, skip_penalty=1.0,
max_sentence_chars=200, budget=None, seed=0, device="cuda",
verbose=False, max_rounds=40):
"""Encode `bits`, generating `batch` candidate sentences per round in parallel.
One round replaces what the CPU path does in up to `batch` sequential passes.
The winner's KV is kept with batch_select_indices, so no sentence is ever
replayed."""
from transformers import DynamicCache
budget = budget or se.Budget()
vocab = se._VOCAB or se.load_vocab()
con = GpuConstraint(tok, bits, skip_penalty, key, device,
vocab_size=model.config.vocab_size)
nbits = len(bits)
eos_ids = sorted(se._eos_ids(tok))
pad_id = eos_ids[0] if eos_ids else 0
gen = torch.Generator(device=device).manual_seed(int(seed))
cache = DynamicCache(config=model.config)
if hasattr(cache, "activate_past_recording"):
cache.activate_past_recording()
tokens, offset = list(prompt), len(prompt)
with torch.inference_mode():
logits1 = model(input_ids=torch.tensor([list(prompt)], device=device),
past_key_values=cache, use_cache=True).logits[:, -1, :]
consumed, stats = 0, {"accepted": 0, "rejected": 0, "fallback": 0, "rounds": 0,
"steps": 0}
eos_t = torch.as_tensor(eos_ids, device=device, dtype=torch.long) if eos_ids else None
while consumed < nbits and stats["rounds"] < max_rounds:
stats["rounds"] += 1
N = int(batch)
base_chars = len(tok.decode(tokens[offset:]))
con.ensure_pos(base_chars + max_sentence_chars + 8, key)
cache.batch_repeat_interleave(N)
logits = logits1.expand(N, -1).contiguous()
rows = [_Row(consumed) for _ in range(N)]
steps = 0
max_steps = max(8, max_sentence_chars) # chars >= tokens, so this bounds it
with torch.inference_mode():
while steps < max_steps and not all(r.done for r in rows):
raw = logits.float()
lse = torch.logsumexp(raw, dim=-1)
logp = raw - lse.unsqueeze(-1)
H = -(logp.exp() * logp).sum(dim=-1)
cons_t = torch.tensor([r.consumed for r in rows], device=device)
cpos_t = torch.tensor([base_chars + len(r.sent) for r in rows],
device=device)
masked, bits_enc = con(cons_t, cpos_t, raw)
scores = antirepeat_rows(masked.clone(),
[tokens[offset:] + r.toks for r in rows],
freq_penalty=0.5, window=64,
no_repeat_ngram=3)
scores = scores + bit_bonus * bits_enc.to(scores.dtype)
if eos_t is not None: # never stop mid-payload
mid = (cons_t < nbits)
if mid.any():
blk = scores[mid]
blk[:, eos_t] = float("-inf")
scores[mid] = blk
choice = sample_rows(scores, temperature, top_p, gen)
ex = (lse - raw.gather(1, choice.unsqueeze(1)).squeeze(1)) - H
gained = bits_enc.gather(1, choice.unsqueeze(1)).squeeze(1)
ch = choice.tolist()
exl = ex.tolist()
gl = gained.tolist()
# Whether a row was finished BEFORE this step. A row that becomes
# done *during* this step still appended a token, and that token
# must be fed like any other or the cache ends up holding a pad
# where the sentence's last character belongs.
was_done = [r.done for r in rows]
appended = []
for i, r in enumerate(rows):
if was_done[i]:
continue
if ch[i] in eos_ids:
r.eos = r.done = r.complete = True
continue
r.ex.append(exl[i])
r.toks.append(ch[i])
r.sent += tok.decode([ch[i]])
r.consumed += int(gl[i])
appended.append(i)
if exl[i] > budget.hard_abort:
r.done = True # aborted: not complete
elif ((se._sentence_done(r.sent)
and se._can_end(r.sent, len(r.toks)))
or len(r.sent) >= max_sentence_chars):
r.complete = r.done = True
# Feed each row's own token; only rows that were already finished
# (or that just chose EOS, which they do not keep) get the pad.
live = torch.zeros(N, dtype=torch.bool, device=device)
if appended:
live[torch.tensor(appended, device=device)] = True
feed = torch.where(live, choice,
torch.full_like(choice, pad_id)).unsqueeze(1)
logits = model(input_ids=feed, past_key_values=cache,
use_cache=True).logits[:, -1, :]
# The next round continues from the distribution that FOLLOWS the
# committed token, so this has to be captured after the forward.
# Capturing it before is an off-by-one: the round would resume by
# re-predicting the token it just emitted, which shows up as
# lowercase sentence starts and doubled punctuation at the seam.
for i in appended:
rows[i].logits = logits[i]
steps += 1
stats["steps"] += steps
# ---- pick the winner, exactly as the CPU path ranks candidates --------
vs = [r.verdict(budget, vocab) for r in rows]
accepted = [i for i, v in enumerate(vs) if v["ok"] and rows[i].toks]
def rank_fallback(i):
v, r = vs[i], rows[i]
return (not (r.complete or r.eos), v["gained"] == 0,
v["caps"] > budget.max_caps, v["coined"], v["shocks"],
v["worst"], -v["gained"])
def rank_accepted(i):
# every one of these already cleared the quality budget, so choose on
# progress first and use quality only to break ties
v = vs[i]
return (-v["gained"], v["coined"], v["shocks"], v["worst"])
if accepted:
w = min(accepted, key=rank_accepted); stats["accepted"] += 1
else:
w = min(range(N), key=rank_fallback); stats["fallback"] += 1
stats["rejected"] += N - 1
cache.batch_select_indices(torch.tensor([w], device=device))
surplus = steps - len(rows[w].toks)
if surplus > 0:
cache.crop(-surplus)
if verbose:
v = vs[w]
print(f" [{'ok ' if v['ok'] else 'FB '}] shocks={v['shocks']} "
f"worst={v['worst']:5.1f} coined={v['coined']} "
f"+{rows[w].consumed - consumed}b {rows[w].sent.strip()[:60]!r}",
flush=True)
if not rows[w].toks:
break
tokens.extend(rows[w].toks)
consumed = rows[w].consumed
logits1 = rows[w].logits.unsqueeze(0)
if rows[w].eos:
break
return tok.decode(tokens[offset:]), stats