Spaces:
Running on Zero
Running on Zero
Fix batched seam: feed each row's own final token, and capture continuation logits after the forward pass
a8e4e2b verified Download gpu_encode.py from infiniteviolets/gn-encoder: direct link, hf CLI and curl.
- Browser
- Download file 19.8 kB
-
https://huggingface.co/spaces/infiniteviolets/gn-encoder/resolve/main/gpu_encode.py
- Command line
-
hf download hf://spaces/infiniteviolets/gn-encoder/gpu_encode.py
-
curl -L -o gpu_encode.py https://huggingface.co/spaces/infiniteviolets/gn-encoder/resolve/main/gpu_encode.py
19.8 kB
| """GPU-resident batched encoder, for ZeroGPU / CUDA Spaces. | |
| The CPU path in sentence_encode.py regenerates a rejected sentence up to | |
| `attempts` times *sequentially*. Measured on real runs, ~82% of sampled tokens | |
| belong to attempts that are later discarded, so almost all of the wall clock is | |
| spent on work that runs one row at a time. | |
| Those attempts are independent, so on a GPU they are one batch: | |
| cache.batch_repeat_interleave(N) fan a batch-1 prefix out to N rows | |
| ... sample N sentences in parallel, each with its own payload position ... | |
| cache.batch_select_indices([w]) keep the winner's KV, drop the rest | |
| cache.crop(-(steps - n_w)) trim the winner's surplus tokens | |
| `batch_select_indices` also removes the rollback/replay logic entirely: the | |
| winner's KV is already in place, so nothing has to be recomputed. | |
| Everything per-step lives on the device -- roles, masking, penalties, anti-repeat | |
| and nucleus sampling -- and only the chosen token ids come back. That matters | |
| because the constraint is O(batch) work: at batch 32 the CPU version would cost | |
| ~64 ms/step against a ~7 ms GPU step, so batching without moving the constraint | |
| would simply relocate the bottleneck. | |
| Hardware (ZeroGPU, 2026): NVIDIA RTX Pro 6000 Blackwell, 96 GB GDDR7, 1.79 TB/s. | |
| `large` (the default) is half a card -- 48 GB, 1x quota; `xlarge` is the full | |
| card -- 96 GB, 2x quota. For a 3B model the KV cache is ~72 KiB/token, so batch | |
| is never memory-bound; see `autosize_batch`. | |
| """ | |
| from __future__ import annotations | |
| import numpy as np | |
| import torch | |
| import sentence_encode as se | |
| # --------------------------------------------------------------------------- # | |
| # batch sizing | |
| # --------------------------------------------------------------------------- # | |
| def model_geometry(model) -> dict: | |
| c = model.config | |
| heads = getattr(c, "num_attention_heads", 1) | |
| kv = getattr(c, "num_key_value_heads", heads) or heads | |
| hd = getattr(c, "head_dim", None) or (getattr(c, "hidden_size", 0) // max(heads, 1)) | |
| return {"layers": getattr(c, "num_hidden_layers", 0), "kv_heads": kv, | |
| "head_dim": hd, "vocab": getattr(c, "vocab_size", 0)} | |
| def autosize_batch(model, seq_budget=1024, reserve=0.30, cap=16, floor=4, | |
| verbose=True) -> int: | |
| """Rows to run in parallel. | |
| Decode is memory-*bandwidth* bound: the weights are read once per step no | |
| matter how many rows ride along, so extra rows are nearly free until the | |
| step turns compute-bound. For a 3B model on this hardware that crossover is | |
| far above any batch we would want, and the KV cache is tiny, so VRAM is not | |
| the binding constraint -- `cap` is. It exists because best-of-N stops paying | |
| off long before the GPU saturates, and because every row still costs one | |
| tokenizer decode and one bookkeeping pass on the CPU each step. | |
| """ | |
| g = model_geometry(model) | |
| esz = 2 if model.dtype in (torch.float16, torch.bfloat16) else 4 | |
| kv_per_tok = 2 * g["layers"] * g["kv_heads"] * g["head_dim"] * esz | |
| # per row: KV for the whole sequence, plus a logits row and the (V, max_c) | |
| # role scratch the constraint allocates per row | |
| per_row = kv_per_tok * seq_budget + g["vocab"] * 4 * 8 | |
| if torch.cuda.is_available(): | |
| free, total = torch.cuda.mem_get_info() | |
| else: | |
| free = total = 8 << 30 | |
| fits = int((free * (1.0 - reserve)) // max(per_row, 1)) | |
| n = max(floor, min(cap, fits)) | |
| if verbose: | |
| print(f"[gpu] {free/2**30:.1f} GiB free of {total/2**30:.1f} GiB | " | |
| f"KV {kv_per_tok/1024:.0f} KiB/token | {per_row/2**20:.1f} MiB/row | " | |
| f"fits {fits} -> batch {n}", flush=True) | |
| return n | |
| # --------------------------------------------------------------------------- # | |
| # device-resident constraint | |
| # --------------------------------------------------------------------------- # | |
| class GpuConstraint: | |
| """se.Encoder's constraint, evaluated for N rows at once on the device. | |
| Each row carries its own payload position and character position, so the | |
| keyed alphabet differs per row; the whole thing is still one set of batched | |
| gathers. Semantics follow Encoder.constrain exactly -- wrong-bit and junk | |
| tokens to -inf, skip characters priced at `skip_penalty`. | |
| """ | |
| def __init__(self, tok, bits, skip_penalty, key, device, max_pos=4096, | |
| vocab_size=None): | |
| base = se.Encoder(tok, bits, skip_penalty, key) | |
| # must match the model's logits width, which can exceed len(tok) | |
| base._build(int(vocab_size or len(tok))) | |
| self.device, self.nbits = device, len(bits) | |
| self.skip_penalty = float(skip_penalty) | |
| self.max_c = base.max_c | |
| self.junk_np = base.junk | |
| d = device | |
| self.clipped = torch.as_tensor(base._clipped.astype(np.int64)).to(d) # (V,C) | |
| self.letter = torch.as_tensor(base._letter_at).to(d) # (V,C) | |
| self.fixed = torch.as_tensor(base._fixed.astype(np.int64)).to(d) # (V,C) | |
| self.junk = torch.as_tensor(base.junk).to(d) # (V,) | |
| self.bits_pad = torch.as_tensor( | |
| np.append(np.asarray(bits, dtype=np.int64), -1)).to(d) # (nbits+1,) | |
| # position -> letter-role table, precomputed once. _letter_roles is a | |
| # SHA-256 schedule on the CPU; hoisting it out of the step loop is what | |
| # lets a step be pure tensor work. | |
| self.max_pos = max_pos | |
| table = np.stack([se._letter_roles(key, p) for p in range(max_pos)]) # (P,26) | |
| self.pos_role = torch.as_tensor(table.astype(np.int64)).to(d) | |
| self.V = self.clipped.shape[0] | |
| def ensure_pos(self, need: int, key): | |
| if need + self.max_c < self.max_pos: | |
| return | |
| grow = int(need + self.max_c + 1024) | |
| table = np.stack([se._letter_roles(key, p) for p in range(grow)]) | |
| self.pos_role = torch.as_tensor(table.astype(np.int64)).to(self.device) | |
| self.max_pos = grow | |
| def __call__(self, consumed: torch.Tensor, char_pos: torch.Tensor, | |
| logits: torch.Tensor): | |
| """consumed/char_pos: (N,) int64. logits: (N,V) float. | |
| Returns (masked (N,V), bits_enc (N,V) int64).""" | |
| N, V, C = logits.shape[0], self.V, self.max_c | |
| done = consumed >= self.nbits | |
| # (N, C, 26) keyed roles for each row's next C character positions | |
| offs = torch.arange(C, device=self.device).unsqueeze(0) # (1,C) | |
| rows = (char_pos.unsqueeze(1) + offs).clamp_(0, self.max_pos - 1) # (N,C) | |
| rmap = self.pos_role[rows] # (N,C,26) | |
| # letters -> their live role; everything else keeps its build-time role. | |
| # Advanced indexing rather than gather-on-an-expanded-view: the latter | |
| # would broadcast a (N,V,C,26) shape, 26x more elements than the answer. | |
| n_ix = torch.arange(N, device=self.device).view(N, 1, 1) | |
| c_ix = torch.arange(C, device=self.device).view(1, 1, C) | |
| role = rmap[n_ix, c_ix, self.clipped.unsqueeze(0)] # (N,V,C) | |
| role = torch.where(self.letter.unsqueeze(0), role, self.fixed.unsqueeze(0)) | |
| enc = (role == 0) | (role == 1) | |
| cum = torch.cumsum(enc.long(), dim=2) - enc.long() # bit offset | |
| bit_pos = consumed.view(N, 1, 1) + cum | |
| within = enc & (bit_pos < self.nbits) | |
| target = self.bits_pad[bit_pos.clamp(0, self.nbits)] | |
| bad = (within & (role != target)).any(dim=2) | self.junk.unsqueeze(0) | |
| out = logits - self.skip_penalty * (role == se.SKIP).sum(dim=2) | |
| out = out.masked_fill(bad, float("-inf")) | |
| bits_enc = within.sum(dim=2).masked_fill(bad, 0) | |
| # rows whose payload is already spent are unconstrained but still may not | |
| # emit junk -- matching Encoder.constrain's rem <= 0 branch | |
| if done.any(): | |
| free = logits.masked_fill(self.junk.unsqueeze(0), float("-inf")) | |
| out = torch.where(done.view(N, 1), free, out) | |
| bits_enc = torch.where(done.view(N, 1), torch.zeros_like(bits_enc), bits_enc) | |
| return out, bits_enc | |
| # --------------------------------------------------------------------------- # | |
| # sampling, on device | |
| # --------------------------------------------------------------------------- # | |
| def sample_rows(scores, temperature, top_p, gen): | |
| """Nucleus sample one token per row from (N,V) scores containing -inf.""" | |
| if temperature <= 1e-6: | |
| return scores.argmax(dim=-1) | |
| z = scores / temperature | |
| probs = torch.softmax(z.float(), dim=-1) | |
| probs = torch.nan_to_num(probs, nan=0.0) | |
| if 0.0 < top_p < 1.0: | |
| sp, si = torch.sort(probs, dim=-1, descending=True) | |
| keep = (torch.cumsum(sp, dim=-1) - sp) < top_p | |
| keep[..., 0] = True | |
| probs = torch.zeros_like(probs).scatter_(-1, si, sp * keep) | |
| tot = probs.sum(dim=-1, keepdim=True) | |
| probs = torch.where(tot > 0, probs / tot.clamp_min(1e-30), | |
| torch.zeros_like(probs)) | |
| dead = (probs.sum(dim=-1) <= 0) | |
| if dead.any(): # fully cornered row: fall back to argmax | |
| probs[dead] = 0.0 | |
| probs[dead, scores[dead].argmax(dim=-1)] = 1.0 | |
| return torch.multinomial(probs, 1, generator=gen).squeeze(-1) | |
| def antirepeat_rows(scores, hist, freq_penalty, window, no_repeat_ngram): | |
| """Per-row frequency penalty and n-gram ban, applied in place on (N,V).""" | |
| if freq_penalty > 0: | |
| for i, h in enumerate(hist): | |
| if not h: | |
| continue | |
| ids, counts = np.unique(np.asarray(h[-window:]), return_counts=True) | |
| t_ids = torch.as_tensor(ids, device=scores.device, dtype=torch.long) | |
| t_cnt = torch.as_tensor(counts, device=scores.device, dtype=scores.dtype) | |
| row = scores[i] | |
| finite = torch.isfinite(row[t_ids]) | |
| row[t_ids] = torch.where(finite, row[t_ids] - freq_penalty * t_cnt, | |
| row[t_ids]) | |
| n = no_repeat_ngram | |
| if n and n >= 1: | |
| for i, h in enumerate(hist): | |
| if len(h) < n - 1: | |
| continue | |
| prefix = tuple(h[-(n - 1):]) if n > 1 else () | |
| banned = [h[j + n - 1] for j in range(len(h) - n + 1) | |
| if tuple(h[j:j + n - 1]) == prefix] | |
| if banned: | |
| scores[i, torch.as_tensor(sorted(set(banned)), | |
| device=scores.device)] = float("-inf") | |
| return scores | |
| # --------------------------------------------------------------------------- # | |
| # batched sentence generation | |
| # --------------------------------------------------------------------------- # | |
| class _Row: | |
| __slots__ = ("sent", "toks", "consumed", "start", "ex", "done", "eos", | |
| "complete", "logits") | |
| def __init__(self, consumed): | |
| self.sent, self.toks, self.consumed = "", [], consumed | |
| self.start = consumed | |
| self.ex, self.done, self.eos, self.complete = [], False, False, False | |
| self.logits = None | |
| def verdict(self, budget, vocab): | |
| exa = np.array(self.ex) if self.ex else np.array([0.0]) | |
| coined = len(se.coined_words(self.sent, vocab)) | |
| caps = se.register_oddity(self.sent) | |
| shocks = int((exa > se.SHOCK_NATS).sum()) | |
| worst = float(exa.max()) | |
| gained = self.consumed - self.start | |
| # A sentence that carries no payload bit makes no progress, so it cannot | |
| # be "acceptable" however clean it reads. Without this a row that emits a | |
| # bare "." scores zero shocks, zero coined words and a low worst token, | |
| # and therefore wins the round -- best-of-N finds that degenerate optimum | |
| # immediately, where the CPU path hid it by stopping at the first | |
| # acceptable candidate rather than the best one. | |
| ok = ((self.complete or self.eos) and gained > 0 | |
| and budget.verdict(shocks, worst, coined, caps)) | |
| return {"shocks": shocks, "worst": worst, "coined": coined, "caps": caps, | |
| "ok": ok, "n": len(self.toks), "gained": gained} | |
| def generate_batched(model, tok, prompt, bits, *, key=None, batch=8, | |
| temperature=0.9, top_p=0.95, bit_bonus=1.0, skip_penalty=1.0, | |
| max_sentence_chars=200, budget=None, seed=0, device="cuda", | |
| verbose=False, max_rounds=40): | |
| """Encode `bits`, generating `batch` candidate sentences per round in parallel. | |
| One round replaces what the CPU path does in up to `batch` sequential passes. | |
| The winner's KV is kept with batch_select_indices, so no sentence is ever | |
| replayed.""" | |
| from transformers import DynamicCache | |
| budget = budget or se.Budget() | |
| vocab = se._VOCAB or se.load_vocab() | |
| con = GpuConstraint(tok, bits, skip_penalty, key, device, | |
| vocab_size=model.config.vocab_size) | |
| nbits = len(bits) | |
| eos_ids = sorted(se._eos_ids(tok)) | |
| pad_id = eos_ids[0] if eos_ids else 0 | |
| gen = torch.Generator(device=device).manual_seed(int(seed)) | |
| cache = DynamicCache(config=model.config) | |
| if hasattr(cache, "activate_past_recording"): | |
| cache.activate_past_recording() | |
| tokens, offset = list(prompt), len(prompt) | |
| with torch.inference_mode(): | |
| logits1 = model(input_ids=torch.tensor([list(prompt)], device=device), | |
| past_key_values=cache, use_cache=True).logits[:, -1, :] | |
| consumed, stats = 0, {"accepted": 0, "rejected": 0, "fallback": 0, "rounds": 0, | |
| "steps": 0} | |
| eos_t = torch.as_tensor(eos_ids, device=device, dtype=torch.long) if eos_ids else None | |
| while consumed < nbits and stats["rounds"] < max_rounds: | |
| stats["rounds"] += 1 | |
| N = int(batch) | |
| base_chars = len(tok.decode(tokens[offset:])) | |
| con.ensure_pos(base_chars + max_sentence_chars + 8, key) | |
| cache.batch_repeat_interleave(N) | |
| logits = logits1.expand(N, -1).contiguous() | |
| rows = [_Row(consumed) for _ in range(N)] | |
| steps = 0 | |
| max_steps = max(8, max_sentence_chars) # chars >= tokens, so this bounds it | |
| with torch.inference_mode(): | |
| while steps < max_steps and not all(r.done for r in rows): | |
| raw = logits.float() | |
| lse = torch.logsumexp(raw, dim=-1) | |
| logp = raw - lse.unsqueeze(-1) | |
| H = -(logp.exp() * logp).sum(dim=-1) | |
| cons_t = torch.tensor([r.consumed for r in rows], device=device) | |
| cpos_t = torch.tensor([base_chars + len(r.sent) for r in rows], | |
| device=device) | |
| masked, bits_enc = con(cons_t, cpos_t, raw) | |
| scores = antirepeat_rows(masked.clone(), | |
| [tokens[offset:] + r.toks for r in rows], | |
| freq_penalty=0.5, window=64, | |
| no_repeat_ngram=3) | |
| scores = scores + bit_bonus * bits_enc.to(scores.dtype) | |
| if eos_t is not None: # never stop mid-payload | |
| mid = (cons_t < nbits) | |
| if mid.any(): | |
| blk = scores[mid] | |
| blk[:, eos_t] = float("-inf") | |
| scores[mid] = blk | |
| choice = sample_rows(scores, temperature, top_p, gen) | |
| ex = (lse - raw.gather(1, choice.unsqueeze(1)).squeeze(1)) - H | |
| gained = bits_enc.gather(1, choice.unsqueeze(1)).squeeze(1) | |
| ch = choice.tolist() | |
| exl = ex.tolist() | |
| gl = gained.tolist() | |
| # Whether a row was finished BEFORE this step. A row that becomes | |
| # done *during* this step still appended a token, and that token | |
| # must be fed like any other or the cache ends up holding a pad | |
| # where the sentence's last character belongs. | |
| was_done = [r.done for r in rows] | |
| appended = [] | |
| for i, r in enumerate(rows): | |
| if was_done[i]: | |
| continue | |
| if ch[i] in eos_ids: | |
| r.eos = r.done = r.complete = True | |
| continue | |
| r.ex.append(exl[i]) | |
| r.toks.append(ch[i]) | |
| r.sent += tok.decode([ch[i]]) | |
| r.consumed += int(gl[i]) | |
| appended.append(i) | |
| if exl[i] > budget.hard_abort: | |
| r.done = True # aborted: not complete | |
| elif ((se._sentence_done(r.sent) | |
| and se._can_end(r.sent, len(r.toks))) | |
| or len(r.sent) >= max_sentence_chars): | |
| r.complete = r.done = True | |
| # Feed each row's own token; only rows that were already finished | |
| # (or that just chose EOS, which they do not keep) get the pad. | |
| live = torch.zeros(N, dtype=torch.bool, device=device) | |
| if appended: | |
| live[torch.tensor(appended, device=device)] = True | |
| feed = torch.where(live, choice, | |
| torch.full_like(choice, pad_id)).unsqueeze(1) | |
| logits = model(input_ids=feed, past_key_values=cache, | |
| use_cache=True).logits[:, -1, :] | |
| # The next round continues from the distribution that FOLLOWS the | |
| # committed token, so this has to be captured after the forward. | |
| # Capturing it before is an off-by-one: the round would resume by | |
| # re-predicting the token it just emitted, which shows up as | |
| # lowercase sentence starts and doubled punctuation at the seam. | |
| for i in appended: | |
| rows[i].logits = logits[i] | |
| steps += 1 | |
| stats["steps"] += steps | |
| # ---- pick the winner, exactly as the CPU path ranks candidates -------- | |
| vs = [r.verdict(budget, vocab) for r in rows] | |
| accepted = [i for i, v in enumerate(vs) if v["ok"] and rows[i].toks] | |
| def rank_fallback(i): | |
| v, r = vs[i], rows[i] | |
| return (not (r.complete or r.eos), v["gained"] == 0, | |
| v["caps"] > budget.max_caps, v["coined"], v["shocks"], | |
| v["worst"], -v["gained"]) | |
| def rank_accepted(i): | |
| # every one of these already cleared the quality budget, so choose on | |
| # progress first and use quality only to break ties | |
| v = vs[i] | |
| return (-v["gained"], v["coined"], v["shocks"], v["worst"]) | |
| if accepted: | |
| w = min(accepted, key=rank_accepted); stats["accepted"] += 1 | |
| else: | |
| w = min(range(N), key=rank_fallback); stats["fallback"] += 1 | |
| stats["rejected"] += N - 1 | |
| cache.batch_select_indices(torch.tensor([w], device=device)) | |
| surplus = steps - len(rows[w].toks) | |
| if surplus > 0: | |
| cache.crop(-surplus) | |
| if verbose: | |
| v = vs[w] | |
| print(f" [{'ok ' if v['ok'] else 'FB '}] shocks={v['shocks']} " | |
| f"worst={v['worst']:5.1f} coined={v['coined']} " | |
| f"+{rows[w].consumed - consumed}b {rows[w].sent.strip()[:60]!r}", | |
| flush=True) | |
| if not rows[w].toks: | |
| break | |
| tokens.extend(rows[w].toks) | |
| consumed = rows[w].consumed | |
| logits1 = rows[w].logits.unsqueeze(0) | |
| if rows[w].eos: | |
| break | |
| return tok.decode(tokens[offset:]), stats | |