Spaces:
Running on Zero
Running on Zero
Fix batched seam: feed each row's own final token, and capture continuation logits after the forward pass
Browse files- gpu_encode.py +22 -5
gpu_encode.py
CHANGED
|
@@ -320,10 +320,15 @@ def generate_batched(model, tok, prompt, bits, *, key=None, batch=8,
|
|
| 320 |
ch = choice.tolist()
|
| 321 |
exl = ex.tolist()
|
| 322 |
gl = gained.tolist()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 323 |
for i, r in enumerate(rows):
|
| 324 |
-
if
|
| 325 |
continue
|
| 326 |
-
r.logits = logits[i]
|
| 327 |
if ch[i] in eos_ids:
|
| 328 |
r.eos = r.done = r.complete = True
|
| 329 |
continue
|
|
@@ -331,6 +336,7 @@ def generate_batched(model, tok, prompt, bits, *, key=None, batch=8,
|
|
| 331 |
r.toks.append(ch[i])
|
| 332 |
r.sent += tok.decode([ch[i]])
|
| 333 |
r.consumed += int(gl[i])
|
|
|
|
| 334 |
if exl[i] > budget.hard_abort:
|
| 335 |
r.done = True # aborted: not complete
|
| 336 |
elif ((se._sentence_done(r.sent)
|
|
@@ -338,11 +344,22 @@ def generate_batched(model, tok, prompt, bits, *, key=None, batch=8,
|
|
| 338 |
or len(r.sent) >= max_sentence_chars):
|
| 339 |
r.complete = r.done = True
|
| 340 |
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 344 |
logits = model(input_ids=feed, past_key_values=cache,
|
| 345 |
use_cache=True).logits[:, -1, :]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 346 |
steps += 1
|
| 347 |
stats["steps"] += steps
|
| 348 |
|
|
|
|
| 320 |
ch = choice.tolist()
|
| 321 |
exl = ex.tolist()
|
| 322 |
gl = gained.tolist()
|
| 323 |
+
# Whether a row was finished BEFORE this step. A row that becomes
|
| 324 |
+
# done *during* this step still appended a token, and that token
|
| 325 |
+
# must be fed like any other or the cache ends up holding a pad
|
| 326 |
+
# where the sentence's last character belongs.
|
| 327 |
+
was_done = [r.done for r in rows]
|
| 328 |
+
appended = []
|
| 329 |
for i, r in enumerate(rows):
|
| 330 |
+
if was_done[i]:
|
| 331 |
continue
|
|
|
|
| 332 |
if ch[i] in eos_ids:
|
| 333 |
r.eos = r.done = r.complete = True
|
| 334 |
continue
|
|
|
|
| 336 |
r.toks.append(ch[i])
|
| 337 |
r.sent += tok.decode([ch[i]])
|
| 338 |
r.consumed += int(gl[i])
|
| 339 |
+
appended.append(i)
|
| 340 |
if exl[i] > budget.hard_abort:
|
| 341 |
r.done = True # aborted: not complete
|
| 342 |
elif ((se._sentence_done(r.sent)
|
|
|
|
| 344 |
or len(r.sent) >= max_sentence_chars):
|
| 345 |
r.complete = r.done = True
|
| 346 |
|
| 347 |
+
# Feed each row's own token; only rows that were already finished
|
| 348 |
+
# (or that just chose EOS, which they do not keep) get the pad.
|
| 349 |
+
live = torch.zeros(N, dtype=torch.bool, device=device)
|
| 350 |
+
if appended:
|
| 351 |
+
live[torch.tensor(appended, device=device)] = True
|
| 352 |
+
feed = torch.where(live, choice,
|
| 353 |
+
torch.full_like(choice, pad_id)).unsqueeze(1)
|
| 354 |
logits = model(input_ids=feed, past_key_values=cache,
|
| 355 |
use_cache=True).logits[:, -1, :]
|
| 356 |
+
# The next round continues from the distribution that FOLLOWS the
|
| 357 |
+
# committed token, so this has to be captured after the forward.
|
| 358 |
+
# Capturing it before is an off-by-one: the round would resume by
|
| 359 |
+
# re-predicting the token it just emitted, which shows up as
|
| 360 |
+
# lowercase sentence starts and doubled punctuation at the seam.
|
| 361 |
+
for i in appended:
|
| 362 |
+
rows[i].logits = logits[i]
|
| 363 |
steps += 1
|
| 364 |
stats["steps"] += steps
|
| 365 |
|