infiniteviolets commited on
Commit
a8e4e2b
·
verified ·
1 Parent(s): 61e06ea

Fix batched seam: feed each row's own final token, and capture continuation logits after the forward pass

Browse files
Files changed (1) hide show
  1. gpu_encode.py +22 -5
gpu_encode.py CHANGED
@@ -320,10 +320,15 @@ def generate_batched(model, tok, prompt, bits, *, key=None, batch=8,
320
  ch = choice.tolist()
321
  exl = ex.tolist()
322
  gl = gained.tolist()
 
 
 
 
 
 
323
  for i, r in enumerate(rows):
324
- if r.done:
325
  continue
326
- r.logits = logits[i]
327
  if ch[i] in eos_ids:
328
  r.eos = r.done = r.complete = True
329
  continue
@@ -331,6 +336,7 @@ def generate_batched(model, tok, prompt, bits, *, key=None, batch=8,
331
  r.toks.append(ch[i])
332
  r.sent += tok.decode([ch[i]])
333
  r.consumed += int(gl[i])
 
334
  if exl[i] > budget.hard_abort:
335
  r.done = True # aborted: not complete
336
  elif ((se._sentence_done(r.sent)
@@ -338,11 +344,22 @@ def generate_batched(model, tok, prompt, bits, *, key=None, batch=8,
338
  or len(r.sent) >= max_sentence_chars):
339
  r.complete = r.done = True
340
 
341
- feed = torch.where(
342
- torch.tensor([r.done for r in rows], device=device),
343
- torch.full_like(choice, pad_id), choice).unsqueeze(1)
 
 
 
 
344
  logits = model(input_ids=feed, past_key_values=cache,
345
  use_cache=True).logits[:, -1, :]
 
 
 
 
 
 
 
346
  steps += 1
347
  stats["steps"] += steps
348
 
 
320
  ch = choice.tolist()
321
  exl = ex.tolist()
322
  gl = gained.tolist()
323
+ # Whether a row was finished BEFORE this step. A row that becomes
324
+ # done *during* this step still appended a token, and that token
325
+ # must be fed like any other or the cache ends up holding a pad
326
+ # where the sentence's last character belongs.
327
+ was_done = [r.done for r in rows]
328
+ appended = []
329
  for i, r in enumerate(rows):
330
+ if was_done[i]:
331
  continue
 
332
  if ch[i] in eos_ids:
333
  r.eos = r.done = r.complete = True
334
  continue
 
336
  r.toks.append(ch[i])
337
  r.sent += tok.decode([ch[i]])
338
  r.consumed += int(gl[i])
339
+ appended.append(i)
340
  if exl[i] > budget.hard_abort:
341
  r.done = True # aborted: not complete
342
  elif ((se._sentence_done(r.sent)
 
344
  or len(r.sent) >= max_sentence_chars):
345
  r.complete = r.done = True
346
 
347
+ # Feed each row's own token; only rows that were already finished
348
+ # (or that just chose EOS, which they do not keep) get the pad.
349
+ live = torch.zeros(N, dtype=torch.bool, device=device)
350
+ if appended:
351
+ live[torch.tensor(appended, device=device)] = True
352
+ feed = torch.where(live, choice,
353
+ torch.full_like(choice, pad_id)).unsqueeze(1)
354
  logits = model(input_ids=feed, past_key_values=cache,
355
  use_cache=True).logits[:, -1, :]
356
+ # The next round continues from the distribution that FOLLOWS the
357
+ # committed token, so this has to be captured after the forward.
358
+ # Capturing it before is an off-by-one: the round would resume by
359
+ # re-predicting the token it just emitted, which shows up as
360
+ # lowercase sentence starts and doubled punctuation at the seam.
361
+ for i in appended:
362
+ rows[i].logits = logits[i]
363
  steps += 1
364
  stats["steps"] += steps
365