Cion-lab commited on
Commit
51eb041
·
verified ·
1 Parent(s): 7aab8c2

preflight: P2 fast-fails after the first dead cell (each repeat bills a GPU session)

Browse files
Files changed (1) hide show
  1. train/preflight.py +8 -0
train/preflight.py CHANGED
@@ -326,6 +326,14 @@ def p2_throughput(args, R):
326
  cell["tail"] = r["out"][-1500:]
327
  cell["err"] = r["err"][-800:]
328
  out[f"L{layers}_s{seq}"] = cell
 
 
 
 
 
 
 
 
329
  R["P2"] = out
330
  got = [c for c in out.values() if isinstance(c.get("tok_per_s"), (int, float))]
331
  # Pass means all four cells ran AND the frozen cell (22L/seq2048) is reported, because that is the
 
326
  cell["tail"] = r["out"][-1500:]
327
  cell["err"] = r["err"][-800:]
328
  out[f"L{layers}_s{seq}"] = cell
329
+ if r["rc"] != 0:
330
+ # Fast-fail: a systematic defect (import, OOM, batch-shape) repeats identically in all
331
+ # four cells, and each repetition bills a GPU session. Stop at the first one, report it,
332
+ # fix, then re-run. Cheapest possible way to be wrong.
333
+ R["P2_aborted_after"] = f"L{layers}_s{seq}"
334
+ break
335
+ if R.get("P2_aborted_after"):
336
+ break
337
  R["P2"] = out
338
  got = [c for c in out.values() if isinstance(c.get("tok_per_s"), (int, float))]
339
  # Pass means all four cells ran AND the frozen cell (22L/seq2048) is reported, because that is the