preflight: P2 fast-fails after the first dead cell (each repeat bills a GPU session)
Browse files- train/preflight.py +8 -0
train/preflight.py
CHANGED
|
@@ -326,6 +326,14 @@ def p2_throughput(args, R):
|
|
| 326 |
cell["tail"] = r["out"][-1500:]
|
| 327 |
cell["err"] = r["err"][-800:]
|
| 328 |
out[f"L{layers}_s{seq}"] = cell
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 329 |
R["P2"] = out
|
| 330 |
got = [c for c in out.values() if isinstance(c.get("tok_per_s"), (int, float))]
|
| 331 |
# Pass means all four cells ran AND the frozen cell (22L/seq2048) is reported, because that is the
|
|
|
|
| 326 |
cell["tail"] = r["out"][-1500:]
|
| 327 |
cell["err"] = r["err"][-800:]
|
| 328 |
out[f"L{layers}_s{seq}"] = cell
|
| 329 |
+
if r["rc"] != 0:
|
| 330 |
+
# Fast-fail: a systematic defect (import, OOM, batch-shape) repeats identically in all
|
| 331 |
+
# four cells, and each repetition bills a GPU session. Stop at the first one, report it,
|
| 332 |
+
# fix, then re-run. Cheapest possible way to be wrong.
|
| 333 |
+
R["P2_aborted_after"] = f"L{layers}_s{seq}"
|
| 334 |
+
break
|
| 335 |
+
if R.get("P2_aborted_after"):
|
| 336 |
+
break
|
| 337 |
R["P2"] = out
|
| 338 |
got = [c for c in out.values() if isinstance(c.get("tok_per_s"), (int, float))]
|
| 339 |
# Pass means all four cells ran AND the frozen cell (22L/seq2048) is reported, because that is the
|