Re-check FTZ/DAZ before every optimizer step (2026-10-20)

pin_fp_mode ran once at start; a library imported mid-run can switch flush-to-zero /
denormals-are-zero back on. fit now checks before each opt.step, re-pins, and names the
rank and step. Verified: final parameter hashes unchanged vs 63780b7 on cuda+cpu and cpu+cpu.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

daisychain/cluster.py

@@ -42,6 +42,29 @@ def pin_fp_mode() -> None:
     torch.set_flush_denormal(False)
 
 
+def fp_mode_ok() -> bool:
+    """True if this thread keeps subnormals: FTZ off (a normal*normal product may be subnormal)
+    and DAZ off (a subnormal input is read as itself)."""
+    ftz = torch.tensor([1e-20], dtype=torch.float32) * torch.tensor([1e-20], dtype=torch.float32)
+    daz = torch.tensor([1e-39], dtype=torch.float32) * 1.0
+    return float(ftz) != 0.0 and float(daz) != 0.0
+
+
+def repin_fp_mode(step: int, rank: int) -> bool:
+    """Re-check the FP mode before an optimizer step; re-pin and say so if it was flipped.
+
+    pin_fp_mode runs once at start, but a library imported mid-run (a dataloader worker's
+    dependency, a fast-math extension) can switch FTZ/DAZ on afterwards. The emulator restores
+    MXCSR around EVERY guest call for this reason; here the replica hash would catch the fork
+    later with nothing naming the cause. Returns True if the mode had been flipped."""
+    if fp_mode_ok():
+        return False
+    pin_fp_mode()
+    print("  [fp] rank %d step %d: flush-to-zero/denormals-are-zero was switched ON since the "
+          "last step (a library changed MXCSR); re-pinned subnormals ON" % (rank, step), flush=True)
+    return True
+
+
 def _cuda_usable() -> bool:
     """is_available() alone is not enough: with CUDA_VISIBLE_DEVICES set but EMPTY (a common way
     to say "CPU only") torch reports available=True and device_count()=0, and the node crashed
@@ -314,6 +337,7 @@ class DaisyCluster:
             loss.backward()
             capacity_weighted_allreduce_grads(model, w)
             check_finite_grads(model, s)
+            repin_fp_mode(s, self.rank)                # before the step: the update is where FTZ forks
             if master is not None:
                 with torch.no_grad():
                     for mp, p in zip(master, params):

test_cluster_reductions.py

@@ -112,6 +112,16 @@ def main():
     caps, unit = plan_capacities([{"cores": 8, "capacity": 8.0}, {"cores": 8, "capacity": 150.0}])
     ck("old-format surveys (no unit) are classified, and mixed -> cores", caps == [8.0, 8.0], caps)
 
+    print("repin_fp_mode -- FTZ switched on mid-run is noticed and undone (2026-10-20)")
+    from daisychain.cluster import fp_mode_ok, pin_fp_mode, repin_fp_mode
+    pin_fp_mode()
+    ck("subnormals kept after pin_fp_mode", fp_mode_ok())
+    ck("no flip: repin reports nothing", repin_fp_mode(0, 0) is False)
+    torch.set_flush_denormal(True)                    # what a fast-math library does on load
+    ck("fp_mode_ok sees the flip", not fp_mode_ok())
+    ck("repin reports the flip", repin_fp_mode(7, 0) is True)
+    ck("...and subnormals are back", fp_mode_ok())
+
     print()
     if FAILURES:
         print("FAILED: %d" % len(FAILURES))