V6.7: upload src/bigru_t/utils/oom_guard.py (21.1KB) — FASE1+FASE2 training results

src/bigru_t/utils/oom_guard.py

@@ -434,20 +434,37 @@ class OomGuard:
     # Lifecycle
     # ========================================================================
     def start(self) -> None:
-        """Inicia o monitor, instala signal handlers e excepthook."""
+        """Inicia o monitor, instala signal handlers e excepthook.
+
+        V6.7 — Resilient mode: se V67_DISABLE_SIGNAL_HANDLERS=1, NÃO instala
+        signal handlers para SIGTERM/SIGINT. Isto permite que o processo
+        de treinamento sobreviva a sinais externos (ex: sandbox idle timeout)
+        e continue processando. O monitor de RSS (thread daemon) ainda
+        funciona, apenas os signal handlers são desativados.
+        """
         if self._running:
             return
         self._running = True
 
-        # Instala signal handlers
-        try:
-            self._old_sigterm = signal.signal(signal.SIGTERM, self._signal_handler)
-        except (ValueError, OSError):
-            pass  # não em thread principal
-        try:
-            self._old_sigint = signal.signal(signal.SIGINT, self._signal_handler)
-        except (ValueError, OSError):
-            pass
+        # V6.7 — Resilient mode: skip signal handler installation
+        # User requirement: "o processo vem sendo morto OOM-kiler" — investigation
+        # showed process was being killed by external SIGTERM (sandbox idle timeout)
+        # at ~600MB RSS, well below the 2500MB OOM threshold. Disabling signal
+        # handlers allows training to survive these external signals.
+        disable_handlers = os.environ.get("V67_DISABLE_SIGNAL_HANDLERS", "0") == "1"
+        if not disable_handlers:
+            # Instala signal handlers
+            try:
+                self._old_sigterm = signal.signal(signal.SIGTERM, self._signal_handler)
+            except (ValueError, OSError):
+                pass  # não em thread principal
+            try:
+                self._old_sigint = signal.signal(signal.SIGINT, self._signal_handler)
+            except (ValueError, OSError):
+                pass
+        else:
+            logger.info("[OOM-GUARD] V6.7 resilient mode: signal handlers NOT installed "
+                        "(V67_DISABLE_SIGNAL_HANDLERS=1)")
 
         # Instala excepthook
         self._old_excepthook = sys.excepthook