"""Definitive fix for "treo session" (hung/stuck session) while executing a task. Root causes (all verified against current upstream smolagents/ml-intern: agent_loop.py, local_tools.py, session_manager.py, session.py): 1. BLOCKING SUBPROCESS (#1 mid-task freeze). agent/tools/local_tools.py `_bash_handler` runs `subprocess.run(...)` synchronously inside an async handler. Because the whole backend is a single asyncio event loop, ANY bash command (pip install, a training run, a long shell pipeline) freezes the ENTIRE loop: every session's agent loop, the SSE fan-out, the reaper, and the cancellation path all stop being scheduled. The frontend sees keepalives but zero events, so it shows "Processing…" for hours. MAX_TIMEOUT=36000 (10 h) makes it worse. Fix: run bash via asyncio.create_subprocess_shell so it never blocks the loop, cap MAX_TIMEOUT to a sane 1800 s, and kill the process group on either cancel (Stop) or timeout so no orphan keeps running. 2. NO TURN-LEVEL WATCHDOG. backend/session_manager.py `_run_session` awaits `process_submission` directly with no timeout. Combined with bep40's patch_lossless_context setting max_iterations=-1 (unlimited), a turn can run forever (doom loop, unkillable tool, connection hang). When a turn hangs, is_processing stays True and the idle reaper REFUSES to reap it -> permanent zombie. Fix: wrap process_submission in asyncio.wait with a turn timeout (1 h). On timeout, cancel the turn, send `interrupted` + `turn_complete` terminal SSE events (so the frontend un-freezes), clear the cancel flag so the session stays usable, and let it continue on the next message. Design notes / safety: - Does NOT change max_iterations semantics. It only adds a hard safety net. - Does NOT change the user's task, model, dataset, or training approach. - Idempotent: every mutation is guarded by a marker check. - Runs in the Python build stage AFTER `uv sync` (needs agent/backend present), NEXT TO the other backend patches (patch_lossless_context etc.). """ import ast import os LOCAL_TOOLS = "/app/agent/tools/local_tools.py" SESSION_MANAGER = "/app/backend/session_manager.py" M1 = "ML_INTERN_TURN_WATCHDOG" M2 = "ML_INTERN_TO_THREAD_BASH" M3 = "_cancelled_ok_helper" errors = [] NEW_BASH_HANDLER = '''async def _bash_handler( args: dict[str, Any], session: Any = None, **_kw ) -> tuple[str, bool]: """Run a shell command WITHOUT blocking the event loop. # ML_INTERN_TO_THREAD_BASH Uses asyncio subprocess + a single wait guard so a long/hung command never freezes the whole backend (every session's loop + SSE). On cancel (Stop) or timeout the process group is killed so nothing keeps running silently. """ command = args.get("command", "") if not command: return "No command provided.", False command = wrap_shell_command_with_hub_artifact_bootstrap(command, session) work_dir = args.get("work_dir", ".") timeout = min(args.get("timeout") or DEFAULT_TIMEOUT, MAX_TIMEOUT) cancelled = asyncio.Event() if session is not None: inner = getattr(session, "_cancelled", None) if inner is not None and hasattr(inner, "add_done_callback"): inner.add_done_callback(lambda _e, ev=cancelled: ev.set()) async def _run() -> tuple[str, bool]: # create_subprocess with start_new_session so we can kill the whole # process group on cancel/timeout. try: proc = await asyncio.create_subprocess_shell( command, cwd=work_dir or ".", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, start_new_session=True, ) except Exception as e: return f"bash error (spawn): {e}", False cancel_task = asyncio.ensure_future(cancelled.wait()) def _kill_group() -> None: try: os.killpg(os.getpgid(proc.pid), 9) except Exception: try: proc.kill() except Exception: pass # communicate() both waits for the process and reads its output, so we # use it as the single "process finished" future. comm_task = asyncio.ensure_future(proc.communicate()) # One single guard: wait for process completion OR cancellation, with the # hard per-call deadline applied directly here. asyncio.wait with a # timeout returns (done, pending) and does NOT raise on timeout. done, pending = await asyncio.wait( [comm_task, cancel_task], timeout=timeout, return_when=asyncio.FIRST_COMPLETED, ) for t in pending: t.cancel() if comm_task not in done: # Either cancelled, or genuinely timed out -> kill + report. comm_task.cancel() _kill_group() reason = ( "cancelled" if cancel_task in done else f"timed out after {timeout}s" ) return ( f"Command {reason} and was killed.\\n\\n" "For long-running commands, run in the background:\\n" " nohup > /tmp/output.log 2>&1 & echo $!\\n" "Then poll:\\n" " kill -0 2>/dev/null && echo 'running' || echo 'done'\\n" " tail -n 50 /tmp/output.log" ), False # Process finished naturally -> read results from the comhpleted future. cancel_task.cancel() try: stdout, stderr = comm_task.result() except Exception: stdout, stderr = b"", b"" raw = stdout.decode("utf-8", errors="replace") + stderr.decode( "utf-8", errors="replace" ) output = _strip_ansi(raw) output = _truncate_output(output) if not output.strip(): output = "(no output)" return output, proc.returncode == 0 if proc.returncode is not None else False try: return await _run() except (asyncio.TimeoutError, asyncio.CancelledError): return "Command was cancelled/interrupted.", False except Exception as e: return f"bash error: {e}", False ''' OLD_BASH_HANDLER = '''async def _bash_handler( args: dict[str, Any], session: Any = None, **_kw ) -> tuple[str, bool]: command = args.get("command", "") if not command: return "No command provided.", False command = wrap_shell_command_with_hub_artifact_bootstrap(command, session) work_dir = args.get("work_dir", ".") timeout = min(args.get("timeout") or DEFAULT_TIMEOUT, MAX_TIMEOUT) try: result = subprocess.run( command, shell=True, capture_output=True, text=True, cwd=work_dir, timeout=timeout, ) output = _strip_ansi(result.stdout + result.stderr) output = _truncate_output(output) if not output.strip(): output = "(no output)" return output, result.returncode == 0 except subprocess.TimeoutExpired: return ( f"Command timed out after {timeout}s and was killed.\\n\\n" f"For long-running commands, run in the background and poll:\\n" f" nohup > /tmp/output.log 2>&1 & echo $!\\n" f"Then check status with:\\n" f" kill -0 2>/dev/null && echo 'running' || echo 'done'\\n" f" tail -n 50 /tmp/output.log" ), False except Exception as e: return f"bash error: {e}", False ''' # ========================================================================= # PART 1: local_tools.py -> non-blocking, cancellation-aware bash # ========================================================================= if os.path.exists(LOCAL_TOOLS): with open(LOCAL_TOOLS) as f: c = f.read() if M2 in c: print("OK: local_tools.py already patched (idempotent)") else: if "import asyncio" not in c: c = c.replace( "import subprocess\n", "import asyncio\nimport subprocess\n", 1, ) c = c.replace( "MAX_TIMEOUT = 36000 # 10 hours — needed for long training runs (e.g. PostTrainBench)", "MAX_TIMEOUT = 1800 # 30 min hard cap; long jobs must run in background (ML Intern)", ) if OLD_BASH_HANDLER in c: c = c.replace(OLD_BASH_HANDLER, NEW_BASH_HANDLER, 1) print("OK: _bash_handler replaced with non-blocking cancellable version") else: start = c.find("async def _bash_handler(") end = c.find("async def _read_handler(", start) if start >= 0 and end > start: c = c[:start] + NEW_BASH_HANDLER + "\n\n" + c[end:] print("OK: _bash_handler replaced via region fallback") else: print("FAIL: could not locate _bash_handler in local_tools.py") errors.append("local_tools _bash_handler") with open(LOCAL_TOOLS, "w") as f: f.write(c) try: ast.parse(c) print("OK: local_tools.py syntax OK") except SyntaxError as e: print(f"FAIL: local_tools.py syntax: {e}") errors.append("local_tools syntax") # ========================================================================= # PART 2: session_manager.py -> turn-level watchdog # ========================================================================= if os.path.exists(SESSION_MANAGER): with open(SESSION_MANAGER) as f: c = f.read() if M1 in c: print("OK: session_manager.py already patched (idempotent)") else: # 2a. Add turn-timeout constant before DEFAULT_CONFIG_PATH. anchor = "DEFAULT_CONFIG_PATH = str(PROJECT_ROOT" const_block = ( "# ML_INTERN_TURN_WATCHDOG\n" "TURN_TIMEOUT_S = 3600 # hard per-turn safety net (1 h).\n" " # A single turn (LLM + tools) must never run forever.\n\n" ) if anchor in c and "TURN_TIMEOUT_S" not in c: c = c.replace(anchor, const_block + anchor, 1) print("OK: added TURN_TIMEOUT_S constant") else: print("FAIL: could not find DEFAULT_CONFIG_PATH anchor in session_manager.py") errors.append("session_manager constant") old_block = ''' agent_session.is_processing = True self._touch(agent_session) try: should_continue = await process_submission( session, submission ) finally: agent_session.is_processing = False # Stamp on turn finish too: a turn that ran longer # than the idle window would otherwise be reaped the # instant it completes. self._touch(agent_session) if session.config.save_sessions: await self.refresh_session_usage_metrics(agent_session) await self.persist_session_snapshot(agent_session)''' new_block = ''' agent_session.is_processing = True self._touch(agent_session) try: # ML_INTERN_TURN_WATCHDOG — hard per-turn deadline so # a hung turn can never leave the session stuck. turn_task = asyncio.create_task( process_submission(session, submission) ) done, _pending = await asyncio.wait( {turn_task}, timeout=TURN_TIMEOUT_S ) if not done: # Watchdog fired: force-cancel the stuck turn. turn_task.cancel() try: await turn_task except asyncio.CancelledError: pass # Clear the cancel flag so the session stays # usable for the NEXT turn (not a permanent # zombie). Only when a resettable Event exists. if _cancelled_ok(session): session._cancelled.clear() # Terminal SSE events -> frontend un-freezes. await session.send_event( Event( event_type="interrupted", data={ "error": ( "Single turn exceeded the " f"{TURN_TIMEOUT_S}s safety limit " "and was stopped." ) }, ) ) await session.send_event( Event( event_type="turn_complete", data={ "message": ( "Turn was auto-stopped after " f"{TURN_TIMEOUT_S}s. The task may " "need smaller steps or background " "work." ), "auto_stopped": True, }, ) ) should_continue = True else: should_continue = turn_task.result() finally: agent_session.is_processing = False # Stamp on turn finish too: a turn that ran longer # than the idle window would otherwise be reaped the # instant it completes. self._touch(agent_session) if session.config.save_sessions: await self.refresh_session_usage_metrics(agent_session) await self.persist_session_snapshot(agent_session)''' if old_block in c: c = c.replace(old_block, new_block, 1) print("OK: wrapped process_submission in turn watchdog") else: print("FAIL: could not find process_submission block in session_manager.py") idx = c.find("should_continue = await process_submission") if idx >= 0: print("DEBUG context:") print(c[idx - 400 : idx + 200]) errors.append("session_manager watchdog") with open(SESSION_MANAGER, "w") as f: f.write(c) try: ast.parse(c) print("OK: session_manager.py syntax OK") except SyntaxError as e: print(f"FAIL: session_manager.py syntax: {e}") errors.append("session_manager syntax") # ========================================================================= # PART 3: helper for the watchdog (clear resettable cancel flag) # ========================================================================= def ensure_helper() -> None: if not os.path.exists(SESSION_MANAGER): return with open(SESSION_MANAGER) as f: c = f.read() if M3 in c: print("OK: _cancelled_ok helper already present") return anchor = "\n\nlogger = logging.getLogger(__name__)\n" helper = ( "\n\ndef _cancelled_ok(session) -> bool: # _cancelled_ok_helper\n" " # ML_INTERN_TURN_WATCHDOG helper — True if session has a resettable\n" " # _cancelled asyncio.Event (used to clear stop-state after an\n" " # auto-stopped turn so the session stays usable).\n" " e = getattr(session, \"_cancelled\", None)\n" " return e is not None and hasattr(e, \"clear\")\n\n" ) if anchor in c and "_cancelled_ok(session)" in c: c = c.replace(anchor, helper + anchor, 1) with open(SESSION_MANAGER, "w") as f: f.write(c) try: ast.parse(c) print("OK: added _cancelled_ok helper; session_manager.py syntax OK") except SyntaxError as e: print(f"FAIL: session_manager.py syntax after helper: {e}") errors.append("helper syntax") else: print("FAIL: could not place _cancelled_ok helper") errors.append("helper placement") ensure_helper() if errors: print(f"PATCH FAILED: {errors}") raise SystemExit(1) print("DONE: session-hang fix applied (non-blocking bash + turn watchdog)")