Spaces:
Running
Running
Download patch_fix_session_hang.py from bep40/ml-intern: direct link, hf CLI and curl.
- Browser
- Download file 17.7 kB
-
https://huggingface.co/spaces/bep40/ml-intern/resolve/main/patch_fix_session_hang.py
- Command line
-
hf download hf://spaces/bep40/ml-intern/patch_fix_session_hang.py
-
curl -L -o patch_fix_session_hang.py https://huggingface.co/spaces/bep40/ml-intern/resolve/main/patch_fix_session_hang.py
17.7 kB
| """Definitive fix for "treo session" (hung/stuck session) while executing a task. | |
| Root causes (all verified against current upstream smolagents/ml-intern: | |
| agent_loop.py, local_tools.py, session_manager.py, session.py): | |
| 1. BLOCKING SUBPROCESS (#1 mid-task freeze). | |
| agent/tools/local_tools.py `_bash_handler` runs `subprocess.run(...)` | |
| synchronously inside an async handler. Because the whole backend is a | |
| single asyncio event loop, ANY bash command (pip install, a training run, | |
| a long shell pipeline) freezes the ENTIRE loop: every session's agent loop, | |
| the SSE fan-out, the reaper, and the cancellation path all stop being | |
| scheduled. The frontend sees keepalives but zero events, so it shows | |
| "Processing…" for hours. MAX_TIMEOUT=36000 (10 h) makes it worse. | |
| Fix: run bash via asyncio.create_subprocess_shell so it never blocks the | |
| loop, cap MAX_TIMEOUT to a sane 1800 s, and kill the process group on | |
| either cancel (Stop) or timeout so no orphan keeps running. | |
| 2. NO TURN-LEVEL WATCHDOG. | |
| backend/session_manager.py `_run_session` awaits `process_submission` | |
| directly with no timeout. Combined with bep40's patch_lossless_context | |
| setting max_iterations=-1 (unlimited), a turn can run forever (doom loop, | |
| unkillable tool, connection hang). When a turn hangs, is_processing stays | |
| True and the idle reaper REFUSES to reap it -> permanent zombie. | |
| Fix: wrap process_submission in asyncio.wait with a turn timeout (1 h). On | |
| timeout, cancel the turn, send `interrupted` + `turn_complete` terminal SSE | |
| events (so the frontend un-freezes), clear the cancel flag so the session | |
| stays usable, and let it continue on the next message. | |
| Design notes / safety: | |
| - Does NOT change max_iterations semantics. It only adds a hard safety net. | |
| - Does NOT change the user's task, model, dataset, or training approach. | |
| - Idempotent: every mutation is guarded by a marker check. | |
| - Runs in the Python build stage AFTER `uv sync` (needs agent/backend present), | |
| NEXT TO the other backend patches (patch_lossless_context etc.). | |
| """ | |
| import ast | |
| import os | |
| LOCAL_TOOLS = "/app/agent/tools/local_tools.py" | |
| SESSION_MANAGER = "/app/backend/session_manager.py" | |
| M1 = "ML_INTERN_TURN_WATCHDOG" | |
| M2 = "ML_INTERN_TO_THREAD_BASH" | |
| M3 = "_cancelled_ok_helper" | |
| errors = [] | |
| NEW_BASH_HANDLER = '''async def _bash_handler( | |
| args: dict[str, Any], session: Any = None, **_kw | |
| ) -> tuple[str, bool]: | |
| """Run a shell command WITHOUT blocking the event loop. # ML_INTERN_TO_THREAD_BASH | |
| Uses asyncio subprocess + a single wait guard so a long/hung command never | |
| freezes the whole backend (every session's loop + SSE). On cancel (Stop) or | |
| timeout the process group is killed so nothing keeps running silently. | |
| """ | |
| command = args.get("command", "") | |
| if not command: | |
| return "No command provided.", False | |
| command = wrap_shell_command_with_hub_artifact_bootstrap(command, session) | |
| work_dir = args.get("work_dir", ".") | |
| timeout = min(args.get("timeout") or DEFAULT_TIMEOUT, MAX_TIMEOUT) | |
| cancelled = asyncio.Event() | |
| if session is not None: | |
| inner = getattr(session, "_cancelled", None) | |
| if inner is not None and hasattr(inner, "add_done_callback"): | |
| inner.add_done_callback(lambda _e, ev=cancelled: ev.set()) | |
| async def _run() -> tuple[str, bool]: | |
| # create_subprocess with start_new_session so we can kill the whole | |
| # process group on cancel/timeout. | |
| try: | |
| proc = await asyncio.create_subprocess_shell( | |
| command, | |
| cwd=work_dir or ".", | |
| stdout=asyncio.subprocess.PIPE, | |
| stderr=asyncio.subprocess.PIPE, | |
| start_new_session=True, | |
| ) | |
| except Exception as e: | |
| return f"bash error (spawn): {e}", False | |
| cancel_task = asyncio.ensure_future(cancelled.wait()) | |
| def _kill_group() -> None: | |
| try: | |
| os.killpg(os.getpgid(proc.pid), 9) | |
| except Exception: | |
| try: | |
| proc.kill() | |
| except Exception: | |
| pass | |
| # communicate() both waits for the process and reads its output, so we | |
| # use it as the single "process finished" future. | |
| comm_task = asyncio.ensure_future(proc.communicate()) | |
| # One single guard: wait for process completion OR cancellation, with the | |
| # hard per-call deadline applied directly here. asyncio.wait with a | |
| # timeout returns (done, pending) and does NOT raise on timeout. | |
| done, pending = await asyncio.wait( | |
| [comm_task, cancel_task], | |
| timeout=timeout, | |
| return_when=asyncio.FIRST_COMPLETED, | |
| ) | |
| for t in pending: | |
| t.cancel() | |
| if comm_task not in done: | |
| # Either cancelled, or genuinely timed out -> kill + report. | |
| comm_task.cancel() | |
| _kill_group() | |
| reason = ( | |
| "cancelled" | |
| if cancel_task in done | |
| else f"timed out after {timeout}s" | |
| ) | |
| return ( | |
| f"Command {reason} and was killed.\\n\\n" | |
| "For long-running commands, run in the background:\\n" | |
| " nohup <command> > /tmp/output.log 2>&1 & echo $!\\n" | |
| "Then poll:\\n" | |
| " kill -0 <PID> 2>/dev/null && echo 'running' || echo 'done'\\n" | |
| " tail -n 50 /tmp/output.log" | |
| ), False | |
| # Process finished naturally -> read results from the comhpleted future. | |
| cancel_task.cancel() | |
| try: | |
| stdout, stderr = comm_task.result() | |
| except Exception: | |
| stdout, stderr = b"", b"" | |
| raw = stdout.decode("utf-8", errors="replace") + stderr.decode( | |
| "utf-8", errors="replace" | |
| ) | |
| output = _strip_ansi(raw) | |
| output = _truncate_output(output) | |
| if not output.strip(): | |
| output = "(no output)" | |
| return output, proc.returncode == 0 if proc.returncode is not None else False | |
| try: | |
| return await _run() | |
| except (asyncio.TimeoutError, asyncio.CancelledError): | |
| return "Command was cancelled/interrupted.", False | |
| except Exception as e: | |
| return f"bash error: {e}", False | |
| ''' | |
| OLD_BASH_HANDLER = '''async def _bash_handler( | |
| args: dict[str, Any], session: Any = None, **_kw | |
| ) -> tuple[str, bool]: | |
| command = args.get("command", "") | |
| if not command: | |
| return "No command provided.", False | |
| command = wrap_shell_command_with_hub_artifact_bootstrap(command, session) | |
| work_dir = args.get("work_dir", ".") | |
| timeout = min(args.get("timeout") or DEFAULT_TIMEOUT, MAX_TIMEOUT) | |
| try: | |
| result = subprocess.run( | |
| command, | |
| shell=True, | |
| capture_output=True, | |
| text=True, | |
| cwd=work_dir, | |
| timeout=timeout, | |
| ) | |
| output = _strip_ansi(result.stdout + result.stderr) | |
| output = _truncate_output(output) | |
| if not output.strip(): | |
| output = "(no output)" | |
| return output, result.returncode == 0 | |
| except subprocess.TimeoutExpired: | |
| return ( | |
| f"Command timed out after {timeout}s and was killed.\\n\\n" | |
| f"For long-running commands, run in the background and poll:\\n" | |
| f" nohup <command> > /tmp/output.log 2>&1 & echo $!\\n" | |
| f"Then check status with:\\n" | |
| f" kill -0 <PID> 2>/dev/null && echo 'running' || echo 'done'\\n" | |
| f" tail -n 50 /tmp/output.log" | |
| ), False | |
| except Exception as e: | |
| return f"bash error: {e}", False | |
| ''' | |
| # ========================================================================= | |
| # PART 1: local_tools.py -> non-blocking, cancellation-aware bash | |
| # ========================================================================= | |
| if os.path.exists(LOCAL_TOOLS): | |
| with open(LOCAL_TOOLS) as f: | |
| c = f.read() | |
| if M2 in c: | |
| print("OK: local_tools.py already patched (idempotent)") | |
| else: | |
| if "import asyncio" not in c: | |
| c = c.replace( | |
| "import subprocess\n", | |
| "import asyncio\nimport subprocess\n", | |
| 1, | |
| ) | |
| c = c.replace( | |
| "MAX_TIMEOUT = 36000 # 10 hours — needed for long training runs (e.g. PostTrainBench)", | |
| "MAX_TIMEOUT = 1800 # 30 min hard cap; long jobs must run in background (ML Intern)", | |
| ) | |
| if OLD_BASH_HANDLER in c: | |
| c = c.replace(OLD_BASH_HANDLER, NEW_BASH_HANDLER, 1) | |
| print("OK: _bash_handler replaced with non-blocking cancellable version") | |
| else: | |
| start = c.find("async def _bash_handler(") | |
| end = c.find("async def _read_handler(", start) | |
| if start >= 0 and end > start: | |
| c = c[:start] + NEW_BASH_HANDLER + "\n\n" + c[end:] | |
| print("OK: _bash_handler replaced via region fallback") | |
| else: | |
| print("FAIL: could not locate _bash_handler in local_tools.py") | |
| errors.append("local_tools _bash_handler") | |
| with open(LOCAL_TOOLS, "w") as f: | |
| f.write(c) | |
| try: | |
| ast.parse(c) | |
| print("OK: local_tools.py syntax OK") | |
| except SyntaxError as e: | |
| print(f"FAIL: local_tools.py syntax: {e}") | |
| errors.append("local_tools syntax") | |
| # ========================================================================= | |
| # PART 2: session_manager.py -> turn-level watchdog | |
| # ========================================================================= | |
| if os.path.exists(SESSION_MANAGER): | |
| with open(SESSION_MANAGER) as f: | |
| c = f.read() | |
| if M1 in c: | |
| print("OK: session_manager.py already patched (idempotent)") | |
| else: | |
| # 2a. Add turn-timeout constant before DEFAULT_CONFIG_PATH. | |
| anchor = "DEFAULT_CONFIG_PATH = str(PROJECT_ROOT" | |
| const_block = ( | |
| "# ML_INTERN_TURN_WATCHDOG\n" | |
| "TURN_TIMEOUT_S = 3600 # hard per-turn safety net (1 h).\n" | |
| " # A single turn (LLM + tools) must never run forever.\n\n" | |
| ) | |
| if anchor in c and "TURN_TIMEOUT_S" not in c: | |
| c = c.replace(anchor, const_block + anchor, 1) | |
| print("OK: added TURN_TIMEOUT_S constant") | |
| else: | |
| print("FAIL: could not find DEFAULT_CONFIG_PATH anchor in session_manager.py") | |
| errors.append("session_manager constant") | |
| old_block = ''' agent_session.is_processing = True | |
| self._touch(agent_session) | |
| try: | |
| should_continue = await process_submission( | |
| session, submission | |
| ) | |
| finally: | |
| agent_session.is_processing = False | |
| # Stamp on turn finish too: a turn that ran longer | |
| # than the idle window would otherwise be reaped the | |
| # instant it completes. | |
| self._touch(agent_session) | |
| if session.config.save_sessions: | |
| await self.refresh_session_usage_metrics(agent_session) | |
| await self.persist_session_snapshot(agent_session)''' | |
| new_block = ''' agent_session.is_processing = True | |
| self._touch(agent_session) | |
| try: | |
| # ML_INTERN_TURN_WATCHDOG — hard per-turn deadline so | |
| # a hung turn can never leave the session stuck. | |
| turn_task = asyncio.create_task( | |
| process_submission(session, submission) | |
| ) | |
| done, _pending = await asyncio.wait( | |
| {turn_task}, timeout=TURN_TIMEOUT_S | |
| ) | |
| if not done: | |
| # Watchdog fired: force-cancel the stuck turn. | |
| turn_task.cancel() | |
| try: | |
| await turn_task | |
| except asyncio.CancelledError: | |
| pass | |
| # Clear the cancel flag so the session stays | |
| # usable for the NEXT turn (not a permanent | |
| # zombie). Only when a resettable Event exists. | |
| if _cancelled_ok(session): | |
| session._cancelled.clear() | |
| # Terminal SSE events -> frontend un-freezes. | |
| await session.send_event( | |
| Event( | |
| event_type="interrupted", | |
| data={ | |
| "error": ( | |
| "Single turn exceeded the " | |
| f"{TURN_TIMEOUT_S}s safety limit " | |
| "and was stopped." | |
| ) | |
| }, | |
| ) | |
| ) | |
| await session.send_event( | |
| Event( | |
| event_type="turn_complete", | |
| data={ | |
| "message": ( | |
| "Turn was auto-stopped after " | |
| f"{TURN_TIMEOUT_S}s. The task may " | |
| "need smaller steps or background " | |
| "work." | |
| ), | |
| "auto_stopped": True, | |
| }, | |
| ) | |
| ) | |
| should_continue = True | |
| else: | |
| should_continue = turn_task.result() | |
| finally: | |
| agent_session.is_processing = False | |
| # Stamp on turn finish too: a turn that ran longer | |
| # than the idle window would otherwise be reaped the | |
| # instant it completes. | |
| self._touch(agent_session) | |
| if session.config.save_sessions: | |
| await self.refresh_session_usage_metrics(agent_session) | |
| await self.persist_session_snapshot(agent_session)''' | |
| if old_block in c: | |
| c = c.replace(old_block, new_block, 1) | |
| print("OK: wrapped process_submission in turn watchdog") | |
| else: | |
| print("FAIL: could not find process_submission block in session_manager.py") | |
| idx = c.find("should_continue = await process_submission") | |
| if idx >= 0: | |
| print("DEBUG context:") | |
| print(c[idx - 400 : idx + 200]) | |
| errors.append("session_manager watchdog") | |
| with open(SESSION_MANAGER, "w") as f: | |
| f.write(c) | |
| try: | |
| ast.parse(c) | |
| print("OK: session_manager.py syntax OK") | |
| except SyntaxError as e: | |
| print(f"FAIL: session_manager.py syntax: {e}") | |
| errors.append("session_manager syntax") | |
| # ========================================================================= | |
| # PART 3: helper for the watchdog (clear resettable cancel flag) | |
| # ========================================================================= | |
| def ensure_helper() -> None: | |
| if not os.path.exists(SESSION_MANAGER): | |
| return | |
| with open(SESSION_MANAGER) as f: | |
| c = f.read() | |
| if M3 in c: | |
| print("OK: _cancelled_ok helper already present") | |
| return | |
| anchor = "\n\nlogger = logging.getLogger(__name__)\n" | |
| helper = ( | |
| "\n\ndef _cancelled_ok(session) -> bool: # _cancelled_ok_helper\n" | |
| " # ML_INTERN_TURN_WATCHDOG helper — True if session has a resettable\n" | |
| " # _cancelled asyncio.Event (used to clear stop-state after an\n" | |
| " # auto-stopped turn so the session stays usable).\n" | |
| " e = getattr(session, \"_cancelled\", None)\n" | |
| " return e is not None and hasattr(e, \"clear\")\n\n" | |
| ) | |
| if anchor in c and "_cancelled_ok(session)" in c: | |
| c = c.replace(anchor, helper + anchor, 1) | |
| with open(SESSION_MANAGER, "w") as f: | |
| f.write(c) | |
| try: | |
| ast.parse(c) | |
| print("OK: added _cancelled_ok helper; session_manager.py syntax OK") | |
| except SyntaxError as e: | |
| print(f"FAIL: session_manager.py syntax after helper: {e}") | |
| errors.append("helper syntax") | |
| else: | |
| print("FAIL: could not place _cancelled_ok helper") | |
| errors.append("helper placement") | |
| ensure_helper() | |
| if errors: | |
| print(f"PATCH FAILED: {errors}") | |
| raise SystemExit(1) | |
| print("DONE: session-hang fix applied (non-blocking bash + turn watchdog)") |