ml-intern / patch_fix_session_hang.py
bep40's picture
Upload patch_fix_session_hang.py
4fca660 verified
Raw History Blame Contribute Delete
17.7 kB
"""Definitive fix for "treo session" (hung/stuck session) while executing a task.
Root causes (all verified against current upstream smolagents/ml-intern:
agent_loop.py, local_tools.py, session_manager.py, session.py):
1. BLOCKING SUBPROCESS (#1 mid-task freeze).
agent/tools/local_tools.py `_bash_handler` runs `subprocess.run(...)`
synchronously inside an async handler. Because the whole backend is a
single asyncio event loop, ANY bash command (pip install, a training run,
a long shell pipeline) freezes the ENTIRE loop: every session's agent loop,
the SSE fan-out, the reaper, and the cancellation path all stop being
scheduled. The frontend sees keepalives but zero events, so it shows
"Processing…" for hours. MAX_TIMEOUT=36000 (10 h) makes it worse.
Fix: run bash via asyncio.create_subprocess_shell so it never blocks the
loop, cap MAX_TIMEOUT to a sane 1800 s, and kill the process group on
either cancel (Stop) or timeout so no orphan keeps running.
2. NO TURN-LEVEL WATCHDOG.
backend/session_manager.py `_run_session` awaits `process_submission`
directly with no timeout. Combined with bep40's patch_lossless_context
setting max_iterations=-1 (unlimited), a turn can run forever (doom loop,
unkillable tool, connection hang). When a turn hangs, is_processing stays
True and the idle reaper REFUSES to reap it -> permanent zombie.
Fix: wrap process_submission in asyncio.wait with a turn timeout (1 h). On
timeout, cancel the turn, send `interrupted` + `turn_complete` terminal SSE
events (so the frontend un-freezes), clear the cancel flag so the session
stays usable, and let it continue on the next message.
Design notes / safety:
- Does NOT change max_iterations semantics. It only adds a hard safety net.
- Does NOT change the user's task, model, dataset, or training approach.
- Idempotent: every mutation is guarded by a marker check.
- Runs in the Python build stage AFTER `uv sync` (needs agent/backend present),
NEXT TO the other backend patches (patch_lossless_context etc.).
"""
import ast
import os
LOCAL_TOOLS = "/app/agent/tools/local_tools.py"
SESSION_MANAGER = "/app/backend/session_manager.py"
M1 = "ML_INTERN_TURN_WATCHDOG"
M2 = "ML_INTERN_TO_THREAD_BASH"
M3 = "_cancelled_ok_helper"
errors = []
NEW_BASH_HANDLER = '''async def _bash_handler(
args: dict[str, Any], session: Any = None, **_kw
) -> tuple[str, bool]:
"""Run a shell command WITHOUT blocking the event loop. # ML_INTERN_TO_THREAD_BASH
Uses asyncio subprocess + a single wait guard so a long/hung command never
freezes the whole backend (every session's loop + SSE). On cancel (Stop) or
timeout the process group is killed so nothing keeps running silently.
"""
command = args.get("command", "")
if not command:
return "No command provided.", False
command = wrap_shell_command_with_hub_artifact_bootstrap(command, session)
work_dir = args.get("work_dir", ".")
timeout = min(args.get("timeout") or DEFAULT_TIMEOUT, MAX_TIMEOUT)
cancelled = asyncio.Event()
if session is not None:
inner = getattr(session, "_cancelled", None)
if inner is not None and hasattr(inner, "add_done_callback"):
inner.add_done_callback(lambda _e, ev=cancelled: ev.set())
async def _run() -> tuple[str, bool]:
# create_subprocess with start_new_session so we can kill the whole
# process group on cancel/timeout.
try:
proc = await asyncio.create_subprocess_shell(
command,
cwd=work_dir or ".",
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
start_new_session=True,
)
except Exception as e:
return f"bash error (spawn): {e}", False
cancel_task = asyncio.ensure_future(cancelled.wait())
def _kill_group() -> None:
try:
os.killpg(os.getpgid(proc.pid), 9)
except Exception:
try:
proc.kill()
except Exception:
pass
# communicate() both waits for the process and reads its output, so we
# use it as the single "process finished" future.
comm_task = asyncio.ensure_future(proc.communicate())
# One single guard: wait for process completion OR cancellation, with the
# hard per-call deadline applied directly here. asyncio.wait with a
# timeout returns (done, pending) and does NOT raise on timeout.
done, pending = await asyncio.wait(
[comm_task, cancel_task],
timeout=timeout,
return_when=asyncio.FIRST_COMPLETED,
)
for t in pending:
t.cancel()
if comm_task not in done:
# Either cancelled, or genuinely timed out -> kill + report.
comm_task.cancel()
_kill_group()
reason = (
"cancelled"
if cancel_task in done
else f"timed out after {timeout}s"
)
return (
f"Command {reason} and was killed.\\n\\n"
"For long-running commands, run in the background:\\n"
" nohup <command> > /tmp/output.log 2>&1 & echo $!\\n"
"Then poll:\\n"
" kill -0 <PID> 2>/dev/null && echo 'running' || echo 'done'\\n"
" tail -n 50 /tmp/output.log"
), False
# Process finished naturally -> read results from the comhpleted future.
cancel_task.cancel()
try:
stdout, stderr = comm_task.result()
except Exception:
stdout, stderr = b"", b""
raw = stdout.decode("utf-8", errors="replace") + stderr.decode(
"utf-8", errors="replace"
)
output = _strip_ansi(raw)
output = _truncate_output(output)
if not output.strip():
output = "(no output)"
return output, proc.returncode == 0 if proc.returncode is not None else False
try:
return await _run()
except (asyncio.TimeoutError, asyncio.CancelledError):
return "Command was cancelled/interrupted.", False
except Exception as e:
return f"bash error: {e}", False
'''
OLD_BASH_HANDLER = '''async def _bash_handler(
args: dict[str, Any], session: Any = None, **_kw
) -> tuple[str, bool]:
command = args.get("command", "")
if not command:
return "No command provided.", False
command = wrap_shell_command_with_hub_artifact_bootstrap(command, session)
work_dir = args.get("work_dir", ".")
timeout = min(args.get("timeout") or DEFAULT_TIMEOUT, MAX_TIMEOUT)
try:
result = subprocess.run(
command,
shell=True,
capture_output=True,
text=True,
cwd=work_dir,
timeout=timeout,
)
output = _strip_ansi(result.stdout + result.stderr)
output = _truncate_output(output)
if not output.strip():
output = "(no output)"
return output, result.returncode == 0
except subprocess.TimeoutExpired:
return (
f"Command timed out after {timeout}s and was killed.\\n\\n"
f"For long-running commands, run in the background and poll:\\n"
f" nohup <command> > /tmp/output.log 2>&1 & echo $!\\n"
f"Then check status with:\\n"
f" kill -0 <PID> 2>/dev/null && echo 'running' || echo 'done'\\n"
f" tail -n 50 /tmp/output.log"
), False
except Exception as e:
return f"bash error: {e}", False
'''
# =========================================================================
# PART 1: local_tools.py -> non-blocking, cancellation-aware bash
# =========================================================================
if os.path.exists(LOCAL_TOOLS):
with open(LOCAL_TOOLS) as f:
c = f.read()
if M2 in c:
print("OK: local_tools.py already patched (idempotent)")
else:
if "import asyncio" not in c:
c = c.replace(
"import subprocess\n",
"import asyncio\nimport subprocess\n",
1,
)
c = c.replace(
"MAX_TIMEOUT = 36000 # 10 hours — needed for long training runs (e.g. PostTrainBench)",
"MAX_TIMEOUT = 1800 # 30 min hard cap; long jobs must run in background (ML Intern)",
)
if OLD_BASH_HANDLER in c:
c = c.replace(OLD_BASH_HANDLER, NEW_BASH_HANDLER, 1)
print("OK: _bash_handler replaced with non-blocking cancellable version")
else:
start = c.find("async def _bash_handler(")
end = c.find("async def _read_handler(", start)
if start >= 0 and end > start:
c = c[:start] + NEW_BASH_HANDLER + "\n\n" + c[end:]
print("OK: _bash_handler replaced via region fallback")
else:
print("FAIL: could not locate _bash_handler in local_tools.py")
errors.append("local_tools _bash_handler")
with open(LOCAL_TOOLS, "w") as f:
f.write(c)
try:
ast.parse(c)
print("OK: local_tools.py syntax OK")
except SyntaxError as e:
print(f"FAIL: local_tools.py syntax: {e}")
errors.append("local_tools syntax")
# =========================================================================
# PART 2: session_manager.py -> turn-level watchdog
# =========================================================================
if os.path.exists(SESSION_MANAGER):
with open(SESSION_MANAGER) as f:
c = f.read()
if M1 in c:
print("OK: session_manager.py already patched (idempotent)")
else:
# 2a. Add turn-timeout constant before DEFAULT_CONFIG_PATH.
anchor = "DEFAULT_CONFIG_PATH = str(PROJECT_ROOT"
const_block = (
"# ML_INTERN_TURN_WATCHDOG\n"
"TURN_TIMEOUT_S = 3600 # hard per-turn safety net (1 h).\n"
" # A single turn (LLM + tools) must never run forever.\n\n"
)
if anchor in c and "TURN_TIMEOUT_S" not in c:
c = c.replace(anchor, const_block + anchor, 1)
print("OK: added TURN_TIMEOUT_S constant")
else:
print("FAIL: could not find DEFAULT_CONFIG_PATH anchor in session_manager.py")
errors.append("session_manager constant")
old_block = ''' agent_session.is_processing = True
self._touch(agent_session)
try:
should_continue = await process_submission(
session, submission
)
finally:
agent_session.is_processing = False
# Stamp on turn finish too: a turn that ran longer
# than the idle window would otherwise be reaped the
# instant it completes.
self._touch(agent_session)
if session.config.save_sessions:
await self.refresh_session_usage_metrics(agent_session)
await self.persist_session_snapshot(agent_session)'''
new_block = ''' agent_session.is_processing = True
self._touch(agent_session)
try:
# ML_INTERN_TURN_WATCHDOG — hard per-turn deadline so
# a hung turn can never leave the session stuck.
turn_task = asyncio.create_task(
process_submission(session, submission)
)
done, _pending = await asyncio.wait(
{turn_task}, timeout=TURN_TIMEOUT_S
)
if not done:
# Watchdog fired: force-cancel the stuck turn.
turn_task.cancel()
try:
await turn_task
except asyncio.CancelledError:
pass
# Clear the cancel flag so the session stays
# usable for the NEXT turn (not a permanent
# zombie). Only when a resettable Event exists.
if _cancelled_ok(session):
session._cancelled.clear()
# Terminal SSE events -> frontend un-freezes.
await session.send_event(
Event(
event_type="interrupted",
data={
"error": (
"Single turn exceeded the "
f"{TURN_TIMEOUT_S}s safety limit "
"and was stopped."
)
},
)
)
await session.send_event(
Event(
event_type="turn_complete",
data={
"message": (
"Turn was auto-stopped after "
f"{TURN_TIMEOUT_S}s. The task may "
"need smaller steps or background "
"work."
),
"auto_stopped": True,
},
)
)
should_continue = True
else:
should_continue = turn_task.result()
finally:
agent_session.is_processing = False
# Stamp on turn finish too: a turn that ran longer
# than the idle window would otherwise be reaped the
# instant it completes.
self._touch(agent_session)
if session.config.save_sessions:
await self.refresh_session_usage_metrics(agent_session)
await self.persist_session_snapshot(agent_session)'''
if old_block in c:
c = c.replace(old_block, new_block, 1)
print("OK: wrapped process_submission in turn watchdog")
else:
print("FAIL: could not find process_submission block in session_manager.py")
idx = c.find("should_continue = await process_submission")
if idx >= 0:
print("DEBUG context:")
print(c[idx - 400 : idx + 200])
errors.append("session_manager watchdog")
with open(SESSION_MANAGER, "w") as f:
f.write(c)
try:
ast.parse(c)
print("OK: session_manager.py syntax OK")
except SyntaxError as e:
print(f"FAIL: session_manager.py syntax: {e}")
errors.append("session_manager syntax")
# =========================================================================
# PART 3: helper for the watchdog (clear resettable cancel flag)
# =========================================================================
def ensure_helper() -> None:
if not os.path.exists(SESSION_MANAGER):
return
with open(SESSION_MANAGER) as f:
c = f.read()
if M3 in c:
print("OK: _cancelled_ok helper already present")
return
anchor = "\n\nlogger = logging.getLogger(__name__)\n"
helper = (
"\n\ndef _cancelled_ok(session) -> bool: # _cancelled_ok_helper\n"
" # ML_INTERN_TURN_WATCHDOG helper — True if session has a resettable\n"
" # _cancelled asyncio.Event (used to clear stop-state after an\n"
" # auto-stopped turn so the session stays usable).\n"
" e = getattr(session, \"_cancelled\", None)\n"
" return e is not None and hasattr(e, \"clear\")\n\n"
)
if anchor in c and "_cancelled_ok(session)" in c:
c = c.replace(anchor, helper + anchor, 1)
with open(SESSION_MANAGER, "w") as f:
f.write(c)
try:
ast.parse(c)
print("OK: added _cancelled_ok helper; session_manager.py syntax OK")
except SyntaxError as e:
print(f"FAIL: session_manager.py syntax after helper: {e}")
errors.append("helper syntax")
else:
print("FAIL: could not place _cancelled_ok helper")
errors.append("helper placement")
ensure_helper()
if errors:
print(f"PATCH FAILED: {errors}")
raise SystemExit(1)
print("DONE: session-hang fix applied (non-blocking bash + turn watchdog)")