Spaces:
Runtime error
Runtime error
Aaron Brown commited on
Commit ·
85e394d
1
Parent(s): ad5992a
debug: add service startup trace logging to /tmp
Browse files
src/open_range/server/environment.py
CHANGED
|
@@ -610,6 +610,18 @@ class RangeEnvironment(Environment[RangeAction, RangeObservation, RangeState]):
|
|
| 610 |
"""Run init commands, start daemon, wait for readiness."""
|
| 611 |
logger.info("Starting service: %s (host=%s)", svc.daemon, svc.host)
|
| 612 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 613 |
# Set env vars
|
| 614 |
env = os.environ.copy()
|
| 615 |
env.update(svc.env_vars)
|
|
@@ -640,6 +652,10 @@ class RangeEnvironment(Environment[RangeAction, RangeObservation, RangeState]):
|
|
| 640 |
else svc.start_command
|
| 641 |
)
|
| 642 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 643 |
# Run init commands synchronously (blocking, no session isolation needed)
|
| 644 |
for cmd in init_commands:
|
| 645 |
try:
|
|
@@ -651,12 +667,15 @@ class RangeEnvironment(Environment[RangeAction, RangeObservation, RangeState]):
|
|
| 651 |
env=env,
|
| 652 |
check=False,
|
| 653 |
)
|
|
|
|
| 654 |
if result.returncode != 0 and result.stderr:
|
|
|
|
| 655 |
logger.debug(
|
| 656 |
"Init cmd stderr for %s: %s",
|
| 657 |
svc.daemon, result.stderr[:200],
|
| 658 |
)
|
| 659 |
except Exception as exc:
|
|
|
|
| 660 |
logger.warning("Init command failed for %s: %s", svc.daemon, exc)
|
| 661 |
|
| 662 |
# Start the daemon using Popen with DEVNULL file descriptors.
|
|
@@ -667,6 +686,9 @@ class RangeEnvironment(Environment[RangeAction, RangeObservation, RangeState]):
|
|
| 667 |
effective_cmd = start_command
|
| 668 |
if not effective_cmd.rstrip().endswith("&"):
|
| 669 |
effective_cmd = f"({effective_cmd}) &"
|
|
|
|
|
|
|
|
|
|
| 670 |
try:
|
| 671 |
proc = sp.Popen(
|
| 672 |
["bash", "-c", effective_cmd],
|
|
@@ -676,22 +698,38 @@ class RangeEnvironment(Environment[RangeAction, RangeObservation, RangeState]):
|
|
| 676 |
env=env,
|
| 677 |
start_new_session=True,
|
| 678 |
)
|
|
|
|
| 679 |
# Wait for bash to exit (it backgrounds the daemon and exits
|
| 680 |
# immediately). The zombie reaper handles cleanup if we lose
|
| 681 |
# the race.
|
| 682 |
try:
|
| 683 |
-
proc.wait(timeout=5)
|
| 684 |
-
|
| 685 |
-
|
|
|
|
|
|
|
|
|
|
| 686 |
except Exception as exc:
|
|
|
|
| 687 |
logger.warning("Start command failed for %s: %s", svc.daemon, exc)
|
| 688 |
return
|
| 689 |
|
| 690 |
# Brief pause to let the daemon initialize before probing readiness
|
| 691 |
-
time.sleep(0.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 692 |
|
| 693 |
# Wait for readiness
|
| 694 |
self._wait_for_readiness(svc)
|
|
|
|
| 695 |
|
| 696 |
def _wait_for_readiness(self, svc: ServiceSpec) -> None:
|
| 697 |
"""Poll the readiness check until success or timeout."""
|
|
|
|
| 610 |
"""Run init commands, start daemon, wait for readiness."""
|
| 611 |
logger.info("Starting service: %s (host=%s)", svc.daemon, svc.host)
|
| 612 |
|
| 613 |
+
# Debug file for diagnosing service startup issues
|
| 614 |
+
_dbg = "/tmp/openrange_svc_debug.log"
|
| 615 |
+
|
| 616 |
+
def _dbg_write(msg: str) -> None:
|
| 617 |
+
try:
|
| 618 |
+
with open(_dbg, "a") as f:
|
| 619 |
+
f.write(f"[{time.time():.3f}] {msg}\n")
|
| 620 |
+
except Exception:
|
| 621 |
+
pass
|
| 622 |
+
|
| 623 |
+
_dbg_write(f"=== START {svc.daemon} (host={svc.host}) ===")
|
| 624 |
+
|
| 625 |
# Set env vars
|
| 626 |
env = os.environ.copy()
|
| 627 |
env.update(svc.env_vars)
|
|
|
|
| 652 |
else svc.start_command
|
| 653 |
)
|
| 654 |
|
| 655 |
+
_dbg_write(f" log_dir={log_dir}")
|
| 656 |
+
_dbg_write(f" init_commands={init_commands}")
|
| 657 |
+
_dbg_write(f" start_command={start_command}")
|
| 658 |
+
|
| 659 |
# Run init commands synchronously (blocking, no session isolation needed)
|
| 660 |
for cmd in init_commands:
|
| 661 |
try:
|
|
|
|
| 667 |
env=env,
|
| 668 |
check=False,
|
| 669 |
)
|
| 670 |
+
_dbg_write(f" init rc={result.returncode} cmd={cmd[:80]}")
|
| 671 |
if result.returncode != 0 and result.stderr:
|
| 672 |
+
_dbg_write(f" init stderr: {result.stderr[:200]}")
|
| 673 |
logger.debug(
|
| 674 |
"Init cmd stderr for %s: %s",
|
| 675 |
svc.daemon, result.stderr[:200],
|
| 676 |
)
|
| 677 |
except Exception as exc:
|
| 678 |
+
_dbg_write(f" init EXCEPTION: {exc}")
|
| 679 |
logger.warning("Init command failed for %s: %s", svc.daemon, exc)
|
| 680 |
|
| 681 |
# Start the daemon using Popen with DEVNULL file descriptors.
|
|
|
|
| 686 |
effective_cmd = start_command
|
| 687 |
if not effective_cmd.rstrip().endswith("&"):
|
| 688 |
effective_cmd = f"({effective_cmd}) &"
|
| 689 |
+
|
| 690 |
+
_dbg_write(f" effective_cmd={effective_cmd}")
|
| 691 |
+
|
| 692 |
try:
|
| 693 |
proc = sp.Popen(
|
| 694 |
["bash", "-c", effective_cmd],
|
|
|
|
| 698 |
env=env,
|
| 699 |
start_new_session=True,
|
| 700 |
)
|
| 701 |
+
_dbg_write(f" Popen pid={proc.pid}")
|
| 702 |
# Wait for bash to exit (it backgrounds the daemon and exits
|
| 703 |
# immediately). The zombie reaper handles cleanup if we lose
|
| 704 |
# the race.
|
| 705 |
try:
|
| 706 |
+
rc = proc.wait(timeout=5)
|
| 707 |
+
_dbg_write(f" bash exited rc={rc}")
|
| 708 |
+
except sp.TimeoutExpired:
|
| 709 |
+
_dbg_write(" bash wait TIMEOUT")
|
| 710 |
+
except ChildProcessError:
|
| 711 |
+
_dbg_write(" bash wait ChildProcessError (reaped by SIGCHLD)")
|
| 712 |
except Exception as exc:
|
| 713 |
+
_dbg_write(f" Popen EXCEPTION: {exc}")
|
| 714 |
logger.warning("Start command failed for %s: %s", svc.daemon, exc)
|
| 715 |
return
|
| 716 |
|
| 717 |
# Brief pause to let the daemon initialize before probing readiness
|
| 718 |
+
time.sleep(0.5)
|
| 719 |
+
|
| 720 |
+
# Check if daemon is running after pause
|
| 721 |
+
try:
|
| 722 |
+
pgrep = sp.run(
|
| 723 |
+
["pgrep", "-x", svc.daemon.split("/")[-1].split()[0]],
|
| 724 |
+
capture_output=True, timeout=3, text=True, check=False,
|
| 725 |
+
)
|
| 726 |
+
_dbg_write(f" pgrep after start: rc={pgrep.returncode} pids={pgrep.stdout.strip()}")
|
| 727 |
+
except Exception:
|
| 728 |
+
_dbg_write(" pgrep failed")
|
| 729 |
|
| 730 |
# Wait for readiness
|
| 731 |
self._wait_for_readiness(svc)
|
| 732 |
+
_dbg_write(f" readiness check done for {svc.daemon}")
|
| 733 |
|
| 734 |
def _wait_for_readiness(self, svc: ServiceSpec) -> None:
|
| 735 |
"""Poll the readiness check until success or timeout."""
|