Aaron Brown commited on
Commit
85e394d
·
1 Parent(s): ad5992a

debug: add service startup trace logging to /tmp

Browse files
src/open_range/server/environment.py CHANGED
@@ -610,6 +610,18 @@ class RangeEnvironment(Environment[RangeAction, RangeObservation, RangeState]):
610
  """Run init commands, start daemon, wait for readiness."""
611
  logger.info("Starting service: %s (host=%s)", svc.daemon, svc.host)
612
 
 
 
 
 
 
 
 
 
 
 
 
 
613
  # Set env vars
614
  env = os.environ.copy()
615
  env.update(svc.env_vars)
@@ -640,6 +652,10 @@ class RangeEnvironment(Environment[RangeAction, RangeObservation, RangeState]):
640
  else svc.start_command
641
  )
642
 
 
 
 
 
643
  # Run init commands synchronously (blocking, no session isolation needed)
644
  for cmd in init_commands:
645
  try:
@@ -651,12 +667,15 @@ class RangeEnvironment(Environment[RangeAction, RangeObservation, RangeState]):
651
  env=env,
652
  check=False,
653
  )
 
654
  if result.returncode != 0 and result.stderr:
 
655
  logger.debug(
656
  "Init cmd stderr for %s: %s",
657
  svc.daemon, result.stderr[:200],
658
  )
659
  except Exception as exc:
 
660
  logger.warning("Init command failed for %s: %s", svc.daemon, exc)
661
 
662
  # Start the daemon using Popen with DEVNULL file descriptors.
@@ -667,6 +686,9 @@ class RangeEnvironment(Environment[RangeAction, RangeObservation, RangeState]):
667
  effective_cmd = start_command
668
  if not effective_cmd.rstrip().endswith("&"):
669
  effective_cmd = f"({effective_cmd}) &"
 
 
 
670
  try:
671
  proc = sp.Popen(
672
  ["bash", "-c", effective_cmd],
@@ -676,22 +698,38 @@ class RangeEnvironment(Environment[RangeAction, RangeObservation, RangeState]):
676
  env=env,
677
  start_new_session=True,
678
  )
 
679
  # Wait for bash to exit (it backgrounds the daemon and exits
680
  # immediately). The zombie reaper handles cleanup if we lose
681
  # the race.
682
  try:
683
- proc.wait(timeout=5)
684
- except (sp.TimeoutExpired, ChildProcessError):
685
- pass
 
 
 
686
  except Exception as exc:
 
687
  logger.warning("Start command failed for %s: %s", svc.daemon, exc)
688
  return
689
 
690
  # Brief pause to let the daemon initialize before probing readiness
691
- time.sleep(0.3)
 
 
 
 
 
 
 
 
 
 
692
 
693
  # Wait for readiness
694
  self._wait_for_readiness(svc)
 
695
 
696
  def _wait_for_readiness(self, svc: ServiceSpec) -> None:
697
  """Poll the readiness check until success or timeout."""
 
610
  """Run init commands, start daemon, wait for readiness."""
611
  logger.info("Starting service: %s (host=%s)", svc.daemon, svc.host)
612
 
613
+ # Debug file for diagnosing service startup issues
614
+ _dbg = "/tmp/openrange_svc_debug.log"
615
+
616
+ def _dbg_write(msg: str) -> None:
617
+ try:
618
+ with open(_dbg, "a") as f:
619
+ f.write(f"[{time.time():.3f}] {msg}\n")
620
+ except Exception:
621
+ pass
622
+
623
+ _dbg_write(f"=== START {svc.daemon} (host={svc.host}) ===")
624
+
625
  # Set env vars
626
  env = os.environ.copy()
627
  env.update(svc.env_vars)
 
652
  else svc.start_command
653
  )
654
 
655
+ _dbg_write(f" log_dir={log_dir}")
656
+ _dbg_write(f" init_commands={init_commands}")
657
+ _dbg_write(f" start_command={start_command}")
658
+
659
  # Run init commands synchronously (blocking, no session isolation needed)
660
  for cmd in init_commands:
661
  try:
 
667
  env=env,
668
  check=False,
669
  )
670
+ _dbg_write(f" init rc={result.returncode} cmd={cmd[:80]}")
671
  if result.returncode != 0 and result.stderr:
672
+ _dbg_write(f" init stderr: {result.stderr[:200]}")
673
  logger.debug(
674
  "Init cmd stderr for %s: %s",
675
  svc.daemon, result.stderr[:200],
676
  )
677
  except Exception as exc:
678
+ _dbg_write(f" init EXCEPTION: {exc}")
679
  logger.warning("Init command failed for %s: %s", svc.daemon, exc)
680
 
681
  # Start the daemon using Popen with DEVNULL file descriptors.
 
686
  effective_cmd = start_command
687
  if not effective_cmd.rstrip().endswith("&"):
688
  effective_cmd = f"({effective_cmd}) &"
689
+
690
+ _dbg_write(f" effective_cmd={effective_cmd}")
691
+
692
  try:
693
  proc = sp.Popen(
694
  ["bash", "-c", effective_cmd],
 
698
  env=env,
699
  start_new_session=True,
700
  )
701
+ _dbg_write(f" Popen pid={proc.pid}")
702
  # Wait for bash to exit (it backgrounds the daemon and exits
703
  # immediately). The zombie reaper handles cleanup if we lose
704
  # the race.
705
  try:
706
+ rc = proc.wait(timeout=5)
707
+ _dbg_write(f" bash exited rc={rc}")
708
+ except sp.TimeoutExpired:
709
+ _dbg_write(" bash wait TIMEOUT")
710
+ except ChildProcessError:
711
+ _dbg_write(" bash wait ChildProcessError (reaped by SIGCHLD)")
712
  except Exception as exc:
713
+ _dbg_write(f" Popen EXCEPTION: {exc}")
714
  logger.warning("Start command failed for %s: %s", svc.daemon, exc)
715
  return
716
 
717
  # Brief pause to let the daemon initialize before probing readiness
718
+ time.sleep(0.5)
719
+
720
+ # Check if daemon is running after pause
721
+ try:
722
+ pgrep = sp.run(
723
+ ["pgrep", "-x", svc.daemon.split("/")[-1].split()[0]],
724
+ capture_output=True, timeout=3, text=True, check=False,
725
+ )
726
+ _dbg_write(f" pgrep after start: rc={pgrep.returncode} pids={pgrep.stdout.strip()}")
727
+ except Exception:
728
+ _dbg_write(" pgrep failed")
729
 
730
  # Wait for readiness
731
  self._wait_for_readiness(svc)
732
+ _dbg_write(f" readiness check done for {svc.daemon}")
733
 
734
  def _wait_for_readiness(self, svc: ServiceSpec) -> None:
735
  """Poll the readiness check until success or timeout."""