noqt commited on
Commit
372f370
·
1 Parent(s): 211a1f0

Handle collected empty workload cgroups

Browse files
src/lumi_eggcracker/supervisor.py CHANGED
@@ -684,6 +684,9 @@ class Supervisor:
684
  events = events_from_fd(cgroup_events_fd)
685
  if events.get("populated") != 0:
686
  return False
 
 
 
687
  lock = self._new_lock(record["run_id"])
688
  with lock:
689
  if record["state"] not in ACTIVE_STATES:
@@ -727,6 +730,14 @@ class Supervisor:
727
  self._watch_once(record, ready)
728
  return
729
  except (JsonInputError, OSError, RuntimeError) as error:
 
 
 
 
 
 
 
 
730
  failure = error
731
  if attempt == 0:
732
  continue
 
684
  events = events_from_fd(cgroup_events_fd)
685
  if events.get("populated") != 0:
686
  return False
687
+ return self._mark_completed(record)
688
+
689
+ def _mark_completed(self, record: dict[str, Any]) -> bool:
690
  lock = self._new_lock(record["run_id"])
691
  with lock:
692
  if record["state"] not in ACTIVE_STATES:
 
730
  self._watch_once(record, ready)
731
  return
732
  except (JsonInputError, OSError, RuntimeError) as error:
733
+ # A collected transient unit is accepted only through the same
734
+ # exact-empty proof used after a direct cgroup kill. It is
735
+ # not inferred from systemd's inactive state.
736
+ if "owned cgroup is unavailable" in str(error):
737
+ _empty_ns, proof = verify_empty(identity_from_run(record))
738
+ if proof.complete:
739
+ self._mark_completed(record)
740
+ return
741
  failure = error
742
  if attempt == 0:
743
  continue
tests/test_supervisor.py CHANGED
@@ -59,6 +59,14 @@ class SupervisorTests(unittest.TestCase):
59
  self.assertTrue(supervisor._complete_allowed(record(), 42))
60
  self.assertEqual("COMPLETED_ALLOWED", saved[0]["state"])
61
 
 
 
 
 
 
 
 
 
62
  def test_status_is_read_only(self) -> None:
63
  supervisor = self._instance()
64
  item = record()
 
59
  self.assertTrue(supervisor._complete_allowed(record(), 42))
60
  self.assertEqual("COMPLETED_ALLOWED", saved[0]["state"])
61
 
62
+ def test_collected_empty_cgroup_allows_normal_completion_with_exact_proof(self) -> None:
63
+ supervisor = self._instance()
64
+ item = record()
65
+ with patch.object(supervisor, "_watch_once", side_effect=JsonInputError("owned cgroup is unavailable")), patch("lumi_eggcracker.supervisor.verify_empty", return_value=(1, EmptyProof(True, 0, 0, []))), patch.object(supervisor, "_store") as stored:
66
+ supervisor._watch(item)
67
+ stored.assert_called_once()
68
+ self.assertEqual("COMPLETED_ALLOWED", item["state"])
69
+
70
  def test_status_is_read_only(self) -> None:
71
  supervisor = self._instance()
72
  item = record()