Spaces:
Running
Running
Handle collected empty workload cgroups
Browse files
src/lumi_eggcracker/supervisor.py
CHANGED
|
@@ -684,6 +684,9 @@ class Supervisor:
|
|
| 684 |
events = events_from_fd(cgroup_events_fd)
|
| 685 |
if events.get("populated") != 0:
|
| 686 |
return False
|
|
|
|
|
|
|
|
|
|
| 687 |
lock = self._new_lock(record["run_id"])
|
| 688 |
with lock:
|
| 689 |
if record["state"] not in ACTIVE_STATES:
|
|
@@ -727,6 +730,14 @@ class Supervisor:
|
|
| 727 |
self._watch_once(record, ready)
|
| 728 |
return
|
| 729 |
except (JsonInputError, OSError, RuntimeError) as error:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 730 |
failure = error
|
| 731 |
if attempt == 0:
|
| 732 |
continue
|
|
|
|
| 684 |
events = events_from_fd(cgroup_events_fd)
|
| 685 |
if events.get("populated") != 0:
|
| 686 |
return False
|
| 687 |
+
return self._mark_completed(record)
|
| 688 |
+
|
| 689 |
+
def _mark_completed(self, record: dict[str, Any]) -> bool:
|
| 690 |
lock = self._new_lock(record["run_id"])
|
| 691 |
with lock:
|
| 692 |
if record["state"] not in ACTIVE_STATES:
|
|
|
|
| 730 |
self._watch_once(record, ready)
|
| 731 |
return
|
| 732 |
except (JsonInputError, OSError, RuntimeError) as error:
|
| 733 |
+
# A collected transient unit is accepted only through the same
|
| 734 |
+
# exact-empty proof used after a direct cgroup kill. It is
|
| 735 |
+
# not inferred from systemd's inactive state.
|
| 736 |
+
if "owned cgroup is unavailable" in str(error):
|
| 737 |
+
_empty_ns, proof = verify_empty(identity_from_run(record))
|
| 738 |
+
if proof.complete:
|
| 739 |
+
self._mark_completed(record)
|
| 740 |
+
return
|
| 741 |
failure = error
|
| 742 |
if attempt == 0:
|
| 743 |
continue
|
tests/test_supervisor.py
CHANGED
|
@@ -59,6 +59,14 @@ class SupervisorTests(unittest.TestCase):
|
|
| 59 |
self.assertTrue(supervisor._complete_allowed(record(), 42))
|
| 60 |
self.assertEqual("COMPLETED_ALLOWED", saved[0]["state"])
|
| 61 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
def test_status_is_read_only(self) -> None:
|
| 63 |
supervisor = self._instance()
|
| 64 |
item = record()
|
|
|
|
| 59 |
self.assertTrue(supervisor._complete_allowed(record(), 42))
|
| 60 |
self.assertEqual("COMPLETED_ALLOWED", saved[0]["state"])
|
| 61 |
|
| 62 |
+
def test_collected_empty_cgroup_allows_normal_completion_with_exact_proof(self) -> None:
|
| 63 |
+
supervisor = self._instance()
|
| 64 |
+
item = record()
|
| 65 |
+
with patch.object(supervisor, "_watch_once", side_effect=JsonInputError("owned cgroup is unavailable")), patch("lumi_eggcracker.supervisor.verify_empty", return_value=(1, EmptyProof(True, 0, 0, []))), patch.object(supervisor, "_store") as stored:
|
| 66 |
+
supervisor._watch(item)
|
| 67 |
+
stored.assert_called_once()
|
| 68 |
+
self.assertEqual("COMPLETED_ALLOWED", item["state"])
|
| 69 |
+
|
| 70 |
def test_status_is_read_only(self) -> None:
|
| 71 |
supervisor = self._instance()
|
| 72 |
item = record()
|