File size: 6,852 Bytes
c8ba9c6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
"""CPU process test: orphan SFT ranks continue excluding base training."""

import json
import os
from pathlib import Path
import signal
import subprocess
import sys
import tempfile
import time

import pytest

from run_40b import process_lock


ROOT = Path(__file__).resolve().parent
WORKSPACE = ROOT.parent


def _identity(pid):
    try:
        fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()
        return fields[0], int(fields[19])
    except (OSError, ValueError, IndexError):
        return None


def _wait(predicate, seconds=20):
    deadline = time.monotonic() + seconds
    while time.monotonic() < deadline:
        if predicate():
            return
        time.sleep(0.02)
    raise AssertionError("Timed out waiting for the CPU lock-test process")


def _kill(identity):
    pid, created = identity
    current = _identity(pid)
    if current is not None and current[0] != "Z" and current[1] == created:
        os.kill(pid, signal.SIGKILL)


def _dead(identity):
    current = _identity(identity[0])
    return current is None or current[0] == "Z" or current[1] != identity[1]


def _exclusive_available(path):
    try:
        with process_lock(path):
            return True
    except BlockingIOError:
        return False


def test_detached_shared_ranks_exclude_base_after_both_ancestors_die():
    """Exercise the real worker helper and base lock across a fatal orphan tree.

    Fake torchrun deliberately closes inherited descriptors and starts workers
    in separate sessions, matching the installed elastic subprocess launcher.
    Either surviving rank must independently prevent an exclusive base/handoff
    lock, even after the guardian and torchrun process have both been SIGKILLed.
    """
    temporary_root = WORKSPACE / "tmp"
    temporary_root.mkdir(exist_ok=True)
    with tempfile.TemporaryDirectory(prefix="posttrain-worker-lock-", dir=temporary_root) as folder:
        directory = Path(folder)
        pipeline = directory / "pipeline.lock"
        pipeline.write_text("Existing lock inode must be preserved.\n")
        initial_inode = pipeline.stat().st_ino
        initial_bytes = pipeline.read_bytes()
        helper = directory / "tree.py"
        helper.write_text(
            "import json, os, pathlib, subprocess, sys, time\n"
            "mode, root = sys.argv[1], pathlib.Path(sys.argv[2])\n"
            "def ready(name):\n"
            "    fields = pathlib.Path('/proc/self/stat').read_text().rsplit(')', 1)[1].split()\n"
            "    target = root / (name + '.json')\n"
            "    temporary = target.with_suffix('.tmp')\n"
            "    temporary.write_text(json.dumps([os.getpid(), int(fields[19])]))\n"
            "    temporary.replace(target)\n"
            "if mode == 'guardian':\n"
            "    child = subprocess.Popen([sys.executable, __file__, 'agent', str(root)],\n"
            "                             start_new_session=True, close_fds=True)\n"
            "    ready('guardian')\n"
            "    sys.exit(child.wait())\n"
            "elif mode == 'agent':\n"
            "    ready('agent')\n"
            "    children = [subprocess.Popen([sys.executable, __file__, 'rank' + str(i), str(root)],\n"
            "                                 start_new_session=True, close_fds=True) for i in range(2)]\n"
            "    sys.exit(max(child.wait() for child in children))\n"
            "else:\n"
            "    from posttrain import shared_pipeline_lock\n"
            "    with shared_pipeline_lock(root / 'pipeline.lock'):\n"
            "        ready(mode)\n"
            "        while True: time.sleep(0.05)\n"
        )
        environment = os.environ.copy()
        environment.update(PYTHONPATH=str(ROOT), PYTHONDONTWRITEBYTECODE="1",
                           CUDA_VISIBLE_DEVICES="", TMPDIR=str(temporary_root))
        identities = {}
        with (directory / "process.log").open("w+") as log:
            guardian = subprocess.Popen([sys.executable, str(helper), "guardian", str(directory)],
                                        env=environment, stdout=log, stderr=log)
            try:
                _wait(lambda: all((directory / f"{name}.json").exists()
                                  for name in ("guardian", "agent", "rank0", "rank1")))
                identities = {name: json.loads((directory / f"{name}.json").read_text())
                              for name in ("guardian", "agent", "rank0", "rank1")}
                assert len({os.getsid(value[0]) for value in identities.values()}) == 4
                with pytest.raises(BlockingIOError), process_lock(pipeline):
                    pass
                guardian.kill()
                guardian.wait(timeout=5)
                assert not _dead(identities["agent"])
                with pytest.raises(BlockingIOError), process_lock(pipeline):
                    pass
                _kill(identities["agent"])
                _wait(lambda: _dead(identities["agent"]))
                assert all(not _dead(identities[name]) for name in ("rank0", "rank1"))
                with pytest.raises(BlockingIOError), process_lock(pipeline):
                    pass
                _kill(identities["rank0"])
                _wait(lambda: _dead(identities["rank0"]))
                assert not _dead(identities["rank1"])
                with pytest.raises(BlockingIOError), process_lock(pipeline):
                    pass
                _kill(identities["rank1"])
                _wait(lambda: _dead(identities["rank1"]))
                # A killed process's main task may be a zombie while other
                # threads are still closing the shared descriptor table.
                _wait(lambda: _exclusive_available(pipeline), seconds=5)
                with process_lock(pipeline):
                    assert pipeline.stat().st_ino == initial_inode
                    assert pipeline.read_bytes() == initial_bytes
            except Exception as error:
                log.flush()
                log.seek(0)
                error.add_note(log.read())
                raise
            finally:
                # Recover ready identities even if setup failed midway. Creation
                # times ensure cleanup never signals an unrelated reused PID.
                for name in ("guardian", "agent", "rank0", "rank1"):
                    ready = directory / f"{name}.json"
                    if ready.exists():
                        identities[name] = json.loads(ready.read_text())
                for identity in identities.values():
                    _kill(identity)
                if guardian.poll() is None:
                    guardian.kill()
                guardian.wait(timeout=5)
                for identity in identities.values():
                    _wait(lambda identity=identity: _dead(identity), seconds=5)