File size: 2,669 Bytes
5d9f970
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
import importlib.util
from collections import deque
from pathlib import Path

spec=importlib.util.spec_from_file_location('supervisor',Path(__file__).parents[1]/'scripts/data/run_causal_video_audit.py')
m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)


def test_rate_ignores_old_reused_count_and_resets_on_restart():
    h=deque()
    assert m.recent_rate(h,0,0) is None
    assert m.recent_rate(h,30,5) is None
    assert m.recent_rate(h,60,10)==10/60
    assert m.recent_rate(h,70,0) is None
    assert m.recent_rate(h,130,6)==.1


def test_rate_sliding_window_is_bounded():
    h=deque()
    for t in range(0,3600,10):
        rate=m.recent_rate(h,t,t//10)
    assert len(h)<=62 and rate==.1


def test_physical_gpu_rollup_counts_work_not_memory_twice():
    tasks=[dict(gpu=gpu,pid=i+1,state='审核中',checked=10,planned=100,
                accepted=8,reused=2,error=None) for i,gpu in enumerate([0,1,2,0,1,2])]
    cards=m.physical_workers(tasks,{g:dict(utilization=80,memory_mib=50000) for g in [0,1,2]})
    assert len(cards)==3
    assert sum(c['checked'] for c in cards)==60
    assert all(c['memory_mib']==50000 and c['process_count']==2 for c in cards)
    assert cards[0]['pid']=='1,4'
    tasks[3].update(pid=None,state='失败',error='test failure')
    cards=m.physical_workers(tasks,{})
    assert '失败' in cards[0]['state'] and cards[0]['error']=='test failure'


def test_keeper_lock_waits_for_release_but_has_timeout(monkeypatch):
    import pytest
    calls=[]
    def lock(*args):
        calls.append(args)
        if len(calls)<3:
            raise BlockingIOError()
    monkeypatch.setattr(m.fcntl,'flock',lock)
    monkeypatch.setattr(m.time,'sleep',lambda _: None)
    m.acquire_keeper_lock(17,timeout=1)
    assert len(calls)==3
    def busy(*args):
        raise BlockingIOError()
    monkeypatch.setattr(m.fcntl,'flock',busy)
    with pytest.raises(RuntimeError,match='Stop the GPU keeper'):
        m.acquire_keeper_lock(17,timeout=0)


def test_status_stale_handle_preserves_counts_and_timestamp(monkeypatch):
    import errno
    import pytest
    old=dict(checked=123,updated_at=100)
    def stale(*args):
        raise OSError(errno.ESTALE,'Stale file handle')
    monkeypatch.setattr(m,'read',stale)
    assert m.read_status(Path('status.json'),old) is old
    new=dict(checked=125,updated_at=120)
    monkeypatch.setattr(m,'read',lambda *args:new)
    assert m.read_status(Path('status.json'),old) is new
    def disk_error(*args):
        raise OSError(errno.EIO,'Real disk error')
    monkeypatch.setattr(m,'read',disk_error)
    with pytest.raises(OSError):
        m.read_status(Path('status.json'),old)