code-Movens-Zero_train / scale /script /test_watch_scale_previews.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
13cc860 verified
Raw History Blame Contribute Delete
7.55 kB
import json
import os
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
import watch_scale_previews as auto
import preview_gpu_lease as gpu
class AutoPreviewTests(unittest.TestCase):
def test_simple_suite_is_separate_and_has_three_cases(self):
with tempfile.TemporaryDirectory() as tmp:
regular=auto.Watcher(Path(tmp))
simple=auto.Watcher(Path(tmp),simple_text=True)
self.assertNotEqual(regular.base,simple.base)
self.assertEqual(len(regular.case_ids),4)
self.assertEqual(len(simple.case_ids),3)
self.assertEqual(simple.state['expected_cases'],3)
simple.bootstrap()
self.assertEqual(simple.state['models'],[])
def test_retains_two_complete_previews_and_protects_partial(self):
with tempfile.TemporaryDirectory() as tmp:
w = auto.Watcher(Path(tmp))
jobs = []
for step in (500, 1000, 1500, 2000):
job = w.base / 'jobs/S/hash' / f'ckpt_{step}__root40_v4'
jobs.append(job)
for i in range(4):
case = job / 'text' / str(i)
case.mkdir(parents=True)
(case / 'render_status.json').write_text(json.dumps({'status': 'rendering' if step == 2000 else 'ready'}))
(case / 'comparison.mp4').write_bytes(b'video')
w.link_asset(f'{step}_{i}.mp4', case / 'comparison.mp4')
w.state['models'] = [dict(size='S', step=1500, cases=[dict(video=f'1500_{i}.mp4') for i in range(4)])]
w.prune_history()
self.assertFalse(jobs[0].exists())
self.assertTrue(all(p.exists() for p in jobs[1:]))
self.assertFalse((w.web / '500_0.mp4').is_symlink())
self.assertTrue((w.base / 'retired_records/S/hash/ckpt_500__root40_v4/text/0/render_status.json').exists())
self.assertEqual(len(w.assets), 12)
def test_latest_complete_skips_partial_and_unconfirmed(self):
with tempfile.TemporaryDirectory() as tmp:
run = Path(tmp)
required = ['model.safetensors', 'optimizer.bin', 'scheduler.bin', 'custom_checkpoint_0.pkl',
'custom_checkpoint_1.pkl', 'random_states_0.pkl', 'random_states_1.pkl']
for step in (500, 1000, 1500):
directory = run / 'checkpoints' / f'ckpt_{step}'
directory.mkdir(parents=True)
for name in required:
if step == 1000 and name == 'random_states_1.pkl':
continue
(directory / name).write_bytes(b'x')
self.assertEqual(auto.latest_complete(run, {500, 1000}, 2)[0], 500)
self.assertEqual(auto.latest_complete(run, {500, 1000, 1500}, 2)[0], 1500)
self.assertIsNone(auto.latest_complete(run, set(), 2))
def test_coalesces_and_distinguishes_restarted_run(self):
entry = dict(run='a', step=1000)
self.assertFalse(auto.needs_update(entry, Path('a'), 1000))
self.assertFalse(auto.needs_update(entry, Path('a'), 500))
self.assertTrue(auto.needs_update(entry, Path('a'), 2000))
self.assertTrue(auto.needs_update(entry, Path('b'), 500))
def test_publish_is_all_or_nothing_and_urls_are_immutable(self):
with tempfile.TemporaryDirectory() as tmp:
w = auto.Watcher(Path(tmp))
w.state['models'] = [dict(size='S', step=1500, cases=[])]
w.write('starting')
previous = w.state_path.read_bytes()
with self.assertRaises(ValueError):
w.publish(dict(size='S', step=2000, cases=[dict(ready=True, video='missing.mp4')]))
self.assertEqual(w.state_path.read_bytes(), previous)
cases = []
for i in range(4):
source = w.base / f'output{i}.mp4'
source.write_bytes(b'video')
w.link_asset(f'S_2000_{i}.mp4', source)
cases.append(dict(id=str(i), ready=True, video=f'S_2000_{i}.mp4'))
w.publish(dict(size='S', run='run', step=2000, cases=cases))
self.assertEqual(json.loads(w.state_path.read_text())['models'][0]['step'], 2000)
self.assertEqual(len(json.loads(w.asset_path.read_text())), 4)
with self.assertRaises(ValueError):
w.link_asset('S_2000_0.mp4', w.base / 'output1.mp4')
def test_release_only_temporary_weight_link(self):
with tempfile.TemporaryDirectory() as tmp:
group = Path(tmp)
w = auto.Watcher(group)
original = group / 'runs/model.safetensors'
original.parent.mkdir()
original.write_bytes(b'original weights')
pinned = w.base / 'jobs/S/run/ckpt_2000/snapshot/checkpoints/ckpt_2000/model.safetensors'
pinned.parent.mkdir(parents=True)
os.link(original, pinned)
w.state['pending_weight'] = str(pinned)
w.release_weight()
self.assertFalse(pinned.exists())
self.assertEqual(original.read_bytes(), b'original weights')
w.state['pending_weight'] = str(original)
with self.assertRaises(ValueError):
w.release_weight()
self.assertTrue(original.exists())
def test_keeper_never_stops_a_foreign_gpu_process(self):
with patch.object(gpu, 'gpu_processes', return_value={1, 99}), patch.object(gpu, 'keeper_pids', return_value=[1]), \
patch.object(gpu, 'stop_keeper') as stop:
with self.assertRaisesRegex(RuntimeError, 'non-keeper'):
with gpu.lease_gpu3():
self.fail('Must not enter occupied GPU')
stop.assert_not_called()
def test_keeper_restored_on_inference_failure(self):
with patch.object(gpu, 'gpu_processes', return_value=set()), patch.object(gpu, 'keeper_pids', return_value=[]), \
patch.object(gpu, 'stop_keeper'), patch.object(gpu, 'start_keeper') as start, \
patch.object(gpu, 'restore_keeper') as restore, patch.object(gpu.subprocess, 'check_output', return_value='0'):
with self.assertRaisesRegex(RuntimeError, 'inference failed'):
with gpu.lease_gpu3():
raise RuntimeError('inference failed')
start.assert_not_called()
restore.assert_called_once()
def test_restore_checks_only_preview_gpu(self):
with patch.object(gpu, 'stop_keeper') as stop, \
patch.object(gpu, 'start_keeper') as start, \
patch.object(gpu.subprocess, 'check_output', return_value='43000\n') as query:
gpu.restore_keeper()
stop.assert_called_once()
start.assert_called_once_with()
self.assertIn('--id=2', query.call_args.args[0])
def test_private_keeper_never_uses_shared_locks_or_other_gpus(self):
with patch.object(gpu.subprocess, 'run') as run:
gpu.start_keeper()
env = run.call_args.kwargs['env']
self.assertEqual(env['MONITOR_GPU_INDICES'], '2')
self.assertEqual(env['MONITOR_LOCK_FILE'], str(gpu.PID_FILES[1]))
self.assertEqual(env['MONITOR_SUPERVISOR_PID_FILE'], str(gpu.PID_FILES[0]))
self.assertTrue(env['MONITOR_SUPERVISOR_LOCK_FILE'].startswith(gpu.KEEPER_PREFIX))
self.assertNotIn('CUDA_VISIBLE_DEVICES', env)
if __name__ == '__main__':
unittest.main()