import json import os from pathlib import Path import tempfile import unittest from unittest.mock import patch import watch_scale_previews as auto import preview_gpu_lease as gpu class AutoPreviewTests(unittest.TestCase): def test_simple_suite_is_separate_and_has_three_cases(self): with tempfile.TemporaryDirectory() as tmp: regular=auto.Watcher(Path(tmp)) simple=auto.Watcher(Path(tmp),simple_text=True) self.assertNotEqual(regular.base,simple.base) self.assertEqual(len(regular.case_ids),4) self.assertEqual(len(simple.case_ids),3) self.assertEqual(simple.state['expected_cases'],3) simple.bootstrap() self.assertEqual(simple.state['models'],[]) def test_retains_two_complete_previews_and_protects_partial(self): with tempfile.TemporaryDirectory() as tmp: w = auto.Watcher(Path(tmp)) jobs = [] for step in (500, 1000, 1500, 2000): job = w.base / 'jobs/S/hash' / f'ckpt_{step}__root40_v4' jobs.append(job) for i in range(4): case = job / 'text' / str(i) case.mkdir(parents=True) (case / 'render_status.json').write_text(json.dumps({'status': 'rendering' if step == 2000 else 'ready'})) (case / 'comparison.mp4').write_bytes(b'video') w.link_asset(f'{step}_{i}.mp4', case / 'comparison.mp4') w.state['models'] = [dict(size='S', step=1500, cases=[dict(video=f'1500_{i}.mp4') for i in range(4)])] w.prune_history() self.assertFalse(jobs[0].exists()) self.assertTrue(all(p.exists() for p in jobs[1:])) self.assertFalse((w.web / '500_0.mp4').is_symlink()) self.assertTrue((w.base / 'retired_records/S/hash/ckpt_500__root40_v4/text/0/render_status.json').exists()) self.assertEqual(len(w.assets), 12) def test_latest_complete_skips_partial_and_unconfirmed(self): with tempfile.TemporaryDirectory() as tmp: run = Path(tmp) required = ['model.safetensors', 'optimizer.bin', 'scheduler.bin', 'custom_checkpoint_0.pkl', 'custom_checkpoint_1.pkl', 'random_states_0.pkl', 'random_states_1.pkl'] for step in (500, 1000, 1500): directory = run / 'checkpoints' / f'ckpt_{step}' directory.mkdir(parents=True) for name in required: if step == 1000 and name == 'random_states_1.pkl': continue (directory / name).write_bytes(b'x') self.assertEqual(auto.latest_complete(run, {500, 1000}, 2)[0], 500) self.assertEqual(auto.latest_complete(run, {500, 1000, 1500}, 2)[0], 1500) self.assertIsNone(auto.latest_complete(run, set(), 2)) def test_coalesces_and_distinguishes_restarted_run(self): entry = dict(run='a', step=1000) self.assertFalse(auto.needs_update(entry, Path('a'), 1000)) self.assertFalse(auto.needs_update(entry, Path('a'), 500)) self.assertTrue(auto.needs_update(entry, Path('a'), 2000)) self.assertTrue(auto.needs_update(entry, Path('b'), 500)) def test_publish_is_all_or_nothing_and_urls_are_immutable(self): with tempfile.TemporaryDirectory() as tmp: w = auto.Watcher(Path(tmp)) w.state['models'] = [dict(size='S', step=1500, cases=[])] w.write('starting') previous = w.state_path.read_bytes() with self.assertRaises(ValueError): w.publish(dict(size='S', step=2000, cases=[dict(ready=True, video='missing.mp4')])) self.assertEqual(w.state_path.read_bytes(), previous) cases = [] for i in range(4): source = w.base / f'output{i}.mp4' source.write_bytes(b'video') w.link_asset(f'S_2000_{i}.mp4', source) cases.append(dict(id=str(i), ready=True, video=f'S_2000_{i}.mp4')) w.publish(dict(size='S', run='run', step=2000, cases=cases)) self.assertEqual(json.loads(w.state_path.read_text())['models'][0]['step'], 2000) self.assertEqual(len(json.loads(w.asset_path.read_text())), 4) with self.assertRaises(ValueError): w.link_asset('S_2000_0.mp4', w.base / 'output1.mp4') def test_release_only_temporary_weight_link(self): with tempfile.TemporaryDirectory() as tmp: group = Path(tmp) w = auto.Watcher(group) original = group / 'runs/model.safetensors' original.parent.mkdir() original.write_bytes(b'original weights') pinned = w.base / 'jobs/S/run/ckpt_2000/snapshot/checkpoints/ckpt_2000/model.safetensors' pinned.parent.mkdir(parents=True) os.link(original, pinned) w.state['pending_weight'] = str(pinned) w.release_weight() self.assertFalse(pinned.exists()) self.assertEqual(original.read_bytes(), b'original weights') w.state['pending_weight'] = str(original) with self.assertRaises(ValueError): w.release_weight() self.assertTrue(original.exists()) def test_keeper_never_stops_a_foreign_gpu_process(self): with patch.object(gpu, 'gpu_processes', return_value={1, 99}), patch.object(gpu, 'keeper_pids', return_value=[1]), \ patch.object(gpu, 'stop_keeper') as stop: with self.assertRaisesRegex(RuntimeError, 'non-keeper'): with gpu.lease_gpu3(): self.fail('Must not enter occupied GPU') stop.assert_not_called() def test_keeper_restored_on_inference_failure(self): with patch.object(gpu, 'gpu_processes', return_value=set()), patch.object(gpu, 'keeper_pids', return_value=[]), \ patch.object(gpu, 'stop_keeper'), patch.object(gpu, 'start_keeper') as start, \ patch.object(gpu, 'restore_keeper') as restore, patch.object(gpu.subprocess, 'check_output', return_value='0'): with self.assertRaisesRegex(RuntimeError, 'inference failed'): with gpu.lease_gpu3(): raise RuntimeError('inference failed') start.assert_not_called() restore.assert_called_once() def test_restore_checks_only_preview_gpu(self): with patch.object(gpu, 'stop_keeper') as stop, \ patch.object(gpu, 'start_keeper') as start, \ patch.object(gpu.subprocess, 'check_output', return_value='43000\n') as query: gpu.restore_keeper() stop.assert_called_once() start.assert_called_once_with() self.assertIn('--id=2', query.call_args.args[0]) def test_private_keeper_never_uses_shared_locks_or_other_gpus(self): with patch.object(gpu.subprocess, 'run') as run: gpu.start_keeper() env = run.call_args.kwargs['env'] self.assertEqual(env['MONITOR_GPU_INDICES'], '2') self.assertEqual(env['MONITOR_LOCK_FILE'], str(gpu.PID_FILES[1])) self.assertEqual(env['MONITOR_SUPERVISOR_PID_FILE'], str(gpu.PID_FILES[0])) self.assertTrue(env['MONITOR_SUPERVISOR_LOCK_FILE'].startswith(gpu.KEEPER_PREFIX)) self.assertNotIn('CUDA_VISIBLE_DEVICES', env) if __name__ == '__main__': unittest.main()