Download scale/script/test_watch_scale_previews.py from Tinghong-Ye22/code-Movens-Zero_train: direct link, hf CLI and curl.
- Browser
- Download file 7.55 kB
-
https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/scale/script/test_watch_scale_previews.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Movens-Zero_train/scale/script/test_watch_scale_previews.py
-
curl -L -o test_watch_scale_previews.py https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/scale/script/test_watch_scale_previews.py
7.55 kB
| import json | |
| import os | |
| from pathlib import Path | |
| import tempfile | |
| import unittest | |
| from unittest.mock import patch | |
| import watch_scale_previews as auto | |
| import preview_gpu_lease as gpu | |
| class AutoPreviewTests(unittest.TestCase): | |
| def test_simple_suite_is_separate_and_has_three_cases(self): | |
| with tempfile.TemporaryDirectory() as tmp: | |
| regular=auto.Watcher(Path(tmp)) | |
| simple=auto.Watcher(Path(tmp),simple_text=True) | |
| self.assertNotEqual(regular.base,simple.base) | |
| self.assertEqual(len(regular.case_ids),4) | |
| self.assertEqual(len(simple.case_ids),3) | |
| self.assertEqual(simple.state['expected_cases'],3) | |
| simple.bootstrap() | |
| self.assertEqual(simple.state['models'],[]) | |
| def test_retains_two_complete_previews_and_protects_partial(self): | |
| with tempfile.TemporaryDirectory() as tmp: | |
| w = auto.Watcher(Path(tmp)) | |
| jobs = [] | |
| for step in (500, 1000, 1500, 2000): | |
| job = w.base / 'jobs/S/hash' / f'ckpt_{step}__root40_v4' | |
| jobs.append(job) | |
| for i in range(4): | |
| case = job / 'text' / str(i) | |
| case.mkdir(parents=True) | |
| (case / 'render_status.json').write_text(json.dumps({'status': 'rendering' if step == 2000 else 'ready'})) | |
| (case / 'comparison.mp4').write_bytes(b'video') | |
| w.link_asset(f'{step}_{i}.mp4', case / 'comparison.mp4') | |
| w.state['models'] = [dict(size='S', step=1500, cases=[dict(video=f'1500_{i}.mp4') for i in range(4)])] | |
| w.prune_history() | |
| self.assertFalse(jobs[0].exists()) | |
| self.assertTrue(all(p.exists() for p in jobs[1:])) | |
| self.assertFalse((w.web / '500_0.mp4').is_symlink()) | |
| self.assertTrue((w.base / 'retired_records/S/hash/ckpt_500__root40_v4/text/0/render_status.json').exists()) | |
| self.assertEqual(len(w.assets), 12) | |
| def test_latest_complete_skips_partial_and_unconfirmed(self): | |
| with tempfile.TemporaryDirectory() as tmp: | |
| run = Path(tmp) | |
| required = ['model.safetensors', 'optimizer.bin', 'scheduler.bin', 'custom_checkpoint_0.pkl', | |
| 'custom_checkpoint_1.pkl', 'random_states_0.pkl', 'random_states_1.pkl'] | |
| for step in (500, 1000, 1500): | |
| directory = run / 'checkpoints' / f'ckpt_{step}' | |
| directory.mkdir(parents=True) | |
| for name in required: | |
| if step == 1000 and name == 'random_states_1.pkl': | |
| continue | |
| (directory / name).write_bytes(b'x') | |
| self.assertEqual(auto.latest_complete(run, {500, 1000}, 2)[0], 500) | |
| self.assertEqual(auto.latest_complete(run, {500, 1000, 1500}, 2)[0], 1500) | |
| self.assertIsNone(auto.latest_complete(run, set(), 2)) | |
| def test_coalesces_and_distinguishes_restarted_run(self): | |
| entry = dict(run='a', step=1000) | |
| self.assertFalse(auto.needs_update(entry, Path('a'), 1000)) | |
| self.assertFalse(auto.needs_update(entry, Path('a'), 500)) | |
| self.assertTrue(auto.needs_update(entry, Path('a'), 2000)) | |
| self.assertTrue(auto.needs_update(entry, Path('b'), 500)) | |
| def test_publish_is_all_or_nothing_and_urls_are_immutable(self): | |
| with tempfile.TemporaryDirectory() as tmp: | |
| w = auto.Watcher(Path(tmp)) | |
| w.state['models'] = [dict(size='S', step=1500, cases=[])] | |
| w.write('starting') | |
| previous = w.state_path.read_bytes() | |
| with self.assertRaises(ValueError): | |
| w.publish(dict(size='S', step=2000, cases=[dict(ready=True, video='missing.mp4')])) | |
| self.assertEqual(w.state_path.read_bytes(), previous) | |
| cases = [] | |
| for i in range(4): | |
| source = w.base / f'output{i}.mp4' | |
| source.write_bytes(b'video') | |
| w.link_asset(f'S_2000_{i}.mp4', source) | |
| cases.append(dict(id=str(i), ready=True, video=f'S_2000_{i}.mp4')) | |
| w.publish(dict(size='S', run='run', step=2000, cases=cases)) | |
| self.assertEqual(json.loads(w.state_path.read_text())['models'][0]['step'], 2000) | |
| self.assertEqual(len(json.loads(w.asset_path.read_text())), 4) | |
| with self.assertRaises(ValueError): | |
| w.link_asset('S_2000_0.mp4', w.base / 'output1.mp4') | |
| def test_release_only_temporary_weight_link(self): | |
| with tempfile.TemporaryDirectory() as tmp: | |
| group = Path(tmp) | |
| w = auto.Watcher(group) | |
| original = group / 'runs/model.safetensors' | |
| original.parent.mkdir() | |
| original.write_bytes(b'original weights') | |
| pinned = w.base / 'jobs/S/run/ckpt_2000/snapshot/checkpoints/ckpt_2000/model.safetensors' | |
| pinned.parent.mkdir(parents=True) | |
| os.link(original, pinned) | |
| w.state['pending_weight'] = str(pinned) | |
| w.release_weight() | |
| self.assertFalse(pinned.exists()) | |
| self.assertEqual(original.read_bytes(), b'original weights') | |
| w.state['pending_weight'] = str(original) | |
| with self.assertRaises(ValueError): | |
| w.release_weight() | |
| self.assertTrue(original.exists()) | |
| def test_keeper_never_stops_a_foreign_gpu_process(self): | |
| with patch.object(gpu, 'gpu_processes', return_value={1, 99}), patch.object(gpu, 'keeper_pids', return_value=[1]), \ | |
| patch.object(gpu, 'stop_keeper') as stop: | |
| with self.assertRaisesRegex(RuntimeError, 'non-keeper'): | |
| with gpu.lease_gpu3(): | |
| self.fail('Must not enter occupied GPU') | |
| stop.assert_not_called() | |
| def test_keeper_restored_on_inference_failure(self): | |
| with patch.object(gpu, 'gpu_processes', return_value=set()), patch.object(gpu, 'keeper_pids', return_value=[]), \ | |
| patch.object(gpu, 'stop_keeper'), patch.object(gpu, 'start_keeper') as start, \ | |
| patch.object(gpu, 'restore_keeper') as restore, patch.object(gpu.subprocess, 'check_output', return_value='0'): | |
| with self.assertRaisesRegex(RuntimeError, 'inference failed'): | |
| with gpu.lease_gpu3(): | |
| raise RuntimeError('inference failed') | |
| start.assert_not_called() | |
| restore.assert_called_once() | |
| def test_restore_checks_only_preview_gpu(self): | |
| with patch.object(gpu, 'stop_keeper') as stop, \ | |
| patch.object(gpu, 'start_keeper') as start, \ | |
| patch.object(gpu.subprocess, 'check_output', return_value='43000\n') as query: | |
| gpu.restore_keeper() | |
| stop.assert_called_once() | |
| start.assert_called_once_with() | |
| self.assertIn('--id=2', query.call_args.args[0]) | |
| def test_private_keeper_never_uses_shared_locks_or_other_gpus(self): | |
| with patch.object(gpu.subprocess, 'run') as run: | |
| gpu.start_keeper() | |
| env = run.call_args.kwargs['env'] | |
| self.assertEqual(env['MONITOR_GPU_INDICES'], '2') | |
| self.assertEqual(env['MONITOR_LOCK_FILE'], str(gpu.PID_FILES[1])) | |
| self.assertEqual(env['MONITOR_SUPERVISOR_PID_FILE'], str(gpu.PID_FILES[0])) | |
| self.assertTrue(env['MONITOR_SUPERVISOR_LOCK_FILE'].startswith(gpu.KEEPER_PREFIX)) | |
| self.assertNotIn('CUDA_VISIBLE_DEVICES', env) | |
| if __name__ == '__main__': | |
| unittest.main() | |