Download scale/script/test_dlc_local_monitor.py from Tinghong-Ye22/code-Movens-Zero_train: direct link, hf CLI and curl.
- Browser
- Download file 3.87 kB
-
https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/scale/script/test_dlc_local_monitor.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Movens-Zero_train/scale/script/test_dlc_local_monitor.py
-
curl -L -o test_dlc_local_monitor.py https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/scale/script/test_dlc_local_monitor.py
3.87 kB
| import hashlib | |
| import json | |
| from pathlib import Path | |
| import tempfile | |
| import time | |
| import unittest | |
| from unittest.mock import patch | |
| import dlc_local_storage as storage | |
| import dlc_local_telemetry as telemetry | |
| import dlc_local_dashboard as dashboard | |
| import watch_dlc_local_previews as preview | |
| class MonitorTests(unittest.TestCase): | |
| def test_tail_drops_partial_json_and_is_bounded(self): | |
| with tempfile.TemporaryDirectory() as tmp: | |
| p=Path(tmp)/'metrics';p.write_bytes(b'old-line\n{"step":1}\n{"step":2}\n{"partial":') | |
| self.assertEqual(telemetry.tail_complete(p,30),b'{"step":2}\n') | |
| def test_export_and_dashboard_show_latest_not_first_historical_step(self): | |
| with tempfile.TemporaryDirectory() as tmp: | |
| local=Path(tmp)/'local';monitor=Path(tmp)/'monitor';relative=Path('group') | |
| for size in 'SML': | |
| run=local/relative/'runs'/size;run.mkdir(parents=True) | |
| storage.save(local/relative/'runs'/f'latest_{size}.json',dict(run_dir=str(run))) | |
| cfg=dict(world_size_expected=2,train=dict(max_training_steps=1000000,train_batch_size=24, | |
| gradient_accumulation_steps=1,num_workers=8)) | |
| storage.save(run/'run_config.json',cfg) | |
| records=[dict(step=n,time=time.time(),metrics={'train/loss':.3,'train/loss_group/flow':.2}) for n in (299999,300000)] | |
| (run/'metrics.jsonl').write_text(''.join(json.dumps(r)+'\n' for r in records)+'{"incomplete":') | |
| (run/'training.log').write_text('running\n') | |
| result=telemetry.export_once(local,relative,monitor,storage.save) | |
| self.assertEqual(result['sizes'],dict(S=300000,M=300000,L=300000)) | |
| with patch.object(dashboard,'MONITOR',monitor),patch.object(dashboard,'CACHE',Path(tmp)/'cache'): | |
| r=dashboard.LiveRun(monitor/'live','S',default_backend='dlc',compact=True) | |
| r.update();s=r.summary(dict(processes=[],gpus=[]),time.time()) | |
| self.assertEqual(s['step'],300000);self.assertEqual(s['flow'],.2) | |
| self.assertEqual(s['indexing_bytes'],0);self.assertIsNone(s['error']) | |
| def test_consume_only_committed_backup_pin_survives_source_unlink(self): | |
| with tempfile.TemporaryDirectory() as tmp: | |
| root=Path(tmp);archive=root/'archive';group=root/'group';snap=archive/'S/ckpt_500' | |
| weights=snap/'checkpoints/ckpt_500/model.safetensors';weights.parent.mkdir(parents=True);weights.write_bytes(b'weights') | |
| files={'checkpoints/ckpt_500/model.safetensors':dict(bytes=7)} | |
| for name in ('run_config.json','dataset_statistics.json'): | |
| (snap/name).write_text('{}');files[name]=dict(sha256=hashlib.sha256(b'{}').hexdigest()) | |
| storage.save(archive/'latest_S.json',dict(step=500,snapshot=str(snap))) | |
| with patch.object(preview,'BACKUP',archive): | |
| w=preview.Watcher(group,simple_text=True) | |
| with self.assertRaises(FileNotFoundError):w.scan('S') | |
| storage.save(snap/'COMPLETE.json',dict(owner='movens-dlc-local-v1',size='S',step=500,run_name='test',files=files)) | |
| run,step,ckpt=w.scan('S');self.assertEqual(step,500) | |
| weights.unlink();self.assertEqual((ckpt/'model.safetensors').read_bytes(),b'weights') | |
| w.release_source();self.assertFalse((ckpt/'model.safetensors').exists()) | |
| def test_idle_keeper_yields_to_foreign_task(self): | |
| with patch.object(preview.gpu,'gpu_processes',return_value={123,456}),patch.object(preview.gpu,'keeper_pids',return_value=[123]),patch.object(preview.gpu,'stop_keeper') as stop,patch.object(preview.gpu,'restore_keeper') as restore: | |
| preview.protect_idle_gpu();stop.assert_called_once();restore.assert_not_called() | |
| if __name__=='__main__':unittest.main() | |