import hashlib import json from pathlib import Path import tempfile import time import unittest from unittest.mock import patch import dlc_local_storage as storage import dlc_local_telemetry as telemetry import dlc_local_dashboard as dashboard import watch_dlc_local_previews as preview class MonitorTests(unittest.TestCase): def test_tail_drops_partial_json_and_is_bounded(self): with tempfile.TemporaryDirectory() as tmp: p=Path(tmp)/'metrics';p.write_bytes(b'old-line\n{"step":1}\n{"step":2}\n{"partial":') self.assertEqual(telemetry.tail_complete(p,30),b'{"step":2}\n') def test_export_and_dashboard_show_latest_not_first_historical_step(self): with tempfile.TemporaryDirectory() as tmp: local=Path(tmp)/'local';monitor=Path(tmp)/'monitor';relative=Path('group') for size in 'SML': run=local/relative/'runs'/size;run.mkdir(parents=True) storage.save(local/relative/'runs'/f'latest_{size}.json',dict(run_dir=str(run))) cfg=dict(world_size_expected=2,train=dict(max_training_steps=1000000,train_batch_size=24, gradient_accumulation_steps=1,num_workers=8)) storage.save(run/'run_config.json',cfg) records=[dict(step=n,time=time.time(),metrics={'train/loss':.3,'train/loss_group/flow':.2}) for n in (299999,300000)] (run/'metrics.jsonl').write_text(''.join(json.dumps(r)+'\n' for r in records)+'{"incomplete":') (run/'training.log').write_text('running\n') result=telemetry.export_once(local,relative,monitor,storage.save) self.assertEqual(result['sizes'],dict(S=300000,M=300000,L=300000)) with patch.object(dashboard,'MONITOR',monitor),patch.object(dashboard,'CACHE',Path(tmp)/'cache'): r=dashboard.LiveRun(monitor/'live','S',default_backend='dlc',compact=True) r.update();s=r.summary(dict(processes=[],gpus=[]),time.time()) self.assertEqual(s['step'],300000);self.assertEqual(s['flow'],.2) self.assertEqual(s['indexing_bytes'],0);self.assertIsNone(s['error']) def test_consume_only_committed_backup_pin_survives_source_unlink(self): with tempfile.TemporaryDirectory() as tmp: root=Path(tmp);archive=root/'archive';group=root/'group';snap=archive/'S/ckpt_500' weights=snap/'checkpoints/ckpt_500/model.safetensors';weights.parent.mkdir(parents=True);weights.write_bytes(b'weights') files={'checkpoints/ckpt_500/model.safetensors':dict(bytes=7)} for name in ('run_config.json','dataset_statistics.json'): (snap/name).write_text('{}');files[name]=dict(sha256=hashlib.sha256(b'{}').hexdigest()) storage.save(archive/'latest_S.json',dict(step=500,snapshot=str(snap))) with patch.object(preview,'BACKUP',archive): w=preview.Watcher(group,simple_text=True) with self.assertRaises(FileNotFoundError):w.scan('S') storage.save(snap/'COMPLETE.json',dict(owner='movens-dlc-local-v1',size='S',step=500,run_name='test',files=files)) run,step,ckpt=w.scan('S');self.assertEqual(step,500) weights.unlink();self.assertEqual((ckpt/'model.safetensors').read_bytes(),b'weights') w.release_source();self.assertFalse((ckpt/'model.safetensors').exists()) def test_idle_keeper_yields_to_foreign_task(self): with patch.object(preview.gpu,'gpu_processes',return_value={123,456}),patch.object(preview.gpu,'keeper_pids',return_value=[123]),patch.object(preview.gpu,'stop_keeper') as stop,patch.object(preview.gpu,'restore_keeper') as restore: preview.protect_idle_gpu();stop.assert_called_once();restore.assert_not_called() if __name__=='__main__':unittest.main()