code-Movens-Zero_train / scale /script /test_dlc_local_monitor.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
13cc860 verified
Raw History Blame Contribute Delete
3.87 kB
import hashlib
import json
from pathlib import Path
import tempfile
import time
import unittest
from unittest.mock import patch
import dlc_local_storage as storage
import dlc_local_telemetry as telemetry
import dlc_local_dashboard as dashboard
import watch_dlc_local_previews as preview
class MonitorTests(unittest.TestCase):
def test_tail_drops_partial_json_and_is_bounded(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp)/'metrics';p.write_bytes(b'old-line\n{"step":1}\n{"step":2}\n{"partial":')
self.assertEqual(telemetry.tail_complete(p,30),b'{"step":2}\n')
def test_export_and_dashboard_show_latest_not_first_historical_step(self):
with tempfile.TemporaryDirectory() as tmp:
local=Path(tmp)/'local';monitor=Path(tmp)/'monitor';relative=Path('group')
for size in 'SML':
run=local/relative/'runs'/size;run.mkdir(parents=True)
storage.save(local/relative/'runs'/f'latest_{size}.json',dict(run_dir=str(run)))
cfg=dict(world_size_expected=2,train=dict(max_training_steps=1000000,train_batch_size=24,
gradient_accumulation_steps=1,num_workers=8))
storage.save(run/'run_config.json',cfg)
records=[dict(step=n,time=time.time(),metrics={'train/loss':.3,'train/loss_group/flow':.2}) for n in (299999,300000)]
(run/'metrics.jsonl').write_text(''.join(json.dumps(r)+'\n' for r in records)+'{"incomplete":')
(run/'training.log').write_text('running\n')
result=telemetry.export_once(local,relative,monitor,storage.save)
self.assertEqual(result['sizes'],dict(S=300000,M=300000,L=300000))
with patch.object(dashboard,'MONITOR',monitor),patch.object(dashboard,'CACHE',Path(tmp)/'cache'):
r=dashboard.LiveRun(monitor/'live','S',default_backend='dlc',compact=True)
r.update();s=r.summary(dict(processes=[],gpus=[]),time.time())
self.assertEqual(s['step'],300000);self.assertEqual(s['flow'],.2)
self.assertEqual(s['indexing_bytes'],0);self.assertIsNone(s['error'])
def test_consume_only_committed_backup_pin_survives_source_unlink(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);archive=root/'archive';group=root/'group';snap=archive/'S/ckpt_500'
weights=snap/'checkpoints/ckpt_500/model.safetensors';weights.parent.mkdir(parents=True);weights.write_bytes(b'weights')
files={'checkpoints/ckpt_500/model.safetensors':dict(bytes=7)}
for name in ('run_config.json','dataset_statistics.json'):
(snap/name).write_text('{}');files[name]=dict(sha256=hashlib.sha256(b'{}').hexdigest())
storage.save(archive/'latest_S.json',dict(step=500,snapshot=str(snap)))
with patch.object(preview,'BACKUP',archive):
w=preview.Watcher(group,simple_text=True)
with self.assertRaises(FileNotFoundError):w.scan('S')
storage.save(snap/'COMPLETE.json',dict(owner='movens-dlc-local-v1',size='S',step=500,run_name='test',files=files))
run,step,ckpt=w.scan('S');self.assertEqual(step,500)
weights.unlink();self.assertEqual((ckpt/'model.safetensors').read_bytes(),b'weights')
w.release_source();self.assertFalse((ckpt/'model.safetensors').exists())
def test_idle_keeper_yields_to_foreign_task(self):
with patch.object(preview.gpu,'gpu_processes',return_value={123,456}),patch.object(preview.gpu,'keeper_pids',return_value=[123]),patch.object(preview.gpu,'stop_keeper') as stop,patch.object(preview.gpu,'restore_keeper') as restore:
preview.protect_idle_gpu();stop.assert_called_once();restore.assert_not_called()
if __name__=='__main__':unittest.main()