robottt-pi05-base-comparison / code /tests /test_completion_watch.py
jisue's picture
Automate completion and error wakeups to the existing Codex session
e2d338b verified
Raw History Blame Contribute Delete
3.31 kB
import hashlib
import importlib.util
import json
import pathlib
import tempfile
import unittest
from unittest.mock import patch
SPEC = importlib.util.spec_from_file_location('comparison_watch', pathlib.Path(__file__).resolve().parents[1] / 'scripts/watch_comparison.py')
watch = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(watch)
class CompletionWatchTests(unittest.TestCase):
def put(self, root, name, value):
p = root / name; p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(json.dumps(value)); return p
def test_live_training_and_error(self):
with tempfile.TemporaryDirectory() as d:
root = pathlib.Path(d)
self.put(root, 'comparison-status.json', dict(stage='train-reference', state='running', pid=7, updated_unix_time=1))
self.put(root, 'runs/comparison-reference/status.json', dict(updated_unix_time=990))
self.assertEqual(watch.events(root, 1000, alive=lambda _: True), ([], False))
found, _ = watch.events(root, 3000, alive=lambda _: True)
self.assertTrue(found[0][0].startswith('driver-stalled:'))
self.put(root, 'comparison-status.json', dict(state='failed', updated_unix_time=3000))
found, _ = watch.events(root, 3001, alive=lambda _: False)
self.assertEqual(len(found), 1)
self.assertTrue(found[0][0].startswith('driver-error:'))
def test_pair_requires_matching_identities(self):
with tempfile.TemporaryDirectory() as d:
root = pathlib.Path(d)
self.put(root, 'libero-eval-plan.json', {'episodes': [1, 2]})
self.put(root, 'comparison-status.json', dict(state='running', pid=7, updated_unix_time=100))
for kind in ('lite', 'reference'):
config = self.put(root, 'comparison-' + kind + '.json', dict(updates=2))
self.put(root, 'runs/comparison-' + kind + '/completed.json', dict(status='completed', updates=2, checkpoint={'sha256': kind}))
self.put(root, 'evaluations/libero-' + kind + '/summary.json', dict(complete=True, attempts=2, identity=dict(checkpoint_sha256=kind, run_config_sha256=hashlib.sha256(config.read_bytes()).hexdigest())))
found, complete = watch.events(root, 101, alive=lambda _: True)
self.assertTrue(complete)
self.assertEqual(len(found), 2)
summary = root / 'evaluations/libero-reference/summary.json'
value = json.loads(summary.read_text()); value['identity']['checkpoint_sha256'] = 'stale'; summary.write_text(json.dumps(value))
self.assertFalse(watch.events(root, 101, alive=lambda _: True)[1])
def test_retry_failed_queue_and_deduplicate_success(self):
with tempfile.TemporaryDirectory() as d:
root = pathlib.Path(d); calls = []
def send(config, message):
calls.append(message)
return dict(queued=len(calls) > 1)
with patch.object(watch, 'events', return_value=([('event-id', 'completion')], False)):
watch.check(root, {}, send)
watch.check(root, {}, send)
watch.check(root, {}, send)
self.assertEqual(len(calls), 2)
if __name__ == '__main__':
unittest.main()