devils-agent / tests /test_synthetic.py
devildasdf's picture
Upload experimental BAIM code, research checkpoints and measured evaluations
795f737 verified
Raw
History Blame Contribute Delete
858 Bytes
import unittest
from baim.synthetic import generate, render, TRAIN_TEMPLATES, TEST_TEMPLATES
class SyntheticTests(unittest.TestCase):
def test_split_templates_are_disjoint_and_reproducible(self):
self.assertFalse(set(TRAIN_TEMPLATES) & set(TEST_TEMPLATES))
self.assertEqual(list(generate('train',8,1)),list(generate('train',8,1)))
self.assertNotEqual(list(generate('train',8,1)),list(generate('train',8,2)))
def test_target_label_matches_role(self):
for row in generate('train',100,9):
role = row['elements'][row['target']]['role']
expected = {'C':{'button','link'},'T':{'textbox'},'O':{'combobox'}}[row['action']]
self.assertIn(role,expected)
self.assertIn(row['elements'][row['target']]['name'],row['goal'])
self.assertIn('<!doctype html>',render(row))