File size: 6,465 Bytes
a64600b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 | import json
from pathlib import Path
import tempfile
import unittest
import numpy as np
from PIL import Image
from suite.visualize_v2 import (aggregate_dose_sweeps, audit_dose, build_artifacts, collect_dose,
load_run, prepare_media, select_panels)
def job(ident, task='a', alpha=.2, mode='raw', **extra):
value = dict(id=ident, study='control_v2', experiment='v2_calibrated',
model='meissonic', task=dict(id=task, split='test'), seed=3, alpha=alpha,
method_id='confidence_duration', arm='confidence',
control_schedule='duration', reference_copy=True,
calibration_mode=mode, **extra)
if mode in ('raw', 'mapped'):
value['requested_effect'] = .2
return value
def result(j, l1=.3, direct=.5, exposure=.25):
return dict(status='complete', job=j, metrics=dict(target_reference_l1=l1,
common_anchor_target_reference_l1=.5,
control_audit=dict(direct_copy_fraction=direct, exposure_per_target_step=exposure)))
class VisualizeV2Tests(unittest.TestCase):
def test_fixed_selection_retains_missing_unreachable_and_does_not_replace(self):
expected = [job('a'), job('b', task='b'), job('c', task='c'),
job('unreachable', alpha=None, mode='mapped', planning_status='unreachable')]
rows = [result(expected[1]), result(expected[2])]
panels, selected = select_panels(expected, rows)
self.assertEqual(selected, {'meissonic': ['a', 'b']})
self.assertEqual(len(panels), 6)
first = panels[0]['cells']
self.assertEqual(first[0]['status'], 'missing')
self.assertEqual(first[7]['status'], 'unreachable')
self.assertTrue(all(p['task'] != 'c' for p in panels))
def test_trace_dose_uses_target_count_and_steps_as_denominators(self):
with tempfile.TemporaryDirectory() as d:
trace = [dict(target_count=4, target_direct_copy_count=1, target_temporary_count=3),
dict(target_count=4, target_direct_copy_count=2, target_temporary_count=1)]
(Path(d) / 'trace.jsonl').write_text(''.join(json.dumps(r) + '\n' for r in trace))
out = audit_dose(dict(metrics={}, _source_folder=d))
self.assertEqual(out['direct_copy_fraction'], .75)
self.assertEqual(out['exposure_per_target_step'], .5)
self.assertEqual(out['source'], 'trace.jsonl')
def test_endpoint_dose_aggregates_seeds_not_alpha_points(self):
rows = []
for seed in (3, 4):
rows.append(result(job(f'endpoint{seed}', alpha=1., mode='anchor', seed_marker=seed), l1=.1 if seed == 3 else .3))
rows[-1]['job']['seed'] = seed
rows.append(result(job(f'raw{seed}', alpha=1., mode='raw'), l1=.2))
rows[-1]['job']['seed'] = seed
points, means, summary = collect_dose(rows)
self.assertEqual(len(points), 4)
self.assertEqual(len(means), 1)
self.assertEqual(means[0]['seeds'], [3, 4])
self.assertAlmostEqual(means[0]['improvement_from_common_confidence_alpha0'], .3)
self.assertEqual(summary['endpoint_task_method_points'], 1)
def test_saved_common_anchor_is_checked_against_available_alpha0(self):
zero = result(job('zero', alpha=0., mode='anchor'), l1=.5)
bad = result(job('bad', alpha=1., mode='anchor'), l1=.1)
bad['metrics']['common_anchor_target_reference_l1'] = .6
points, means, summary = collect_dose([zero, bad])
self.assertEqual(len(points), 1)
self.assertEqual(len(means), 0)
self.assertEqual(summary['skipped']['stored_anchor_differs_from_anchor_result'], 1)
def test_sweep_dose_gives_tasks_equal_weight_after_seed_aggregation(self):
rows = [result(job('a3'), l1=.1), result(job('a4'), l1=.1),
result(job('b3', task='b'), l1=.5)]
rows[1]['job']['seed'] = 4
points = collect_dose(rows)[0]
tasks, summaries = aggregate_dose_sweeps(points)
self.assertEqual(len(tasks), 2)
self.assertEqual(len(summaries), 1)
self.assertEqual(summaries[0]['task_count'], 2)
self.assertAlmostEqual(summaries[0]['improvement_from_common_confidence_alpha0'], .2)
def test_momask_media_is_rendered_from_saved_joints_only(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
folder, output = root / 'job', root / 'export'
folder.mkdir()
output.mkdir()
xyz = np.zeros((3, 22, 3), dtype=float)
xyz[:, :, 1] = np.linspace(0, 1.7, 22)
np.save(folder / 'output.joints.npy', xyz)
row = result(job('motion'))
row['job']['model'] = 'momask'
(folder / 'result.json').write_text(json.dumps(row))
row.update(_source_folder=str(folder), _source_result=str(folder / 'result.json'))
media = prepare_media(row, output)
self.assertTrue((output / media['preview']).exists())
self.assertTrue((output / media['animation']).exists())
self.assertEqual(media['source_joints'], str(folder / 'output.joints.npy'))
def test_export_uses_locked_slots_and_real_saved_media(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
run, output = root / 'run', root / 'showcase'
run.mkdir()
expected = [job('raw'), job('mapped', alpha=None, mode='mapped', planning_status='unreachable')]
(run / 'expected_slots.json').write_text(json.dumps(dict(expected_slots=expected)))
folder = run / 'jobs/raw'
folder.mkdir(parents=True)
(folder / 'result.json').write_text(json.dumps(result(expected[0])))
Image.new('RGB', (16, 16), '#234567').save(folder / 'output.png')
manifest = build_artifacts(run, output)
self.assertEqual(manifest['panels'], 3)
self.assertEqual(manifest['statuses']['complete'], 1)
self.assertEqual(manifest['statuses']['unreachable'], 1)
data = json.loads((output / 'manifest.json').read_text())
self.assertFalse(data['selection_rule']['substitution_allowed'])
self.assertEqual(data['selection_rule']['seed'], 3)
self.assertTrue((output / 'index.html').exists())
self.assertEqual(len(list((output / 'figures').glob('*.pdf'))), 4)
if __name__ == '__main__':
unittest.main()
|