File size: 6,465 Bytes
a64600b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
import json
from pathlib import Path
import tempfile
import unittest

import numpy as np
from PIL import Image

from suite.visualize_v2 import (aggregate_dose_sweeps, audit_dose, build_artifacts, collect_dose,
                                load_run, prepare_media, select_panels)


def job(ident, task='a', alpha=.2, mode='raw', **extra):
    value = dict(id=ident, study='control_v2', experiment='v2_calibrated',
        model='meissonic', task=dict(id=task, split='test'), seed=3, alpha=alpha,
        method_id='confidence_duration', arm='confidence',
        control_schedule='duration', reference_copy=True,
        calibration_mode=mode, **extra)
    if mode in ('raw', 'mapped'):
        value['requested_effect'] = .2
    return value


def result(j, l1=.3, direct=.5, exposure=.25):
    return dict(status='complete', job=j, metrics=dict(target_reference_l1=l1,
        common_anchor_target_reference_l1=.5,
        control_audit=dict(direct_copy_fraction=direct, exposure_per_target_step=exposure)))


class VisualizeV2Tests(unittest.TestCase):
    def test_fixed_selection_retains_missing_unreachable_and_does_not_replace(self):
        expected = [job('a'), job('b', task='b'), job('c', task='c'),
                    job('unreachable', alpha=None, mode='mapped', planning_status='unreachable')]
        rows = [result(expected[1]), result(expected[2])]
        panels, selected = select_panels(expected, rows)
        self.assertEqual(selected, {'meissonic': ['a', 'b']})
        self.assertEqual(len(panels), 6)
        first = panels[0]['cells']
        self.assertEqual(first[0]['status'], 'missing')
        self.assertEqual(first[7]['status'], 'unreachable')
        self.assertTrue(all(p['task'] != 'c' for p in panels))

    def test_trace_dose_uses_target_count_and_steps_as_denominators(self):
        with tempfile.TemporaryDirectory() as d:
            trace = [dict(target_count=4, target_direct_copy_count=1, target_temporary_count=3),
                     dict(target_count=4, target_direct_copy_count=2, target_temporary_count=1)]
            (Path(d) / 'trace.jsonl').write_text(''.join(json.dumps(r) + '\n' for r in trace))
            out = audit_dose(dict(metrics={}, _source_folder=d))
            self.assertEqual(out['direct_copy_fraction'], .75)
            self.assertEqual(out['exposure_per_target_step'], .5)
            self.assertEqual(out['source'], 'trace.jsonl')

    def test_endpoint_dose_aggregates_seeds_not_alpha_points(self):
        rows = []
        for seed in (3, 4):
            rows.append(result(job(f'endpoint{seed}', alpha=1., mode='anchor', seed_marker=seed), l1=.1 if seed == 3 else .3))
            rows[-1]['job']['seed'] = seed
            rows.append(result(job(f'raw{seed}', alpha=1., mode='raw'), l1=.2))
            rows[-1]['job']['seed'] = seed
        points, means, summary = collect_dose(rows)
        self.assertEqual(len(points), 4)
        self.assertEqual(len(means), 1)
        self.assertEqual(means[0]['seeds'], [3, 4])
        self.assertAlmostEqual(means[0]['improvement_from_common_confidence_alpha0'], .3)
        self.assertEqual(summary['endpoint_task_method_points'], 1)

    def test_saved_common_anchor_is_checked_against_available_alpha0(self):
        zero = result(job('zero', alpha=0., mode='anchor'), l1=.5)
        bad = result(job('bad', alpha=1., mode='anchor'), l1=.1)
        bad['metrics']['common_anchor_target_reference_l1'] = .6
        points, means, summary = collect_dose([zero, bad])
        self.assertEqual(len(points), 1)
        self.assertEqual(len(means), 0)
        self.assertEqual(summary['skipped']['stored_anchor_differs_from_anchor_result'], 1)

    def test_sweep_dose_gives_tasks_equal_weight_after_seed_aggregation(self):
        rows = [result(job('a3'), l1=.1), result(job('a4'), l1=.1),
                result(job('b3', task='b'), l1=.5)]
        rows[1]['job']['seed'] = 4
        points = collect_dose(rows)[0]
        tasks, summaries = aggregate_dose_sweeps(points)
        self.assertEqual(len(tasks), 2)
        self.assertEqual(len(summaries), 1)
        self.assertEqual(summaries[0]['task_count'], 2)
        self.assertAlmostEqual(summaries[0]['improvement_from_common_confidence_alpha0'], .2)

    def test_momask_media_is_rendered_from_saved_joints_only(self):
        with tempfile.TemporaryDirectory() as d:
            root = Path(d)
            folder, output = root / 'job', root / 'export'
            folder.mkdir()
            output.mkdir()
            xyz = np.zeros((3, 22, 3), dtype=float)
            xyz[:, :, 1] = np.linspace(0, 1.7, 22)
            np.save(folder / 'output.joints.npy', xyz)
            row = result(job('motion'))
            row['job']['model'] = 'momask'
            (folder / 'result.json').write_text(json.dumps(row))
            row.update(_source_folder=str(folder), _source_result=str(folder / 'result.json'))
            media = prepare_media(row, output)
            self.assertTrue((output / media['preview']).exists())
            self.assertTrue((output / media['animation']).exists())
            self.assertEqual(media['source_joints'], str(folder / 'output.joints.npy'))

    def test_export_uses_locked_slots_and_real_saved_media(self):
        with tempfile.TemporaryDirectory() as d:
            root = Path(d)
            run, output = root / 'run', root / 'showcase'
            run.mkdir()
            expected = [job('raw'), job('mapped', alpha=None, mode='mapped', planning_status='unreachable')]
            (run / 'expected_slots.json').write_text(json.dumps(dict(expected_slots=expected)))
            folder = run / 'jobs/raw'
            folder.mkdir(parents=True)
            (folder / 'result.json').write_text(json.dumps(result(expected[0])))
            Image.new('RGB', (16, 16), '#234567').save(folder / 'output.png')
            manifest = build_artifacts(run, output)
            self.assertEqual(manifest['panels'], 3)
            self.assertEqual(manifest['statuses']['complete'], 1)
            self.assertEqual(manifest['statuses']['unreachable'], 1)
            data = json.loads((output / 'manifest.json').read_text())
            self.assertFalse(data['selection_rule']['substitution_allowed'])
            self.assertEqual(data['selection_rule']['seed'], 3)
            self.assertTrue((output / 'index.html').exists())
            self.assertEqual(len(list((output / 'figures').glob('*.pdf'))), 4)


if __name__ == '__main__':
    unittest.main()