snf_experiments / method_v2 /evidence /code /suite /tests /test_visualize_v2.py
CharlesZhang-USTC's picture
Add Control v2 incremental experiment results (evidence, visuals, 42 fixed samples)
a64600b verified
Raw History Blame Contribute Delete
6.47 kB
import json
from pathlib import Path
import tempfile
import unittest
import numpy as np
from PIL import Image
from suite.visualize_v2 import (aggregate_dose_sweeps, audit_dose, build_artifacts, collect_dose,
load_run, prepare_media, select_panels)
def job(ident, task='a', alpha=.2, mode='raw', **extra):
value = dict(id=ident, study='control_v2', experiment='v2_calibrated',
model='meissonic', task=dict(id=task, split='test'), seed=3, alpha=alpha,
method_id='confidence_duration', arm='confidence',
control_schedule='duration', reference_copy=True,
calibration_mode=mode, **extra)
if mode in ('raw', 'mapped'):
value['requested_effect'] = .2
return value
def result(j, l1=.3, direct=.5, exposure=.25):
return dict(status='complete', job=j, metrics=dict(target_reference_l1=l1,
common_anchor_target_reference_l1=.5,
control_audit=dict(direct_copy_fraction=direct, exposure_per_target_step=exposure)))
class VisualizeV2Tests(unittest.TestCase):
def test_fixed_selection_retains_missing_unreachable_and_does_not_replace(self):
expected = [job('a'), job('b', task='b'), job('c', task='c'),
job('unreachable', alpha=None, mode='mapped', planning_status='unreachable')]
rows = [result(expected[1]), result(expected[2])]
panels, selected = select_panels(expected, rows)
self.assertEqual(selected, {'meissonic': ['a', 'b']})
self.assertEqual(len(panels), 6)
first = panels[0]['cells']
self.assertEqual(first[0]['status'], 'missing')
self.assertEqual(first[7]['status'], 'unreachable')
self.assertTrue(all(p['task'] != 'c' for p in panels))
def test_trace_dose_uses_target_count_and_steps_as_denominators(self):
with tempfile.TemporaryDirectory() as d:
trace = [dict(target_count=4, target_direct_copy_count=1, target_temporary_count=3),
dict(target_count=4, target_direct_copy_count=2, target_temporary_count=1)]
(Path(d) / 'trace.jsonl').write_text(''.join(json.dumps(r) + '\n' for r in trace))
out = audit_dose(dict(metrics={}, _source_folder=d))
self.assertEqual(out['direct_copy_fraction'], .75)
self.assertEqual(out['exposure_per_target_step'], .5)
self.assertEqual(out['source'], 'trace.jsonl')
def test_endpoint_dose_aggregates_seeds_not_alpha_points(self):
rows = []
for seed in (3, 4):
rows.append(result(job(f'endpoint{seed}', alpha=1., mode='anchor', seed_marker=seed), l1=.1 if seed == 3 else .3))
rows[-1]['job']['seed'] = seed
rows.append(result(job(f'raw{seed}', alpha=1., mode='raw'), l1=.2))
rows[-1]['job']['seed'] = seed
points, means, summary = collect_dose(rows)
self.assertEqual(len(points), 4)
self.assertEqual(len(means), 1)
self.assertEqual(means[0]['seeds'], [3, 4])
self.assertAlmostEqual(means[0]['improvement_from_common_confidence_alpha0'], .3)
self.assertEqual(summary['endpoint_task_method_points'], 1)
def test_saved_common_anchor_is_checked_against_available_alpha0(self):
zero = result(job('zero', alpha=0., mode='anchor'), l1=.5)
bad = result(job('bad', alpha=1., mode='anchor'), l1=.1)
bad['metrics']['common_anchor_target_reference_l1'] = .6
points, means, summary = collect_dose([zero, bad])
self.assertEqual(len(points), 1)
self.assertEqual(len(means), 0)
self.assertEqual(summary['skipped']['stored_anchor_differs_from_anchor_result'], 1)
def test_sweep_dose_gives_tasks_equal_weight_after_seed_aggregation(self):
rows = [result(job('a3'), l1=.1), result(job('a4'), l1=.1),
result(job('b3', task='b'), l1=.5)]
rows[1]['job']['seed'] = 4
points = collect_dose(rows)[0]
tasks, summaries = aggregate_dose_sweeps(points)
self.assertEqual(len(tasks), 2)
self.assertEqual(len(summaries), 1)
self.assertEqual(summaries[0]['task_count'], 2)
self.assertAlmostEqual(summaries[0]['improvement_from_common_confidence_alpha0'], .2)
def test_momask_media_is_rendered_from_saved_joints_only(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
folder, output = root / 'job', root / 'export'
folder.mkdir()
output.mkdir()
xyz = np.zeros((3, 22, 3), dtype=float)
xyz[:, :, 1] = np.linspace(0, 1.7, 22)
np.save(folder / 'output.joints.npy', xyz)
row = result(job('motion'))
row['job']['model'] = 'momask'
(folder / 'result.json').write_text(json.dumps(row))
row.update(_source_folder=str(folder), _source_result=str(folder / 'result.json'))
media = prepare_media(row, output)
self.assertTrue((output / media['preview']).exists())
self.assertTrue((output / media['animation']).exists())
self.assertEqual(media['source_joints'], str(folder / 'output.joints.npy'))
def test_export_uses_locked_slots_and_real_saved_media(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
run, output = root / 'run', root / 'showcase'
run.mkdir()
expected = [job('raw'), job('mapped', alpha=None, mode='mapped', planning_status='unreachable')]
(run / 'expected_slots.json').write_text(json.dumps(dict(expected_slots=expected)))
folder = run / 'jobs/raw'
folder.mkdir(parents=True)
(folder / 'result.json').write_text(json.dumps(result(expected[0])))
Image.new('RGB', (16, 16), '#234567').save(folder / 'output.png')
manifest = build_artifacts(run, output)
self.assertEqual(manifest['panels'], 3)
self.assertEqual(manifest['statuses']['complete'], 1)
self.assertEqual(manifest['statuses']['unreachable'], 1)
data = json.loads((output / 'manifest.json').read_text())
self.assertFalse(data['selection_rule']['substitution_allowed'])
self.assertEqual(data['selection_rule']['seed'], 3)
self.assertTrue((output / 'index.html').exists())
self.assertEqual(len(list((output / 'figures').glob('*.pdf'))), 4)
if __name__ == '__main__':
unittest.main()