Add Control v2 incremental experiment results (evidence, visuals, 42 fixed samples)
a64600b verified Download method_v2/evidence/code/suite/tests/test_visualize_v2.py from CharlesZhang-USTC/snf_experiments: direct link, hf CLI and curl.
- Browser
- Download file 6.47 kB
-
https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/method_v2/evidence/code/suite/tests/test_visualize_v2.py
- Command line
-
hf download hf://CharlesZhang-USTC/snf_experiments/method_v2/evidence/code/suite/tests/test_visualize_v2.py
-
curl -L -o test_visualize_v2.py https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/method_v2/evidence/code/suite/tests/test_visualize_v2.py
6.47 kB
| import json | |
| from pathlib import Path | |
| import tempfile | |
| import unittest | |
| import numpy as np | |
| from PIL import Image | |
| from suite.visualize_v2 import (aggregate_dose_sweeps, audit_dose, build_artifacts, collect_dose, | |
| load_run, prepare_media, select_panels) | |
| def job(ident, task='a', alpha=.2, mode='raw', **extra): | |
| value = dict(id=ident, study='control_v2', experiment='v2_calibrated', | |
| model='meissonic', task=dict(id=task, split='test'), seed=3, alpha=alpha, | |
| method_id='confidence_duration', arm='confidence', | |
| control_schedule='duration', reference_copy=True, | |
| calibration_mode=mode, **extra) | |
| if mode in ('raw', 'mapped'): | |
| value['requested_effect'] = .2 | |
| return value | |
| def result(j, l1=.3, direct=.5, exposure=.25): | |
| return dict(status='complete', job=j, metrics=dict(target_reference_l1=l1, | |
| common_anchor_target_reference_l1=.5, | |
| control_audit=dict(direct_copy_fraction=direct, exposure_per_target_step=exposure))) | |
| class VisualizeV2Tests(unittest.TestCase): | |
| def test_fixed_selection_retains_missing_unreachable_and_does_not_replace(self): | |
| expected = [job('a'), job('b', task='b'), job('c', task='c'), | |
| job('unreachable', alpha=None, mode='mapped', planning_status='unreachable')] | |
| rows = [result(expected[1]), result(expected[2])] | |
| panels, selected = select_panels(expected, rows) | |
| self.assertEqual(selected, {'meissonic': ['a', 'b']}) | |
| self.assertEqual(len(panels), 6) | |
| first = panels[0]['cells'] | |
| self.assertEqual(first[0]['status'], 'missing') | |
| self.assertEqual(first[7]['status'], 'unreachable') | |
| self.assertTrue(all(p['task'] != 'c' for p in panels)) | |
| def test_trace_dose_uses_target_count_and_steps_as_denominators(self): | |
| with tempfile.TemporaryDirectory() as d: | |
| trace = [dict(target_count=4, target_direct_copy_count=1, target_temporary_count=3), | |
| dict(target_count=4, target_direct_copy_count=2, target_temporary_count=1)] | |
| (Path(d) / 'trace.jsonl').write_text(''.join(json.dumps(r) + '\n' for r in trace)) | |
| out = audit_dose(dict(metrics={}, _source_folder=d)) | |
| self.assertEqual(out['direct_copy_fraction'], .75) | |
| self.assertEqual(out['exposure_per_target_step'], .5) | |
| self.assertEqual(out['source'], 'trace.jsonl') | |
| def test_endpoint_dose_aggregates_seeds_not_alpha_points(self): | |
| rows = [] | |
| for seed in (3, 4): | |
| rows.append(result(job(f'endpoint{seed}', alpha=1., mode='anchor', seed_marker=seed), l1=.1 if seed == 3 else .3)) | |
| rows[-1]['job']['seed'] = seed | |
| rows.append(result(job(f'raw{seed}', alpha=1., mode='raw'), l1=.2)) | |
| rows[-1]['job']['seed'] = seed | |
| points, means, summary = collect_dose(rows) | |
| self.assertEqual(len(points), 4) | |
| self.assertEqual(len(means), 1) | |
| self.assertEqual(means[0]['seeds'], [3, 4]) | |
| self.assertAlmostEqual(means[0]['improvement_from_common_confidence_alpha0'], .3) | |
| self.assertEqual(summary['endpoint_task_method_points'], 1) | |
| def test_saved_common_anchor_is_checked_against_available_alpha0(self): | |
| zero = result(job('zero', alpha=0., mode='anchor'), l1=.5) | |
| bad = result(job('bad', alpha=1., mode='anchor'), l1=.1) | |
| bad['metrics']['common_anchor_target_reference_l1'] = .6 | |
| points, means, summary = collect_dose([zero, bad]) | |
| self.assertEqual(len(points), 1) | |
| self.assertEqual(len(means), 0) | |
| self.assertEqual(summary['skipped']['stored_anchor_differs_from_anchor_result'], 1) | |
| def test_sweep_dose_gives_tasks_equal_weight_after_seed_aggregation(self): | |
| rows = [result(job('a3'), l1=.1), result(job('a4'), l1=.1), | |
| result(job('b3', task='b'), l1=.5)] | |
| rows[1]['job']['seed'] = 4 | |
| points = collect_dose(rows)[0] | |
| tasks, summaries = aggregate_dose_sweeps(points) | |
| self.assertEqual(len(tasks), 2) | |
| self.assertEqual(len(summaries), 1) | |
| self.assertEqual(summaries[0]['task_count'], 2) | |
| self.assertAlmostEqual(summaries[0]['improvement_from_common_confidence_alpha0'], .2) | |
| def test_momask_media_is_rendered_from_saved_joints_only(self): | |
| with tempfile.TemporaryDirectory() as d: | |
| root = Path(d) | |
| folder, output = root / 'job', root / 'export' | |
| folder.mkdir() | |
| output.mkdir() | |
| xyz = np.zeros((3, 22, 3), dtype=float) | |
| xyz[:, :, 1] = np.linspace(0, 1.7, 22) | |
| np.save(folder / 'output.joints.npy', xyz) | |
| row = result(job('motion')) | |
| row['job']['model'] = 'momask' | |
| (folder / 'result.json').write_text(json.dumps(row)) | |
| row.update(_source_folder=str(folder), _source_result=str(folder / 'result.json')) | |
| media = prepare_media(row, output) | |
| self.assertTrue((output / media['preview']).exists()) | |
| self.assertTrue((output / media['animation']).exists()) | |
| self.assertEqual(media['source_joints'], str(folder / 'output.joints.npy')) | |
| def test_export_uses_locked_slots_and_real_saved_media(self): | |
| with tempfile.TemporaryDirectory() as d: | |
| root = Path(d) | |
| run, output = root / 'run', root / 'showcase' | |
| run.mkdir() | |
| expected = [job('raw'), job('mapped', alpha=None, mode='mapped', planning_status='unreachable')] | |
| (run / 'expected_slots.json').write_text(json.dumps(dict(expected_slots=expected))) | |
| folder = run / 'jobs/raw' | |
| folder.mkdir(parents=True) | |
| (folder / 'result.json').write_text(json.dumps(result(expected[0]))) | |
| Image.new('RGB', (16, 16), '#234567').save(folder / 'output.png') | |
| manifest = build_artifacts(run, output) | |
| self.assertEqual(manifest['panels'], 3) | |
| self.assertEqual(manifest['statuses']['complete'], 1) | |
| self.assertEqual(manifest['statuses']['unreachable'], 1) | |
| data = json.loads((output / 'manifest.json').read_text()) | |
| self.assertFalse(data['selection_rule']['substitution_allowed']) | |
| self.assertEqual(data['selection_rule']['seed'], 3) | |
| self.assertTrue((output / 'index.html').exists()) | |
| self.assertEqual(len(list((output / 'figures').glob('*.pdf'))), 4) | |
| if __name__ == '__main__': | |
| unittest.main() | |