| """Public construction and scoring controls; never read the sealed suite.""" |
| from copy import deepcopy |
| import random |
|
|
| import pytest |
|
|
| from experiments import morphology_data as data |
|
|
|
|
| @pytest.fixture(scope='module') |
| def execute(): |
| return data.independent_executor() |
|
|
|
|
| @pytest.mark.parametrize('spec', data.FAMILIES + data.EXTENDED) |
| def test_public_constructions_have_two_independent_semantic_checks(spec, execute): |
| rows = [] |
| for index in range(2): |
| rows.extend(data.make_group(random.Random(673 + index), spec, index, |
| 'public-test', 'main')) |
| report = data.validate_rows(rows, execute) |
| assert report['episodes'] == 16 |
| assert report['individually_necessary_edit_checks'] == spec[3] * 8 |
| assert sum(bool(r['edits']) for r in rows) == 8 |
| assert {r['noop_kind'] for r in rows if r['is_noop']} == {'negation', 'already_repaired'} |
|
|
|
|
| def test_pair_disambiguation_is_beyond_local_byte_windows(): |
| rows = data.make_group(random.Random(3731), data.FAMILIES[6], 0, 'public-test') |
| left, right = rows[0], rows[2] |
| a, b = bytes.fromhex(left['binary']), bytes.fromhex(right['binary']) |
| assert a[:-8] == b[:-8] |
| assert a[-8:-4] != b[-8:-4] |
| assert {e['offset'] for e in left['edits']}.isdisjoint(e['offset'] for e in right['edits']) |
| for branch in left['branch_source_positions']: |
| for position in branch: |
| assert a[max(0, position - 16):position + 17] == b[max(0, position - 16):position + 17] |
|
|
|
|
| def test_all_multi_edit_subsets_supply_order_independent_labels(): |
| row = data.make_group(random.Random(5511), data.FAMILIES[6], 0, 'public-test')[0] |
| states = data.expand_subsets(row) |
| assert len(states) == 8 |
| assert sorted(len(state['edits']) for state in states) == [0, 1, 1, 1, 2, 2, 2, 3] |
| assert len({state['binary'] for state in states}) == 8 |
| for state in states: |
| current, target = bytes.fromhex(state['binary']), bytes.fromhex(state['target']) |
| assert state['edits'] == [dict(offset=i, value=y) |
| for i, (x, y) in enumerate(zip(current, target)) if x != y] |
|
|
|
|
| @pytest.mark.parametrize('kind', ('dead_value', 'irrelevant_bytes')) |
| def test_irrelevant_perturbation_preserves_patch_and_local_windows(kind, execute): |
| rows = data.make_group(random.Random(3417), data.FAMILIES[7], 0, 'public-test', nuisance_kind=kind) |
| for first, second in zip(rows[:4], rows[4:]): |
| assert first['invariance_pair_id'] == second['invariance_pair_id'] |
| assert first['binary'] != second['binary'] |
| assert first['edits'] == second['edits'] |
| a, b = bytes.fromhex(first['binary']), bytes.fromhex(second['binary']) |
| for inputs, _ in first['cases']: |
| assert execute(a, inputs) == execute(b, inputs) |
| for branch in first['branch_source_positions']: |
| for pos in branch: |
| assert a[max(0, pos - 16):pos + 17] == b[max(0, pos - 16):pos + 17] |
|
|
|
|
| def test_corrupt_reference_cannot_pass_independent_validation(execute): |
| rows = data.make_group(random.Random(1861), data.FAMILIES[4], 0, 'public-test') |
| bad = deepcopy(rows[0]) |
| bad['cases'][0][1] += 1 |
| with pytest.raises(ValueError, match='oracle'): |
| data.validate_rows([bad], execute) |
|
|
|
|
| def synthetic_label_control(row, extra_dead=False): |
| |
| from lightnest.iterative_patch import ByteEdit, PatchDecision |
| edits = list(reversed(row['edits'])) |
| if extra_dead: |
| required = {e['offset'] for e in edits} |
| offset = next(pos for branch in row['branch_source_positions'] for pos in branch |
| if pos not in required) |
| initial = bytes.fromhex(row['binary']) |
| edits.append(dict(offset=offset, value=initial[offset] ^ 64)) |
|
|
| def predict(request, current, step=0, history=()): |
| if step >= len(edits): |
| return PatchDecision(stop=True, reason='model_stop') |
| edit = edits[step] |
| offset = edit['offset'] |
| return PatchDecision(edits=(ByteEdit(offset, current[offset:offset + 1], |
| bytes([edit['value']])),)) |
| return predict |
|
|
|
|
| def test_six_edits_need_stop_and_dead_changes_fail_despite_matching_outputs(): |
| pytest.importorskip('torch') |
| from experiments.architecture_eval import score_episode |
| row = data.make_group(random.Random(3519), data.EXTENDED[6], 0, 'public-test')[0] |
| success = score_episode(synthetic_label_control(row), row) |
| assert success['strict_success'] and success['accepted_steps'] == 6 |
| assert not success['raw_one_step_success'] |
| incomplete = score_episode(synthetic_label_control(row), row, max_steps=6) |
| assert incomplete['exact_final'] and not incomplete['strict_success'] |
| extra = score_episode(synthetic_label_control(row, extra_dead=True), row) |
| assert extra['semantic_final'] and not extra['strict_success'] |
| assert not extra['exact_final'] |
|
|
|
|
| def test_all_stop_scores_zero_edit_and_zero_pair_credit(): |
| pytest.importorskip('torch') |
| from experiments.architecture_eval import score_episode |
| from lightnest.iterative_patch import PatchDecision |
| rows = data.make_group(random.Random(4351), data.FAMILIES[3], 0, 'public-test') |
| scores = [score_episode(lambda *a, **kw: PatchDecision(stop=True, reason='model_stop'), row) |
| for row in rows] |
| summary = data.summarize_scored(rows, scores)['all'] |
| assert summary['edit_total'] == 4 and summary['noop_total'] == 4 |
| assert summary['edit_strict_rate'] == 0 and summary['noop_strict_rate'] == 1 |
| assert summary['pair_total'] == 2 and summary['pair_both_strict'] == 0 |
| assert summary['invariance_predicted_delta_agreement'] == summary['invariance_total'] == 4 |
| assert summary['invariance_edit_both_strict'] == 0 |
| assert summary['false_edit_on_correct'] == 0 |
| with pytest.raises(ValueError, match='identifiers/order'): |
| data.summarize_scored(rows, scores[::-1]) |
| with pytest.raises(ValueError, match='missing'): |
| data.summarize_scored(rows, scores[:-1]) |
|
|
|
|
| def test_paired_summary_requires_both_variants(): |
| pytest.importorskip('torch') |
| from experiments.architecture_eval import score_episode |
| rows = data.make_group(random.Random(4353), data.FAMILIES[3], 0, 'public-test') |
| scores = [score_episode(synthetic_label_control(row), row) for row in rows] |
| summary = data.summarize_scored(rows, scores)['all'] |
| assert summary['edit_strict'] == 4 and summary['noop_strict'] == 4 |
| assert summary['pair_both_strict'] == 2 |
| assert summary['invariance_both_strict'] == 4 |
| with pytest.raises(ValueError, match='incomplete return-choice pair'): |
| data.summarize_scored(rows[:2], scores[:2]) |
|
|
|
|
| def test_freeze_refuses_existing_evidence_before_reading_old_or_sealed_data(tmp_path): |
| (tmp_path / 'preserve.txt').write_text('existing evidence') |
| with pytest.raises(ValueError, match='refuse to replace'): |
| data.freeze(tmp_path) |
| assert (tmp_path / 'preserve.txt').read_text() == 'existing evidence' |
|
|
