File size: 4,256 Bytes
1831ecd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
import copy
from pathlib import Path
import pytest
from common import read_json,write_json,is_validation_group
from benchmarks.bfcl_tasks import validate_sample,replay_sample,parent_correction_sample
from benchmarks.bfcl_data import add_replay,readiness
from learning.trajectories import validate_pool
from helpers import generated_fixture,make_sample,row


def test_guided_success_preserves_bound_failure(monkeypatch,tmp_path,config):
    f=generated_fixture(monkeypatch,tmp_path,config)
    s=make_sample(f)
    assert s['pair_kind']=='guided_positive'
    assert s['failure_labels']
    assert s['verification']['first_failure']['episode']==str(f['bad'])
    assert 'rejected' not in s
    assert validate_sample(s,{('bfcl',f['parent']['task_id']):f['parent']})
    validate_pool([s],[f['parent']])
    assert readiness([s],set(),config)['correction_samples']==0


def test_same_prompt_is_a_correction_pair(monkeypatch,tmp_path,config):
    f=generated_fixture(monkeypatch,tmp_path,config,source='student_retry',same_prompt=True)
    s=make_sample(f)
    assert s['pair_kind']=='same_prompt' and s['rejected']
    assert validate_sample(s)
    assert readiness([s],set(),config)['correction_samples']==1


def test_non_harness_guidance_change_cannot_be_laundered(monkeypatch,tmp_path,config):
    f=generated_fixture(monkeypatch,tmp_path,config,source='student_retry')
    with pytest.raises(ValueError,match='Only an explicit temporary Harness'):
        make_sample(f)


@pytest.mark.parametrize('mutation',['target','first_failure','category','group'])
def test_sample_tampering_rejected(monkeypatch,tmp_path,config,mutation):
    f=generated_fixture(monkeypatch,tmp_path,config)
    s=make_sample(f)
    if mutation=='target':s['target']['content']='[]'
    elif mutation=='first_failure':s['verification']['first_failure']=None
    elif mutation=='category':s['category']='parallel'
    else:s['split_group']='0'*64
    with pytest.raises(ValueError):validate_sample(s)


def test_first_failure_integrity_is_not_silently_dropped(monkeypatch,tmp_path,config):
    f=generated_fixture(monkeypatch,tmp_path,config)
    bad=read_json(f['bad']/'outcome.json');bad['task_id']='unrelated'
    write_json(f['bad']/'outcome.json',bad)
    with pytest.raises(ValueError,match='real failed first answer'):make_sample(f)


@pytest.mark.parametrize('mutation',['teacher_backend','correct_response'])
def test_student_failure_cannot_be_fabricated(monkeypatch,tmp_path,config,mutation):
    f=generated_fixture(monkeypatch,tmp_path,config)
    generation=read_json(f['bad']/'static-generation.json')
    if mutation=='teacher_backend':generation['execution_backend']='teacher'
    else:generation['response']=read_json(f['good']/'static-generation.json')['response']
    write_json(f['bad']/'static-generation.json',generation)
    with pytest.raises(ValueError):make_sample(f)


def test_replay_is_explicit_and_cannot_cross_roles():
    r=row();s=replay_sample(r)
    assert s['verification']['executed_success_claim'] is False
    assert s['failure_labels']==[] and 'assistance' not in s
    assert validate_sample(s,{('bfcl',r['task_id']):r})
    validate_pool([s],[r])
    for role in ['selection','audit','meta_eval','sealed_test']:
        with pytest.raises(ValueError):replay_sample(row(role=role))
    forged=copy.deepcopy(s);forged['target']['content']='[]'
    with pytest.raises(ValueError):validate_sample(forged)


def test_replay_fills_healthy_categories_without_inventing_failure_groups(config):
    rows=[row(i,c) for c in ('simple_python','multiple','parallel','parallel_multiple')
          for i in range(100)]
    failure=rows[0]
    samples=add_replay([],rows,{('bfcl',failure['task_id'])},config)
    assert len(samples)==64
    status=readiness(samples,set(),config)
    for coverage in status['category_coverage'].values():
        assert coverage['samples']>=8 and coverage['groups']>=4
        assert coverage['train_groups'] and coverage['validation_groups']
    assert all(s['task']['task_id']!=failure['task_id'] for s in samples)
    assert not status['ready']  # Replay cannot satisfy genuine correction floors.
    assert status['correction_groups']==0
    assert add_replay(samples,rows,{('bfcl',failure['task_id'])},config)==[]