File size: 2,225 Bytes
fcfdbcb
 
 
 
 
 
5423dad
52a9362
fcfdbcb
 
 
95e7452
5423dad
 
fcfdbcb
 
16f323d
fcfdbcb
 
16f323d
 
5473335
 
 
 
 
 
16f323d
fcfdbcb
d59b954
fcfdbcb
16f323d
fcfdbcb
 
 
 
95e7452
16f323d
95e7452
16f323d
52a9362
 
 
 
95e7452
16f323d
95e7452
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
import json
from pathlib import Path

import pytest

from demo.app import run_verification
from demo.examples import EXAMPLE_NAMES, example_metadata
from demo.faultless import NAMES as COMPLETENESS_NAMES


@pytest.mark.parametrize('compiler', ['gcc', 'clang'])
@pytest.mark.parametrize('name', [n for n in EXAMPLE_NAMES if n.startswith('faultless-')])
def test_faultless_frontend_stream(tmp_path, compiler, name, example_rows):
    rows = example_rows
    args = rows[EXAMPLE_NAMES.index(name)].copy()
    args[8] = compiler
    seed_count = args[5]
    updates = list(run_verification(*args, report_directory=tmp_path / 'reports'))
    report = json.loads(Path(updates[-1][3]).read_text())
    assert len(updates) == seed_count + 1
    assert report['completed'] == seed_count
    expected = example_metadata(name)['expected_verdict']
    if name == 'faultless-byte-reinterpretation' and compiler == 'clang':
        # The original is built with gcc, so the escaped frames differ in layout.
        expected = 'inconclusive'
        assert all(r['reason'] == 'escaped_frame_differs' for r in report['results'])
    assert report['verdict'] == expected
    assert [r['seed'] for r in report['results']] == list(range(seed_count))
    if name == 'faultless-linked-list':
        assert report['counts']['mismatch'] == seed_count
    elif name == 'faultless-deep-pointer':
        assert report['counts']['mismatch'] == seed_count
        assert all(r['divergence'] == 'call[1]' for r in report['results'])
    elif name == 'faultless-heap-loop':
        assert report['counts']['agree'] > 0
        assert report['counts']['mismatch'] > 0
    elif name == 'faultless-memory-dependency':
        assert report['counts']['agree'] == seed_count
    elif name == 'faultless-byte-reinterpretation':
        assert report['counts'][expected] == seed_count
    elif name in COMPLETENESS_NAMES:
        assert report['counts'][expected] == seed_count
        if name == 'faultless-my-putchar-count':
            assert all(r['reason'] == 'escaped_frame_differs' for r in report['results'])
    else:
        assert report['counts']['mismatch'] == seed_count
        assert all(r['divergence'] == 'call[0]' for r in report['results'])