File size: 3,358 Bytes
360628f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
"""Tests for guard.py and the two scorers (no training). Each attack must make the scorer refuse (exit 2, no verdict);
the real result sets must still score exactly as before. SELF-TESTED."""
import json
import shutil
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path

HERE = Path(__file__).resolve().parent
ROOT = HERE.parents[1]
V1 = ROOT / "results" / "tiny_engine"
V2 = ROOT / "results" / "tiny_engine" / "002"


def score(script, d):
    p = subprocess.run([sys.executable, str(HERE / script), str(d)], capture_output=True, text=True, cwd=HERE)
    return p.returncode, p.stdout


def copy_set(src):
    d = Path(tempfile.mkdtemp())
    for f in src.glob("seed*.json"):
        shutil.copy(f, d / f.name)
    return d


class RealSets(unittest.TestCase):
    def test_v1_verdicts_unchanged(self):
        code, out = score("score.py", V1)
        self.assertEqual(code, 0)
        self.assertIn("Q1: HELD (6/6", out)
        self.assertIn("Q2: FAILED (1/6", out)
        self.assertIn("Q3: HELD (6/6", out)

    def test_v2_verdicts_unchanged(self):
        code, out = score("score2.py", V2)
        self.assertEqual(code, 0)
        for q in ("Q1", "Q2", "Q3", "Q4"):
            self.assertIn(f"{q}: HELD (6/6", out)


class Attacks(unittest.TestCase):
    def refused(self, script, d):
        code, out = score(script, d)
        self.assertEqual(code, 2, out)
        self.assertIn("INVALID RESULT SET", out)
        self.assertNotIn("HELD", out)
        self.assertNotIn("FAILED", out)

    def test_five_copies_of_one_seed(self):
        """Codex's counterexample, PR #163."""
        d = Path(tempfile.mkdtemp())
        for i in range(1, 6):
            shutil.copy(V2 / "seed1.json", d / f"seed{i}.json")
        self.refused("score2.py", d)

    def test_six_copies_with_seed_field_rewritten(self):
        d = Path(tempfile.mkdtemp())
        r = json.loads((V2 / "seed1.json").read_text())
        for i in range(1, 7):
            (d / f"seed{i}.json").write_text(json.dumps(dict(r, seed=i)))
        self.refused("score2.py", d)

    def test_missing_result(self):
        d = copy_set(V2)
        (d / "seed6.json").unlink()
        self.refused("score2.py", d)

    def test_extra_seed(self):
        d = copy_set(V2)
        shutil.copy(V2 / "seed1.json", d / "seed7.json")
        self.refused("score2.py", d)

    def test_wrong_config(self):
        d = copy_set(V2)
        r = json.loads((d / "seed3.json").read_text())
        r["cfg"]["steps"] = 100
        (d / "seed3.json").write_text(json.dumps(r))
        self.refused("score2.py", d)

    def test_v1_results_rejected_by_v2_scorer(self):
        self.refused("score2.py", V1)

    def test_bad_score_values(self):
        for bad in (None, "0.99", 1.5, -0.1, float("nan"), True):
            d = copy_set(V2)
            r = json.loads((d / "seed2.json").read_text())
            r["scores"]["copy_L256"] = bad
            (d / "seed2.json").write_text(json.dumps(r))
            with self.subTest(bad=bad):
                self.refused("score2.py", d)

    def test_seed_field_mismatch(self):
        d = copy_set(V2)
        r = json.loads((d / "seed4.json").read_text())
        r["seed"] = 9
        (d / "seed4.json").write_text(json.dumps(r))
        self.refused("score2.py", d)


if __name__ == "__main__":
    unittest.main()