File size: 3,358 Bytes
360628f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 | """Tests for guard.py and the two scorers (no training). Each attack must make the scorer refuse (exit 2, no verdict);
the real result sets must still score exactly as before. SELF-TESTED."""
import json
import shutil
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
HERE = Path(__file__).resolve().parent
ROOT = HERE.parents[1]
V1 = ROOT / "results" / "tiny_engine"
V2 = ROOT / "results" / "tiny_engine" / "002"
def score(script, d):
p = subprocess.run([sys.executable, str(HERE / script), str(d)], capture_output=True, text=True, cwd=HERE)
return p.returncode, p.stdout
def copy_set(src):
d = Path(tempfile.mkdtemp())
for f in src.glob("seed*.json"):
shutil.copy(f, d / f.name)
return d
class RealSets(unittest.TestCase):
def test_v1_verdicts_unchanged(self):
code, out = score("score.py", V1)
self.assertEqual(code, 0)
self.assertIn("Q1: HELD (6/6", out)
self.assertIn("Q2: FAILED (1/6", out)
self.assertIn("Q3: HELD (6/6", out)
def test_v2_verdicts_unchanged(self):
code, out = score("score2.py", V2)
self.assertEqual(code, 0)
for q in ("Q1", "Q2", "Q3", "Q4"):
self.assertIn(f"{q}: HELD (6/6", out)
class Attacks(unittest.TestCase):
def refused(self, script, d):
code, out = score(script, d)
self.assertEqual(code, 2, out)
self.assertIn("INVALID RESULT SET", out)
self.assertNotIn("HELD", out)
self.assertNotIn("FAILED", out)
def test_five_copies_of_one_seed(self):
"""Codex's counterexample, PR #163."""
d = Path(tempfile.mkdtemp())
for i in range(1, 6):
shutil.copy(V2 / "seed1.json", d / f"seed{i}.json")
self.refused("score2.py", d)
def test_six_copies_with_seed_field_rewritten(self):
d = Path(tempfile.mkdtemp())
r = json.loads((V2 / "seed1.json").read_text())
for i in range(1, 7):
(d / f"seed{i}.json").write_text(json.dumps(dict(r, seed=i)))
self.refused("score2.py", d)
def test_missing_result(self):
d = copy_set(V2)
(d / "seed6.json").unlink()
self.refused("score2.py", d)
def test_extra_seed(self):
d = copy_set(V2)
shutil.copy(V2 / "seed1.json", d / "seed7.json")
self.refused("score2.py", d)
def test_wrong_config(self):
d = copy_set(V2)
r = json.loads((d / "seed3.json").read_text())
r["cfg"]["steps"] = 100
(d / "seed3.json").write_text(json.dumps(r))
self.refused("score2.py", d)
def test_v1_results_rejected_by_v2_scorer(self):
self.refused("score2.py", V1)
def test_bad_score_values(self):
for bad in (None, "0.99", 1.5, -0.1, float("nan"), True):
d = copy_set(V2)
r = json.loads((d / "seed2.json").read_text())
r["scores"]["copy_L256"] = bad
(d / "seed2.json").write_text(json.dumps(r))
with self.subTest(bad=bad):
self.refused("score2.py", d)
def test_seed_field_mismatch(self):
d = copy_set(V2)
r = json.loads((d / "seed4.json").read_text())
r["seed"] = 9
(d / "seed4.json").write_text(json.dumps(r))
self.refused("score2.py", d)
if __name__ == "__main__":
unittest.main()
|