File size: 4,792 Bytes
387a4cf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 | """Regression fixtures for the treatment, published formulas and data boundary."""
import random
from copy import deepcopy
import numpy as np
from rqgm.archive import Archive
from rqgm.models import UtilityRecord, WorkspaceNode
from rqstudy.data import clean, prepare
from rqstudy.policies import (
archive_view,
choose_parent,
choose_thompson,
reference_choice,
)
from rqstudy.sandbox import Worker
from rqstudy.workspace import ROOT, apply_changes
def tree():
archive = Archive(WorkspaceNode("root", deepcopy(ROOT), None, 0))
archive.add_node(WorkspaceNode("older", deepcopy(ROOT), "root", 1))
archive.add_node(WorkspaceNode("latest", deepcopy(ROOT), "older", 2))
for node, outcomes in (
("root", [0] * 16),
("older", [0] * 4),
("latest", [1] * 20),
):
for i, outcome in enumerate(outcomes):
archive.add_record(
UtilityRecord(
f"{node}{i}", node, "reviewer", "human", outcome, {}, None, None
)
)
return archive
def test_node_and_clade_evidence_differ():
view = archive_view(tree(), 1, 40, 64)
assert view["nodes"][0]["own_s"] == 0
assert view["nodes"][0]["clade_s"] == 20
# Compare decisions with identical RNG streams on deliberately mixed lineages.
choices = [
(
choose_thompson(view, np.random.default_rng(s), clade=False),
choose_thompson(view, np.random.default_rng(s), clade=True),
)
for s in range(500)
]
assert any(a != b for a, b in choices)
def test_latest_selector_and_older_parent_inheritance():
a = tree()
a.nodes["latest"].workspace["selector"] = (
"def select_parent(archive_snapshot, rng):\n return 'older'"
)
selected = choose_parent(
"editable", a, archive_view(a, 1, 40, 64), np.random.default_rng(42), Worker()
)
assert (selected["owner"], selected["parent"]) == ("latest", "older")
child = apply_changes(
a.nodes["older"].workspace, {"writer": "Focus on evidence."}, editable=True
)
assert child["selector"] == ROOT["selector"]
assert child["writer"] != ROOT["writer"]
def test_failed_owner_falls_back():
a = tree()
a.nodes["latest"].workspace["selector"] = (
"def select_parent(archive_snapshot, rng):\n return 'absent'"
)
result = choose_parent(
"editable", a, archive_view(a, 1, 40, 64), np.random.default_rng(2), Worker()
)
assert result["owner"] == "older" and result["failures"][0]["owner"] == "latest"
def test_released_and_published_penalties_can_disagree():
v = {
"nodes": [
{"id": "a", "score": 0.5, "children": 6},
{"id": "b", "score": 0.5, "children": 0},
]
}
assert reference_choice(v, 0.21, paper=False) != reference_choice(
v, 0.21, paper=True
)
def test_strip_outcome_headers_preserve_neighbor_text():
assert "science" in clean(
"'Title: science\\nPublished as a conference paper at ICLR 2024\\nSection: proof'"
)
assert "Published as" not in clean(
"Published as a conference paper at ICLR 2024\nproof"
)
def test_sandbox_blocks_private_io():
import pytest
with pytest.raises(ValueError):
Worker().call(
"def instruct(context):\n return open('.env').read()", "instruct", {}
)
def test_concatenated_mastheads_and_blind_review_titles():
assert (
clean("formula2Under review as a conference paper at ICLR 2025 appendix")
== "formula2 appendix"
)
assert (
clean("analyticalPublished as a conference paper at ICLR 2024 noise")
== "analytical noise"
)
assert (
clean(
"Study Anonymous authors Paper under double-blind review\n\nAbstract: proof"
)
== "Study \n\nAbstract: proof"
)
assert "openreview.net" not in clean(
"Citation https: //openreview.net/forum?id=public-id"
)
def test_failed_worker_uniform_root_and_readonly_snapshot():
class FailedWorker:
def call(self, *args):
raise ValueError("Worker timed out")
a = tree()
before = archive_view(a, 1, 40, 64)
selected = choose_parent(
"editable",
a,
before,
np.random.default_rng(42),
FailedWorker(),
fallback_allowance=1,
)
assert (
selected["trusted_root_fallback"]
and selected["owner"] == "root"
and len(selected["failures"]) == 2
)
assert selected["parent"] in a.nodes
assert "secret" not in before
Worker().call(
"def instruct(context):\n context['secret'] = 1\n return 'ok'",
"instruct",
before,
)
assert "secret" not in before
|