File size: 12,724 Bytes
0e8028d
 
 
2719bf4
 
 
0e8028d
2719bf4
 
0e8028d
 
 
 
 
 
986ec09
 
 
2719bf4
 
 
0e8028d
 
 
 
 
 
 
 
 
 
 
 
2719bf4
0e8028d
 
 
986ec09
0e8028d
 
 
 
 
 
 
 
 
2719bf4
 
0e8028d
 
 
 
 
 
 
 
 
2719bf4
 
 
 
0e8028d
 
 
 
 
 
986ec09
 
 
0e8028d
 
 
 
 
 
986ec09
 
0e8028d
 
 
 
2719bf4
 
986ec09
 
 
 
 
2719bf4
 
 
 
 
 
 
 
0e8028d
 
2719bf4
 
 
 
 
 
0e8028d
2719bf4
0e8028d
 
2719bf4
0e8028d
2719bf4
0e8028d
 
 
2719bf4
0e8028d
 
2719bf4
 
 
0e8028d
 
 
 
 
 
 
986ec09
 
 
0e8028d
 
 
 
 
 
2719bf4
 
 
 
 
 
 
 
 
 
 
 
0e8028d
 
 
986ec09
0e8028d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
986ec09
0e8028d
 
 
 
 
 
 
 
 
 
 
 
 
 
986ec09
2719bf4
0e8028d
 
 
 
 
 
 
 
 
 
2719bf4
 
 
0e8028d
 
 
 
 
 
 
986ec09
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
"""Optional Gymnasium entry point.

Install ``borderland-rl[gymnasium]`` and import ``BorderlandEnv`` here.
Observations are current public Transition JSON, including legal_actions, without
operator revision or diagnostics. Actions are JSON
``{"tag": "...", "arguments": {...}}`` objects. The additional tag
``replace_notebook`` takes exactly ``{"text": "..."}`` and obeys the engine's
configured byte limit in generation three or 768-character limit in generation
two. JSON text uses printable ASCII with Unicode escapes.

Default reset starts a complete campaign. ``options={"checkpoint": N}`` starts
a room diagnostic for N in 1..7 and ends when that room is left. For N > 1 a
public-observation LegalReference executes the prefix and writes its legitimate
notebook. This assisted diagnostic is never full-campaign success. Seeds are
operator inputs; neither seeds nor prefix history enter observations or info.
The default ``reward_mode="engine"`` preserves engine reward. For a checkpoint,
``reward_mode="room_exit"`` selects a binary local training surrogate and adds
its versioned contract and separate actor returns to info only.
``options={"split": "train"}`` samples a frozen generation-specific split.
The other split names are ``dev`` and ``test``. The reset seed selects a
reproducible member. A split and ``world_seed`` are mutually exclusive.
"""

from __future__ import annotations

import json
import string

import gymnasium as gym
from gymnasium import spaces

from borderland import Campaign
from borderland.domain import ActionError
from .context import actor_transition, actor_view


class BorderlandEnv(gym.Env):
    """Text/tool environment with bounded attempts and default engine rewards.

    ``seed`` in reset controls world generation through Gymnasium's RNG.
    ``world_seed`` fixes a world for operator diagnostics independently of that RNG.
    A diagnostic ends on leaving its selected room, or on an engine terminal state.
    Invalid actions consume adapter attempts but leave the campaign unchanged.
    """

    metadata = {"render_modes": []}

    def __init__(self, *, world_seed=None, max_steps=512, invalid_attempt_limit=32,
                 generation_version=2, notebook_bytes=768, memory_profile="static"):
        super().__init__()
        if world_seed is not None and type(world_seed) is not int:
            raise ValueError("world_seed must be an integer or None")
        for name, value in (("max_steps", max_steps), ("invalid_attempt_limit", invalid_attempt_limit)):
            if type(value) is not int or value < 1:
                raise ValueError(f"{name} must be a positive integer")
        self._world_seed = world_seed
        self._max_steps = max_steps
        self._invalid_attempt_limit = invalid_attempt_limit
        self._profile(generation_version, notebook_bytes, memory_profile)
        self._generation_version = generation_version
        self._notebook_bytes = notebook_bytes
        self._memory_profile = memory_profile
        charset = string.ascii_letters + string.digits + string.punctuation + " "
        self.action_space = spaces.Text(max_length=16384, min_length=1, charset=charset)
        self.observation_space = spaces.Text(max_length=131072, min_length=1, charset=charset)
        self._campaign = None
        self._finished = True
        self._checkpoint = None
        self._reward_mode = "engine"
        self._actor_engine_return = 0.0
        self._actor_training_return = 0.0
        self._attempts = 0
        self._invalid_attempts = 0

    def reset(self, *, seed=None, options=None):
        if seed is not None and (type(seed) is not int or seed < 0):
            raise ValueError("seed must be a nonnegative integer or None")
        if options is not None and (type(options) is not dict or set(options) - {"checkpoint","archive_workload","generation_version","notebook_bytes","memory_profile","split","reward_mode"}):
            raise ValueError("options supports checkpoint, archive_workload, generation_version, notebook_bytes, memory_profile, split and reward_mode")
        checkpoint = None if options is None else options.get("checkpoint")
        if options is not None and "checkpoint" in options:
            if type(checkpoint) is not int or not 1 <= checkpoint <= 7:
                raise ValueError("checkpoint must be an integer from 1 to 7")
        workload = "standard" if options is None else options.get("archive_workload","standard")
        options = options or {}
        reward_mode = options.get("reward_mode", "engine")
        if type(reward_mode) is not str or reward_mode not in ("engine", "room_exit"):
            raise ValueError("reward_mode must be engine or room_exit")
        if reward_mode == "room_exit" and checkpoint is None:
            raise ValueError("room_exit reward requires a checkpoint")
        profile = self._profile(options.get("generation_version", self._generation_version),
                                options.get("notebook_bytes", self._notebook_bytes),
                                options.get("memory_profile", self._memory_profile))
        split = options.get("split")
        if "split" in options and split not in ("train", "dev", "test"):
            raise ValueError("split must be train, dev, or test")
        if split is not None and self._world_seed is not None:
            raise ValueError("world_seed and split are mutually exclusive")
        super().reset(seed=seed)
        world_seed = self._world_seed
        if split is not None:
            from .manifests import manifest

            roots = manifest(split, profile["generation_version"])["root_seeds"]
            world_seed = roots[int(self.np_random.integers(len(roots)))]
        elif world_seed is None:
            world_seed = int(self.np_random.integers(0, 2**63))
        campaign = Campaign.generated(world_seed, archive_workload=workload, **profile)
        transition = campaign.observe()
        if checkpoint is not None and checkpoint > 1:
            from borderland_verify.reference import public_reference

            reference = public_reference(actor_transition(transition))
            for _ in range(240):
                if transition.done or int(transition.observation.phase) >= checkpoint:
                    break
                note = reference.notebook(actor_transition(transition))
                if note is not None:
                    transition = campaign.replace_notebook(note, expected_revision=transition.revision)
                transition = campaign.step(reference.choose(actor_transition(transition)), expected_revision=transition.revision)
                if transition.context_reset:
                    reference = type(reference)()
                if transition.error:
                    raise RuntimeError("public reference could not reach diagnostic checkpoint")
            if transition.done or int(transition.observation.phase) != checkpoint:
                raise RuntimeError("public reference did not reach diagnostic checkpoint")
        self._campaign = campaign
        self._transition = transition
        self._checkpoint = checkpoint
        self._reward_mode = reward_mode
        self._actor_engine_return = 0.0
        self._actor_training_return = 0.0
        self._attempts = 0
        self._invalid_attempts = 0
        self._finished = False
        observation, _, _, _, info = self._result(transition)
        return observation, info

    @staticmethod
    def _profile(generation_version, notebook_bytes, memory_profile="static"):
        if type(generation_version) is not int or generation_version not in (2, 3):
            raise ValueError("generation_version must be 2 or 3")
        if type(notebook_bytes) is not int or notebook_bytes < 0:
            raise ValueError("notebook_bytes must be a nonnegative integer")
        if generation_version == 2 and notebook_bytes != 768:
            raise ValueError("custom byte capacity requires generation three")
        if memory_profile not in ("static", "commissioning") or (generation_version == 2 and memory_profile != "static"):
            raise ValueError("commissioning requires generation three")
        return {"generation_version": generation_version, "notebook_bytes": notebook_bytes, "memory_profile": memory_profile}

    def step(self, action):
        if self._finished or self._campaign is None:
            raise gym.error.ResetNeeded("Call reset before stepping a new episode")
        previous_phase = int(self._transition.observation.phase)
        self._attempts += 1
        error = None
        if not self.action_space.contains(action):
            error = ActionError("malformed", "Expected printable ASCII JSON action text within the action-space bound.")
        else:
            try:
                raw = json.loads(action, parse_constant=self._reject_constant, object_pairs_hook=self._unique_object)
            except (ValueError, RecursionError):
                error = ActionError("malformed", "Expected finite JSON with unique object keys.")
            else:
                if type(raw) is dict and raw.get("tag") == "replace_notebook":
                    if set(raw) != {"tag", "arguments"} or type(raw["arguments"]) is not dict or set(raw["arguments"]) != {"text"}:
                        error = ActionError("malformed", "Notebook action requires exactly one text argument.")
                    else:
                        self._transition = self._campaign.replace_notebook(raw["arguments"]["text"])
                else:
                    self._transition = self._campaign.step(raw)
        if error is None:
            error = self._transition.error
        if error is not None:
            self._invalid_attempts += 1
        return self._result(self._transition, error, previous_phase=previous_phase)

    @staticmethod
    def _reject_constant(value):
        raise ValueError("nonfinite JSON")

    @staticmethod
    def _unique_object(pairs):
        result = {}
        for key, value in pairs:
            if key in result:
                raise ValueError("duplicate JSON key")
            result[key] = value
        return result

    def _result(self, transition, error=None, *, previous_phase=None):
        data = actor_view(transition)
        data["error"] = None if error is None else error.to_dict()
        room_complete = self._checkpoint is not None and int(transition.observation.phase) != self._checkpoint
        terminated = bool(transition.terminated or room_complete)
        reason = None
        if not terminated and self._invalid_attempts >= self._invalid_attempt_limit:
            reason = "invalid_attempt_limit"
        elif not terminated and self._attempts >= self._max_steps:
            reason = "attempt_limit"
        truncated = bool(transition.truncated or reason is not None)
        self._finished = terminated or truncated
        info = {"outcome": transition.outcome,
                "generation_version": self._campaign._world.generation_version,
                "memory_profile": self._campaign._world.memory_profile,
                "context_reset": transition.context_reset, "error": data["error"],
                "scope": "campaign" if self._checkpoint is None else "room_diagnostic",
                "checkpoint": self._checkpoint, "assisted_prefix": self._checkpoint is not None and self._checkpoint > 1,
                "room_complete": room_complete or (self._checkpoint == 7 and transition.outcome == "escaped"),
                "full_campaign_success": self._checkpoint is None and transition.outcome == "escaped",
                "truncation_reason": reason}
        observation = json.dumps(data, ensure_ascii=True, allow_nan=False, separators=(",", ":"), sort_keys=True)
        reward = float(transition.reward)
        if self._reward_mode == "room_exit":
            reward = float(error is None and not truncated and previous_phase == self._checkpoint and (
                (self._checkpoint < 7 and int(transition.observation.phase) == self._checkpoint + 1
                 and transition.outcome == "running")
                or (self._checkpoint == 7 and transition.outcome == "escaped")))
            if previous_phase is not None:
                self._actor_engine_return += float(transition.reward)
                self._actor_training_return += reward
            info.update(reward_mode="room_exit", reward_contract="local_surrogate.room_exit.v1",
                        engine_reward=float(transition.reward),
                        actor_engine_return=self._actor_engine_return,
                        actor_training_return=self._actor_training_return)
        return observation, reward, terminated, truncated, info