Spaces:
Running
Running
File size: 12,724 Bytes
0e8028d 2719bf4 0e8028d 2719bf4 0e8028d 986ec09 2719bf4 0e8028d 2719bf4 0e8028d 986ec09 0e8028d 2719bf4 0e8028d 2719bf4 0e8028d 986ec09 0e8028d 986ec09 0e8028d 2719bf4 986ec09 2719bf4 0e8028d 2719bf4 0e8028d 2719bf4 0e8028d 2719bf4 0e8028d 2719bf4 0e8028d 2719bf4 0e8028d 2719bf4 0e8028d 986ec09 0e8028d 2719bf4 0e8028d 986ec09 0e8028d 986ec09 0e8028d 986ec09 2719bf4 0e8028d 2719bf4 0e8028d 986ec09 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 | """Optional Gymnasium entry point.
Install ``borderland-rl[gymnasium]`` and import ``BorderlandEnv`` here.
Observations are current public Transition JSON, including legal_actions, without
operator revision or diagnostics. Actions are JSON
``{"tag": "...", "arguments": {...}}`` objects. The additional tag
``replace_notebook`` takes exactly ``{"text": "..."}`` and obeys the engine's
configured byte limit in generation three or 768-character limit in generation
two. JSON text uses printable ASCII with Unicode escapes.
Default reset starts a complete campaign. ``options={"checkpoint": N}`` starts
a room diagnostic for N in 1..7 and ends when that room is left. For N > 1 a
public-observation LegalReference executes the prefix and writes its legitimate
notebook. This assisted diagnostic is never full-campaign success. Seeds are
operator inputs; neither seeds nor prefix history enter observations or info.
The default ``reward_mode="engine"`` preserves engine reward. For a checkpoint,
``reward_mode="room_exit"`` selects a binary local training surrogate and adds
its versioned contract and separate actor returns to info only.
``options={"split": "train"}`` samples a frozen generation-specific split.
The other split names are ``dev`` and ``test``. The reset seed selects a
reproducible member. A split and ``world_seed`` are mutually exclusive.
"""
from __future__ import annotations
import json
import string
import gymnasium as gym
from gymnasium import spaces
from borderland import Campaign
from borderland.domain import ActionError
from .context import actor_transition, actor_view
class BorderlandEnv(gym.Env):
"""Text/tool environment with bounded attempts and default engine rewards.
``seed`` in reset controls world generation through Gymnasium's RNG.
``world_seed`` fixes a world for operator diagnostics independently of that RNG.
A diagnostic ends on leaving its selected room, or on an engine terminal state.
Invalid actions consume adapter attempts but leave the campaign unchanged.
"""
metadata = {"render_modes": []}
def __init__(self, *, world_seed=None, max_steps=512, invalid_attempt_limit=32,
generation_version=2, notebook_bytes=768, memory_profile="static"):
super().__init__()
if world_seed is not None and type(world_seed) is not int:
raise ValueError("world_seed must be an integer or None")
for name, value in (("max_steps", max_steps), ("invalid_attempt_limit", invalid_attempt_limit)):
if type(value) is not int or value < 1:
raise ValueError(f"{name} must be a positive integer")
self._world_seed = world_seed
self._max_steps = max_steps
self._invalid_attempt_limit = invalid_attempt_limit
self._profile(generation_version, notebook_bytes, memory_profile)
self._generation_version = generation_version
self._notebook_bytes = notebook_bytes
self._memory_profile = memory_profile
charset = string.ascii_letters + string.digits + string.punctuation + " "
self.action_space = spaces.Text(max_length=16384, min_length=1, charset=charset)
self.observation_space = spaces.Text(max_length=131072, min_length=1, charset=charset)
self._campaign = None
self._finished = True
self._checkpoint = None
self._reward_mode = "engine"
self._actor_engine_return = 0.0
self._actor_training_return = 0.0
self._attempts = 0
self._invalid_attempts = 0
def reset(self, *, seed=None, options=None):
if seed is not None and (type(seed) is not int or seed < 0):
raise ValueError("seed must be a nonnegative integer or None")
if options is not None and (type(options) is not dict or set(options) - {"checkpoint","archive_workload","generation_version","notebook_bytes","memory_profile","split","reward_mode"}):
raise ValueError("options supports checkpoint, archive_workload, generation_version, notebook_bytes, memory_profile, split and reward_mode")
checkpoint = None if options is None else options.get("checkpoint")
if options is not None and "checkpoint" in options:
if type(checkpoint) is not int or not 1 <= checkpoint <= 7:
raise ValueError("checkpoint must be an integer from 1 to 7")
workload = "standard" if options is None else options.get("archive_workload","standard")
options = options or {}
reward_mode = options.get("reward_mode", "engine")
if type(reward_mode) is not str or reward_mode not in ("engine", "room_exit"):
raise ValueError("reward_mode must be engine or room_exit")
if reward_mode == "room_exit" and checkpoint is None:
raise ValueError("room_exit reward requires a checkpoint")
profile = self._profile(options.get("generation_version", self._generation_version),
options.get("notebook_bytes", self._notebook_bytes),
options.get("memory_profile", self._memory_profile))
split = options.get("split")
if "split" in options and split not in ("train", "dev", "test"):
raise ValueError("split must be train, dev, or test")
if split is not None and self._world_seed is not None:
raise ValueError("world_seed and split are mutually exclusive")
super().reset(seed=seed)
world_seed = self._world_seed
if split is not None:
from .manifests import manifest
roots = manifest(split, profile["generation_version"])["root_seeds"]
world_seed = roots[int(self.np_random.integers(len(roots)))]
elif world_seed is None:
world_seed = int(self.np_random.integers(0, 2**63))
campaign = Campaign.generated(world_seed, archive_workload=workload, **profile)
transition = campaign.observe()
if checkpoint is not None and checkpoint > 1:
from borderland_verify.reference import public_reference
reference = public_reference(actor_transition(transition))
for _ in range(240):
if transition.done or int(transition.observation.phase) >= checkpoint:
break
note = reference.notebook(actor_transition(transition))
if note is not None:
transition = campaign.replace_notebook(note, expected_revision=transition.revision)
transition = campaign.step(reference.choose(actor_transition(transition)), expected_revision=transition.revision)
if transition.context_reset:
reference = type(reference)()
if transition.error:
raise RuntimeError("public reference could not reach diagnostic checkpoint")
if transition.done or int(transition.observation.phase) != checkpoint:
raise RuntimeError("public reference did not reach diagnostic checkpoint")
self._campaign = campaign
self._transition = transition
self._checkpoint = checkpoint
self._reward_mode = reward_mode
self._actor_engine_return = 0.0
self._actor_training_return = 0.0
self._attempts = 0
self._invalid_attempts = 0
self._finished = False
observation, _, _, _, info = self._result(transition)
return observation, info
@staticmethod
def _profile(generation_version, notebook_bytes, memory_profile="static"):
if type(generation_version) is not int or generation_version not in (2, 3):
raise ValueError("generation_version must be 2 or 3")
if type(notebook_bytes) is not int or notebook_bytes < 0:
raise ValueError("notebook_bytes must be a nonnegative integer")
if generation_version == 2 and notebook_bytes != 768:
raise ValueError("custom byte capacity requires generation three")
if memory_profile not in ("static", "commissioning") or (generation_version == 2 and memory_profile != "static"):
raise ValueError("commissioning requires generation three")
return {"generation_version": generation_version, "notebook_bytes": notebook_bytes, "memory_profile": memory_profile}
def step(self, action):
if self._finished or self._campaign is None:
raise gym.error.ResetNeeded("Call reset before stepping a new episode")
previous_phase = int(self._transition.observation.phase)
self._attempts += 1
error = None
if not self.action_space.contains(action):
error = ActionError("malformed", "Expected printable ASCII JSON action text within the action-space bound.")
else:
try:
raw = json.loads(action, parse_constant=self._reject_constant, object_pairs_hook=self._unique_object)
except (ValueError, RecursionError):
error = ActionError("malformed", "Expected finite JSON with unique object keys.")
else:
if type(raw) is dict and raw.get("tag") == "replace_notebook":
if set(raw) != {"tag", "arguments"} or type(raw["arguments"]) is not dict or set(raw["arguments"]) != {"text"}:
error = ActionError("malformed", "Notebook action requires exactly one text argument.")
else:
self._transition = self._campaign.replace_notebook(raw["arguments"]["text"])
else:
self._transition = self._campaign.step(raw)
if error is None:
error = self._transition.error
if error is not None:
self._invalid_attempts += 1
return self._result(self._transition, error, previous_phase=previous_phase)
@staticmethod
def _reject_constant(value):
raise ValueError("nonfinite JSON")
@staticmethod
def _unique_object(pairs):
result = {}
for key, value in pairs:
if key in result:
raise ValueError("duplicate JSON key")
result[key] = value
return result
def _result(self, transition, error=None, *, previous_phase=None):
data = actor_view(transition)
data["error"] = None if error is None else error.to_dict()
room_complete = self._checkpoint is not None and int(transition.observation.phase) != self._checkpoint
terminated = bool(transition.terminated or room_complete)
reason = None
if not terminated and self._invalid_attempts >= self._invalid_attempt_limit:
reason = "invalid_attempt_limit"
elif not terminated and self._attempts >= self._max_steps:
reason = "attempt_limit"
truncated = bool(transition.truncated or reason is not None)
self._finished = terminated or truncated
info = {"outcome": transition.outcome,
"generation_version": self._campaign._world.generation_version,
"memory_profile": self._campaign._world.memory_profile,
"context_reset": transition.context_reset, "error": data["error"],
"scope": "campaign" if self._checkpoint is None else "room_diagnostic",
"checkpoint": self._checkpoint, "assisted_prefix": self._checkpoint is not None and self._checkpoint > 1,
"room_complete": room_complete or (self._checkpoint == 7 and transition.outcome == "escaped"),
"full_campaign_success": self._checkpoint is None and transition.outcome == "escaped",
"truncation_reason": reason}
observation = json.dumps(data, ensure_ascii=True, allow_nan=False, separators=(",", ":"), sort_keys=True)
reward = float(transition.reward)
if self._reward_mode == "room_exit":
reward = float(error is None and not truncated and previous_phase == self._checkpoint and (
(self._checkpoint < 7 and int(transition.observation.phase) == self._checkpoint + 1
and transition.outcome == "running")
or (self._checkpoint == 7 and transition.outcome == "escaped")))
if previous_phase is not None:
self._actor_engine_return += float(transition.reward)
self._actor_training_return += reward
info.update(reward_mode="room_exit", reward_contract="local_surrogate.room_exit.v1",
engine_reward=float(transition.reward),
actor_engine_return=self._actor_engine_return,
actor_training_return=self._actor_training_return)
return observation, reward, terminated, truncated, info
|