FatUnc6969's picture
Export optional checkpoint completion reward
986ec09 verified
Raw History Blame Contribute Delete
12.7 kB
"""Optional Gymnasium entry point.
Install ``borderland-rl[gymnasium]`` and import ``BorderlandEnv`` here.
Observations are current public Transition JSON, including legal_actions, without
operator revision or diagnostics. Actions are JSON
``{"tag": "...", "arguments": {...}}`` objects. The additional tag
``replace_notebook`` takes exactly ``{"text": "..."}`` and obeys the engine's
configured byte limit in generation three or 768-character limit in generation
two. JSON text uses printable ASCII with Unicode escapes.
Default reset starts a complete campaign. ``options={"checkpoint": N}`` starts
a room diagnostic for N in 1..7 and ends when that room is left. For N > 1 a
public-observation LegalReference executes the prefix and writes its legitimate
notebook. This assisted diagnostic is never full-campaign success. Seeds are
operator inputs; neither seeds nor prefix history enter observations or info.
The default ``reward_mode="engine"`` preserves engine reward. For a checkpoint,
``reward_mode="room_exit"`` selects a binary local training surrogate and adds
its versioned contract and separate actor returns to info only.
``options={"split": "train"}`` samples a frozen generation-specific split.
The other split names are ``dev`` and ``test``. The reset seed selects a
reproducible member. A split and ``world_seed`` are mutually exclusive.
"""
from __future__ import annotations
import json
import string
import gymnasium as gym
from gymnasium import spaces
from borderland import Campaign
from borderland.domain import ActionError
from .context import actor_transition, actor_view
class BorderlandEnv(gym.Env):
"""Text/tool environment with bounded attempts and default engine rewards.
``seed`` in reset controls world generation through Gymnasium's RNG.
``world_seed`` fixes a world for operator diagnostics independently of that RNG.
A diagnostic ends on leaving its selected room, or on an engine terminal state.
Invalid actions consume adapter attempts but leave the campaign unchanged.
"""
metadata = {"render_modes": []}
def __init__(self, *, world_seed=None, max_steps=512, invalid_attempt_limit=32,
generation_version=2, notebook_bytes=768, memory_profile="static"):
super().__init__()
if world_seed is not None and type(world_seed) is not int:
raise ValueError("world_seed must be an integer or None")
for name, value in (("max_steps", max_steps), ("invalid_attempt_limit", invalid_attempt_limit)):
if type(value) is not int or value < 1:
raise ValueError(f"{name} must be a positive integer")
self._world_seed = world_seed
self._max_steps = max_steps
self._invalid_attempt_limit = invalid_attempt_limit
self._profile(generation_version, notebook_bytes, memory_profile)
self._generation_version = generation_version
self._notebook_bytes = notebook_bytes
self._memory_profile = memory_profile
charset = string.ascii_letters + string.digits + string.punctuation + " "
self.action_space = spaces.Text(max_length=16384, min_length=1, charset=charset)
self.observation_space = spaces.Text(max_length=131072, min_length=1, charset=charset)
self._campaign = None
self._finished = True
self._checkpoint = None
self._reward_mode = "engine"
self._actor_engine_return = 0.0
self._actor_training_return = 0.0
self._attempts = 0
self._invalid_attempts = 0
def reset(self, *, seed=None, options=None):
if seed is not None and (type(seed) is not int or seed < 0):
raise ValueError("seed must be a nonnegative integer or None")
if options is not None and (type(options) is not dict or set(options) - {"checkpoint","archive_workload","generation_version","notebook_bytes","memory_profile","split","reward_mode"}):
raise ValueError("options supports checkpoint, archive_workload, generation_version, notebook_bytes, memory_profile, split and reward_mode")
checkpoint = None if options is None else options.get("checkpoint")
if options is not None and "checkpoint" in options:
if type(checkpoint) is not int or not 1 <= checkpoint <= 7:
raise ValueError("checkpoint must be an integer from 1 to 7")
workload = "standard" if options is None else options.get("archive_workload","standard")
options = options or {}
reward_mode = options.get("reward_mode", "engine")
if type(reward_mode) is not str or reward_mode not in ("engine", "room_exit"):
raise ValueError("reward_mode must be engine or room_exit")
if reward_mode == "room_exit" and checkpoint is None:
raise ValueError("room_exit reward requires a checkpoint")
profile = self._profile(options.get("generation_version", self._generation_version),
options.get("notebook_bytes", self._notebook_bytes),
options.get("memory_profile", self._memory_profile))
split = options.get("split")
if "split" in options and split not in ("train", "dev", "test"):
raise ValueError("split must be train, dev, or test")
if split is not None and self._world_seed is not None:
raise ValueError("world_seed and split are mutually exclusive")
super().reset(seed=seed)
world_seed = self._world_seed
if split is not None:
from .manifests import manifest
roots = manifest(split, profile["generation_version"])["root_seeds"]
world_seed = roots[int(self.np_random.integers(len(roots)))]
elif world_seed is None:
world_seed = int(self.np_random.integers(0, 2**63))
campaign = Campaign.generated(world_seed, archive_workload=workload, **profile)
transition = campaign.observe()
if checkpoint is not None and checkpoint > 1:
from borderland_verify.reference import public_reference
reference = public_reference(actor_transition(transition))
for _ in range(240):
if transition.done or int(transition.observation.phase) >= checkpoint:
break
note = reference.notebook(actor_transition(transition))
if note is not None:
transition = campaign.replace_notebook(note, expected_revision=transition.revision)
transition = campaign.step(reference.choose(actor_transition(transition)), expected_revision=transition.revision)
if transition.context_reset:
reference = type(reference)()
if transition.error:
raise RuntimeError("public reference could not reach diagnostic checkpoint")
if transition.done or int(transition.observation.phase) != checkpoint:
raise RuntimeError("public reference did not reach diagnostic checkpoint")
self._campaign = campaign
self._transition = transition
self._checkpoint = checkpoint
self._reward_mode = reward_mode
self._actor_engine_return = 0.0
self._actor_training_return = 0.0
self._attempts = 0
self._invalid_attempts = 0
self._finished = False
observation, _, _, _, info = self._result(transition)
return observation, info
@staticmethod
def _profile(generation_version, notebook_bytes, memory_profile="static"):
if type(generation_version) is not int or generation_version not in (2, 3):
raise ValueError("generation_version must be 2 or 3")
if type(notebook_bytes) is not int or notebook_bytes < 0:
raise ValueError("notebook_bytes must be a nonnegative integer")
if generation_version == 2 and notebook_bytes != 768:
raise ValueError("custom byte capacity requires generation three")
if memory_profile not in ("static", "commissioning") or (generation_version == 2 and memory_profile != "static"):
raise ValueError("commissioning requires generation three")
return {"generation_version": generation_version, "notebook_bytes": notebook_bytes, "memory_profile": memory_profile}
def step(self, action):
if self._finished or self._campaign is None:
raise gym.error.ResetNeeded("Call reset before stepping a new episode")
previous_phase = int(self._transition.observation.phase)
self._attempts += 1
error = None
if not self.action_space.contains(action):
error = ActionError("malformed", "Expected printable ASCII JSON action text within the action-space bound.")
else:
try:
raw = json.loads(action, parse_constant=self._reject_constant, object_pairs_hook=self._unique_object)
except (ValueError, RecursionError):
error = ActionError("malformed", "Expected finite JSON with unique object keys.")
else:
if type(raw) is dict and raw.get("tag") == "replace_notebook":
if set(raw) != {"tag", "arguments"} or type(raw["arguments"]) is not dict or set(raw["arguments"]) != {"text"}:
error = ActionError("malformed", "Notebook action requires exactly one text argument.")
else:
self._transition = self._campaign.replace_notebook(raw["arguments"]["text"])
else:
self._transition = self._campaign.step(raw)
if error is None:
error = self._transition.error
if error is not None:
self._invalid_attempts += 1
return self._result(self._transition, error, previous_phase=previous_phase)
@staticmethod
def _reject_constant(value):
raise ValueError("nonfinite JSON")
@staticmethod
def _unique_object(pairs):
result = {}
for key, value in pairs:
if key in result:
raise ValueError("duplicate JSON key")
result[key] = value
return result
def _result(self, transition, error=None, *, previous_phase=None):
data = actor_view(transition)
data["error"] = None if error is None else error.to_dict()
room_complete = self._checkpoint is not None and int(transition.observation.phase) != self._checkpoint
terminated = bool(transition.terminated or room_complete)
reason = None
if not terminated and self._invalid_attempts >= self._invalid_attempt_limit:
reason = "invalid_attempt_limit"
elif not terminated and self._attempts >= self._max_steps:
reason = "attempt_limit"
truncated = bool(transition.truncated or reason is not None)
self._finished = terminated or truncated
info = {"outcome": transition.outcome,
"generation_version": self._campaign._world.generation_version,
"memory_profile": self._campaign._world.memory_profile,
"context_reset": transition.context_reset, "error": data["error"],
"scope": "campaign" if self._checkpoint is None else "room_diagnostic",
"checkpoint": self._checkpoint, "assisted_prefix": self._checkpoint is not None and self._checkpoint > 1,
"room_complete": room_complete or (self._checkpoint == 7 and transition.outcome == "escaped"),
"full_campaign_success": self._checkpoint is None and transition.outcome == "escaped",
"truncation_reason": reason}
observation = json.dumps(data, ensure_ascii=True, allow_nan=False, separators=(",", ":"), sort_keys=True)
reward = float(transition.reward)
if self._reward_mode == "room_exit":
reward = float(error is None and not truncated and previous_phase == self._checkpoint and (
(self._checkpoint < 7 and int(transition.observation.phase) == self._checkpoint + 1
and transition.outcome == "running")
or (self._checkpoint == 7 and transition.outcome == "escaped")))
if previous_phase is not None:
self._actor_engine_return += float(transition.reward)
self._actor_training_return += reward
info.update(reward_mode="room_exit", reward_contract="local_surrogate.room_exit.v1",
engine_reward=float(transition.reward),
actor_engine_return=self._actor_engine_return,
actor_training_return=self._actor_training_return)
return observation, reward, terminated, truncated, info