import collections import hashlib import json import math import random import stat import tempfile from contextlib import contextmanager from importlib.resources import files from pathlib import Path from borderland import Action, Campaign from borderland.replay import source_hash as engine_hash from borderland_eval.manifests import digest, source_hash from borderland_verify.reference import LegalReference CONFIG = dict(method='hierarchical non-LLM REINFORCE with scripted public-history prior', episodes=480, learning_rate=0.16, baseline_decay=0.9, gamma=1.0, max_steps=240, training_seeds=[41, 42, 43, 44, 45], learned_phases=[1, 6, 7], dimensions=12, reward='unmodified engine reward, one on escape and zero otherwise', prior='LegalReference supplies all navigation, evidence interpretation, notebook compression, circuit order, and action timing. Learned policy replaces latch choice, each connect bus, and final valve side. No reference prefix checkpoint.', features='one-hot phase x public notebook bus x candidate side; no comparison to a correctness label', workloads=['compact']) class SidePolicy: def __init__(self, weights=None, neutral_phases=()): self.weights = [0.0] * 12 if weights is None else list(weights) if len(self.weights) != 12 or any(type(value) not in (int, float) or not math.isfinite(value) for value in self.weights): raise ValueError('weights must contain 12 finite numbers') self.neutral_phases = set(neutral_phases) if not self.neutral_phases.issubset({1, 6, 7}): raise ValueError('neutral phases must be 1, 6, or 7') def distribution(self, observation, actions): phase = observation['phase'] bus = json.loads(observation['notebook']['text'])['bus'] assert phase in (1, 6, 7) and bus in ('west', 'east') base = ((1, 6, 7).index(phase) * 2 + ('west', 'east').index(bus)) * 2 indices = [] for action in actions: arguments = action.arguments side = ('west', 'east')[arguments['value'] - 2] if phase == 1 else arguments['bus'] if phase == 6 else arguments['side'] indices.append(base + ('west', 'east').index(side)) logits = [0.0 if phase in self.neutral_phases else self.weights[index] for index in indices] exps = [math.exp(value - max(logits)) for value in logits] return [value / sum(exps) for value in exps], indices def choose(self, observation, actions, rng): probabilities, indices = self.distribution(observation, actions) selected = 0 if rng.random() < probabilities[0] else 1 gradient = {index: -probability for index, probability in zip(indices, probabilities)} gradient[indices[selected]] += 1.0 return actions[selected], gradient, probabilities def replacements(action, phase): if phase == 1 and action.tag == 'set_latch' and action.arguments['value'] >= 2: return [Action('set_latch', dict(value=value)) for value in (2, 3)] if phase == 6 and action.tag == 'connect': return [Action('connect', dict(circuit=action.arguments['circuit'], bus=side)) for side in ('west', 'east')] if phase == 7 and action.tag == 'open_valve': return [Action('open_valve', dict(side=side)) for side in ('west', 'east')] return None def rollout(policy, seed, rng, record=False): campaign = Campaign.generated(seed, archive_workload='compact') transition = campaign.reset() prior = LegalReference() decisions, trajectory, trace = [], [], [] phases = set() failures = [] for step in range(CONFIG['max_steps']): if transition.done: break note = prior.notebook(transition) if note is not None: transition = campaign.replace_notebook(note) observation = transition.observation.to_dict() phases.add(observation['phase']) action = prior.choose(transition) options = replacements(action, observation['phase']) if options and all(option in transition.legal_actions for option in options): action, gradient, probabilities = policy.choose(observation, options, rng) decisions.append(dict(phase=observation['phase'], notebook=observation['notebook']['text'], action=action.to_dict(), probabilities=probabilities, gradient=gradient, step=step)) transition = campaign.step(action) trajectory.append(dict(step=step, phase=observation['phase'], action=action.to_dict(), reward=transition.reward, outcome=transition.outcome, error=None if transition.error is None else transition.error.to_dict())) if record: trace.append(transition.to_dict()) if transition.error: failures.append(transition.error.to_dict()) break result = dict(operator_seed=seed, outcome=transition.outcome, escaped=transition.outcome == 'escaped', engine_return=sum(row['reward'] for row in trajectory), steps=len(trajectory), truncated=not transition.done, errors=failures, reached_phases=sorted(phases), decisions=decisions, reward_trajectory=trajectory) if record: result['trace'] = trace return result def update(policy, row, baseline): gradient = collections.Counter() for decision in row['decisions']: for index, value in decision['gradient'].items(): gradient[index] += (row['engine_return'] - baseline) * value for index, value in gradient.items(): policy.weights[index] += CONFIG['learning_rate'] * value return math.sqrt(sum(value * value for value in gradient.values())) def evaluate(policy, seeds, rng_seed, record=False): before = digest(policy.weights) rows = [rollout(policy, seed, random.Random(rng_seed + index), record) for index, seed in enumerate(seeds)] assert before == digest(policy.weights) return dict(escapes=sum(row['escaped'] for row in rows), denominator=len(rows), error_episodes=sum(bool(row['errors']) for row in rows), truncated=sum(row['truncated'] for row in rows), outcome_counts=dict(collections.Counter(row['outcome'] for row in rows)), episodes=rows, frozen_checksum=before) def load_split(split): if split not in ('train', 'dev', 'test'): raise ValueError('split must be train, dev, or test') payload = files('borderland_eval').joinpath('manifests', 'v2', split + '.json').read_bytes() return json.loads(payload)['root_seeds'], hashlib.sha256(payload).hexdigest() def _output_directory(output_dir, episodes, seed): if type(episodes) is not int or not 1 <= episodes <= 1000000: raise ValueError('episodes must be an integer between 1 and 1000000') if type(seed) is not int or not 0 <= seed < 2**63: raise ValueError('seed must be an integer between 0 and 2**63 - 1') if not isinstance(output_dir, (str, Path)) or not str(output_dir).strip(): raise ValueError('output_dir must be a nonempty path') output = Path(output_dir).resolve() for package in Path(__file__).resolve().parent.parent.glob('borderland*'): if package.is_dir() and output.is_relative_to(package.resolve()): raise ValueError('output_dir must be outside installed packages') for name in ('weights.json', 'checkpoint-weights.json', 'training.json', 'trajectories.jsonl'): try: mode = (output / name).lstat().st_mode except FileNotFoundError: continue if not stat.S_ISREG(mode): raise ValueError(f'output artifact must be a regular file: {name}') return output @contextmanager def _artifact_file(path, progress=False): temporary = None try: with tempfile.NamedTemporaryFile(mode='w', dir=path.parent, delete=False) as artifact: temporary = Path(artifact.name) if progress: temporary.replace(path) yield artifact if not progress: temporary.replace(path) finally: if temporary is not None: temporary.unlink(missing_ok=True) def train_hierarchical(output_dir, episodes=480, seed=41, *, train_worlds=None, dev_worlds=None): from .checkpoints import checkpoint, training_worlds output = _output_directory(output_dir, episodes, seed) if (train_worlds is None) != (dev_worlds is None): raise ValueError('provide both train_worlds and dev_worlds') if train_worlds is None: train_seeds, train_hash = load_split('train') dev_seeds, dev_hash = load_split('dev') else: train_seeds, dev_seeds = train_worlds, dev_worlds train_hash, dev_hash = digest(train_seeds), digest(dev_seeds) train_seeds, dev_seeds = training_worlds(train_seeds, dev_seeds) policy = SidePolicy() before = evaluate(policy, dev_seeds, 97000) initial = digest(policy.weights) rng = random.Random(seed) baseline = 0.0 rows = [] output.mkdir(parents=True, exist_ok=True) with _artifact_file(output / 'trajectories.jsonl', progress=True) as progress: for episode in range(episodes): row = rollout(policy, train_seeds[episode % len(train_seeds)], rng) norm = update(policy, row, baseline) baseline = CONFIG['baseline_decay'] * baseline + (1 - CONFIG['baseline_decay']) * row['engine_return'] rows.append(dict(episode=episode, **row, gradient_norm=norm, ema_baseline=baseline)) progress.write(json.dumps(rows[-1]) + '\n') progress.flush() after = evaluate(policy, dev_seeds, 97000) ablations = {str(phase): evaluate(SidePolicy(policy.weights, [phase]), dev_seeds, 97000) for phase in CONFIG['learned_phases']} result = dict(learner=CONFIG['method'], compressor='LegalReference public-history notebook', training_seed=seed, initial_checksum=initial, final_checksum=digest(policy.weights), weights=policy.weights, before=before, after=after, ablations=ablations, training=rows, rl_updates=len(rows), nonzero_rl_updates=sum(row['gradient_norm'] > 0 for row in rows), imitation_updates=0, config=CONFIG, configuration_hash=digest(CONFIG), run_configuration=dict(episodes=episodes, seed=seed, evaluation_rng_seed=97000), run_configuration_hash=digest(dict(episodes=episodes, seed=seed, evaluation_rng_seed=97000)), split_hashes=dict(train=train_hash, dev=dev_hash), source_hash=source_hash(), engine_hash=engine_hash(), learner_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), limits=['LegalReference scripts all decisions except polarity choices in games 1, 6, and 7.', 'Full initial reset, compact workload only, no imitation updates or model calls.', 'Before and after use frozen weights on the same dev worlds and action RNG streams.', 'This hybrid is not a fully learned solver or evidence of Beam training.']) with _artifact_file(output / 'weights.json') as artifact: run = dict(episodes=episodes, training_seed=seed, train_worlds=train_seeds, dev_worlds=dev_seeds, evaluation_rng_seed=97000, split_hashes=dict(train=train_hash, dev=dev_hash), optimizer=CONFIG) artifact.write(json.dumps(checkpoint('hierarchical', policy.weights, run)) + '\n') with _artifact_file(output / 'training.json') as artifact: artifact.write(json.dumps(result, indent=2) + '\n') return result