borderland-rl / python /borderland_eval /hierarchical_training.py
FatUnc6969's picture
Publish reviewed research tools and evidence audit
0e8028d verified
Raw History Blame Contribute Delete
11.9 kB
import collections
import hashlib
import json
import math
import random
import stat
import tempfile
from contextlib import contextmanager
from importlib.resources import files
from pathlib import Path
from borderland import Action, Campaign
from borderland.replay import source_hash as engine_hash
from borderland_eval.manifests import digest, source_hash
from borderland_verify.reference import LegalReference
CONFIG = dict(method='hierarchical non-LLM REINFORCE with scripted public-history prior',
episodes=480, learning_rate=0.16, baseline_decay=0.9, gamma=1.0,
max_steps=240, training_seeds=[41, 42, 43, 44, 45],
learned_phases=[1, 6, 7], dimensions=12,
reward='unmodified engine reward, one on escape and zero otherwise',
prior='LegalReference supplies all navigation, evidence interpretation, notebook compression, circuit order, and action timing. Learned policy replaces latch choice, each connect bus, and final valve side. No reference prefix checkpoint.',
features='one-hot phase x public notebook bus x candidate side; no comparison to a correctness label',
workloads=['compact'])
class SidePolicy:
def __init__(self, weights=None, neutral_phases=()):
self.weights = [0.0] * 12 if weights is None else list(weights)
if len(self.weights) != 12 or any(type(value) not in (int, float) or not math.isfinite(value) for value in self.weights):
raise ValueError('weights must contain 12 finite numbers')
self.neutral_phases = set(neutral_phases)
if not self.neutral_phases.issubset({1, 6, 7}):
raise ValueError('neutral phases must be 1, 6, or 7')
def distribution(self, observation, actions):
phase = observation['phase']
bus = json.loads(observation['notebook']['text'])['bus']
assert phase in (1, 6, 7) and bus in ('west', 'east')
base = ((1, 6, 7).index(phase) * 2 + ('west', 'east').index(bus)) * 2
indices = []
for action in actions:
arguments = action.arguments
side = ('west', 'east')[arguments['value'] - 2] if phase == 1 else arguments['bus'] if phase == 6 else arguments['side']
indices.append(base + ('west', 'east').index(side))
logits = [0.0 if phase in self.neutral_phases else self.weights[index] for index in indices]
exps = [math.exp(value - max(logits)) for value in logits]
return [value / sum(exps) for value in exps], indices
def choose(self, observation, actions, rng):
probabilities, indices = self.distribution(observation, actions)
selected = 0 if rng.random() < probabilities[0] else 1
gradient = {index: -probability for index, probability in zip(indices, probabilities)}
gradient[indices[selected]] += 1.0
return actions[selected], gradient, probabilities
def replacements(action, phase):
if phase == 1 and action.tag == 'set_latch' and action.arguments['value'] >= 2:
return [Action('set_latch', dict(value=value)) for value in (2, 3)]
if phase == 6 and action.tag == 'connect':
return [Action('connect', dict(circuit=action.arguments['circuit'], bus=side)) for side in ('west', 'east')]
if phase == 7 and action.tag == 'open_valve':
return [Action('open_valve', dict(side=side)) for side in ('west', 'east')]
return None
def rollout(policy, seed, rng, record=False):
campaign = Campaign.generated(seed, archive_workload='compact')
transition = campaign.reset()
prior = LegalReference()
decisions, trajectory, trace = [], [], []
phases = set()
failures = []
for step in range(CONFIG['max_steps']):
if transition.done:
break
note = prior.notebook(transition)
if note is not None:
transition = campaign.replace_notebook(note)
observation = transition.observation.to_dict()
phases.add(observation['phase'])
action = prior.choose(transition)
options = replacements(action, observation['phase'])
if options and all(option in transition.legal_actions for option in options):
action, gradient, probabilities = policy.choose(observation, options, rng)
decisions.append(dict(phase=observation['phase'], notebook=observation['notebook']['text'],
action=action.to_dict(), probabilities=probabilities, gradient=gradient,
step=step))
transition = campaign.step(action)
trajectory.append(dict(step=step, phase=observation['phase'], action=action.to_dict(), reward=transition.reward,
outcome=transition.outcome, error=None if transition.error is None else transition.error.to_dict()))
if record:
trace.append(transition.to_dict())
if transition.error:
failures.append(transition.error.to_dict())
break
result = dict(operator_seed=seed, outcome=transition.outcome, escaped=transition.outcome == 'escaped',
engine_return=sum(row['reward'] for row in trajectory), steps=len(trajectory),
truncated=not transition.done, errors=failures, reached_phases=sorted(phases),
decisions=decisions, reward_trajectory=trajectory)
if record:
result['trace'] = trace
return result
def update(policy, row, baseline):
gradient = collections.Counter()
for decision in row['decisions']:
for index, value in decision['gradient'].items():
gradient[index] += (row['engine_return'] - baseline) * value
for index, value in gradient.items():
policy.weights[index] += CONFIG['learning_rate'] * value
return math.sqrt(sum(value * value for value in gradient.values()))
def evaluate(policy, seeds, rng_seed, record=False):
before = digest(policy.weights)
rows = [rollout(policy, seed, random.Random(rng_seed + index), record) for index, seed in enumerate(seeds)]
assert before == digest(policy.weights)
return dict(escapes=sum(row['escaped'] for row in rows), denominator=len(rows),
error_episodes=sum(bool(row['errors']) for row in rows),
truncated=sum(row['truncated'] for row in rows),
outcome_counts=dict(collections.Counter(row['outcome'] for row in rows)), episodes=rows,
frozen_checksum=before)
def load_split(split):
if split not in ('train', 'dev', 'test'):
raise ValueError('split must be train, dev, or test')
payload = files('borderland_eval').joinpath('manifests', 'v2', split + '.json').read_bytes()
return json.loads(payload)['root_seeds'], hashlib.sha256(payload).hexdigest()
def _output_directory(output_dir, episodes, seed):
if type(episodes) is not int or not 1 <= episodes <= 1000000:
raise ValueError('episodes must be an integer between 1 and 1000000')
if type(seed) is not int or not 0 <= seed < 2**63:
raise ValueError('seed must be an integer between 0 and 2**63 - 1')
if not isinstance(output_dir, (str, Path)) or not str(output_dir).strip():
raise ValueError('output_dir must be a nonempty path')
output = Path(output_dir).resolve()
for package in Path(__file__).resolve().parent.parent.glob('borderland*'):
if package.is_dir() and output.is_relative_to(package.resolve()):
raise ValueError('output_dir must be outside installed packages')
for name in ('weights.json', 'checkpoint-weights.json', 'training.json', 'trajectories.jsonl'):
try:
mode = (output / name).lstat().st_mode
except FileNotFoundError:
continue
if not stat.S_ISREG(mode):
raise ValueError(f'output artifact must be a regular file: {name}')
return output
@contextmanager
def _artifact_file(path, progress=False):
temporary = None
try:
with tempfile.NamedTemporaryFile(mode='w', dir=path.parent, delete=False) as artifact:
temporary = Path(artifact.name)
if progress:
temporary.replace(path)
yield artifact
if not progress:
temporary.replace(path)
finally:
if temporary is not None:
temporary.unlink(missing_ok=True)
def train_hierarchical(output_dir, episodes=480, seed=41, *, train_worlds=None, dev_worlds=None):
from .checkpoints import checkpoint, training_worlds
output = _output_directory(output_dir, episodes, seed)
if (train_worlds is None) != (dev_worlds is None):
raise ValueError('provide both train_worlds and dev_worlds')
if train_worlds is None:
train_seeds, train_hash = load_split('train')
dev_seeds, dev_hash = load_split('dev')
else:
train_seeds, dev_seeds = train_worlds, dev_worlds
train_hash, dev_hash = digest(train_seeds), digest(dev_seeds)
train_seeds, dev_seeds = training_worlds(train_seeds, dev_seeds)
policy = SidePolicy()
before = evaluate(policy, dev_seeds, 97000)
initial = digest(policy.weights)
rng = random.Random(seed)
baseline = 0.0
rows = []
output.mkdir(parents=True, exist_ok=True)
with _artifact_file(output / 'trajectories.jsonl', progress=True) as progress:
for episode in range(episodes):
row = rollout(policy, train_seeds[episode % len(train_seeds)], rng)
norm = update(policy, row, baseline)
baseline = CONFIG['baseline_decay'] * baseline + (1 - CONFIG['baseline_decay']) * row['engine_return']
rows.append(dict(episode=episode, **row, gradient_norm=norm, ema_baseline=baseline))
progress.write(json.dumps(rows[-1]) + '\n')
progress.flush()
after = evaluate(policy, dev_seeds, 97000)
ablations = {str(phase): evaluate(SidePolicy(policy.weights, [phase]), dev_seeds, 97000)
for phase in CONFIG['learned_phases']}
result = dict(learner=CONFIG['method'], compressor='LegalReference public-history notebook',
training_seed=seed, initial_checksum=initial, final_checksum=digest(policy.weights),
weights=policy.weights, before=before, after=after, ablations=ablations, training=rows,
rl_updates=len(rows), nonzero_rl_updates=sum(row['gradient_norm'] > 0 for row in rows),
imitation_updates=0, config=CONFIG, configuration_hash=digest(CONFIG),
run_configuration=dict(episodes=episodes, seed=seed, evaluation_rng_seed=97000),
run_configuration_hash=digest(dict(episodes=episodes, seed=seed, evaluation_rng_seed=97000)),
split_hashes=dict(train=train_hash, dev=dev_hash), source_hash=source_hash(), engine_hash=engine_hash(),
learner_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
limits=['LegalReference scripts all decisions except polarity choices in games 1, 6, and 7.',
'Full initial reset, compact workload only, no imitation updates or model calls.',
'Before and after use frozen weights on the same dev worlds and action RNG streams.',
'This hybrid is not a fully learned solver or evidence of Beam training.'])
with _artifact_file(output / 'weights.json') as artifact:
run = dict(episodes=episodes, training_seed=seed, train_worlds=train_seeds, dev_worlds=dev_seeds,
evaluation_rng_seed=97000, split_hashes=dict(train=train_hash, dev=dev_hash),
optimizer=CONFIG)
artifact.write(json.dumps(checkpoint('hierarchical', policy.weights, run)) + '\n')
with _artifact_file(output / 'training.json') as artifact:
artifact.write(json.dumps(result, indent=2) + '\n')
return result