Spaces:
Running
Running
File size: 11,884 Bytes
0e8028d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 | import collections
import hashlib
import json
import math
import random
import stat
import tempfile
from contextlib import contextmanager
from importlib.resources import files
from pathlib import Path
from borderland import Action, Campaign
from borderland.replay import source_hash as engine_hash
from borderland_eval.manifests import digest, source_hash
from borderland_verify.reference import LegalReference
CONFIG = dict(method='hierarchical non-LLM REINFORCE with scripted public-history prior',
episodes=480, learning_rate=0.16, baseline_decay=0.9, gamma=1.0,
max_steps=240, training_seeds=[41, 42, 43, 44, 45],
learned_phases=[1, 6, 7], dimensions=12,
reward='unmodified engine reward, one on escape and zero otherwise',
prior='LegalReference supplies all navigation, evidence interpretation, notebook compression, circuit order, and action timing. Learned policy replaces latch choice, each connect bus, and final valve side. No reference prefix checkpoint.',
features='one-hot phase x public notebook bus x candidate side; no comparison to a correctness label',
workloads=['compact'])
class SidePolicy:
def __init__(self, weights=None, neutral_phases=()):
self.weights = [0.0] * 12 if weights is None else list(weights)
if len(self.weights) != 12 or any(type(value) not in (int, float) or not math.isfinite(value) for value in self.weights):
raise ValueError('weights must contain 12 finite numbers')
self.neutral_phases = set(neutral_phases)
if not self.neutral_phases.issubset({1, 6, 7}):
raise ValueError('neutral phases must be 1, 6, or 7')
def distribution(self, observation, actions):
phase = observation['phase']
bus = json.loads(observation['notebook']['text'])['bus']
assert phase in (1, 6, 7) and bus in ('west', 'east')
base = ((1, 6, 7).index(phase) * 2 + ('west', 'east').index(bus)) * 2
indices = []
for action in actions:
arguments = action.arguments
side = ('west', 'east')[arguments['value'] - 2] if phase == 1 else arguments['bus'] if phase == 6 else arguments['side']
indices.append(base + ('west', 'east').index(side))
logits = [0.0 if phase in self.neutral_phases else self.weights[index] for index in indices]
exps = [math.exp(value - max(logits)) for value in logits]
return [value / sum(exps) for value in exps], indices
def choose(self, observation, actions, rng):
probabilities, indices = self.distribution(observation, actions)
selected = 0 if rng.random() < probabilities[0] else 1
gradient = {index: -probability for index, probability in zip(indices, probabilities)}
gradient[indices[selected]] += 1.0
return actions[selected], gradient, probabilities
def replacements(action, phase):
if phase == 1 and action.tag == 'set_latch' and action.arguments['value'] >= 2:
return [Action('set_latch', dict(value=value)) for value in (2, 3)]
if phase == 6 and action.tag == 'connect':
return [Action('connect', dict(circuit=action.arguments['circuit'], bus=side)) for side in ('west', 'east')]
if phase == 7 and action.tag == 'open_valve':
return [Action('open_valve', dict(side=side)) for side in ('west', 'east')]
return None
def rollout(policy, seed, rng, record=False):
campaign = Campaign.generated(seed, archive_workload='compact')
transition = campaign.reset()
prior = LegalReference()
decisions, trajectory, trace = [], [], []
phases = set()
failures = []
for step in range(CONFIG['max_steps']):
if transition.done:
break
note = prior.notebook(transition)
if note is not None:
transition = campaign.replace_notebook(note)
observation = transition.observation.to_dict()
phases.add(observation['phase'])
action = prior.choose(transition)
options = replacements(action, observation['phase'])
if options and all(option in transition.legal_actions for option in options):
action, gradient, probabilities = policy.choose(observation, options, rng)
decisions.append(dict(phase=observation['phase'], notebook=observation['notebook']['text'],
action=action.to_dict(), probabilities=probabilities, gradient=gradient,
step=step))
transition = campaign.step(action)
trajectory.append(dict(step=step, phase=observation['phase'], action=action.to_dict(), reward=transition.reward,
outcome=transition.outcome, error=None if transition.error is None else transition.error.to_dict()))
if record:
trace.append(transition.to_dict())
if transition.error:
failures.append(transition.error.to_dict())
break
result = dict(operator_seed=seed, outcome=transition.outcome, escaped=transition.outcome == 'escaped',
engine_return=sum(row['reward'] for row in trajectory), steps=len(trajectory),
truncated=not transition.done, errors=failures, reached_phases=sorted(phases),
decisions=decisions, reward_trajectory=trajectory)
if record:
result['trace'] = trace
return result
def update(policy, row, baseline):
gradient = collections.Counter()
for decision in row['decisions']:
for index, value in decision['gradient'].items():
gradient[index] += (row['engine_return'] - baseline) * value
for index, value in gradient.items():
policy.weights[index] += CONFIG['learning_rate'] * value
return math.sqrt(sum(value * value for value in gradient.values()))
def evaluate(policy, seeds, rng_seed, record=False):
before = digest(policy.weights)
rows = [rollout(policy, seed, random.Random(rng_seed + index), record) for index, seed in enumerate(seeds)]
assert before == digest(policy.weights)
return dict(escapes=sum(row['escaped'] for row in rows), denominator=len(rows),
error_episodes=sum(bool(row['errors']) for row in rows),
truncated=sum(row['truncated'] for row in rows),
outcome_counts=dict(collections.Counter(row['outcome'] for row in rows)), episodes=rows,
frozen_checksum=before)
def load_split(split):
if split not in ('train', 'dev', 'test'):
raise ValueError('split must be train, dev, or test')
payload = files('borderland_eval').joinpath('manifests', 'v2', split + '.json').read_bytes()
return json.loads(payload)['root_seeds'], hashlib.sha256(payload).hexdigest()
def _output_directory(output_dir, episodes, seed):
if type(episodes) is not int or not 1 <= episodes <= 1000000:
raise ValueError('episodes must be an integer between 1 and 1000000')
if type(seed) is not int or not 0 <= seed < 2**63:
raise ValueError('seed must be an integer between 0 and 2**63 - 1')
if not isinstance(output_dir, (str, Path)) or not str(output_dir).strip():
raise ValueError('output_dir must be a nonempty path')
output = Path(output_dir).resolve()
for package in Path(__file__).resolve().parent.parent.glob('borderland*'):
if package.is_dir() and output.is_relative_to(package.resolve()):
raise ValueError('output_dir must be outside installed packages')
for name in ('weights.json', 'checkpoint-weights.json', 'training.json', 'trajectories.jsonl'):
try:
mode = (output / name).lstat().st_mode
except FileNotFoundError:
continue
if not stat.S_ISREG(mode):
raise ValueError(f'output artifact must be a regular file: {name}')
return output
@contextmanager
def _artifact_file(path, progress=False):
temporary = None
try:
with tempfile.NamedTemporaryFile(mode='w', dir=path.parent, delete=False) as artifact:
temporary = Path(artifact.name)
if progress:
temporary.replace(path)
yield artifact
if not progress:
temporary.replace(path)
finally:
if temporary is not None:
temporary.unlink(missing_ok=True)
def train_hierarchical(output_dir, episodes=480, seed=41, *, train_worlds=None, dev_worlds=None):
from .checkpoints import checkpoint, training_worlds
output = _output_directory(output_dir, episodes, seed)
if (train_worlds is None) != (dev_worlds is None):
raise ValueError('provide both train_worlds and dev_worlds')
if train_worlds is None:
train_seeds, train_hash = load_split('train')
dev_seeds, dev_hash = load_split('dev')
else:
train_seeds, dev_seeds = train_worlds, dev_worlds
train_hash, dev_hash = digest(train_seeds), digest(dev_seeds)
train_seeds, dev_seeds = training_worlds(train_seeds, dev_seeds)
policy = SidePolicy()
before = evaluate(policy, dev_seeds, 97000)
initial = digest(policy.weights)
rng = random.Random(seed)
baseline = 0.0
rows = []
output.mkdir(parents=True, exist_ok=True)
with _artifact_file(output / 'trajectories.jsonl', progress=True) as progress:
for episode in range(episodes):
row = rollout(policy, train_seeds[episode % len(train_seeds)], rng)
norm = update(policy, row, baseline)
baseline = CONFIG['baseline_decay'] * baseline + (1 - CONFIG['baseline_decay']) * row['engine_return']
rows.append(dict(episode=episode, **row, gradient_norm=norm, ema_baseline=baseline))
progress.write(json.dumps(rows[-1]) + '\n')
progress.flush()
after = evaluate(policy, dev_seeds, 97000)
ablations = {str(phase): evaluate(SidePolicy(policy.weights, [phase]), dev_seeds, 97000)
for phase in CONFIG['learned_phases']}
result = dict(learner=CONFIG['method'], compressor='LegalReference public-history notebook',
training_seed=seed, initial_checksum=initial, final_checksum=digest(policy.weights),
weights=policy.weights, before=before, after=after, ablations=ablations, training=rows,
rl_updates=len(rows), nonzero_rl_updates=sum(row['gradient_norm'] > 0 for row in rows),
imitation_updates=0, config=CONFIG, configuration_hash=digest(CONFIG),
run_configuration=dict(episodes=episodes, seed=seed, evaluation_rng_seed=97000),
run_configuration_hash=digest(dict(episodes=episodes, seed=seed, evaluation_rng_seed=97000)),
split_hashes=dict(train=train_hash, dev=dev_hash), source_hash=source_hash(), engine_hash=engine_hash(),
learner_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
limits=['LegalReference scripts all decisions except polarity choices in games 1, 6, and 7.',
'Full initial reset, compact workload only, no imitation updates or model calls.',
'Before and after use frozen weights on the same dev worlds and action RNG streams.',
'This hybrid is not a fully learned solver or evidence of Beam training.'])
with _artifact_file(output / 'weights.json') as artifact:
run = dict(episodes=episodes, training_seed=seed, train_worlds=train_seeds, dev_worlds=dev_seeds,
evaluation_rng_seed=97000, split_hashes=dict(train=train_hash, dev=dev_hash),
optimizer=CONFIG)
artifact.write(json.dumps(checkpoint('hierarchical', policy.weights, run)) + '\n')
with _artifact_file(output / 'training.json') as artifact:
artifact.write(json.dumps(result, indent=2) + '\n')
return result
|