Spaces:
Running
Running
Download python/borderland_eval/hierarchical_training.py from FatUnc6969/borderland-rl: direct link, hf CLI and curl.
- Browser
- Download file 11.9 kB
-
https://huggingface.co/spaces/FatUnc6969/borderland-rl/resolve/main/python/borderland_eval/hierarchical_training.py
- Command line
-
hf download hf://spaces/FatUnc6969/borderland-rl/python/borderland_eval/hierarchical_training.py
-
curl -L -o hierarchical_training.py https://huggingface.co/spaces/FatUnc6969/borderland-rl/resolve/main/python/borderland_eval/hierarchical_training.py
11.9 kB
| import collections | |
| import hashlib | |
| import json | |
| import math | |
| import random | |
| import stat | |
| import tempfile | |
| from contextlib import contextmanager | |
| from importlib.resources import files | |
| from pathlib import Path | |
| from borderland import Action, Campaign | |
| from borderland.replay import source_hash as engine_hash | |
| from borderland_eval.manifests import digest, source_hash | |
| from borderland_verify.reference import LegalReference | |
| CONFIG = dict(method='hierarchical non-LLM REINFORCE with scripted public-history prior', | |
| episodes=480, learning_rate=0.16, baseline_decay=0.9, gamma=1.0, | |
| max_steps=240, training_seeds=[41, 42, 43, 44, 45], | |
| learned_phases=[1, 6, 7], dimensions=12, | |
| reward='unmodified engine reward, one on escape and zero otherwise', | |
| prior='LegalReference supplies all navigation, evidence interpretation, notebook compression, circuit order, and action timing. Learned policy replaces latch choice, each connect bus, and final valve side. No reference prefix checkpoint.', | |
| features='one-hot phase x public notebook bus x candidate side; no comparison to a correctness label', | |
| workloads=['compact']) | |
| class SidePolicy: | |
| def __init__(self, weights=None, neutral_phases=()): | |
| self.weights = [0.0] * 12 if weights is None else list(weights) | |
| if len(self.weights) != 12 or any(type(value) not in (int, float) or not math.isfinite(value) for value in self.weights): | |
| raise ValueError('weights must contain 12 finite numbers') | |
| self.neutral_phases = set(neutral_phases) | |
| if not self.neutral_phases.issubset({1, 6, 7}): | |
| raise ValueError('neutral phases must be 1, 6, or 7') | |
| def distribution(self, observation, actions): | |
| phase = observation['phase'] | |
| bus = json.loads(observation['notebook']['text'])['bus'] | |
| assert phase in (1, 6, 7) and bus in ('west', 'east') | |
| base = ((1, 6, 7).index(phase) * 2 + ('west', 'east').index(bus)) * 2 | |
| indices = [] | |
| for action in actions: | |
| arguments = action.arguments | |
| side = ('west', 'east')[arguments['value'] - 2] if phase == 1 else arguments['bus'] if phase == 6 else arguments['side'] | |
| indices.append(base + ('west', 'east').index(side)) | |
| logits = [0.0 if phase in self.neutral_phases else self.weights[index] for index in indices] | |
| exps = [math.exp(value - max(logits)) for value in logits] | |
| return [value / sum(exps) for value in exps], indices | |
| def choose(self, observation, actions, rng): | |
| probabilities, indices = self.distribution(observation, actions) | |
| selected = 0 if rng.random() < probabilities[0] else 1 | |
| gradient = {index: -probability for index, probability in zip(indices, probabilities)} | |
| gradient[indices[selected]] += 1.0 | |
| return actions[selected], gradient, probabilities | |
| def replacements(action, phase): | |
| if phase == 1 and action.tag == 'set_latch' and action.arguments['value'] >= 2: | |
| return [Action('set_latch', dict(value=value)) for value in (2, 3)] | |
| if phase == 6 and action.tag == 'connect': | |
| return [Action('connect', dict(circuit=action.arguments['circuit'], bus=side)) for side in ('west', 'east')] | |
| if phase == 7 and action.tag == 'open_valve': | |
| return [Action('open_valve', dict(side=side)) for side in ('west', 'east')] | |
| return None | |
| def rollout(policy, seed, rng, record=False): | |
| campaign = Campaign.generated(seed, archive_workload='compact') | |
| transition = campaign.reset() | |
| prior = LegalReference() | |
| decisions, trajectory, trace = [], [], [] | |
| phases = set() | |
| failures = [] | |
| for step in range(CONFIG['max_steps']): | |
| if transition.done: | |
| break | |
| note = prior.notebook(transition) | |
| if note is not None: | |
| transition = campaign.replace_notebook(note) | |
| observation = transition.observation.to_dict() | |
| phases.add(observation['phase']) | |
| action = prior.choose(transition) | |
| options = replacements(action, observation['phase']) | |
| if options and all(option in transition.legal_actions for option in options): | |
| action, gradient, probabilities = policy.choose(observation, options, rng) | |
| decisions.append(dict(phase=observation['phase'], notebook=observation['notebook']['text'], | |
| action=action.to_dict(), probabilities=probabilities, gradient=gradient, | |
| step=step)) | |
| transition = campaign.step(action) | |
| trajectory.append(dict(step=step, phase=observation['phase'], action=action.to_dict(), reward=transition.reward, | |
| outcome=transition.outcome, error=None if transition.error is None else transition.error.to_dict())) | |
| if record: | |
| trace.append(transition.to_dict()) | |
| if transition.error: | |
| failures.append(transition.error.to_dict()) | |
| break | |
| result = dict(operator_seed=seed, outcome=transition.outcome, escaped=transition.outcome == 'escaped', | |
| engine_return=sum(row['reward'] for row in trajectory), steps=len(trajectory), | |
| truncated=not transition.done, errors=failures, reached_phases=sorted(phases), | |
| decisions=decisions, reward_trajectory=trajectory) | |
| if record: | |
| result['trace'] = trace | |
| return result | |
| def update(policy, row, baseline): | |
| gradient = collections.Counter() | |
| for decision in row['decisions']: | |
| for index, value in decision['gradient'].items(): | |
| gradient[index] += (row['engine_return'] - baseline) * value | |
| for index, value in gradient.items(): | |
| policy.weights[index] += CONFIG['learning_rate'] * value | |
| return math.sqrt(sum(value * value for value in gradient.values())) | |
| def evaluate(policy, seeds, rng_seed, record=False): | |
| before = digest(policy.weights) | |
| rows = [rollout(policy, seed, random.Random(rng_seed + index), record) for index, seed in enumerate(seeds)] | |
| assert before == digest(policy.weights) | |
| return dict(escapes=sum(row['escaped'] for row in rows), denominator=len(rows), | |
| error_episodes=sum(bool(row['errors']) for row in rows), | |
| truncated=sum(row['truncated'] for row in rows), | |
| outcome_counts=dict(collections.Counter(row['outcome'] for row in rows)), episodes=rows, | |
| frozen_checksum=before) | |
| def load_split(split): | |
| if split not in ('train', 'dev', 'test'): | |
| raise ValueError('split must be train, dev, or test') | |
| payload = files('borderland_eval').joinpath('manifests', 'v2', split + '.json').read_bytes() | |
| return json.loads(payload)['root_seeds'], hashlib.sha256(payload).hexdigest() | |
| def _output_directory(output_dir, episodes, seed): | |
| if type(episodes) is not int or not 1 <= episodes <= 1000000: | |
| raise ValueError('episodes must be an integer between 1 and 1000000') | |
| if type(seed) is not int or not 0 <= seed < 2**63: | |
| raise ValueError('seed must be an integer between 0 and 2**63 - 1') | |
| if not isinstance(output_dir, (str, Path)) or not str(output_dir).strip(): | |
| raise ValueError('output_dir must be a nonempty path') | |
| output = Path(output_dir).resolve() | |
| for package in Path(__file__).resolve().parent.parent.glob('borderland*'): | |
| if package.is_dir() and output.is_relative_to(package.resolve()): | |
| raise ValueError('output_dir must be outside installed packages') | |
| for name in ('weights.json', 'checkpoint-weights.json', 'training.json', 'trajectories.jsonl'): | |
| try: | |
| mode = (output / name).lstat().st_mode | |
| except FileNotFoundError: | |
| continue | |
| if not stat.S_ISREG(mode): | |
| raise ValueError(f'output artifact must be a regular file: {name}') | |
| return output | |
| def _artifact_file(path, progress=False): | |
| temporary = None | |
| try: | |
| with tempfile.NamedTemporaryFile(mode='w', dir=path.parent, delete=False) as artifact: | |
| temporary = Path(artifact.name) | |
| if progress: | |
| temporary.replace(path) | |
| yield artifact | |
| if not progress: | |
| temporary.replace(path) | |
| finally: | |
| if temporary is not None: | |
| temporary.unlink(missing_ok=True) | |
| def train_hierarchical(output_dir, episodes=480, seed=41, *, train_worlds=None, dev_worlds=None): | |
| from .checkpoints import checkpoint, training_worlds | |
| output = _output_directory(output_dir, episodes, seed) | |
| if (train_worlds is None) != (dev_worlds is None): | |
| raise ValueError('provide both train_worlds and dev_worlds') | |
| if train_worlds is None: | |
| train_seeds, train_hash = load_split('train') | |
| dev_seeds, dev_hash = load_split('dev') | |
| else: | |
| train_seeds, dev_seeds = train_worlds, dev_worlds | |
| train_hash, dev_hash = digest(train_seeds), digest(dev_seeds) | |
| train_seeds, dev_seeds = training_worlds(train_seeds, dev_seeds) | |
| policy = SidePolicy() | |
| before = evaluate(policy, dev_seeds, 97000) | |
| initial = digest(policy.weights) | |
| rng = random.Random(seed) | |
| baseline = 0.0 | |
| rows = [] | |
| output.mkdir(parents=True, exist_ok=True) | |
| with _artifact_file(output / 'trajectories.jsonl', progress=True) as progress: | |
| for episode in range(episodes): | |
| row = rollout(policy, train_seeds[episode % len(train_seeds)], rng) | |
| norm = update(policy, row, baseline) | |
| baseline = CONFIG['baseline_decay'] * baseline + (1 - CONFIG['baseline_decay']) * row['engine_return'] | |
| rows.append(dict(episode=episode, **row, gradient_norm=norm, ema_baseline=baseline)) | |
| progress.write(json.dumps(rows[-1]) + '\n') | |
| progress.flush() | |
| after = evaluate(policy, dev_seeds, 97000) | |
| ablations = {str(phase): evaluate(SidePolicy(policy.weights, [phase]), dev_seeds, 97000) | |
| for phase in CONFIG['learned_phases']} | |
| result = dict(learner=CONFIG['method'], compressor='LegalReference public-history notebook', | |
| training_seed=seed, initial_checksum=initial, final_checksum=digest(policy.weights), | |
| weights=policy.weights, before=before, after=after, ablations=ablations, training=rows, | |
| rl_updates=len(rows), nonzero_rl_updates=sum(row['gradient_norm'] > 0 for row in rows), | |
| imitation_updates=0, config=CONFIG, configuration_hash=digest(CONFIG), | |
| run_configuration=dict(episodes=episodes, seed=seed, evaluation_rng_seed=97000), | |
| run_configuration_hash=digest(dict(episodes=episodes, seed=seed, evaluation_rng_seed=97000)), | |
| split_hashes=dict(train=train_hash, dev=dev_hash), source_hash=source_hash(), engine_hash=engine_hash(), | |
| learner_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), | |
| limits=['LegalReference scripts all decisions except polarity choices in games 1, 6, and 7.', | |
| 'Full initial reset, compact workload only, no imitation updates or model calls.', | |
| 'Before and after use frozen weights on the same dev worlds and action RNG streams.', | |
| 'This hybrid is not a fully learned solver or evidence of Beam training.']) | |
| with _artifact_file(output / 'weights.json') as artifact: | |
| run = dict(episodes=episodes, training_seed=seed, train_worlds=train_seeds, dev_worlds=dev_seeds, | |
| evaluation_rng_seed=97000, split_hashes=dict(train=train_hash, dev=dev_hash), | |
| optimizer=CONFIG) | |
| artifact.write(json.dumps(checkpoint('hierarchical', policy.weights, run)) + '\n') | |
| with _artifact_file(output / 'training.json') as artifact: | |
| artifact.write(json.dumps(result, indent=2) + '\n') | |
| return result | |