File size: 11,884 Bytes
0e8028d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
import collections
import hashlib
import json
import math
import random
import stat
import tempfile
from contextlib import contextmanager
from importlib.resources import files
from pathlib import Path

from borderland import Action, Campaign
from borderland.replay import source_hash as engine_hash
from borderland_eval.manifests import digest, source_hash
from borderland_verify.reference import LegalReference


CONFIG = dict(method='hierarchical non-LLM REINFORCE with scripted public-history prior',
              episodes=480, learning_rate=0.16, baseline_decay=0.9, gamma=1.0,
              max_steps=240, training_seeds=[41, 42, 43, 44, 45],
              learned_phases=[1, 6, 7], dimensions=12,
              reward='unmodified engine reward, one on escape and zero otherwise',
              prior='LegalReference supplies all navigation, evidence interpretation, notebook compression, circuit order, and action timing. Learned policy replaces latch choice, each connect bus, and final valve side. No reference prefix checkpoint.',
              features='one-hot phase x public notebook bus x candidate side; no comparison to a correctness label',
              workloads=['compact'])


class SidePolicy:
    def __init__(self, weights=None, neutral_phases=()):
        self.weights = [0.0] * 12 if weights is None else list(weights)
        if len(self.weights) != 12 or any(type(value) not in (int, float) or not math.isfinite(value) for value in self.weights):
            raise ValueError('weights must contain 12 finite numbers')
        self.neutral_phases = set(neutral_phases)
        if not self.neutral_phases.issubset({1, 6, 7}):
            raise ValueError('neutral phases must be 1, 6, or 7')

    def distribution(self, observation, actions):
        phase = observation['phase']
        bus = json.loads(observation['notebook']['text'])['bus']
        assert phase in (1, 6, 7) and bus in ('west', 'east')
        base = ((1, 6, 7).index(phase) * 2 + ('west', 'east').index(bus)) * 2
        indices = []
        for action in actions:
            arguments = action.arguments
            side = ('west', 'east')[arguments['value'] - 2] if phase == 1 else arguments['bus'] if phase == 6 else arguments['side']
            indices.append(base + ('west', 'east').index(side))
        logits = [0.0 if phase in self.neutral_phases else self.weights[index] for index in indices]
        exps = [math.exp(value - max(logits)) for value in logits]
        return [value / sum(exps) for value in exps], indices

    def choose(self, observation, actions, rng):
        probabilities, indices = self.distribution(observation, actions)
        selected = 0 if rng.random() < probabilities[0] else 1
        gradient = {index: -probability for index, probability in zip(indices, probabilities)}
        gradient[indices[selected]] += 1.0
        return actions[selected], gradient, probabilities


def replacements(action, phase):
    if phase == 1 and action.tag == 'set_latch' and action.arguments['value'] >= 2:
        return [Action('set_latch', dict(value=value)) for value in (2, 3)]
    if phase == 6 and action.tag == 'connect':
        return [Action('connect', dict(circuit=action.arguments['circuit'], bus=side)) for side in ('west', 'east')]
    if phase == 7 and action.tag == 'open_valve':
        return [Action('open_valve', dict(side=side)) for side in ('west', 'east')]
    return None


def rollout(policy, seed, rng, record=False):
    campaign = Campaign.generated(seed, archive_workload='compact')
    transition = campaign.reset()
    prior = LegalReference()
    decisions, trajectory, trace = [], [], []
    phases = set()
    failures = []
    for step in range(CONFIG['max_steps']):
        if transition.done:
            break
        note = prior.notebook(transition)
        if note is not None:
            transition = campaign.replace_notebook(note)
        observation = transition.observation.to_dict()
        phases.add(observation['phase'])
        action = prior.choose(transition)
        options = replacements(action, observation['phase'])
        if options and all(option in transition.legal_actions for option in options):
            action, gradient, probabilities = policy.choose(observation, options, rng)
            decisions.append(dict(phase=observation['phase'], notebook=observation['notebook']['text'],
                                  action=action.to_dict(), probabilities=probabilities, gradient=gradient,
                                  step=step))
        transition = campaign.step(action)
        trajectory.append(dict(step=step, phase=observation['phase'], action=action.to_dict(), reward=transition.reward,
                               outcome=transition.outcome, error=None if transition.error is None else transition.error.to_dict()))
        if record:
            trace.append(transition.to_dict())
        if transition.error:
            failures.append(transition.error.to_dict())
            break
    result = dict(operator_seed=seed, outcome=transition.outcome, escaped=transition.outcome == 'escaped',
                  engine_return=sum(row['reward'] for row in trajectory), steps=len(trajectory),
                  truncated=not transition.done, errors=failures, reached_phases=sorted(phases),
                  decisions=decisions, reward_trajectory=trajectory)
    if record:
        result['trace'] = trace
    return result


def update(policy, row, baseline):
    gradient = collections.Counter()
    for decision in row['decisions']:
        for index, value in decision['gradient'].items():
            gradient[index] += (row['engine_return'] - baseline) * value
    for index, value in gradient.items():
        policy.weights[index] += CONFIG['learning_rate'] * value
    return math.sqrt(sum(value * value for value in gradient.values()))


def evaluate(policy, seeds, rng_seed, record=False):
    before = digest(policy.weights)
    rows = [rollout(policy, seed, random.Random(rng_seed + index), record) for index, seed in enumerate(seeds)]
    assert before == digest(policy.weights)
    return dict(escapes=sum(row['escaped'] for row in rows), denominator=len(rows),
                error_episodes=sum(bool(row['errors']) for row in rows),
                truncated=sum(row['truncated'] for row in rows),
                outcome_counts=dict(collections.Counter(row['outcome'] for row in rows)), episodes=rows,
                frozen_checksum=before)


def load_split(split):
    if split not in ('train', 'dev', 'test'):
        raise ValueError('split must be train, dev, or test')
    payload = files('borderland_eval').joinpath('manifests', 'v2', split + '.json').read_bytes()
    return json.loads(payload)['root_seeds'], hashlib.sha256(payload).hexdigest()


def _output_directory(output_dir, episodes, seed):
    if type(episodes) is not int or not 1 <= episodes <= 1000000:
        raise ValueError('episodes must be an integer between 1 and 1000000')
    if type(seed) is not int or not 0 <= seed < 2**63:
        raise ValueError('seed must be an integer between 0 and 2**63 - 1')
    if not isinstance(output_dir, (str, Path)) or not str(output_dir).strip():
        raise ValueError('output_dir must be a nonempty path')
    output = Path(output_dir).resolve()
    for package in Path(__file__).resolve().parent.parent.glob('borderland*'):
        if package.is_dir() and output.is_relative_to(package.resolve()):
            raise ValueError('output_dir must be outside installed packages')
    for name in ('weights.json', 'checkpoint-weights.json', 'training.json', 'trajectories.jsonl'):
        try:
            mode = (output / name).lstat().st_mode
        except FileNotFoundError:
            continue
        if not stat.S_ISREG(mode):
            raise ValueError(f'output artifact must be a regular file: {name}')
    return output


@contextmanager
def _artifact_file(path, progress=False):
    temporary = None
    try:
        with tempfile.NamedTemporaryFile(mode='w', dir=path.parent, delete=False) as artifact:
            temporary = Path(artifact.name)
            if progress:
                temporary.replace(path)
            yield artifact
        if not progress:
            temporary.replace(path)
    finally:
        if temporary is not None:
            temporary.unlink(missing_ok=True)


def train_hierarchical(output_dir, episodes=480, seed=41, *, train_worlds=None, dev_worlds=None):
    from .checkpoints import checkpoint, training_worlds

    output = _output_directory(output_dir, episodes, seed)
    if (train_worlds is None) != (dev_worlds is None):
        raise ValueError('provide both train_worlds and dev_worlds')
    if train_worlds is None:
        train_seeds, train_hash = load_split('train')
        dev_seeds, dev_hash = load_split('dev')
    else:
        train_seeds, dev_seeds = train_worlds, dev_worlds
        train_hash, dev_hash = digest(train_seeds), digest(dev_seeds)
    train_seeds, dev_seeds = training_worlds(train_seeds, dev_seeds)
    policy = SidePolicy()
    before = evaluate(policy, dev_seeds, 97000)
    initial = digest(policy.weights)
    rng = random.Random(seed)
    baseline = 0.0
    rows = []
    output.mkdir(parents=True, exist_ok=True)
    with _artifact_file(output / 'trajectories.jsonl', progress=True) as progress:
        for episode in range(episodes):
            row = rollout(policy, train_seeds[episode % len(train_seeds)], rng)
            norm = update(policy, row, baseline)
            baseline = CONFIG['baseline_decay'] * baseline + (1 - CONFIG['baseline_decay']) * row['engine_return']
            rows.append(dict(episode=episode, **row, gradient_norm=norm, ema_baseline=baseline))
            progress.write(json.dumps(rows[-1]) + '\n')
            progress.flush()
    after = evaluate(policy, dev_seeds, 97000)
    ablations = {str(phase): evaluate(SidePolicy(policy.weights, [phase]), dev_seeds, 97000)
                 for phase in CONFIG['learned_phases']}
    result = dict(learner=CONFIG['method'], compressor='LegalReference public-history notebook',
                  training_seed=seed, initial_checksum=initial, final_checksum=digest(policy.weights),
                  weights=policy.weights, before=before, after=after, ablations=ablations, training=rows,
                  rl_updates=len(rows), nonzero_rl_updates=sum(row['gradient_norm'] > 0 for row in rows),
                  imitation_updates=0, config=CONFIG, configuration_hash=digest(CONFIG),
                  run_configuration=dict(episodes=episodes, seed=seed, evaluation_rng_seed=97000),
                  run_configuration_hash=digest(dict(episodes=episodes, seed=seed, evaluation_rng_seed=97000)),
                  split_hashes=dict(train=train_hash, dev=dev_hash), source_hash=source_hash(), engine_hash=engine_hash(),
                  learner_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
                  limits=['LegalReference scripts all decisions except polarity choices in games 1, 6, and 7.',
                          'Full initial reset, compact workload only, no imitation updates or model calls.',
                          'Before and after use frozen weights on the same dev worlds and action RNG streams.',
                          'This hybrid is not a fully learned solver or evidence of Beam training.'])
    with _artifact_file(output / 'weights.json') as artifact:
        run = dict(episodes=episodes, training_seed=seed, train_worlds=train_seeds, dev_worlds=dev_seeds,
                   evaluation_rng_seed=97000, split_hashes=dict(train=train_hash, dev=dev_hash),
                   optimizer=CONFIG)
        artifact.write(json.dumps(checkpoint('hierarchical', policy.weights, run)) + '\n')
    with _artifact_file(output / 'training.json') as artifact:
        artifact.write(json.dumps(result, indent=2) + '\n')
    return result