File size: 15,475 Bytes
0e8028d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
"""Sparse linear REINFORCE using only public transitions and legal actions."""

import hashlib
import json
import math
import random
from collections import Counter
from pathlib import Path

from .context import scripted_notebook, wire
from .manifests import canonical, digest, manifest, source_hash


DIMENSIONS = 4096


def _flatten(value, prefix="board", depth=0):
    if depth > 4:
        return
    if isinstance(value, dict):
        for key, item in sorted(value.items()):
            yield from _flatten(item, prefix + "." + str(key), depth + 1)
    elif isinstance(value, (list, tuple)):
        for item in value[:12]:
            yield from _flatten(item, prefix, depth + 1)
    elif isinstance(value, (str, int, bool)) or value is None:
        yield prefix + "=" + str(value)


def features(observation, action):
    tokens = ["bias", "phase=" + str(observation["phase"]),
              "charge_bin=" + str(observation["charge"] // 10)]
    tokens.extend(_flatten(observation["board"]))
    tokens.extend(_flatten(observation["team"], "team"))
    tokens.extend("inventory=" + str(item) for item in observation["inventory"])
    tokens.extend("note=" + token for token in observation["notebook"]["text"].replace('"', ' ').replace(':', ' ').replace(',', ' ').split()[:128])
    action_text = canonical(action)
    indices = Counter(int.from_bytes(hashlib.sha256((action_text + "\0" + token).encode()).digest()[:8], "big") % DIMENSIONS for token in tokens)
    scale = math.sqrt(sum(value * value for value in indices.values()))
    return {index: value / scale for index, value in indices.items()}


class LinearPolicy:
    def __init__(self, weights=None):
        self.weights = list(weights) if weights is not None else [0.0] * DIMENSIONS
        if len(self.weights) != DIMENSIONS or any(type(value) not in (int, float) or not math.isfinite(value) for value in self.weights):
            raise ValueError('weights must contain 4096 finite numbers')

    def distribution(self, observation, actions):
        vectors = [features(observation, action) for action in actions]
        logits = [sum(self.weights[index] * value for index, value in vector.items()) for vector in vectors]
        if not all(math.isfinite(value) for value in logits):
            raise ValueError('policy logits overflowed')
        high = max(logits)
        exps = [math.exp(logit - high) for logit in logits]
        total = sum(exps)
        return [value / total for value in exps], vectors

    def sample(self, observation, actions, rng):
        probabilities, vectors = self.distribution(observation, actions)
        threshold = rng.random()
        chosen = len(actions) - 1
        cumulative = 0.0
        for index, probability in enumerate(probabilities):
            cumulative += probability
            if threshold <= cumulative:
                chosen = index
                break
        gradient = Counter(vectors[chosen])
        for probability, vector in zip(probabilities, vectors):
            for index, value in vector.items():
                gradient[index] -= probability * value
        return actions[chosen], dict(gradient)

    def checksum(self):
        return digest(self.weights)


def reinforce_update(policy, trajectory, baseline=0.0, learning_rate=0.08, gamma=0.99):
    total_return = 0.0
    gradient = Counter()
    for step in reversed(trajectory):
        total_return = step["reward"] + gamma * total_return
        for index, derivative in step["gradient"].items():
            gradient[index] += (total_return - baseline) * derivative
    for index, derivative in gradient.items():
        policy.weights[index] += learning_rate * derivative
    return {"gradient_norm": math.sqrt(sum(value * value for value in gradient.values())),
            "discounted_return": total_return, "steps": len(trajectory)}


def _potential(observation):
    return 0.15 * (int(observation["phase"]) - 1) + 0.10 * bool(observation["board"].get("gate_open"))


def _rollout(policy, world_seed, rng, max_steps=160):
    from borderland import Campaign

    campaign = Campaign.generated(world_seed)
    transition = campaign.reset()
    trajectory = []
    shaping_total = 0.0
    for _ in range(max_steps):
        if transition.done:
            break
        public = wire(transition)
        observation = public["observation"]
        note = scripted_notebook(observation, observation["notebook"]["text"], observation["notebook"]["capacity_chars"])
        if note != observation["notebook"]["text"]:
            transition = campaign.replace_notebook(note)
            public = wire(transition)
            observation = public["observation"]
        action, gradient = policy.sample(observation, public["legal_actions"], rng)
        before = _potential(observation)
        transition = campaign.step(action)
        shaping = 0.99 * _potential(wire(transition)["observation"]) - before - 0.002
        terminal = 1.0 if transition.outcome == "escaped" else -1.0 if transition.done else 0.0
        trajectory.append({"gradient": gradient, "reward": terminal + shaping,
                           "action": action, "terminal_reward": terminal, "potential_shaping": shaping})
        shaping_total += shaping
    if not transition.done and trajectory:
        trajectory[-1]["reward"] -= 1.0
        trajectory[-1]["evaluator_timeout_penalty"] = -1.0
    return trajectory, {"verified_escape": transition.outcome == "escaped", "outcome": transition.outcome,
                        "steps": len(trajectory), "shaping_return": shaping_total,
                        "training_return": sum(step["reward"] for step in trajectory),
                        "evaluator_truncated": not transition.done}


def _evaluate(policy, seeds, checkpoint=False, rng_seed=97000):
    checksum = policy.checksum()
    rows = []
    for index, seed in enumerate(seeds):
        runner = _checkpoint_rollout if checkpoint else _rollout
        _, row = runner(policy, seed, random.Random(rng_seed + index))
        rows.append({"operator_seed": seed, **row})
    assert checksum == policy.checksum(), "evaluation must never update policy"
    result = {"episodes": rows, "denominator": len(rows),
              "escape_rate": sum(row["verified_escape"] for row in rows) / len(rows),
              "frozen_checksum": checksum}
    if checkpoint:
        result["enumerated_valve_expectation"] = _checkpoint_expected(policy, seeds)
    assert checksum == policy.checksum(), "evaluation must keep policy frozen"
    return result


def _checkpoint_state(world_seed):
    from borderland import Campaign
    from borderland_verify.reference import LegalReference

    campaign = Campaign.generated(world_seed)
    reference = LegalReference()
    transition = campaign.reset()
    prefix_steps = 0
    for _ in range(256):
        if transition.done:
            raise RuntimeError("reference did not reach valve checkpoint")
        note = reference.notebook(transition)
        if note is not None and note != wire(transition)["observation"]["notebook"]["text"]:
            transition = campaign.replace_notebook(note)
        action = reference.choose(transition)
        if wire(action)["tag"] == "open_valve":
            break
        transition = campaign.step(action)
        prefix_steps += 1
    else:
        raise RuntimeError("checkpoint prefix budget")
    public = wire(transition)
    actions = [action for action in public["legal_actions"] if action["tag"] == "open_valve"]
    if len(actions) != 2:
        raise RuntimeError("checkpoint needs exactly two legal valves")
    return campaign, transition, actions, prefix_steps


def _checkpoint_rollout(policy, world_seed, rng):
    campaign, transition, actions, prefix_steps = _checkpoint_state(world_seed)
    action, gradient = policy.sample(wire(transition)["observation"], actions, rng)
    transition = campaign.step(action)
    if not transition.done:
        launches = [action for action in wire(transition)["legal_actions"] if action["tag"] == "launch"]
        if not launches:
            raise RuntimeError("checkpoint launch unavailable")
        transition = campaign.step(launches[0])
    reward = 1.0 if transition.outcome == "escaped" else -1.0
    return [{"gradient": gradient, "reward": reward, "action": action}], {
        "verified_escape": transition.outcome == "escaped", "outcome": transition.outcome,
        "steps": 1, "training_return": reward, "reference_prefix_steps": prefix_steps,
        "evaluator_truncated": not transition.done}


def _checkpoint_expected(policy, seeds):
    rows = []
    for seed in seeds:
        _, transition, actions, _ = _checkpoint_state(seed)
        probabilities, _ = policy.distribution(wire(transition)["observation"], actions)
        outcomes = []
        for action in actions:
            campaign, _, _, _ = _checkpoint_state(seed)
            result = campaign.step(action)
            if not result.done:
                launch = next(action for action in wire(result)["legal_actions"] if action["tag"] == "launch")
                result = campaign.step(launch)
            outcomes.append(int(result.outcome == "escaped"))
        rows.append({"operator_seed": seed, "actions": actions, "probabilities": probabilities,
                     "verified_returns": outcomes,
                     "expected_escape": sum(probability * value for probability, value in zip(probabilities, outcomes)),
                     "argmax_escape": outcomes[max(range(len(actions)), key=lambda index: probabilities[index])]})
    return {"episodes": rows, "expected_escape_rate": sum(row["expected_escape"] for row in rows) / len(rows),
            "argmax_escape_rate": sum(row["argmax_escape"] for row in rows) / len(rows),
            "method": "Each legal valve reexecuted from a fresh real-engine reference prefix; weighted by frozen student probabilities."}


def train_local(output_dir, episodes=128, seed=42, *, train_worlds=None, dev_worlds=None):
    from .checkpoints import checkpoint, training_worlds
    from .hierarchical_training import _output_directory

    output = _output_directory(output_dir, episodes, seed)
    if (train_worlds is None) != (dev_worlds is None):
        raise ValueError('provide both train_worlds and dev_worlds')
    frozen = {split: manifest(split) for split in ('train', 'dev')} if train_worlds is None else {
        'train': dict(root_seeds=train_worlds, split='ad_hoc_train'),
        'dev': dict(root_seeds=dev_worlds, split='ad_hoc_dev')}
    train_seeds = frozen["train"]["root_seeds"]
    train_seeds, dev_seeds = training_worlds(train_seeds, frozen['dev']['root_seeds'])
    output.mkdir(parents=True, exist_ok=True)
    run = dict(episodes=episodes, training_seed=seed, train_worlds=train_seeds, dev_worlds=dev_seeds,
               evaluation_rng_seed=97000, split_hashes={key: digest(value) for key, value in frozen.items()},
               optimizer=dict(full_lr=0.08, full_gamma=0.99, valve_lr=0.4, valve_gamma=1.0, ema_decay=0.9))
    policy = LinearPolicy()
    initial = policy.checksum()
    before = _evaluate(policy, dev_seeds)
    rng = random.Random(seed)
    baseline = 0.0
    logs = []
    for episode in range(episodes):
        trajectory, row = _rollout(policy, train_seeds[episode % len(train_seeds)], rng)
        update = reinforce_update(policy, trajectory, baseline)
        baseline = 0.9 * baseline + 0.1 * update["discounted_return"]
        logs.append({"episode": episode, **row, **update, "ema_baseline": baseline,
                     "reward_trajectory": [{key: value for key, value in step.items() if key != "gradient"}
                                           for step in trajectory]})
    after = _evaluate(policy, dev_seeds)
    (output / "weights.json").write_text(json.dumps(checkpoint('sparse_full', policy.weights, run)))
    valve_policy = LinearPolicy()
    checkpoint_before = _evaluate(valve_policy, dev_seeds, checkpoint=True)
    checkpoint_logs = []
    checkpoint_baseline = 0.0
    checkpoint_rng = random.Random(seed)
    for episode in range(episodes):
        trajectory, row = _checkpoint_rollout(valve_policy, train_seeds[episode % len(train_seeds)], checkpoint_rng)
        update = reinforce_update(valve_policy, trajectory, checkpoint_baseline, learning_rate=0.4, gamma=1.0)
        checkpoint_baseline = 0.9 * checkpoint_baseline + 0.1 * update["discounted_return"]
        checkpoint_logs.append({"episode": episode, **row, **update, "ema_baseline": checkpoint_baseline,
                                "reward_trajectory": [{key: value for key, value in step.items() if key != "gradient"}
                                                      for step in trajectory]})
    checkpoint_after = _evaluate(valve_policy, dev_seeds, checkpoint=True)
    (output / "checkpoint-weights.json").write_text(json.dumps(checkpoint('sparse_valve', valve_policy.weights, run)))
    result = {"learner": "non-LLM sparse linear softmax REINFORCE", "compressor": "scripted public-record compressor",
              "feature_hash": "SHA-256(action canonical JSON + NUL + public feature), first 8 bytes big endian modulo 4096; L2 normalized",
              "initial_checksum": initial, "final_checksum": policy.checksum(), "rl_updates": len(logs),
              "nonzero_rl_updates": sum(row["gradient_norm"] > 0 for row in logs),
              "imitation_updates": 0, "optimizer": {"learning_rate": 0.08, "gamma": 0.99, "ema_decay": 0.9},
              "full_campaign": {"before": before, "after": after, "training": logs,
                                "improvement_observed": after["escape_rate"] > before["escape_rate"],
                                "learning_claim": "Development control comparison only. A gain does not establish held-out improvement."},
              "valve_checkpoint_curriculum": {"scope": "Real campaign reference prefix, two legal valve choices, scripted launch. Student controls only valve; notebook preserves acquired public facts.",
                                              "before": checkpoint_before, "after": checkpoint_after,
                                              "training": checkpoint_logs, "rl_updates": len(checkpoint_logs),
                                              "optimizer": {"learning_rate": 0.4, "gamma": 1.0, "ema_decay": 0.9},
                                              "improvement_observed": checkpoint_after["escape_rate"] > checkpoint_before["escape_rate"]},
              "manifests": frozen, "manifest_hashes": {key: digest(value) for key, value in frozen.items()},
              "training_seed": seed, "source_hash": source_hash(), "evaluation_split": "dev", "run_configuration": run,
              "configuration_hash": digest({"episodes": episodes, "seed": seed, "dimensions": DIMENSIONS,
                                             "full_lr": 0.08, "full_gamma": 0.99, "checkpoint_lr": 0.4,
                                             "checkpoint_gamma": 1.0, "ema_decay": 0.9}),
              "limits": ["Training reads only train and dev worlds. No final test is run automatically.",
                         "Development before and after comparisons can guide training and are not final held-out evidence.",
                         "No Beam calls or Beam parameter updates occur.",
                         "Checkpoint success is not autonomous full-campaign success."]}
    (output / "training.json").write_text(json.dumps(result, indent=2))
    return result