Spaces:
Running
Running
File size: 15,475 Bytes
0e8028d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 | """Sparse linear REINFORCE using only public transitions and legal actions."""
import hashlib
import json
import math
import random
from collections import Counter
from pathlib import Path
from .context import scripted_notebook, wire
from .manifests import canonical, digest, manifest, source_hash
DIMENSIONS = 4096
def _flatten(value, prefix="board", depth=0):
if depth > 4:
return
if isinstance(value, dict):
for key, item in sorted(value.items()):
yield from _flatten(item, prefix + "." + str(key), depth + 1)
elif isinstance(value, (list, tuple)):
for item in value[:12]:
yield from _flatten(item, prefix, depth + 1)
elif isinstance(value, (str, int, bool)) or value is None:
yield prefix + "=" + str(value)
def features(observation, action):
tokens = ["bias", "phase=" + str(observation["phase"]),
"charge_bin=" + str(observation["charge"] // 10)]
tokens.extend(_flatten(observation["board"]))
tokens.extend(_flatten(observation["team"], "team"))
tokens.extend("inventory=" + str(item) for item in observation["inventory"])
tokens.extend("note=" + token for token in observation["notebook"]["text"].replace('"', ' ').replace(':', ' ').replace(',', ' ').split()[:128])
action_text = canonical(action)
indices = Counter(int.from_bytes(hashlib.sha256((action_text + "\0" + token).encode()).digest()[:8], "big") % DIMENSIONS for token in tokens)
scale = math.sqrt(sum(value * value for value in indices.values()))
return {index: value / scale for index, value in indices.items()}
class LinearPolicy:
def __init__(self, weights=None):
self.weights = list(weights) if weights is not None else [0.0] * DIMENSIONS
if len(self.weights) != DIMENSIONS or any(type(value) not in (int, float) or not math.isfinite(value) for value in self.weights):
raise ValueError('weights must contain 4096 finite numbers')
def distribution(self, observation, actions):
vectors = [features(observation, action) for action in actions]
logits = [sum(self.weights[index] * value for index, value in vector.items()) for vector in vectors]
if not all(math.isfinite(value) for value in logits):
raise ValueError('policy logits overflowed')
high = max(logits)
exps = [math.exp(logit - high) for logit in logits]
total = sum(exps)
return [value / total for value in exps], vectors
def sample(self, observation, actions, rng):
probabilities, vectors = self.distribution(observation, actions)
threshold = rng.random()
chosen = len(actions) - 1
cumulative = 0.0
for index, probability in enumerate(probabilities):
cumulative += probability
if threshold <= cumulative:
chosen = index
break
gradient = Counter(vectors[chosen])
for probability, vector in zip(probabilities, vectors):
for index, value in vector.items():
gradient[index] -= probability * value
return actions[chosen], dict(gradient)
def checksum(self):
return digest(self.weights)
def reinforce_update(policy, trajectory, baseline=0.0, learning_rate=0.08, gamma=0.99):
total_return = 0.0
gradient = Counter()
for step in reversed(trajectory):
total_return = step["reward"] + gamma * total_return
for index, derivative in step["gradient"].items():
gradient[index] += (total_return - baseline) * derivative
for index, derivative in gradient.items():
policy.weights[index] += learning_rate * derivative
return {"gradient_norm": math.sqrt(sum(value * value for value in gradient.values())),
"discounted_return": total_return, "steps": len(trajectory)}
def _potential(observation):
return 0.15 * (int(observation["phase"]) - 1) + 0.10 * bool(observation["board"].get("gate_open"))
def _rollout(policy, world_seed, rng, max_steps=160):
from borderland import Campaign
campaign = Campaign.generated(world_seed)
transition = campaign.reset()
trajectory = []
shaping_total = 0.0
for _ in range(max_steps):
if transition.done:
break
public = wire(transition)
observation = public["observation"]
note = scripted_notebook(observation, observation["notebook"]["text"], observation["notebook"]["capacity_chars"])
if note != observation["notebook"]["text"]:
transition = campaign.replace_notebook(note)
public = wire(transition)
observation = public["observation"]
action, gradient = policy.sample(observation, public["legal_actions"], rng)
before = _potential(observation)
transition = campaign.step(action)
shaping = 0.99 * _potential(wire(transition)["observation"]) - before - 0.002
terminal = 1.0 if transition.outcome == "escaped" else -1.0 if transition.done else 0.0
trajectory.append({"gradient": gradient, "reward": terminal + shaping,
"action": action, "terminal_reward": terminal, "potential_shaping": shaping})
shaping_total += shaping
if not transition.done and trajectory:
trajectory[-1]["reward"] -= 1.0
trajectory[-1]["evaluator_timeout_penalty"] = -1.0
return trajectory, {"verified_escape": transition.outcome == "escaped", "outcome": transition.outcome,
"steps": len(trajectory), "shaping_return": shaping_total,
"training_return": sum(step["reward"] for step in trajectory),
"evaluator_truncated": not transition.done}
def _evaluate(policy, seeds, checkpoint=False, rng_seed=97000):
checksum = policy.checksum()
rows = []
for index, seed in enumerate(seeds):
runner = _checkpoint_rollout if checkpoint else _rollout
_, row = runner(policy, seed, random.Random(rng_seed + index))
rows.append({"operator_seed": seed, **row})
assert checksum == policy.checksum(), "evaluation must never update policy"
result = {"episodes": rows, "denominator": len(rows),
"escape_rate": sum(row["verified_escape"] for row in rows) / len(rows),
"frozen_checksum": checksum}
if checkpoint:
result["enumerated_valve_expectation"] = _checkpoint_expected(policy, seeds)
assert checksum == policy.checksum(), "evaluation must keep policy frozen"
return result
def _checkpoint_state(world_seed):
from borderland import Campaign
from borderland_verify.reference import LegalReference
campaign = Campaign.generated(world_seed)
reference = LegalReference()
transition = campaign.reset()
prefix_steps = 0
for _ in range(256):
if transition.done:
raise RuntimeError("reference did not reach valve checkpoint")
note = reference.notebook(transition)
if note is not None and note != wire(transition)["observation"]["notebook"]["text"]:
transition = campaign.replace_notebook(note)
action = reference.choose(transition)
if wire(action)["tag"] == "open_valve":
break
transition = campaign.step(action)
prefix_steps += 1
else:
raise RuntimeError("checkpoint prefix budget")
public = wire(transition)
actions = [action for action in public["legal_actions"] if action["tag"] == "open_valve"]
if len(actions) != 2:
raise RuntimeError("checkpoint needs exactly two legal valves")
return campaign, transition, actions, prefix_steps
def _checkpoint_rollout(policy, world_seed, rng):
campaign, transition, actions, prefix_steps = _checkpoint_state(world_seed)
action, gradient = policy.sample(wire(transition)["observation"], actions, rng)
transition = campaign.step(action)
if not transition.done:
launches = [action for action in wire(transition)["legal_actions"] if action["tag"] == "launch"]
if not launches:
raise RuntimeError("checkpoint launch unavailable")
transition = campaign.step(launches[0])
reward = 1.0 if transition.outcome == "escaped" else -1.0
return [{"gradient": gradient, "reward": reward, "action": action}], {
"verified_escape": transition.outcome == "escaped", "outcome": transition.outcome,
"steps": 1, "training_return": reward, "reference_prefix_steps": prefix_steps,
"evaluator_truncated": not transition.done}
def _checkpoint_expected(policy, seeds):
rows = []
for seed in seeds:
_, transition, actions, _ = _checkpoint_state(seed)
probabilities, _ = policy.distribution(wire(transition)["observation"], actions)
outcomes = []
for action in actions:
campaign, _, _, _ = _checkpoint_state(seed)
result = campaign.step(action)
if not result.done:
launch = next(action for action in wire(result)["legal_actions"] if action["tag"] == "launch")
result = campaign.step(launch)
outcomes.append(int(result.outcome == "escaped"))
rows.append({"operator_seed": seed, "actions": actions, "probabilities": probabilities,
"verified_returns": outcomes,
"expected_escape": sum(probability * value for probability, value in zip(probabilities, outcomes)),
"argmax_escape": outcomes[max(range(len(actions)), key=lambda index: probabilities[index])]})
return {"episodes": rows, "expected_escape_rate": sum(row["expected_escape"] for row in rows) / len(rows),
"argmax_escape_rate": sum(row["argmax_escape"] for row in rows) / len(rows),
"method": "Each legal valve reexecuted from a fresh real-engine reference prefix; weighted by frozen student probabilities."}
def train_local(output_dir, episodes=128, seed=42, *, train_worlds=None, dev_worlds=None):
from .checkpoints import checkpoint, training_worlds
from .hierarchical_training import _output_directory
output = _output_directory(output_dir, episodes, seed)
if (train_worlds is None) != (dev_worlds is None):
raise ValueError('provide both train_worlds and dev_worlds')
frozen = {split: manifest(split) for split in ('train', 'dev')} if train_worlds is None else {
'train': dict(root_seeds=train_worlds, split='ad_hoc_train'),
'dev': dict(root_seeds=dev_worlds, split='ad_hoc_dev')}
train_seeds = frozen["train"]["root_seeds"]
train_seeds, dev_seeds = training_worlds(train_seeds, frozen['dev']['root_seeds'])
output.mkdir(parents=True, exist_ok=True)
run = dict(episodes=episodes, training_seed=seed, train_worlds=train_seeds, dev_worlds=dev_seeds,
evaluation_rng_seed=97000, split_hashes={key: digest(value) for key, value in frozen.items()},
optimizer=dict(full_lr=0.08, full_gamma=0.99, valve_lr=0.4, valve_gamma=1.0, ema_decay=0.9))
policy = LinearPolicy()
initial = policy.checksum()
before = _evaluate(policy, dev_seeds)
rng = random.Random(seed)
baseline = 0.0
logs = []
for episode in range(episodes):
trajectory, row = _rollout(policy, train_seeds[episode % len(train_seeds)], rng)
update = reinforce_update(policy, trajectory, baseline)
baseline = 0.9 * baseline + 0.1 * update["discounted_return"]
logs.append({"episode": episode, **row, **update, "ema_baseline": baseline,
"reward_trajectory": [{key: value for key, value in step.items() if key != "gradient"}
for step in trajectory]})
after = _evaluate(policy, dev_seeds)
(output / "weights.json").write_text(json.dumps(checkpoint('sparse_full', policy.weights, run)))
valve_policy = LinearPolicy()
checkpoint_before = _evaluate(valve_policy, dev_seeds, checkpoint=True)
checkpoint_logs = []
checkpoint_baseline = 0.0
checkpoint_rng = random.Random(seed)
for episode in range(episodes):
trajectory, row = _checkpoint_rollout(valve_policy, train_seeds[episode % len(train_seeds)], checkpoint_rng)
update = reinforce_update(valve_policy, trajectory, checkpoint_baseline, learning_rate=0.4, gamma=1.0)
checkpoint_baseline = 0.9 * checkpoint_baseline + 0.1 * update["discounted_return"]
checkpoint_logs.append({"episode": episode, **row, **update, "ema_baseline": checkpoint_baseline,
"reward_trajectory": [{key: value for key, value in step.items() if key != "gradient"}
for step in trajectory]})
checkpoint_after = _evaluate(valve_policy, dev_seeds, checkpoint=True)
(output / "checkpoint-weights.json").write_text(json.dumps(checkpoint('sparse_valve', valve_policy.weights, run)))
result = {"learner": "non-LLM sparse linear softmax REINFORCE", "compressor": "scripted public-record compressor",
"feature_hash": "SHA-256(action canonical JSON + NUL + public feature), first 8 bytes big endian modulo 4096; L2 normalized",
"initial_checksum": initial, "final_checksum": policy.checksum(), "rl_updates": len(logs),
"nonzero_rl_updates": sum(row["gradient_norm"] > 0 for row in logs),
"imitation_updates": 0, "optimizer": {"learning_rate": 0.08, "gamma": 0.99, "ema_decay": 0.9},
"full_campaign": {"before": before, "after": after, "training": logs,
"improvement_observed": after["escape_rate"] > before["escape_rate"],
"learning_claim": "Development control comparison only. A gain does not establish held-out improvement."},
"valve_checkpoint_curriculum": {"scope": "Real campaign reference prefix, two legal valve choices, scripted launch. Student controls only valve; notebook preserves acquired public facts.",
"before": checkpoint_before, "after": checkpoint_after,
"training": checkpoint_logs, "rl_updates": len(checkpoint_logs),
"optimizer": {"learning_rate": 0.4, "gamma": 1.0, "ema_decay": 0.9},
"improvement_observed": checkpoint_after["escape_rate"] > checkpoint_before["escape_rate"]},
"manifests": frozen, "manifest_hashes": {key: digest(value) for key, value in frozen.items()},
"training_seed": seed, "source_hash": source_hash(), "evaluation_split": "dev", "run_configuration": run,
"configuration_hash": digest({"episodes": episodes, "seed": seed, "dimensions": DIMENSIONS,
"full_lr": 0.08, "full_gamma": 0.99, "checkpoint_lr": 0.4,
"checkpoint_gamma": 1.0, "ema_decay": 0.9}),
"limits": ["Training reads only train and dev worlds. No final test is run automatically.",
"Development before and after comparisons can guide training and are not final held-out evidence.",
"No Beam calls or Beam parameter updates occur.",
"Checkpoint success is not autonomous full-campaign success."]}
(output / "training.json").write_text(json.dumps(result, indent=2))
return result
|