FatUnc6969's picture
Publish reviewed research tools and evidence audit
0e8028d verified
Raw History Blame Contribute Delete
15.5 kB
"""Sparse linear REINFORCE using only public transitions and legal actions."""
import hashlib
import json
import math
import random
from collections import Counter
from pathlib import Path
from .context import scripted_notebook, wire
from .manifests import canonical, digest, manifest, source_hash
DIMENSIONS = 4096
def _flatten(value, prefix="board", depth=0):
if depth > 4:
return
if isinstance(value, dict):
for key, item in sorted(value.items()):
yield from _flatten(item, prefix + "." + str(key), depth + 1)
elif isinstance(value, (list, tuple)):
for item in value[:12]:
yield from _flatten(item, prefix, depth + 1)
elif isinstance(value, (str, int, bool)) or value is None:
yield prefix + "=" + str(value)
def features(observation, action):
tokens = ["bias", "phase=" + str(observation["phase"]),
"charge_bin=" + str(observation["charge"] // 10)]
tokens.extend(_flatten(observation["board"]))
tokens.extend(_flatten(observation["team"], "team"))
tokens.extend("inventory=" + str(item) for item in observation["inventory"])
tokens.extend("note=" + token for token in observation["notebook"]["text"].replace('"', ' ').replace(':', ' ').replace(',', ' ').split()[:128])
action_text = canonical(action)
indices = Counter(int.from_bytes(hashlib.sha256((action_text + "\0" + token).encode()).digest()[:8], "big") % DIMENSIONS for token in tokens)
scale = math.sqrt(sum(value * value for value in indices.values()))
return {index: value / scale for index, value in indices.items()}
class LinearPolicy:
def __init__(self, weights=None):
self.weights = list(weights) if weights is not None else [0.0] * DIMENSIONS
if len(self.weights) != DIMENSIONS or any(type(value) not in (int, float) or not math.isfinite(value) for value in self.weights):
raise ValueError('weights must contain 4096 finite numbers')
def distribution(self, observation, actions):
vectors = [features(observation, action) for action in actions]
logits = [sum(self.weights[index] * value for index, value in vector.items()) for vector in vectors]
if not all(math.isfinite(value) for value in logits):
raise ValueError('policy logits overflowed')
high = max(logits)
exps = [math.exp(logit - high) for logit in logits]
total = sum(exps)
return [value / total for value in exps], vectors
def sample(self, observation, actions, rng):
probabilities, vectors = self.distribution(observation, actions)
threshold = rng.random()
chosen = len(actions) - 1
cumulative = 0.0
for index, probability in enumerate(probabilities):
cumulative += probability
if threshold <= cumulative:
chosen = index
break
gradient = Counter(vectors[chosen])
for probability, vector in zip(probabilities, vectors):
for index, value in vector.items():
gradient[index] -= probability * value
return actions[chosen], dict(gradient)
def checksum(self):
return digest(self.weights)
def reinforce_update(policy, trajectory, baseline=0.0, learning_rate=0.08, gamma=0.99):
total_return = 0.0
gradient = Counter()
for step in reversed(trajectory):
total_return = step["reward"] + gamma * total_return
for index, derivative in step["gradient"].items():
gradient[index] += (total_return - baseline) * derivative
for index, derivative in gradient.items():
policy.weights[index] += learning_rate * derivative
return {"gradient_norm": math.sqrt(sum(value * value for value in gradient.values())),
"discounted_return": total_return, "steps": len(trajectory)}
def _potential(observation):
return 0.15 * (int(observation["phase"]) - 1) + 0.10 * bool(observation["board"].get("gate_open"))
def _rollout(policy, world_seed, rng, max_steps=160):
from borderland import Campaign
campaign = Campaign.generated(world_seed)
transition = campaign.reset()
trajectory = []
shaping_total = 0.0
for _ in range(max_steps):
if transition.done:
break
public = wire(transition)
observation = public["observation"]
note = scripted_notebook(observation, observation["notebook"]["text"], observation["notebook"]["capacity_chars"])
if note != observation["notebook"]["text"]:
transition = campaign.replace_notebook(note)
public = wire(transition)
observation = public["observation"]
action, gradient = policy.sample(observation, public["legal_actions"], rng)
before = _potential(observation)
transition = campaign.step(action)
shaping = 0.99 * _potential(wire(transition)["observation"]) - before - 0.002
terminal = 1.0 if transition.outcome == "escaped" else -1.0 if transition.done else 0.0
trajectory.append({"gradient": gradient, "reward": terminal + shaping,
"action": action, "terminal_reward": terminal, "potential_shaping": shaping})
shaping_total += shaping
if not transition.done and trajectory:
trajectory[-1]["reward"] -= 1.0
trajectory[-1]["evaluator_timeout_penalty"] = -1.0
return trajectory, {"verified_escape": transition.outcome == "escaped", "outcome": transition.outcome,
"steps": len(trajectory), "shaping_return": shaping_total,
"training_return": sum(step["reward"] for step in trajectory),
"evaluator_truncated": not transition.done}
def _evaluate(policy, seeds, checkpoint=False, rng_seed=97000):
checksum = policy.checksum()
rows = []
for index, seed in enumerate(seeds):
runner = _checkpoint_rollout if checkpoint else _rollout
_, row = runner(policy, seed, random.Random(rng_seed + index))
rows.append({"operator_seed": seed, **row})
assert checksum == policy.checksum(), "evaluation must never update policy"
result = {"episodes": rows, "denominator": len(rows),
"escape_rate": sum(row["verified_escape"] for row in rows) / len(rows),
"frozen_checksum": checksum}
if checkpoint:
result["enumerated_valve_expectation"] = _checkpoint_expected(policy, seeds)
assert checksum == policy.checksum(), "evaluation must keep policy frozen"
return result
def _checkpoint_state(world_seed):
from borderland import Campaign
from borderland_verify.reference import LegalReference
campaign = Campaign.generated(world_seed)
reference = LegalReference()
transition = campaign.reset()
prefix_steps = 0
for _ in range(256):
if transition.done:
raise RuntimeError("reference did not reach valve checkpoint")
note = reference.notebook(transition)
if note is not None and note != wire(transition)["observation"]["notebook"]["text"]:
transition = campaign.replace_notebook(note)
action = reference.choose(transition)
if wire(action)["tag"] == "open_valve":
break
transition = campaign.step(action)
prefix_steps += 1
else:
raise RuntimeError("checkpoint prefix budget")
public = wire(transition)
actions = [action for action in public["legal_actions"] if action["tag"] == "open_valve"]
if len(actions) != 2:
raise RuntimeError("checkpoint needs exactly two legal valves")
return campaign, transition, actions, prefix_steps
def _checkpoint_rollout(policy, world_seed, rng):
campaign, transition, actions, prefix_steps = _checkpoint_state(world_seed)
action, gradient = policy.sample(wire(transition)["observation"], actions, rng)
transition = campaign.step(action)
if not transition.done:
launches = [action for action in wire(transition)["legal_actions"] if action["tag"] == "launch"]
if not launches:
raise RuntimeError("checkpoint launch unavailable")
transition = campaign.step(launches[0])
reward = 1.0 if transition.outcome == "escaped" else -1.0
return [{"gradient": gradient, "reward": reward, "action": action}], {
"verified_escape": transition.outcome == "escaped", "outcome": transition.outcome,
"steps": 1, "training_return": reward, "reference_prefix_steps": prefix_steps,
"evaluator_truncated": not transition.done}
def _checkpoint_expected(policy, seeds):
rows = []
for seed in seeds:
_, transition, actions, _ = _checkpoint_state(seed)
probabilities, _ = policy.distribution(wire(transition)["observation"], actions)
outcomes = []
for action in actions:
campaign, _, _, _ = _checkpoint_state(seed)
result = campaign.step(action)
if not result.done:
launch = next(action for action in wire(result)["legal_actions"] if action["tag"] == "launch")
result = campaign.step(launch)
outcomes.append(int(result.outcome == "escaped"))
rows.append({"operator_seed": seed, "actions": actions, "probabilities": probabilities,
"verified_returns": outcomes,
"expected_escape": sum(probability * value for probability, value in zip(probabilities, outcomes)),
"argmax_escape": outcomes[max(range(len(actions)), key=lambda index: probabilities[index])]})
return {"episodes": rows, "expected_escape_rate": sum(row["expected_escape"] for row in rows) / len(rows),
"argmax_escape_rate": sum(row["argmax_escape"] for row in rows) / len(rows),
"method": "Each legal valve reexecuted from a fresh real-engine reference prefix; weighted by frozen student probabilities."}
def train_local(output_dir, episodes=128, seed=42, *, train_worlds=None, dev_worlds=None):
from .checkpoints import checkpoint, training_worlds
from .hierarchical_training import _output_directory
output = _output_directory(output_dir, episodes, seed)
if (train_worlds is None) != (dev_worlds is None):
raise ValueError('provide both train_worlds and dev_worlds')
frozen = {split: manifest(split) for split in ('train', 'dev')} if train_worlds is None else {
'train': dict(root_seeds=train_worlds, split='ad_hoc_train'),
'dev': dict(root_seeds=dev_worlds, split='ad_hoc_dev')}
train_seeds = frozen["train"]["root_seeds"]
train_seeds, dev_seeds = training_worlds(train_seeds, frozen['dev']['root_seeds'])
output.mkdir(parents=True, exist_ok=True)
run = dict(episodes=episodes, training_seed=seed, train_worlds=train_seeds, dev_worlds=dev_seeds,
evaluation_rng_seed=97000, split_hashes={key: digest(value) for key, value in frozen.items()},
optimizer=dict(full_lr=0.08, full_gamma=0.99, valve_lr=0.4, valve_gamma=1.0, ema_decay=0.9))
policy = LinearPolicy()
initial = policy.checksum()
before = _evaluate(policy, dev_seeds)
rng = random.Random(seed)
baseline = 0.0
logs = []
for episode in range(episodes):
trajectory, row = _rollout(policy, train_seeds[episode % len(train_seeds)], rng)
update = reinforce_update(policy, trajectory, baseline)
baseline = 0.9 * baseline + 0.1 * update["discounted_return"]
logs.append({"episode": episode, **row, **update, "ema_baseline": baseline,
"reward_trajectory": [{key: value for key, value in step.items() if key != "gradient"}
for step in trajectory]})
after = _evaluate(policy, dev_seeds)
(output / "weights.json").write_text(json.dumps(checkpoint('sparse_full', policy.weights, run)))
valve_policy = LinearPolicy()
checkpoint_before = _evaluate(valve_policy, dev_seeds, checkpoint=True)
checkpoint_logs = []
checkpoint_baseline = 0.0
checkpoint_rng = random.Random(seed)
for episode in range(episodes):
trajectory, row = _checkpoint_rollout(valve_policy, train_seeds[episode % len(train_seeds)], checkpoint_rng)
update = reinforce_update(valve_policy, trajectory, checkpoint_baseline, learning_rate=0.4, gamma=1.0)
checkpoint_baseline = 0.9 * checkpoint_baseline + 0.1 * update["discounted_return"]
checkpoint_logs.append({"episode": episode, **row, **update, "ema_baseline": checkpoint_baseline,
"reward_trajectory": [{key: value for key, value in step.items() if key != "gradient"}
for step in trajectory]})
checkpoint_after = _evaluate(valve_policy, dev_seeds, checkpoint=True)
(output / "checkpoint-weights.json").write_text(json.dumps(checkpoint('sparse_valve', valve_policy.weights, run)))
result = {"learner": "non-LLM sparse linear softmax REINFORCE", "compressor": "scripted public-record compressor",
"feature_hash": "SHA-256(action canonical JSON + NUL + public feature), first 8 bytes big endian modulo 4096; L2 normalized",
"initial_checksum": initial, "final_checksum": policy.checksum(), "rl_updates": len(logs),
"nonzero_rl_updates": sum(row["gradient_norm"] > 0 for row in logs),
"imitation_updates": 0, "optimizer": {"learning_rate": 0.08, "gamma": 0.99, "ema_decay": 0.9},
"full_campaign": {"before": before, "after": after, "training": logs,
"improvement_observed": after["escape_rate"] > before["escape_rate"],
"learning_claim": "Development control comparison only. A gain does not establish held-out improvement."},
"valve_checkpoint_curriculum": {"scope": "Real campaign reference prefix, two legal valve choices, scripted launch. Student controls only valve; notebook preserves acquired public facts.",
"before": checkpoint_before, "after": checkpoint_after,
"training": checkpoint_logs, "rl_updates": len(checkpoint_logs),
"optimizer": {"learning_rate": 0.4, "gamma": 1.0, "ema_decay": 0.9},
"improvement_observed": checkpoint_after["escape_rate"] > checkpoint_before["escape_rate"]},
"manifests": frozen, "manifest_hashes": {key: digest(value) for key, value in frozen.items()},
"training_seed": seed, "source_hash": source_hash(), "evaluation_split": "dev", "run_configuration": run,
"configuration_hash": digest({"episodes": episodes, "seed": seed, "dimensions": DIMENSIONS,
"full_lr": 0.08, "full_gamma": 0.99, "checkpoint_lr": 0.4,
"checkpoint_gamma": 1.0, "ema_decay": 0.9}),
"limits": ["Training reads only train and dev worlds. No final test is run automatically.",
"Development before and after comparisons can guide training and are not final held-out evidence.",
"No Beam calls or Beam parameter updates occur.",
"Checkpoint success is not autonomous full-campaign success."]}
(output / "training.json").write_text(json.dumps(result, indent=2))
return result