"""Sparse linear REINFORCE using only public transitions and legal actions.""" import hashlib import json import math import random from collections import Counter from pathlib import Path from .context import scripted_notebook, wire from .manifests import canonical, digest, manifest, source_hash DIMENSIONS = 4096 def _flatten(value, prefix="board", depth=0): if depth > 4: return if isinstance(value, dict): for key, item in sorted(value.items()): yield from _flatten(item, prefix + "." + str(key), depth + 1) elif isinstance(value, (list, tuple)): for item in value[:12]: yield from _flatten(item, prefix, depth + 1) elif isinstance(value, (str, int, bool)) or value is None: yield prefix + "=" + str(value) def features(observation, action): tokens = ["bias", "phase=" + str(observation["phase"]), "charge_bin=" + str(observation["charge"] // 10)] tokens.extend(_flatten(observation["board"])) tokens.extend(_flatten(observation["team"], "team")) tokens.extend("inventory=" + str(item) for item in observation["inventory"]) tokens.extend("note=" + token for token in observation["notebook"]["text"].replace('"', ' ').replace(':', ' ').replace(',', ' ').split()[:128]) action_text = canonical(action) indices = Counter(int.from_bytes(hashlib.sha256((action_text + "\0" + token).encode()).digest()[:8], "big") % DIMENSIONS for token in tokens) scale = math.sqrt(sum(value * value for value in indices.values())) return {index: value / scale for index, value in indices.items()} class LinearPolicy: def __init__(self, weights=None): self.weights = list(weights) if weights is not None else [0.0] * DIMENSIONS if len(self.weights) != DIMENSIONS or any(type(value) not in (int, float) or not math.isfinite(value) for value in self.weights): raise ValueError('weights must contain 4096 finite numbers') def distribution(self, observation, actions): vectors = [features(observation, action) for action in actions] logits = [sum(self.weights[index] * value for index, value in vector.items()) for vector in vectors] if not all(math.isfinite(value) for value in logits): raise ValueError('policy logits overflowed') high = max(logits) exps = [math.exp(logit - high) for logit in logits] total = sum(exps) return [value / total for value in exps], vectors def sample(self, observation, actions, rng): probabilities, vectors = self.distribution(observation, actions) threshold = rng.random() chosen = len(actions) - 1 cumulative = 0.0 for index, probability in enumerate(probabilities): cumulative += probability if threshold <= cumulative: chosen = index break gradient = Counter(vectors[chosen]) for probability, vector in zip(probabilities, vectors): for index, value in vector.items(): gradient[index] -= probability * value return actions[chosen], dict(gradient) def checksum(self): return digest(self.weights) def reinforce_update(policy, trajectory, baseline=0.0, learning_rate=0.08, gamma=0.99): total_return = 0.0 gradient = Counter() for step in reversed(trajectory): total_return = step["reward"] + gamma * total_return for index, derivative in step["gradient"].items(): gradient[index] += (total_return - baseline) * derivative for index, derivative in gradient.items(): policy.weights[index] += learning_rate * derivative return {"gradient_norm": math.sqrt(sum(value * value for value in gradient.values())), "discounted_return": total_return, "steps": len(trajectory)} def _potential(observation): return 0.15 * (int(observation["phase"]) - 1) + 0.10 * bool(observation["board"].get("gate_open")) def _rollout(policy, world_seed, rng, max_steps=160): from borderland import Campaign campaign = Campaign.generated(world_seed) transition = campaign.reset() trajectory = [] shaping_total = 0.0 for _ in range(max_steps): if transition.done: break public = wire(transition) observation = public["observation"] note = scripted_notebook(observation, observation["notebook"]["text"], observation["notebook"]["capacity_chars"]) if note != observation["notebook"]["text"]: transition = campaign.replace_notebook(note) public = wire(transition) observation = public["observation"] action, gradient = policy.sample(observation, public["legal_actions"], rng) before = _potential(observation) transition = campaign.step(action) shaping = 0.99 * _potential(wire(transition)["observation"]) - before - 0.002 terminal = 1.0 if transition.outcome == "escaped" else -1.0 if transition.done else 0.0 trajectory.append({"gradient": gradient, "reward": terminal + shaping, "action": action, "terminal_reward": terminal, "potential_shaping": shaping}) shaping_total += shaping if not transition.done and trajectory: trajectory[-1]["reward"] -= 1.0 trajectory[-1]["evaluator_timeout_penalty"] = -1.0 return trajectory, {"verified_escape": transition.outcome == "escaped", "outcome": transition.outcome, "steps": len(trajectory), "shaping_return": shaping_total, "training_return": sum(step["reward"] for step in trajectory), "evaluator_truncated": not transition.done} def _evaluate(policy, seeds, checkpoint=False, rng_seed=97000): checksum = policy.checksum() rows = [] for index, seed in enumerate(seeds): runner = _checkpoint_rollout if checkpoint else _rollout _, row = runner(policy, seed, random.Random(rng_seed + index)) rows.append({"operator_seed": seed, **row}) assert checksum == policy.checksum(), "evaluation must never update policy" result = {"episodes": rows, "denominator": len(rows), "escape_rate": sum(row["verified_escape"] for row in rows) / len(rows), "frozen_checksum": checksum} if checkpoint: result["enumerated_valve_expectation"] = _checkpoint_expected(policy, seeds) assert checksum == policy.checksum(), "evaluation must keep policy frozen" return result def _checkpoint_state(world_seed): from borderland import Campaign from borderland_verify.reference import LegalReference campaign = Campaign.generated(world_seed) reference = LegalReference() transition = campaign.reset() prefix_steps = 0 for _ in range(256): if transition.done: raise RuntimeError("reference did not reach valve checkpoint") note = reference.notebook(transition) if note is not None and note != wire(transition)["observation"]["notebook"]["text"]: transition = campaign.replace_notebook(note) action = reference.choose(transition) if wire(action)["tag"] == "open_valve": break transition = campaign.step(action) prefix_steps += 1 else: raise RuntimeError("checkpoint prefix budget") public = wire(transition) actions = [action for action in public["legal_actions"] if action["tag"] == "open_valve"] if len(actions) != 2: raise RuntimeError("checkpoint needs exactly two legal valves") return campaign, transition, actions, prefix_steps def _checkpoint_rollout(policy, world_seed, rng): campaign, transition, actions, prefix_steps = _checkpoint_state(world_seed) action, gradient = policy.sample(wire(transition)["observation"], actions, rng) transition = campaign.step(action) if not transition.done: launches = [action for action in wire(transition)["legal_actions"] if action["tag"] == "launch"] if not launches: raise RuntimeError("checkpoint launch unavailable") transition = campaign.step(launches[0]) reward = 1.0 if transition.outcome == "escaped" else -1.0 return [{"gradient": gradient, "reward": reward, "action": action}], { "verified_escape": transition.outcome == "escaped", "outcome": transition.outcome, "steps": 1, "training_return": reward, "reference_prefix_steps": prefix_steps, "evaluator_truncated": not transition.done} def _checkpoint_expected(policy, seeds): rows = [] for seed in seeds: _, transition, actions, _ = _checkpoint_state(seed) probabilities, _ = policy.distribution(wire(transition)["observation"], actions) outcomes = [] for action in actions: campaign, _, _, _ = _checkpoint_state(seed) result = campaign.step(action) if not result.done: launch = next(action for action in wire(result)["legal_actions"] if action["tag"] == "launch") result = campaign.step(launch) outcomes.append(int(result.outcome == "escaped")) rows.append({"operator_seed": seed, "actions": actions, "probabilities": probabilities, "verified_returns": outcomes, "expected_escape": sum(probability * value for probability, value in zip(probabilities, outcomes)), "argmax_escape": outcomes[max(range(len(actions)), key=lambda index: probabilities[index])]}) return {"episodes": rows, "expected_escape_rate": sum(row["expected_escape"] for row in rows) / len(rows), "argmax_escape_rate": sum(row["argmax_escape"] for row in rows) / len(rows), "method": "Each legal valve reexecuted from a fresh real-engine reference prefix; weighted by frozen student probabilities."} def train_local(output_dir, episodes=128, seed=42, *, train_worlds=None, dev_worlds=None): from .checkpoints import checkpoint, training_worlds from .hierarchical_training import _output_directory output = _output_directory(output_dir, episodes, seed) if (train_worlds is None) != (dev_worlds is None): raise ValueError('provide both train_worlds and dev_worlds') frozen = {split: manifest(split) for split in ('train', 'dev')} if train_worlds is None else { 'train': dict(root_seeds=train_worlds, split='ad_hoc_train'), 'dev': dict(root_seeds=dev_worlds, split='ad_hoc_dev')} train_seeds = frozen["train"]["root_seeds"] train_seeds, dev_seeds = training_worlds(train_seeds, frozen['dev']['root_seeds']) output.mkdir(parents=True, exist_ok=True) run = dict(episodes=episodes, training_seed=seed, train_worlds=train_seeds, dev_worlds=dev_seeds, evaluation_rng_seed=97000, split_hashes={key: digest(value) for key, value in frozen.items()}, optimizer=dict(full_lr=0.08, full_gamma=0.99, valve_lr=0.4, valve_gamma=1.0, ema_decay=0.9)) policy = LinearPolicy() initial = policy.checksum() before = _evaluate(policy, dev_seeds) rng = random.Random(seed) baseline = 0.0 logs = [] for episode in range(episodes): trajectory, row = _rollout(policy, train_seeds[episode % len(train_seeds)], rng) update = reinforce_update(policy, trajectory, baseline) baseline = 0.9 * baseline + 0.1 * update["discounted_return"] logs.append({"episode": episode, **row, **update, "ema_baseline": baseline, "reward_trajectory": [{key: value for key, value in step.items() if key != "gradient"} for step in trajectory]}) after = _evaluate(policy, dev_seeds) (output / "weights.json").write_text(json.dumps(checkpoint('sparse_full', policy.weights, run))) valve_policy = LinearPolicy() checkpoint_before = _evaluate(valve_policy, dev_seeds, checkpoint=True) checkpoint_logs = [] checkpoint_baseline = 0.0 checkpoint_rng = random.Random(seed) for episode in range(episodes): trajectory, row = _checkpoint_rollout(valve_policy, train_seeds[episode % len(train_seeds)], checkpoint_rng) update = reinforce_update(valve_policy, trajectory, checkpoint_baseline, learning_rate=0.4, gamma=1.0) checkpoint_baseline = 0.9 * checkpoint_baseline + 0.1 * update["discounted_return"] checkpoint_logs.append({"episode": episode, **row, **update, "ema_baseline": checkpoint_baseline, "reward_trajectory": [{key: value for key, value in step.items() if key != "gradient"} for step in trajectory]}) checkpoint_after = _evaluate(valve_policy, dev_seeds, checkpoint=True) (output / "checkpoint-weights.json").write_text(json.dumps(checkpoint('sparse_valve', valve_policy.weights, run))) result = {"learner": "non-LLM sparse linear softmax REINFORCE", "compressor": "scripted public-record compressor", "feature_hash": "SHA-256(action canonical JSON + NUL + public feature), first 8 bytes big endian modulo 4096; L2 normalized", "initial_checksum": initial, "final_checksum": policy.checksum(), "rl_updates": len(logs), "nonzero_rl_updates": sum(row["gradient_norm"] > 0 for row in logs), "imitation_updates": 0, "optimizer": {"learning_rate": 0.08, "gamma": 0.99, "ema_decay": 0.9}, "full_campaign": {"before": before, "after": after, "training": logs, "improvement_observed": after["escape_rate"] > before["escape_rate"], "learning_claim": "Development control comparison only. A gain does not establish held-out improvement."}, "valve_checkpoint_curriculum": {"scope": "Real campaign reference prefix, two legal valve choices, scripted launch. Student controls only valve; notebook preserves acquired public facts.", "before": checkpoint_before, "after": checkpoint_after, "training": checkpoint_logs, "rl_updates": len(checkpoint_logs), "optimizer": {"learning_rate": 0.4, "gamma": 1.0, "ema_decay": 0.9}, "improvement_observed": checkpoint_after["escape_rate"] > checkpoint_before["escape_rate"]}, "manifests": frozen, "manifest_hashes": {key: digest(value) for key, value in frozen.items()}, "training_seed": seed, "source_hash": source_hash(), "evaluation_split": "dev", "run_configuration": run, "configuration_hash": digest({"episodes": episodes, "seed": seed, "dimensions": DIMENSIONS, "full_lr": 0.08, "full_gamma": 0.99, "checkpoint_lr": 0.4, "checkpoint_gamma": 1.0, "ema_decay": 0.9}), "limits": ["Training reads only train and dev worlds. No final test is run automatically.", "Development before and after comparisons can guide training and are not final held-out evidence.", "No Beam calls or Beam parameter updates occur.", "Checkpoint success is not autonomous full-campaign success."]} (output / "training.json").write_text(json.dumps(result, indent=2)) return result