Spaces:
Running
Running
Download python/borderland_eval/training.py from FatUnc6969/borderland-rl: direct link, hf CLI and curl.
- Browser
- Download file 15.5 kB
-
https://huggingface.co/spaces/FatUnc6969/borderland-rl/resolve/main/python/borderland_eval/training.py
- Command line
-
hf download hf://spaces/FatUnc6969/borderland-rl/python/borderland_eval/training.py
-
curl -L -o training.py https://huggingface.co/spaces/FatUnc6969/borderland-rl/resolve/main/python/borderland_eval/training.py
15.5 kB
| """Sparse linear REINFORCE using only public transitions and legal actions.""" | |
| import hashlib | |
| import json | |
| import math | |
| import random | |
| from collections import Counter | |
| from pathlib import Path | |
| from .context import scripted_notebook, wire | |
| from .manifests import canonical, digest, manifest, source_hash | |
| DIMENSIONS = 4096 | |
| def _flatten(value, prefix="board", depth=0): | |
| if depth > 4: | |
| return | |
| if isinstance(value, dict): | |
| for key, item in sorted(value.items()): | |
| yield from _flatten(item, prefix + "." + str(key), depth + 1) | |
| elif isinstance(value, (list, tuple)): | |
| for item in value[:12]: | |
| yield from _flatten(item, prefix, depth + 1) | |
| elif isinstance(value, (str, int, bool)) or value is None: | |
| yield prefix + "=" + str(value) | |
| def features(observation, action): | |
| tokens = ["bias", "phase=" + str(observation["phase"]), | |
| "charge_bin=" + str(observation["charge"] // 10)] | |
| tokens.extend(_flatten(observation["board"])) | |
| tokens.extend(_flatten(observation["team"], "team")) | |
| tokens.extend("inventory=" + str(item) for item in observation["inventory"]) | |
| tokens.extend("note=" + token for token in observation["notebook"]["text"].replace('"', ' ').replace(':', ' ').replace(',', ' ').split()[:128]) | |
| action_text = canonical(action) | |
| indices = Counter(int.from_bytes(hashlib.sha256((action_text + "\0" + token).encode()).digest()[:8], "big") % DIMENSIONS for token in tokens) | |
| scale = math.sqrt(sum(value * value for value in indices.values())) | |
| return {index: value / scale for index, value in indices.items()} | |
| class LinearPolicy: | |
| def __init__(self, weights=None): | |
| self.weights = list(weights) if weights is not None else [0.0] * DIMENSIONS | |
| if len(self.weights) != DIMENSIONS or any(type(value) not in (int, float) or not math.isfinite(value) for value in self.weights): | |
| raise ValueError('weights must contain 4096 finite numbers') | |
| def distribution(self, observation, actions): | |
| vectors = [features(observation, action) for action in actions] | |
| logits = [sum(self.weights[index] * value for index, value in vector.items()) for vector in vectors] | |
| if not all(math.isfinite(value) for value in logits): | |
| raise ValueError('policy logits overflowed') | |
| high = max(logits) | |
| exps = [math.exp(logit - high) for logit in logits] | |
| total = sum(exps) | |
| return [value / total for value in exps], vectors | |
| def sample(self, observation, actions, rng): | |
| probabilities, vectors = self.distribution(observation, actions) | |
| threshold = rng.random() | |
| chosen = len(actions) - 1 | |
| cumulative = 0.0 | |
| for index, probability in enumerate(probabilities): | |
| cumulative += probability | |
| if threshold <= cumulative: | |
| chosen = index | |
| break | |
| gradient = Counter(vectors[chosen]) | |
| for probability, vector in zip(probabilities, vectors): | |
| for index, value in vector.items(): | |
| gradient[index] -= probability * value | |
| return actions[chosen], dict(gradient) | |
| def checksum(self): | |
| return digest(self.weights) | |
| def reinforce_update(policy, trajectory, baseline=0.0, learning_rate=0.08, gamma=0.99): | |
| total_return = 0.0 | |
| gradient = Counter() | |
| for step in reversed(trajectory): | |
| total_return = step["reward"] + gamma * total_return | |
| for index, derivative in step["gradient"].items(): | |
| gradient[index] += (total_return - baseline) * derivative | |
| for index, derivative in gradient.items(): | |
| policy.weights[index] += learning_rate * derivative | |
| return {"gradient_norm": math.sqrt(sum(value * value for value in gradient.values())), | |
| "discounted_return": total_return, "steps": len(trajectory)} | |
| def _potential(observation): | |
| return 0.15 * (int(observation["phase"]) - 1) + 0.10 * bool(observation["board"].get("gate_open")) | |
| def _rollout(policy, world_seed, rng, max_steps=160): | |
| from borderland import Campaign | |
| campaign = Campaign.generated(world_seed) | |
| transition = campaign.reset() | |
| trajectory = [] | |
| shaping_total = 0.0 | |
| for _ in range(max_steps): | |
| if transition.done: | |
| break | |
| public = wire(transition) | |
| observation = public["observation"] | |
| note = scripted_notebook(observation, observation["notebook"]["text"], observation["notebook"]["capacity_chars"]) | |
| if note != observation["notebook"]["text"]: | |
| transition = campaign.replace_notebook(note) | |
| public = wire(transition) | |
| observation = public["observation"] | |
| action, gradient = policy.sample(observation, public["legal_actions"], rng) | |
| before = _potential(observation) | |
| transition = campaign.step(action) | |
| shaping = 0.99 * _potential(wire(transition)["observation"]) - before - 0.002 | |
| terminal = 1.0 if transition.outcome == "escaped" else -1.0 if transition.done else 0.0 | |
| trajectory.append({"gradient": gradient, "reward": terminal + shaping, | |
| "action": action, "terminal_reward": terminal, "potential_shaping": shaping}) | |
| shaping_total += shaping | |
| if not transition.done and trajectory: | |
| trajectory[-1]["reward"] -= 1.0 | |
| trajectory[-1]["evaluator_timeout_penalty"] = -1.0 | |
| return trajectory, {"verified_escape": transition.outcome == "escaped", "outcome": transition.outcome, | |
| "steps": len(trajectory), "shaping_return": shaping_total, | |
| "training_return": sum(step["reward"] for step in trajectory), | |
| "evaluator_truncated": not transition.done} | |
| def _evaluate(policy, seeds, checkpoint=False, rng_seed=97000): | |
| checksum = policy.checksum() | |
| rows = [] | |
| for index, seed in enumerate(seeds): | |
| runner = _checkpoint_rollout if checkpoint else _rollout | |
| _, row = runner(policy, seed, random.Random(rng_seed + index)) | |
| rows.append({"operator_seed": seed, **row}) | |
| assert checksum == policy.checksum(), "evaluation must never update policy" | |
| result = {"episodes": rows, "denominator": len(rows), | |
| "escape_rate": sum(row["verified_escape"] for row in rows) / len(rows), | |
| "frozen_checksum": checksum} | |
| if checkpoint: | |
| result["enumerated_valve_expectation"] = _checkpoint_expected(policy, seeds) | |
| assert checksum == policy.checksum(), "evaluation must keep policy frozen" | |
| return result | |
| def _checkpoint_state(world_seed): | |
| from borderland import Campaign | |
| from borderland_verify.reference import LegalReference | |
| campaign = Campaign.generated(world_seed) | |
| reference = LegalReference() | |
| transition = campaign.reset() | |
| prefix_steps = 0 | |
| for _ in range(256): | |
| if transition.done: | |
| raise RuntimeError("reference did not reach valve checkpoint") | |
| note = reference.notebook(transition) | |
| if note is not None and note != wire(transition)["observation"]["notebook"]["text"]: | |
| transition = campaign.replace_notebook(note) | |
| action = reference.choose(transition) | |
| if wire(action)["tag"] == "open_valve": | |
| break | |
| transition = campaign.step(action) | |
| prefix_steps += 1 | |
| else: | |
| raise RuntimeError("checkpoint prefix budget") | |
| public = wire(transition) | |
| actions = [action for action in public["legal_actions"] if action["tag"] == "open_valve"] | |
| if len(actions) != 2: | |
| raise RuntimeError("checkpoint needs exactly two legal valves") | |
| return campaign, transition, actions, prefix_steps | |
| def _checkpoint_rollout(policy, world_seed, rng): | |
| campaign, transition, actions, prefix_steps = _checkpoint_state(world_seed) | |
| action, gradient = policy.sample(wire(transition)["observation"], actions, rng) | |
| transition = campaign.step(action) | |
| if not transition.done: | |
| launches = [action for action in wire(transition)["legal_actions"] if action["tag"] == "launch"] | |
| if not launches: | |
| raise RuntimeError("checkpoint launch unavailable") | |
| transition = campaign.step(launches[0]) | |
| reward = 1.0 if transition.outcome == "escaped" else -1.0 | |
| return [{"gradient": gradient, "reward": reward, "action": action}], { | |
| "verified_escape": transition.outcome == "escaped", "outcome": transition.outcome, | |
| "steps": 1, "training_return": reward, "reference_prefix_steps": prefix_steps, | |
| "evaluator_truncated": not transition.done} | |
| def _checkpoint_expected(policy, seeds): | |
| rows = [] | |
| for seed in seeds: | |
| _, transition, actions, _ = _checkpoint_state(seed) | |
| probabilities, _ = policy.distribution(wire(transition)["observation"], actions) | |
| outcomes = [] | |
| for action in actions: | |
| campaign, _, _, _ = _checkpoint_state(seed) | |
| result = campaign.step(action) | |
| if not result.done: | |
| launch = next(action for action in wire(result)["legal_actions"] if action["tag"] == "launch") | |
| result = campaign.step(launch) | |
| outcomes.append(int(result.outcome == "escaped")) | |
| rows.append({"operator_seed": seed, "actions": actions, "probabilities": probabilities, | |
| "verified_returns": outcomes, | |
| "expected_escape": sum(probability * value for probability, value in zip(probabilities, outcomes)), | |
| "argmax_escape": outcomes[max(range(len(actions)), key=lambda index: probabilities[index])]}) | |
| return {"episodes": rows, "expected_escape_rate": sum(row["expected_escape"] for row in rows) / len(rows), | |
| "argmax_escape_rate": sum(row["argmax_escape"] for row in rows) / len(rows), | |
| "method": "Each legal valve reexecuted from a fresh real-engine reference prefix; weighted by frozen student probabilities."} | |
| def train_local(output_dir, episodes=128, seed=42, *, train_worlds=None, dev_worlds=None): | |
| from .checkpoints import checkpoint, training_worlds | |
| from .hierarchical_training import _output_directory | |
| output = _output_directory(output_dir, episodes, seed) | |
| if (train_worlds is None) != (dev_worlds is None): | |
| raise ValueError('provide both train_worlds and dev_worlds') | |
| frozen = {split: manifest(split) for split in ('train', 'dev')} if train_worlds is None else { | |
| 'train': dict(root_seeds=train_worlds, split='ad_hoc_train'), | |
| 'dev': dict(root_seeds=dev_worlds, split='ad_hoc_dev')} | |
| train_seeds = frozen["train"]["root_seeds"] | |
| train_seeds, dev_seeds = training_worlds(train_seeds, frozen['dev']['root_seeds']) | |
| output.mkdir(parents=True, exist_ok=True) | |
| run = dict(episodes=episodes, training_seed=seed, train_worlds=train_seeds, dev_worlds=dev_seeds, | |
| evaluation_rng_seed=97000, split_hashes={key: digest(value) for key, value in frozen.items()}, | |
| optimizer=dict(full_lr=0.08, full_gamma=0.99, valve_lr=0.4, valve_gamma=1.0, ema_decay=0.9)) | |
| policy = LinearPolicy() | |
| initial = policy.checksum() | |
| before = _evaluate(policy, dev_seeds) | |
| rng = random.Random(seed) | |
| baseline = 0.0 | |
| logs = [] | |
| for episode in range(episodes): | |
| trajectory, row = _rollout(policy, train_seeds[episode % len(train_seeds)], rng) | |
| update = reinforce_update(policy, trajectory, baseline) | |
| baseline = 0.9 * baseline + 0.1 * update["discounted_return"] | |
| logs.append({"episode": episode, **row, **update, "ema_baseline": baseline, | |
| "reward_trajectory": [{key: value for key, value in step.items() if key != "gradient"} | |
| for step in trajectory]}) | |
| after = _evaluate(policy, dev_seeds) | |
| (output / "weights.json").write_text(json.dumps(checkpoint('sparse_full', policy.weights, run))) | |
| valve_policy = LinearPolicy() | |
| checkpoint_before = _evaluate(valve_policy, dev_seeds, checkpoint=True) | |
| checkpoint_logs = [] | |
| checkpoint_baseline = 0.0 | |
| checkpoint_rng = random.Random(seed) | |
| for episode in range(episodes): | |
| trajectory, row = _checkpoint_rollout(valve_policy, train_seeds[episode % len(train_seeds)], checkpoint_rng) | |
| update = reinforce_update(valve_policy, trajectory, checkpoint_baseline, learning_rate=0.4, gamma=1.0) | |
| checkpoint_baseline = 0.9 * checkpoint_baseline + 0.1 * update["discounted_return"] | |
| checkpoint_logs.append({"episode": episode, **row, **update, "ema_baseline": checkpoint_baseline, | |
| "reward_trajectory": [{key: value for key, value in step.items() if key != "gradient"} | |
| for step in trajectory]}) | |
| checkpoint_after = _evaluate(valve_policy, dev_seeds, checkpoint=True) | |
| (output / "checkpoint-weights.json").write_text(json.dumps(checkpoint('sparse_valve', valve_policy.weights, run))) | |
| result = {"learner": "non-LLM sparse linear softmax REINFORCE", "compressor": "scripted public-record compressor", | |
| "feature_hash": "SHA-256(action canonical JSON + NUL + public feature), first 8 bytes big endian modulo 4096; L2 normalized", | |
| "initial_checksum": initial, "final_checksum": policy.checksum(), "rl_updates": len(logs), | |
| "nonzero_rl_updates": sum(row["gradient_norm"] > 0 for row in logs), | |
| "imitation_updates": 0, "optimizer": {"learning_rate": 0.08, "gamma": 0.99, "ema_decay": 0.9}, | |
| "full_campaign": {"before": before, "after": after, "training": logs, | |
| "improvement_observed": after["escape_rate"] > before["escape_rate"], | |
| "learning_claim": "Development control comparison only. A gain does not establish held-out improvement."}, | |
| "valve_checkpoint_curriculum": {"scope": "Real campaign reference prefix, two legal valve choices, scripted launch. Student controls only valve; notebook preserves acquired public facts.", | |
| "before": checkpoint_before, "after": checkpoint_after, | |
| "training": checkpoint_logs, "rl_updates": len(checkpoint_logs), | |
| "optimizer": {"learning_rate": 0.4, "gamma": 1.0, "ema_decay": 0.9}, | |
| "improvement_observed": checkpoint_after["escape_rate"] > checkpoint_before["escape_rate"]}, | |
| "manifests": frozen, "manifest_hashes": {key: digest(value) for key, value in frozen.items()}, | |
| "training_seed": seed, "source_hash": source_hash(), "evaluation_split": "dev", "run_configuration": run, | |
| "configuration_hash": digest({"episodes": episodes, "seed": seed, "dimensions": DIMENSIONS, | |
| "full_lr": 0.08, "full_gamma": 0.99, "checkpoint_lr": 0.4, | |
| "checkpoint_gamma": 1.0, "ema_decay": 0.9}), | |
| "limits": ["Training reads only train and dev worlds. No final test is run automatically.", | |
| "Development before and after comparisons can guide training and are not final held-out evidence.", | |
| "No Beam calls or Beam parameter updates occur.", | |
| "Checkpoint success is not autonomous full-campaign success."]} | |
| (output / "training.json").write_text(json.dumps(result, indent=2)) | |
| return result | |