Spaces:
Running
Running
Download python/borderland_eval/archive_learning.py from FatUnc6969/borderland-rl: direct link, hf CLI and curl.
- Browser
- Download file 6.22 kB
-
https://huggingface.co/spaces/FatUnc6969/borderland-rl/resolve/main/python/borderland_eval/archive_learning.py
- Command line
-
hf download hf://spaces/FatUnc6969/borderland-rl/python/borderland_eval/archive_learning.py
-
curl -L -o archive_learning.py https://huggingface.co/spaces/FatUnc6969/borderland-rl/resolve/main/python/borderland_eval/archive_learning.py
6.22 kB
| from __future__ import annotations | |
| import math | |
| import random | |
| from dataclasses import dataclass | |
| from borderland_eval.archive_cases import BoundExceeded, digest | |
| FEATURES = ('measured0_candidate0', 'measured0_candidate1', | |
| 'measured1_candidate0', 'measured1_candidate1', 'baseline') | |
| SCHEDULE = ((32, 1), (48, 2), (64, 4), (112, 8)) | |
| def ownership(episode): | |
| if type(episode) is not int or not 0 <= episode < 256: | |
| raise ValueError('episode outside fixed schedule') | |
| for count, owned in SCHEDULE: | |
| if episode < count: | |
| return owned | |
| episode -= count | |
| def features(evidence, record): | |
| vector = [0.0] * 5 | |
| if record.supersedes is None: | |
| vector[4] = 1.0 | |
| else: | |
| bit = evidence.bit(record.applies_channel) | |
| if type(bit) is not int or bit not in (0, 1) or type(record.applies_bit) is not int or record.applies_bit not in (0, 1): | |
| raise ValueError('public bit feature outside domain') | |
| vector[2 * bit + record.applies_bit] = 1.0 | |
| return tuple(vector) | |
| class Decision: | |
| service: str | |
| candidates: tuple[str, ...] | |
| vectors: tuple[tuple[float, ...], ...] | |
| probabilities: tuple[float, ...] | |
| selected: int | |
| def gradient(self): | |
| return tuple(self.vectors[self.selected][i] - sum(p * v[i] for p, v in zip(self.probabilities, self.vectors)) for i in range(5)) | |
| class ArchivePolicy: | |
| def __init__(self, weights=None): | |
| self.weights = tuple([0.0] * 5 if weights is None else weights) | |
| if len(self.weights) != 5 or not all(math.isfinite(v) for v in self.weights): | |
| raise ValueError('five finite weights required') | |
| def decide(self, evidence, service, rng, greedy=False): | |
| candidates = list(evidence.candidates(service)) | |
| if len(candidates) != 3 or sum(record.supersedes is None for record in candidates) != 1: | |
| raise ValueError('all three revisions including baseline required') | |
| rng.shuffle(candidates) | |
| vectors = tuple(features(evidence, record) for record in candidates) | |
| scores = [sum(w * x for w, x in zip(self.weights, vector)) for vector in vectors] | |
| exponentials = [math.exp(score - max(scores)) for score in scores] | |
| probabilities = tuple(value / sum(exponentials) for value in exponentials) | |
| if greedy: | |
| maximum = max(scores) | |
| selected = next(index for index, score in enumerate(scores) if score == maximum) | |
| else: | |
| selected = rng.choices(range(len(candidates)), weights=probabilities)[0] | |
| return Decision(service, tuple(record.id for record in candidates), vectors, probabilities, selected) | |
| def update(self, decisions, reward, previous_baseline): | |
| if reward not in (0, 1) or not 0 <= previous_baseline <= 1: | |
| raise ValueError('actual binary exit return and previous mean required') | |
| before = self.weights | |
| score_gradient = tuple(sum(decision.gradient()[i] for decision in decisions) for i in range(5)) | |
| gradient = tuple((reward - previous_baseline) * value for value in score_gradient) | |
| self.weights = tuple(weight + .15 * value for weight, value in zip(before, gradient)) | |
| if not all(math.isfinite(value) for value in self.weights): | |
| raise ValueError('nonfinite update') | |
| return dict(before=before, after=self.weights, previous_baseline=previous_baseline, | |
| reward=reward, learning_rate=.15, gamma=1, | |
| score_gradient=score_gradient, objective_gradient=gradient, | |
| gradient_norm=math.sqrt(sum(value * value for value in gradient)), | |
| before_hash=digest(before), after_hash=digest(self.weights)) | |
| def execute_docket(execution, evidence, selected, owned): | |
| records = {record.id: record for record in evidence.records} | |
| suffix_start = len(execution.commands) | |
| for service in evidence.services: | |
| record = records[selected[service]] | |
| role = 'scripted_owned_executor' if service in owned else 'public_teacher' | |
| if record.supersedes is not None: | |
| execution.command('action', dict(tag='compare_seal', arguments=dict(left_id=record.supersedes, right_id=record.id)), role) | |
| execution.command('action', dict(tag='file_record', arguments=dict(record_id=record.id)), role) | |
| if len(execution.commands) - suffix_start >= 24: | |
| raise BoundExceeded('suffix_cap') | |
| transition = execution.command('action', dict(tag='bind_docket', arguments=dict(record_ids=sorted(selected.values()))), 'fixed_bind_executor') | |
| reward = int(int(transition.observation.phase) == 4) | |
| return reward | |
| def trial(execution, evidence, policy, owned_count, rng, greedy=False): | |
| services = list(evidence.services) | |
| rng.shuffle(services) | |
| owned = tuple(services[:owned_count]) | |
| decisions = tuple(policy.decide(evidence, service, rng, greedy) for service in owned) | |
| selected = {decision.service: decision.candidates[decision.selected] for decision in decisions} | |
| decision_hash = digest([decision.__dict__ for decision in decisions]) | |
| execution.trial_receipt = dict(owned=owned, decisions=[decision.__dict__ for decision in decisions], | |
| owned_batch_hash_before_teacher=decision_hash) | |
| for service in services: | |
| if service not in selected: | |
| candidates = evidence.candidates(service) | |
| selected[service] = next(record.id for record in candidates if record.supersedes is not None and record.applies_bit == evidence.bit(record.applies_channel)) | |
| execution.trial_receipt['selected'] = dict(selected) | |
| reward = execute_docket(execution, evidence, selected, owned) | |
| return dict(reward=reward, owned=owned, decisions=[decision.__dict__ for decision in decisions], | |
| owned_batch_hash_before_teacher=decision_hash, selected=selected, | |
| assistance=dict(prefix='public_reference', evidence='supplied_measurements_and_all_packets', | |
| proof_and_filing='scripted_every_stage', teacher_services=sorted(set(services) - set(owned))), | |
| phase=int(execution.campaign.observe().observation.phase)), decisions | |