from __future__ import annotations import math import random from dataclasses import dataclass from borderland_eval.archive_cases import BoundExceeded, digest FEATURES = ('measured0_candidate0', 'measured0_candidate1', 'measured1_candidate0', 'measured1_candidate1', 'baseline') SCHEDULE = ((32, 1), (48, 2), (64, 4), (112, 8)) def ownership(episode): if type(episode) is not int or not 0 <= episode < 256: raise ValueError('episode outside fixed schedule') for count, owned in SCHEDULE: if episode < count: return owned episode -= count def features(evidence, record): vector = [0.0] * 5 if record.supersedes is None: vector[4] = 1.0 else: bit = evidence.bit(record.applies_channel) if type(bit) is not int or bit not in (0, 1) or type(record.applies_bit) is not int or record.applies_bit not in (0, 1): raise ValueError('public bit feature outside domain') vector[2 * bit + record.applies_bit] = 1.0 return tuple(vector) @dataclass(frozen=True) class Decision: service: str candidates: tuple[str, ...] vectors: tuple[tuple[float, ...], ...] probabilities: tuple[float, ...] selected: int def gradient(self): return tuple(self.vectors[self.selected][i] - sum(p * v[i] for p, v in zip(self.probabilities, self.vectors)) for i in range(5)) class ArchivePolicy: def __init__(self, weights=None): self.weights = tuple([0.0] * 5 if weights is None else weights) if len(self.weights) != 5 or not all(math.isfinite(v) for v in self.weights): raise ValueError('five finite weights required') def decide(self, evidence, service, rng, greedy=False): candidates = list(evidence.candidates(service)) if len(candidates) != 3 or sum(record.supersedes is None for record in candidates) != 1: raise ValueError('all three revisions including baseline required') rng.shuffle(candidates) vectors = tuple(features(evidence, record) for record in candidates) scores = [sum(w * x for w, x in zip(self.weights, vector)) for vector in vectors] exponentials = [math.exp(score - max(scores)) for score in scores] probabilities = tuple(value / sum(exponentials) for value in exponentials) if greedy: maximum = max(scores) selected = next(index for index, score in enumerate(scores) if score == maximum) else: selected = rng.choices(range(len(candidates)), weights=probabilities)[0] return Decision(service, tuple(record.id for record in candidates), vectors, probabilities, selected) def update(self, decisions, reward, previous_baseline): if reward not in (0, 1) or not 0 <= previous_baseline <= 1: raise ValueError('actual binary exit return and previous mean required') before = self.weights score_gradient = tuple(sum(decision.gradient()[i] for decision in decisions) for i in range(5)) gradient = tuple((reward - previous_baseline) * value for value in score_gradient) self.weights = tuple(weight + .15 * value for weight, value in zip(before, gradient)) if not all(math.isfinite(value) for value in self.weights): raise ValueError('nonfinite update') return dict(before=before, after=self.weights, previous_baseline=previous_baseline, reward=reward, learning_rate=.15, gamma=1, score_gradient=score_gradient, objective_gradient=gradient, gradient_norm=math.sqrt(sum(value * value for value in gradient)), before_hash=digest(before), after_hash=digest(self.weights)) def execute_docket(execution, evidence, selected, owned): records = {record.id: record for record in evidence.records} suffix_start = len(execution.commands) for service in evidence.services: record = records[selected[service]] role = 'scripted_owned_executor' if service in owned else 'public_teacher' if record.supersedes is not None: execution.command('action', dict(tag='compare_seal', arguments=dict(left_id=record.supersedes, right_id=record.id)), role) execution.command('action', dict(tag='file_record', arguments=dict(record_id=record.id)), role) if len(execution.commands) - suffix_start >= 24: raise BoundExceeded('suffix_cap') transition = execution.command('action', dict(tag='bind_docket', arguments=dict(record_ids=sorted(selected.values()))), 'fixed_bind_executor') reward = int(int(transition.observation.phase) == 4) return reward def trial(execution, evidence, policy, owned_count, rng, greedy=False): services = list(evidence.services) rng.shuffle(services) owned = tuple(services[:owned_count]) decisions = tuple(policy.decide(evidence, service, rng, greedy) for service in owned) selected = {decision.service: decision.candidates[decision.selected] for decision in decisions} decision_hash = digest([decision.__dict__ for decision in decisions]) execution.trial_receipt = dict(owned=owned, decisions=[decision.__dict__ for decision in decisions], owned_batch_hash_before_teacher=decision_hash) for service in services: if service not in selected: candidates = evidence.candidates(service) selected[service] = next(record.id for record in candidates if record.supersedes is not None and record.applies_bit == evidence.bit(record.applies_channel)) execution.trial_receipt['selected'] = dict(selected) reward = execute_docket(execution, evidence, selected, owned) return dict(reward=reward, owned=owned, decisions=[decision.__dict__ for decision in decisions], owned_batch_hash_before_teacher=decision_hash, selected=selected, assistance=dict(prefix='public_reference', evidence='supplied_measurements_and_all_packets', proof_and_filing='scripted_every_stage', teacher_services=sorted(set(services) - set(owned))), phase=int(execution.campaign.observe().observation.phase)), decisions