Spaces:
Running
Running
File size: 6,216 Bytes
a8aea51 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | from __future__ import annotations
import math
import random
from dataclasses import dataclass
from borderland_eval.archive_cases import BoundExceeded, digest
FEATURES = ('measured0_candidate0', 'measured0_candidate1',
'measured1_candidate0', 'measured1_candidate1', 'baseline')
SCHEDULE = ((32, 1), (48, 2), (64, 4), (112, 8))
def ownership(episode):
if type(episode) is not int or not 0 <= episode < 256:
raise ValueError('episode outside fixed schedule')
for count, owned in SCHEDULE:
if episode < count:
return owned
episode -= count
def features(evidence, record):
vector = [0.0] * 5
if record.supersedes is None:
vector[4] = 1.0
else:
bit = evidence.bit(record.applies_channel)
if type(bit) is not int or bit not in (0, 1) or type(record.applies_bit) is not int or record.applies_bit not in (0, 1):
raise ValueError('public bit feature outside domain')
vector[2 * bit + record.applies_bit] = 1.0
return tuple(vector)
@dataclass(frozen=True)
class Decision:
service: str
candidates: tuple[str, ...]
vectors: tuple[tuple[float, ...], ...]
probabilities: tuple[float, ...]
selected: int
def gradient(self):
return tuple(self.vectors[self.selected][i] - sum(p * v[i] for p, v in zip(self.probabilities, self.vectors)) for i in range(5))
class ArchivePolicy:
def __init__(self, weights=None):
self.weights = tuple([0.0] * 5 if weights is None else weights)
if len(self.weights) != 5 or not all(math.isfinite(v) for v in self.weights):
raise ValueError('five finite weights required')
def decide(self, evidence, service, rng, greedy=False):
candidates = list(evidence.candidates(service))
if len(candidates) != 3 or sum(record.supersedes is None for record in candidates) != 1:
raise ValueError('all three revisions including baseline required')
rng.shuffle(candidates)
vectors = tuple(features(evidence, record) for record in candidates)
scores = [sum(w * x for w, x in zip(self.weights, vector)) for vector in vectors]
exponentials = [math.exp(score - max(scores)) for score in scores]
probabilities = tuple(value / sum(exponentials) for value in exponentials)
if greedy:
maximum = max(scores)
selected = next(index for index, score in enumerate(scores) if score == maximum)
else:
selected = rng.choices(range(len(candidates)), weights=probabilities)[0]
return Decision(service, tuple(record.id for record in candidates), vectors, probabilities, selected)
def update(self, decisions, reward, previous_baseline):
if reward not in (0, 1) or not 0 <= previous_baseline <= 1:
raise ValueError('actual binary exit return and previous mean required')
before = self.weights
score_gradient = tuple(sum(decision.gradient()[i] for decision in decisions) for i in range(5))
gradient = tuple((reward - previous_baseline) * value for value in score_gradient)
self.weights = tuple(weight + .15 * value for weight, value in zip(before, gradient))
if not all(math.isfinite(value) for value in self.weights):
raise ValueError('nonfinite update')
return dict(before=before, after=self.weights, previous_baseline=previous_baseline,
reward=reward, learning_rate=.15, gamma=1,
score_gradient=score_gradient, objective_gradient=gradient,
gradient_norm=math.sqrt(sum(value * value for value in gradient)),
before_hash=digest(before), after_hash=digest(self.weights))
def execute_docket(execution, evidence, selected, owned):
records = {record.id: record for record in evidence.records}
suffix_start = len(execution.commands)
for service in evidence.services:
record = records[selected[service]]
role = 'scripted_owned_executor' if service in owned else 'public_teacher'
if record.supersedes is not None:
execution.command('action', dict(tag='compare_seal', arguments=dict(left_id=record.supersedes, right_id=record.id)), role)
execution.command('action', dict(tag='file_record', arguments=dict(record_id=record.id)), role)
if len(execution.commands) - suffix_start >= 24:
raise BoundExceeded('suffix_cap')
transition = execution.command('action', dict(tag='bind_docket', arguments=dict(record_ids=sorted(selected.values()))), 'fixed_bind_executor')
reward = int(int(transition.observation.phase) == 4)
return reward
def trial(execution, evidence, policy, owned_count, rng, greedy=False):
services = list(evidence.services)
rng.shuffle(services)
owned = tuple(services[:owned_count])
decisions = tuple(policy.decide(evidence, service, rng, greedy) for service in owned)
selected = {decision.service: decision.candidates[decision.selected] for decision in decisions}
decision_hash = digest([decision.__dict__ for decision in decisions])
execution.trial_receipt = dict(owned=owned, decisions=[decision.__dict__ for decision in decisions],
owned_batch_hash_before_teacher=decision_hash)
for service in services:
if service not in selected:
candidates = evidence.candidates(service)
selected[service] = next(record.id for record in candidates if record.supersedes is not None and record.applies_bit == evidence.bit(record.applies_channel))
execution.trial_receipt['selected'] = dict(selected)
reward = execute_docket(execution, evidence, selected, owned)
return dict(reward=reward, owned=owned, decisions=[decision.__dict__ for decision in decisions],
owned_batch_hash_before_teacher=decision_hash, selected=selected,
assistance=dict(prefix='public_reference', evidence='supplied_measurements_and_all_packets',
proof_and_filing='scripted_every_stage', teacher_services=sorted(set(services) - set(owned))),
phase=int(execution.campaign.observe().observation.phase)), decisions
|