File size: 6,216 Bytes
a8aea51
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
from __future__ import annotations

import math
import random
from dataclasses import dataclass

from borderland_eval.archive_cases import BoundExceeded, digest


FEATURES = ('measured0_candidate0', 'measured0_candidate1',
            'measured1_candidate0', 'measured1_candidate1', 'baseline')
SCHEDULE = ((32, 1), (48, 2), (64, 4), (112, 8))


def ownership(episode):
    if type(episode) is not int or not 0 <= episode < 256:
        raise ValueError('episode outside fixed schedule')
    for count, owned in SCHEDULE:
        if episode < count:
            return owned
        episode -= count


def features(evidence, record):
    vector = [0.0] * 5
    if record.supersedes is None:
        vector[4] = 1.0
    else:
        bit = evidence.bit(record.applies_channel)
        if type(bit) is not int or bit not in (0, 1) or type(record.applies_bit) is not int or record.applies_bit not in (0, 1):
            raise ValueError('public bit feature outside domain')
        vector[2 * bit + record.applies_bit] = 1.0
    return tuple(vector)


@dataclass(frozen=True)
class Decision:
    service: str
    candidates: tuple[str, ...]
    vectors: tuple[tuple[float, ...], ...]
    probabilities: tuple[float, ...]
    selected: int

    def gradient(self):
        return tuple(self.vectors[self.selected][i] - sum(p * v[i] for p, v in zip(self.probabilities, self.vectors)) for i in range(5))


class ArchivePolicy:
    def __init__(self, weights=None):
        self.weights = tuple([0.0] * 5 if weights is None else weights)
        if len(self.weights) != 5 or not all(math.isfinite(v) for v in self.weights):
            raise ValueError('five finite weights required')

    def decide(self, evidence, service, rng, greedy=False):
        candidates = list(evidence.candidates(service))
        if len(candidates) != 3 or sum(record.supersedes is None for record in candidates) != 1:
            raise ValueError('all three revisions including baseline required')
        rng.shuffle(candidates)
        vectors = tuple(features(evidence, record) for record in candidates)
        scores = [sum(w * x for w, x in zip(self.weights, vector)) for vector in vectors]
        exponentials = [math.exp(score - max(scores)) for score in scores]
        probabilities = tuple(value / sum(exponentials) for value in exponentials)
        if greedy:
            maximum = max(scores)
            selected = next(index for index, score in enumerate(scores) if score == maximum)
        else:
            selected = rng.choices(range(len(candidates)), weights=probabilities)[0]
        return Decision(service, tuple(record.id for record in candidates), vectors, probabilities, selected)

    def update(self, decisions, reward, previous_baseline):
        if reward not in (0, 1) or not 0 <= previous_baseline <= 1:
            raise ValueError('actual binary exit return and previous mean required')
        before = self.weights
        score_gradient = tuple(sum(decision.gradient()[i] for decision in decisions) for i in range(5))
        gradient = tuple((reward - previous_baseline) * value for value in score_gradient)
        self.weights = tuple(weight + .15 * value for weight, value in zip(before, gradient))
        if not all(math.isfinite(value) for value in self.weights):
            raise ValueError('nonfinite update')
        return dict(before=before, after=self.weights, previous_baseline=previous_baseline,
                    reward=reward, learning_rate=.15, gamma=1,
                    score_gradient=score_gradient, objective_gradient=gradient,
                    gradient_norm=math.sqrt(sum(value * value for value in gradient)),
                    before_hash=digest(before), after_hash=digest(self.weights))


def execute_docket(execution, evidence, selected, owned):
    records = {record.id: record for record in evidence.records}
    suffix_start = len(execution.commands)
    for service in evidence.services:
        record = records[selected[service]]
        role = 'scripted_owned_executor' if service in owned else 'public_teacher'
        if record.supersedes is not None:
            execution.command('action', dict(tag='compare_seal', arguments=dict(left_id=record.supersedes, right_id=record.id)), role)
        execution.command('action', dict(tag='file_record', arguments=dict(record_id=record.id)), role)
    if len(execution.commands) - suffix_start >= 24:
        raise BoundExceeded('suffix_cap')
    transition = execution.command('action', dict(tag='bind_docket', arguments=dict(record_ids=sorted(selected.values()))), 'fixed_bind_executor')
    reward = int(int(transition.observation.phase) == 4)
    return reward


def trial(execution, evidence, policy, owned_count, rng, greedy=False):
    services = list(evidence.services)
    rng.shuffle(services)
    owned = tuple(services[:owned_count])
    decisions = tuple(policy.decide(evidence, service, rng, greedy) for service in owned)
    selected = {decision.service: decision.candidates[decision.selected] for decision in decisions}
    decision_hash = digest([decision.__dict__ for decision in decisions])
    execution.trial_receipt = dict(owned=owned, decisions=[decision.__dict__ for decision in decisions],
                                   owned_batch_hash_before_teacher=decision_hash)
    for service in services:
        if service not in selected:
            candidates = evidence.candidates(service)
            selected[service] = next(record.id for record in candidates if record.supersedes is not None and record.applies_bit == evidence.bit(record.applies_channel))
    execution.trial_receipt['selected'] = dict(selected)
    reward = execute_docket(execution, evidence, selected, owned)
    return dict(reward=reward, owned=owned, decisions=[decision.__dict__ for decision in decisions],
                owned_batch_hash_before_teacher=decision_hash, selected=selected,
                assistance=dict(prefix='public_reference', evidence='supplied_measurements_and_all_packets',
                                proof_and_filing='scripted_every_stage', teacher_services=sorted(set(services) - set(owned))),
                phase=int(execution.campaign.observe().observation.phase)), decisions