File size: 9,426 Bytes
2719bf4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
from collections import Counter
import json
from pathlib import Path
from tempfile import TemporaryDirectory

from borderland import Campaign
from borderland.engine import parse_action
from borderland.replay import source_hash, verify_replay
from borderland.worlds import candidate
from borderland_verify.streaming_reference import choose
from .causal_dependencies import CausalDependencyAudit, extract_notebook
from .context import ContextSelector, actor_view


DECLARED_TOOLS = [{"type": "function", "function": {
    "name": "execute_action", "description": "Execute one currently legal campaign action.",
    "parameters": {"type": "object", "properties": {
        "tag": {"type": "string"}, "arguments": {"type": "object"}},
        "required": ["tag", "arguments"], "additionalProperties": False}}},
    {"type": "function", "function": {"name": "replace_notebook",
     "description": "Replace your retained notebook. This is your memory across the Blackout conversation reset.",
     "parameters": {"type": "object", "properties": {"text": {"type": "string"}},
                    "required": ["text"], "additionalProperties": False}}}]


def declared_request(public):
    context = ContextSelector(condition='bounded_notebook',
                              notebook_chars=public['observation']['notebook']['capacity_bytes'])
    context.observe(public)
    return dict(model='offline-protocol-fixture', messages=context.selected(),
                tools=DECLARED_TOOLS, tool_choice='auto', max_tokens=2048)


def run_streaming(seed=16, archive_workload='standard', call_budget=200, notebook_bytes=768):
    if type(seed) is not int or seed < 0:
        raise ValueError('seed must be a nonnegative integer')
    if archive_workload not in ('compact', 'standard', 'extended'):
        raise ValueError('unsupported Archive workload')
    if type(call_budget) is not int or call_budget < 0:
        raise ValueError('call budget must be a nonnegative integer')
    if type(notebook_bytes) is not int or notebook_bytes < 0:
        raise ValueError('notebook capacity must be a nonnegative integer')
    campaign = Campaign(candidate(seed, generation_version=3, archive_workload=archive_workload,
                                  notebook_bytes=notebook_bytes, memory_profile='commissioning'))
    transition = campaign.observe()
    audit = CausalDependencyAudit(campaign._world)
    initial_charge = transition.observation.charge
    phases = {int(transition.observation.phase)}
    phase_calls, note_classes = Counter(), Counter()
    trace = []
    resets = peak_note = peak_request = 0
    circuits = {}
    policy_failure = None
    for index in range(1, call_budget + 1):
        if transition.done:
            break
        public = actor_view(transition)
        phase = public['observation']['phase']
        request = declared_request(public)
        payload_bytes = len(json.dumps(request).encode('utf-8'))
        peak_request = max(peak_request, payload_bytes)
        note = extract_notebook(public['observation']['notebook']['text'])
        note_class = ('empty_supported' if not public['observation']['notebook']['text'] else
                      'supported_v3') if note.scorable else note.reason
        note_classes[note_class] += 1
        phase_calls[phase] += 1
        audit.observe(public, index, context=request['messages'])
        row = dict(call=index, phase_before=phase, command=None, applied=False,
                   declared_request_utf8_bytes=payload_bytes, notebook_class=note_class)
        trace.append(row)
        try:
            command = choose(public)
        except (AssertionError, ValueError, KeyError, TypeError, IndexError) as error:
            policy_failure = type(error).__name__
            row['policy_failure'] = policy_failure
            break
        row['command'] = command
        if command['tool'] == 'replace_notebook':
            after = campaign.replace_notebook(command['arguments']['text'],
                                              expected_revision=transition.revision)
        elif command['tool'] == 'execute_action':
            after = campaign.step(command['arguments'], expected_revision=transition.revision)
            parsed, parse_error = parse_action(command['arguments'], generation_version=3)
            canonical = parsed.to_dict() if not parse_error else command['arguments']
            row['canonical_action'] = canonical
            audit.record_decision(public, canonical, actor_view(after), index)
        else:
            policy_failure = 'unsupported_tool'
            row['policy_failure'] = policy_failure
            break
        transition = after
        peak_note = max(peak_note, len(after.observation.notebook['text'].encode('utf-8')))
        row.update(applied=True, error=after.error.to_dict() if after.error else None,
                   phase_after=int(after.observation.phase), charge_after=after.observation.charge,
                   notebook_utf8_bytes=len(after.observation.notebook['text'].encode('utf-8')),
                   context_reset=bool(after.context_reset), outcome=after.outcome)
        phases.add(int(after.observation.phase))
        resets += int(after.context_reset)
        for circuit in after.observation.board.get('circuits', []):
            circuits[circuit['id']] = dict(status=circuit['status'],
                                          completed_tests=circuit['completed_tests'])
        audit.observe(actor_view(after), index + 1)
    audit.observe(actor_view(transition), len(trace) + 1)
    report = audit.report()
    with TemporaryDirectory() as directory:
        replay_path = Path(directory) / 'streaming.private.jsonl'
        campaign.save_replay(replay_path, 'private')
        replay = verify_replay(replay_path)
    metrics = {key: report[key] for key in ('metric_kind', 'causal_reliance', 'predefined_denominator',
               'eligible_units', 'dependencies_matched', 'completed_units', 'dependency_agreement_rate')}
    dependencies = report['dependencies']
    metrics['all_first_choices_publicly_exposed'] = all(
        row['first_choice'] and row['first_choice']['evidence_exposed'] for row in dependencies)
    metrics['all_first_choices_scorable'] = all(
        row['first_choice'] and row['first_choice']['notebook_scorable'] for row in dependencies)
    metrics['censoring'] = dict(Counter(row['censoring'] for row in dependencies if row['censoring']))
    applied = [row for row in trace if row['applied']]
    commands = [row['command'] for row in applied]
    summary = dict(
        scope='Scripted bounded public control on an explicit known world. No learned control or Beam claim.',
        seed=seed, archive_workload=archive_workload, generation_version=3, memory_profile='commissioning',
        call_budget=call_budget, calls=len(trace), applied_commands=len(applied),
        notebook_writes=sum(c['tool'] == 'replace_notebook' for c in commands),
        gameplay_actions=sum(c['tool'] == 'execute_action' for c in commands),
        rejected_commands=sum(row['error'] is not None for row in applied), policy_failure=policy_failure,
        budget_exhausted=not transition.done and len(trace) == call_budget and policy_failure is None,
        outcome=transition.outcome, completed=transition.done, reached_phases=sorted(phases),
        phase_call_counts=dict(phase_calls), context_resets=resets,
        boarded=list(transition.observation.board.get('boarded', [])), initial_charge=initial_charge,
        remaining_charge=transition.observation.charge, charge_consumed=initial_charge-transition.observation.charge,
        notebook_capacity_bytes=notebook_bytes, peak_notebook_utf8_bytes=peak_note,
        peak_declared_request_utf8_bytes=peak_request,
        declared_request_limit_bytes=32768, declared_request_within_limit=peak_request <= 32768,
        declared_request_envelope=dict(model='offline-protocol-fixture', max_tokens=2048,
            messages=['system', 'user'], tools=DECLARED_TOOLS,
            measurement='UTF-8 bytes of json.dumps; offline envelope, not provider traffic or token count'),
        notebook_classes_by_decision=dict(note_classes),
        notebook_scoring='Prefix v4 is unsupported by the stock audit, not zero retention. Late v3 is scorable.',
        circuits=circuits, successful_commissioning_tests=sum(c['completed_tests'] for c in circuits.values()),
        dependency_metrics=metrics, private_replay_verified=replay['verified'], engine_source_hash=source_hash(),
        policy_inputs='current actor_view JSON only; fresh controller and CausalReference per decision',
        limitations=['Known-case scripted feasibility does not establish learnability, generalization, optimality, '
                     'Beam performance, or a causal notebook-erasure effect.'])
    summary['passed'] = (transition.outcome == 'escaped' and summary['boarded'] == ['Ada', 'Bo', 'Cy']
                         and phases == set(range(1, 8)) and resets == 1 and replay['verified']
                         and not policy_failure and not summary['rejected_commands']
                         and summary['declared_request_within_limit']
                         and metrics['completed_units'] == metrics['predefined_denominator']
                         and all(c == dict(status='live', completed_tests=3) for c in circuits.values()))
    return dict(summary=summary, public_command_trace=trace)