Spaces:
Running
Running
File size: 9,426 Bytes
2719bf4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | from collections import Counter
import json
from pathlib import Path
from tempfile import TemporaryDirectory
from borderland import Campaign
from borderland.engine import parse_action
from borderland.replay import source_hash, verify_replay
from borderland.worlds import candidate
from borderland_verify.streaming_reference import choose
from .causal_dependencies import CausalDependencyAudit, extract_notebook
from .context import ContextSelector, actor_view
DECLARED_TOOLS = [{"type": "function", "function": {
"name": "execute_action", "description": "Execute one currently legal campaign action.",
"parameters": {"type": "object", "properties": {
"tag": {"type": "string"}, "arguments": {"type": "object"}},
"required": ["tag", "arguments"], "additionalProperties": False}}},
{"type": "function", "function": {"name": "replace_notebook",
"description": "Replace your retained notebook. This is your memory across the Blackout conversation reset.",
"parameters": {"type": "object", "properties": {"text": {"type": "string"}},
"required": ["text"], "additionalProperties": False}}}]
def declared_request(public):
context = ContextSelector(condition='bounded_notebook',
notebook_chars=public['observation']['notebook']['capacity_bytes'])
context.observe(public)
return dict(model='offline-protocol-fixture', messages=context.selected(),
tools=DECLARED_TOOLS, tool_choice='auto', max_tokens=2048)
def run_streaming(seed=16, archive_workload='standard', call_budget=200, notebook_bytes=768):
if type(seed) is not int or seed < 0:
raise ValueError('seed must be a nonnegative integer')
if archive_workload not in ('compact', 'standard', 'extended'):
raise ValueError('unsupported Archive workload')
if type(call_budget) is not int or call_budget < 0:
raise ValueError('call budget must be a nonnegative integer')
if type(notebook_bytes) is not int or notebook_bytes < 0:
raise ValueError('notebook capacity must be a nonnegative integer')
campaign = Campaign(candidate(seed, generation_version=3, archive_workload=archive_workload,
notebook_bytes=notebook_bytes, memory_profile='commissioning'))
transition = campaign.observe()
audit = CausalDependencyAudit(campaign._world)
initial_charge = transition.observation.charge
phases = {int(transition.observation.phase)}
phase_calls, note_classes = Counter(), Counter()
trace = []
resets = peak_note = peak_request = 0
circuits = {}
policy_failure = None
for index in range(1, call_budget + 1):
if transition.done:
break
public = actor_view(transition)
phase = public['observation']['phase']
request = declared_request(public)
payload_bytes = len(json.dumps(request).encode('utf-8'))
peak_request = max(peak_request, payload_bytes)
note = extract_notebook(public['observation']['notebook']['text'])
note_class = ('empty_supported' if not public['observation']['notebook']['text'] else
'supported_v3') if note.scorable else note.reason
note_classes[note_class] += 1
phase_calls[phase] += 1
audit.observe(public, index, context=request['messages'])
row = dict(call=index, phase_before=phase, command=None, applied=False,
declared_request_utf8_bytes=payload_bytes, notebook_class=note_class)
trace.append(row)
try:
command = choose(public)
except (AssertionError, ValueError, KeyError, TypeError, IndexError) as error:
policy_failure = type(error).__name__
row['policy_failure'] = policy_failure
break
row['command'] = command
if command['tool'] == 'replace_notebook':
after = campaign.replace_notebook(command['arguments']['text'],
expected_revision=transition.revision)
elif command['tool'] == 'execute_action':
after = campaign.step(command['arguments'], expected_revision=transition.revision)
parsed, parse_error = parse_action(command['arguments'], generation_version=3)
canonical = parsed.to_dict() if not parse_error else command['arguments']
row['canonical_action'] = canonical
audit.record_decision(public, canonical, actor_view(after), index)
else:
policy_failure = 'unsupported_tool'
row['policy_failure'] = policy_failure
break
transition = after
peak_note = max(peak_note, len(after.observation.notebook['text'].encode('utf-8')))
row.update(applied=True, error=after.error.to_dict() if after.error else None,
phase_after=int(after.observation.phase), charge_after=after.observation.charge,
notebook_utf8_bytes=len(after.observation.notebook['text'].encode('utf-8')),
context_reset=bool(after.context_reset), outcome=after.outcome)
phases.add(int(after.observation.phase))
resets += int(after.context_reset)
for circuit in after.observation.board.get('circuits', []):
circuits[circuit['id']] = dict(status=circuit['status'],
completed_tests=circuit['completed_tests'])
audit.observe(actor_view(after), index + 1)
audit.observe(actor_view(transition), len(trace) + 1)
report = audit.report()
with TemporaryDirectory() as directory:
replay_path = Path(directory) / 'streaming.private.jsonl'
campaign.save_replay(replay_path, 'private')
replay = verify_replay(replay_path)
metrics = {key: report[key] for key in ('metric_kind', 'causal_reliance', 'predefined_denominator',
'eligible_units', 'dependencies_matched', 'completed_units', 'dependency_agreement_rate')}
dependencies = report['dependencies']
metrics['all_first_choices_publicly_exposed'] = all(
row['first_choice'] and row['first_choice']['evidence_exposed'] for row in dependencies)
metrics['all_first_choices_scorable'] = all(
row['first_choice'] and row['first_choice']['notebook_scorable'] for row in dependencies)
metrics['censoring'] = dict(Counter(row['censoring'] for row in dependencies if row['censoring']))
applied = [row for row in trace if row['applied']]
commands = [row['command'] for row in applied]
summary = dict(
scope='Scripted bounded public control on an explicit known world. No learned control or Beam claim.',
seed=seed, archive_workload=archive_workload, generation_version=3, memory_profile='commissioning',
call_budget=call_budget, calls=len(trace), applied_commands=len(applied),
notebook_writes=sum(c['tool'] == 'replace_notebook' for c in commands),
gameplay_actions=sum(c['tool'] == 'execute_action' for c in commands),
rejected_commands=sum(row['error'] is not None for row in applied), policy_failure=policy_failure,
budget_exhausted=not transition.done and len(trace) == call_budget and policy_failure is None,
outcome=transition.outcome, completed=transition.done, reached_phases=sorted(phases),
phase_call_counts=dict(phase_calls), context_resets=resets,
boarded=list(transition.observation.board.get('boarded', [])), initial_charge=initial_charge,
remaining_charge=transition.observation.charge, charge_consumed=initial_charge-transition.observation.charge,
notebook_capacity_bytes=notebook_bytes, peak_notebook_utf8_bytes=peak_note,
peak_declared_request_utf8_bytes=peak_request,
declared_request_limit_bytes=32768, declared_request_within_limit=peak_request <= 32768,
declared_request_envelope=dict(model='offline-protocol-fixture', max_tokens=2048,
messages=['system', 'user'], tools=DECLARED_TOOLS,
measurement='UTF-8 bytes of json.dumps; offline envelope, not provider traffic or token count'),
notebook_classes_by_decision=dict(note_classes),
notebook_scoring='Prefix v4 is unsupported by the stock audit, not zero retention. Late v3 is scorable.',
circuits=circuits, successful_commissioning_tests=sum(c['completed_tests'] for c in circuits.values()),
dependency_metrics=metrics, private_replay_verified=replay['verified'], engine_source_hash=source_hash(),
policy_inputs='current actor_view JSON only; fresh controller and CausalReference per decision',
limitations=['Known-case scripted feasibility does not establish learnability, generalization, optimality, '
'Beam performance, or a causal notebook-erasure effect.'])
summary['passed'] = (transition.outcome == 'escaped' and summary['boarded'] == ['Ada', 'Bo', 'Cy']
and phases == set(range(1, 8)) and resets == 1 and replay['verified']
and not policy_failure and not summary['rejected_commands']
and summary['declared_request_within_limit']
and metrics['completed_units'] == metrics['predefined_denominator']
and all(c == dict(status='live', completed_tests=3) for c in circuits.values()))
return dict(summary=summary, public_command_trace=trace)
|