FatUnc6969's picture
Publish reviewed Borderland v8 research artifact
2719bf4 verified
Raw History Blame Contribute Delete
9.43 kB
from collections import Counter
import json
from pathlib import Path
from tempfile import TemporaryDirectory
from borderland import Campaign
from borderland.engine import parse_action
from borderland.replay import source_hash, verify_replay
from borderland.worlds import candidate
from borderland_verify.streaming_reference import choose
from .causal_dependencies import CausalDependencyAudit, extract_notebook
from .context import ContextSelector, actor_view
DECLARED_TOOLS = [{"type": "function", "function": {
"name": "execute_action", "description": "Execute one currently legal campaign action.",
"parameters": {"type": "object", "properties": {
"tag": {"type": "string"}, "arguments": {"type": "object"}},
"required": ["tag", "arguments"], "additionalProperties": False}}},
{"type": "function", "function": {"name": "replace_notebook",
"description": "Replace your retained notebook. This is your memory across the Blackout conversation reset.",
"parameters": {"type": "object", "properties": {"text": {"type": "string"}},
"required": ["text"], "additionalProperties": False}}}]
def declared_request(public):
context = ContextSelector(condition='bounded_notebook',
notebook_chars=public['observation']['notebook']['capacity_bytes'])
context.observe(public)
return dict(model='offline-protocol-fixture', messages=context.selected(),
tools=DECLARED_TOOLS, tool_choice='auto', max_tokens=2048)
def run_streaming(seed=16, archive_workload='standard', call_budget=200, notebook_bytes=768):
if type(seed) is not int or seed < 0:
raise ValueError('seed must be a nonnegative integer')
if archive_workload not in ('compact', 'standard', 'extended'):
raise ValueError('unsupported Archive workload')
if type(call_budget) is not int or call_budget < 0:
raise ValueError('call budget must be a nonnegative integer')
if type(notebook_bytes) is not int or notebook_bytes < 0:
raise ValueError('notebook capacity must be a nonnegative integer')
campaign = Campaign(candidate(seed, generation_version=3, archive_workload=archive_workload,
notebook_bytes=notebook_bytes, memory_profile='commissioning'))
transition = campaign.observe()
audit = CausalDependencyAudit(campaign._world)
initial_charge = transition.observation.charge
phases = {int(transition.observation.phase)}
phase_calls, note_classes = Counter(), Counter()
trace = []
resets = peak_note = peak_request = 0
circuits = {}
policy_failure = None
for index in range(1, call_budget + 1):
if transition.done:
break
public = actor_view(transition)
phase = public['observation']['phase']
request = declared_request(public)
payload_bytes = len(json.dumps(request).encode('utf-8'))
peak_request = max(peak_request, payload_bytes)
note = extract_notebook(public['observation']['notebook']['text'])
note_class = ('empty_supported' if not public['observation']['notebook']['text'] else
'supported_v3') if note.scorable else note.reason
note_classes[note_class] += 1
phase_calls[phase] += 1
audit.observe(public, index, context=request['messages'])
row = dict(call=index, phase_before=phase, command=None, applied=False,
declared_request_utf8_bytes=payload_bytes, notebook_class=note_class)
trace.append(row)
try:
command = choose(public)
except (AssertionError, ValueError, KeyError, TypeError, IndexError) as error:
policy_failure = type(error).__name__
row['policy_failure'] = policy_failure
break
row['command'] = command
if command['tool'] == 'replace_notebook':
after = campaign.replace_notebook(command['arguments']['text'],
expected_revision=transition.revision)
elif command['tool'] == 'execute_action':
after = campaign.step(command['arguments'], expected_revision=transition.revision)
parsed, parse_error = parse_action(command['arguments'], generation_version=3)
canonical = parsed.to_dict() if not parse_error else command['arguments']
row['canonical_action'] = canonical
audit.record_decision(public, canonical, actor_view(after), index)
else:
policy_failure = 'unsupported_tool'
row['policy_failure'] = policy_failure
break
transition = after
peak_note = max(peak_note, len(after.observation.notebook['text'].encode('utf-8')))
row.update(applied=True, error=after.error.to_dict() if after.error else None,
phase_after=int(after.observation.phase), charge_after=after.observation.charge,
notebook_utf8_bytes=len(after.observation.notebook['text'].encode('utf-8')),
context_reset=bool(after.context_reset), outcome=after.outcome)
phases.add(int(after.observation.phase))
resets += int(after.context_reset)
for circuit in after.observation.board.get('circuits', []):
circuits[circuit['id']] = dict(status=circuit['status'],
completed_tests=circuit['completed_tests'])
audit.observe(actor_view(after), index + 1)
audit.observe(actor_view(transition), len(trace) + 1)
report = audit.report()
with TemporaryDirectory() as directory:
replay_path = Path(directory) / 'streaming.private.jsonl'
campaign.save_replay(replay_path, 'private')
replay = verify_replay(replay_path)
metrics = {key: report[key] for key in ('metric_kind', 'causal_reliance', 'predefined_denominator',
'eligible_units', 'dependencies_matched', 'completed_units', 'dependency_agreement_rate')}
dependencies = report['dependencies']
metrics['all_first_choices_publicly_exposed'] = all(
row['first_choice'] and row['first_choice']['evidence_exposed'] for row in dependencies)
metrics['all_first_choices_scorable'] = all(
row['first_choice'] and row['first_choice']['notebook_scorable'] for row in dependencies)
metrics['censoring'] = dict(Counter(row['censoring'] for row in dependencies if row['censoring']))
applied = [row for row in trace if row['applied']]
commands = [row['command'] for row in applied]
summary = dict(
scope='Scripted bounded public control on an explicit known world. No learned control or Beam claim.',
seed=seed, archive_workload=archive_workload, generation_version=3, memory_profile='commissioning',
call_budget=call_budget, calls=len(trace), applied_commands=len(applied),
notebook_writes=sum(c['tool'] == 'replace_notebook' for c in commands),
gameplay_actions=sum(c['tool'] == 'execute_action' for c in commands),
rejected_commands=sum(row['error'] is not None for row in applied), policy_failure=policy_failure,
budget_exhausted=not transition.done and len(trace) == call_budget and policy_failure is None,
outcome=transition.outcome, completed=transition.done, reached_phases=sorted(phases),
phase_call_counts=dict(phase_calls), context_resets=resets,
boarded=list(transition.observation.board.get('boarded', [])), initial_charge=initial_charge,
remaining_charge=transition.observation.charge, charge_consumed=initial_charge-transition.observation.charge,
notebook_capacity_bytes=notebook_bytes, peak_notebook_utf8_bytes=peak_note,
peak_declared_request_utf8_bytes=peak_request,
declared_request_limit_bytes=32768, declared_request_within_limit=peak_request <= 32768,
declared_request_envelope=dict(model='offline-protocol-fixture', max_tokens=2048,
messages=['system', 'user'], tools=DECLARED_TOOLS,
measurement='UTF-8 bytes of json.dumps; offline envelope, not provider traffic or token count'),
notebook_classes_by_decision=dict(note_classes),
notebook_scoring='Prefix v4 is unsupported by the stock audit, not zero retention. Late v3 is scorable.',
circuits=circuits, successful_commissioning_tests=sum(c['completed_tests'] for c in circuits.values()),
dependency_metrics=metrics, private_replay_verified=replay['verified'], engine_source_hash=source_hash(),
policy_inputs='current actor_view JSON only; fresh controller and CausalReference per decision',
limitations=['Known-case scripted feasibility does not establish learnability, generalization, optimality, '
'Beam performance, or a causal notebook-erasure effect.'])
summary['passed'] = (transition.outcome == 'escaped' and summary['boarded'] == ['Ada', 'Bo', 'Cy']
and phases == set(range(1, 8)) and resets == 1 and replay['verified']
and not policy_failure and not summary['rejected_commands']
and summary['declared_request_within_limit']
and metrics['completed_units'] == metrics['predefined_denominator']
and all(c == dict(status='live', completed_tests=3) for c in circuits.values()))
return dict(summary=summary, public_command_trace=trace)