Spaces:
Running
Running
Download python/borderland_eval/streaming.py from FatUnc6969/borderland-rl: direct link, hf CLI and curl.
- Browser
- Download file 9.43 kB
-
https://huggingface.co/spaces/FatUnc6969/borderland-rl/resolve/main/python/borderland_eval/streaming.py
- Command line
-
hf download hf://spaces/FatUnc6969/borderland-rl/python/borderland_eval/streaming.py
-
curl -L -o streaming.py https://huggingface.co/spaces/FatUnc6969/borderland-rl/resolve/main/python/borderland_eval/streaming.py
9.43 kB
| from collections import Counter | |
| import json | |
| from pathlib import Path | |
| from tempfile import TemporaryDirectory | |
| from borderland import Campaign | |
| from borderland.engine import parse_action | |
| from borderland.replay import source_hash, verify_replay | |
| from borderland.worlds import candidate | |
| from borderland_verify.streaming_reference import choose | |
| from .causal_dependencies import CausalDependencyAudit, extract_notebook | |
| from .context import ContextSelector, actor_view | |
| DECLARED_TOOLS = [{"type": "function", "function": { | |
| "name": "execute_action", "description": "Execute one currently legal campaign action.", | |
| "parameters": {"type": "object", "properties": { | |
| "tag": {"type": "string"}, "arguments": {"type": "object"}}, | |
| "required": ["tag", "arguments"], "additionalProperties": False}}}, | |
| {"type": "function", "function": {"name": "replace_notebook", | |
| "description": "Replace your retained notebook. This is your memory across the Blackout conversation reset.", | |
| "parameters": {"type": "object", "properties": {"text": {"type": "string"}}, | |
| "required": ["text"], "additionalProperties": False}}}] | |
| def declared_request(public): | |
| context = ContextSelector(condition='bounded_notebook', | |
| notebook_chars=public['observation']['notebook']['capacity_bytes']) | |
| context.observe(public) | |
| return dict(model='offline-protocol-fixture', messages=context.selected(), | |
| tools=DECLARED_TOOLS, tool_choice='auto', max_tokens=2048) | |
| def run_streaming(seed=16, archive_workload='standard', call_budget=200, notebook_bytes=768): | |
| if type(seed) is not int or seed < 0: | |
| raise ValueError('seed must be a nonnegative integer') | |
| if archive_workload not in ('compact', 'standard', 'extended'): | |
| raise ValueError('unsupported Archive workload') | |
| if type(call_budget) is not int or call_budget < 0: | |
| raise ValueError('call budget must be a nonnegative integer') | |
| if type(notebook_bytes) is not int or notebook_bytes < 0: | |
| raise ValueError('notebook capacity must be a nonnegative integer') | |
| campaign = Campaign(candidate(seed, generation_version=3, archive_workload=archive_workload, | |
| notebook_bytes=notebook_bytes, memory_profile='commissioning')) | |
| transition = campaign.observe() | |
| audit = CausalDependencyAudit(campaign._world) | |
| initial_charge = transition.observation.charge | |
| phases = {int(transition.observation.phase)} | |
| phase_calls, note_classes = Counter(), Counter() | |
| trace = [] | |
| resets = peak_note = peak_request = 0 | |
| circuits = {} | |
| policy_failure = None | |
| for index in range(1, call_budget + 1): | |
| if transition.done: | |
| break | |
| public = actor_view(transition) | |
| phase = public['observation']['phase'] | |
| request = declared_request(public) | |
| payload_bytes = len(json.dumps(request).encode('utf-8')) | |
| peak_request = max(peak_request, payload_bytes) | |
| note = extract_notebook(public['observation']['notebook']['text']) | |
| note_class = ('empty_supported' if not public['observation']['notebook']['text'] else | |
| 'supported_v3') if note.scorable else note.reason | |
| note_classes[note_class] += 1 | |
| phase_calls[phase] += 1 | |
| audit.observe(public, index, context=request['messages']) | |
| row = dict(call=index, phase_before=phase, command=None, applied=False, | |
| declared_request_utf8_bytes=payload_bytes, notebook_class=note_class) | |
| trace.append(row) | |
| try: | |
| command = choose(public) | |
| except (AssertionError, ValueError, KeyError, TypeError, IndexError) as error: | |
| policy_failure = type(error).__name__ | |
| row['policy_failure'] = policy_failure | |
| break | |
| row['command'] = command | |
| if command['tool'] == 'replace_notebook': | |
| after = campaign.replace_notebook(command['arguments']['text'], | |
| expected_revision=transition.revision) | |
| elif command['tool'] == 'execute_action': | |
| after = campaign.step(command['arguments'], expected_revision=transition.revision) | |
| parsed, parse_error = parse_action(command['arguments'], generation_version=3) | |
| canonical = parsed.to_dict() if not parse_error else command['arguments'] | |
| row['canonical_action'] = canonical | |
| audit.record_decision(public, canonical, actor_view(after), index) | |
| else: | |
| policy_failure = 'unsupported_tool' | |
| row['policy_failure'] = policy_failure | |
| break | |
| transition = after | |
| peak_note = max(peak_note, len(after.observation.notebook['text'].encode('utf-8'))) | |
| row.update(applied=True, error=after.error.to_dict() if after.error else None, | |
| phase_after=int(after.observation.phase), charge_after=after.observation.charge, | |
| notebook_utf8_bytes=len(after.observation.notebook['text'].encode('utf-8')), | |
| context_reset=bool(after.context_reset), outcome=after.outcome) | |
| phases.add(int(after.observation.phase)) | |
| resets += int(after.context_reset) | |
| for circuit in after.observation.board.get('circuits', []): | |
| circuits[circuit['id']] = dict(status=circuit['status'], | |
| completed_tests=circuit['completed_tests']) | |
| audit.observe(actor_view(after), index + 1) | |
| audit.observe(actor_view(transition), len(trace) + 1) | |
| report = audit.report() | |
| with TemporaryDirectory() as directory: | |
| replay_path = Path(directory) / 'streaming.private.jsonl' | |
| campaign.save_replay(replay_path, 'private') | |
| replay = verify_replay(replay_path) | |
| metrics = {key: report[key] for key in ('metric_kind', 'causal_reliance', 'predefined_denominator', | |
| 'eligible_units', 'dependencies_matched', 'completed_units', 'dependency_agreement_rate')} | |
| dependencies = report['dependencies'] | |
| metrics['all_first_choices_publicly_exposed'] = all( | |
| row['first_choice'] and row['first_choice']['evidence_exposed'] for row in dependencies) | |
| metrics['all_first_choices_scorable'] = all( | |
| row['first_choice'] and row['first_choice']['notebook_scorable'] for row in dependencies) | |
| metrics['censoring'] = dict(Counter(row['censoring'] for row in dependencies if row['censoring'])) | |
| applied = [row for row in trace if row['applied']] | |
| commands = [row['command'] for row in applied] | |
| summary = dict( | |
| scope='Scripted bounded public control on an explicit known world. No learned control or Beam claim.', | |
| seed=seed, archive_workload=archive_workload, generation_version=3, memory_profile='commissioning', | |
| call_budget=call_budget, calls=len(trace), applied_commands=len(applied), | |
| notebook_writes=sum(c['tool'] == 'replace_notebook' for c in commands), | |
| gameplay_actions=sum(c['tool'] == 'execute_action' for c in commands), | |
| rejected_commands=sum(row['error'] is not None for row in applied), policy_failure=policy_failure, | |
| budget_exhausted=not transition.done and len(trace) == call_budget and policy_failure is None, | |
| outcome=transition.outcome, completed=transition.done, reached_phases=sorted(phases), | |
| phase_call_counts=dict(phase_calls), context_resets=resets, | |
| boarded=list(transition.observation.board.get('boarded', [])), initial_charge=initial_charge, | |
| remaining_charge=transition.observation.charge, charge_consumed=initial_charge-transition.observation.charge, | |
| notebook_capacity_bytes=notebook_bytes, peak_notebook_utf8_bytes=peak_note, | |
| peak_declared_request_utf8_bytes=peak_request, | |
| declared_request_limit_bytes=32768, declared_request_within_limit=peak_request <= 32768, | |
| declared_request_envelope=dict(model='offline-protocol-fixture', max_tokens=2048, | |
| messages=['system', 'user'], tools=DECLARED_TOOLS, | |
| measurement='UTF-8 bytes of json.dumps; offline envelope, not provider traffic or token count'), | |
| notebook_classes_by_decision=dict(note_classes), | |
| notebook_scoring='Prefix v4 is unsupported by the stock audit, not zero retention. Late v3 is scorable.', | |
| circuits=circuits, successful_commissioning_tests=sum(c['completed_tests'] for c in circuits.values()), | |
| dependency_metrics=metrics, private_replay_verified=replay['verified'], engine_source_hash=source_hash(), | |
| policy_inputs='current actor_view JSON only; fresh controller and CausalReference per decision', | |
| limitations=['Known-case scripted feasibility does not establish learnability, generalization, optimality, ' | |
| 'Beam performance, or a causal notebook-erasure effect.']) | |
| summary['passed'] = (transition.outcome == 'escaped' and summary['boarded'] == ['Ada', 'Bo', 'Cy'] | |
| and phases == set(range(1, 8)) and resets == 1 and replay['verified'] | |
| and not policy_failure and not summary['rejected_commands'] | |
| and summary['declared_request_within_limit'] | |
| and metrics['completed_units'] == metrics['predefined_denominator'] | |
| and all(c == dict(status='live', completed_tests=3) for c in circuits.values())) | |
| return dict(summary=summary, public_command_trace=trace) | |