Spaces:
Running
Running
Download python/borderland_eval/dependencies.py from FatUnc6969/borderland-rl: direct link, hf CLI and curl.
- Browser
- Download file 25 kB
-
https://huggingface.co/spaces/FatUnc6969/borderland-rl/resolve/main/python/borderland_eval/dependencies.py
- Command line
-
hf download hf://spaces/FatUnc6969/borderland-rl/python/borderland_eval/dependencies.py
-
curl -L -o dependencies.py https://huggingface.co/spaces/FatUnc6969/borderland-rl/resolve/main/python/borderland_eval/dependencies.py
25 kB
| """Operator-only dependency accounting. Never attach this data to policy inputs. | |
| Notebook scoring accepts an empty note or a JSON object containing LegalReference's | |
| bus, valve, order, witness, seal, track and record_<subject> fields. It also accepts | |
| scripted_notebook's experimental_bus and authenticated_records mapping. Other prose | |
| or JSON schemas are unscorable. Equality is evidence of retention or action agreement, | |
| not evidence of a policy's private reasoning or causal reliance on its notebook. | |
| """ | |
| from collections import Counter | |
| from copy import deepcopy | |
| from dataclasses import asdict, dataclass, field | |
| from itertools import permutations | |
| import json | |
| from .context import wire | |
| class Dependency: | |
| name: str | |
| source_phase: int | |
| target_phase: int | |
| facts: tuple[str, ...] | |
| decisions: tuple[str, ...] | |
| required: tuple[str, ...] | |
| class EvidenceAcquisition: | |
| step: int | |
| phase: int | |
| channel: str | |
| DESIGNATED_CHANNELS = {"rule_sensor", "archive_current_record"} | |
| DEPENDENCIES = ( | |
| Dependency("wiring_to_bus", 1, 6, ("bus",), ("connect",), ("alpha", "beta", "gamma")), | |
| Dependency("wiring_to_valve", 1, 7, ("valve",), ("open_valve",), ("valve",)), | |
| Dependency("restoration_to_order", 3, 6, ("order",), ("energize",), ("alpha", "beta", "gamma")), | |
| Dependency("authorization_to_witness", 3, 5, ("witness",), ("escort",), ("escort",)), | |
| Dependency("authorization_to_cross_check", 3, 5, ("record_authorization",), ("cross_check",), ("check",)), | |
| Dependency("authorization_to_checkpoint", 3, 5, ("seal",), ("test_checkpoint",), ("checkpoint",)), | |
| Dependency("authorization_to_exit_seal", 3, 7, ("seal",), ("arm_exit",), ("seal",)), | |
| Dependency("egress_to_track", 3, 7, ("track",), ("align_track", "launch"), ("launch",)), | |
| ) | |
| def _record_facts(record): | |
| if not isinstance(record, dict) or record.get("authenticated") is not True: | |
| return {} | |
| subject, statement = record.get("subject"), record.get("statement") | |
| if not isinstance(subject, str) or subject not in {"restoration", "authorization", "egress"} or not isinstance(statement, str): | |
| return {} | |
| facts = {"record_" + subject: record.get("id")} | |
| if subject == "restoration": | |
| order = statement.split(">") | |
| if sorted(order) == ["alpha", "beta", "gamma"]: | |
| facts["order"] = order | |
| elif subject == "authorization" and statement.count("/") == 1: | |
| facts["witness"], facts["seal"] = statement.split("/") | |
| elif subject == "egress" and statement in {"0", "1", "2"}: | |
| facts["track"] = int(statement) | |
| return facts | |
| def _valid_fact(key, value): | |
| if key in {"bus", "valve"}: | |
| return value in ("west", "east") | |
| if key == "order": | |
| return isinstance(value, list) and len(value) == 3 and all(isinstance(x, str) for x in value) and sorted(value) == ["alpha", "beta", "gamma"] | |
| if key == "witness": | |
| return value in ("Iris", "Juno", "Kade") | |
| if key == "seal": | |
| return value in ("amber", "cobalt", "silver") | |
| if key == "track": | |
| return type(value) is int and value in (0, 1, 2) | |
| return key in {"record_restoration", "record_authorization", "record_egress"} and isinstance(value, str) and bool(value) | |
| class NotebookFacts: | |
| scorable: bool | |
| values: dict = field(default_factory=dict) | |
| reason: str | None = None | |
| def retains(self, key, expected): | |
| values = self.values.get(key, []) | |
| return bool(values) and all(value == expected for value in values) | |
| def extract_notebook(text): | |
| """Parse only explicit supported facts. Conflicting duplicates are not retention.""" | |
| if text == "": | |
| return NotebookFacts(True) | |
| try: | |
| parsed = json.loads(text) | |
| except (ValueError, TypeError): | |
| return NotebookFacts(False, reason="unsupported_prose_or_invalid_json") | |
| if not isinstance(parsed, dict): | |
| return NotebookFacts(False, reason="unsupported_json_schema") | |
| values = {} | |
| recognized = not parsed | |
| malformed = False | |
| def add(key, value): | |
| nonlocal malformed | |
| if _valid_fact(key, value): | |
| values.setdefault(key, []).append(value) | |
| else: | |
| malformed = True | |
| for key in ("bus", "valve", "order", "witness", "seal", "track", | |
| "record_restoration", "record_authorization", "record_egress"): | |
| if key in parsed: | |
| recognized = True | |
| add(key, parsed[key]) | |
| if "experimental_bus" in parsed: | |
| recognized = True | |
| add("bus", parsed["experimental_bus"]) | |
| add("valve", parsed["experimental_bus"]) | |
| if "authenticated_records" in parsed: | |
| recognized = True | |
| records = parsed["authenticated_records"] | |
| if isinstance(records, dict): | |
| for subject, record in records.items(): | |
| facts = _record_facts(record) | |
| if not facts or record.get("subject") != subject: | |
| malformed = True | |
| for key, value in facts.items(): | |
| add(key, value) | |
| else: | |
| malformed = True | |
| scorable = recognized and not malformed | |
| return NotebookFacts(scorable, values, None if scorable else "unsupported_or_malformed_fact_schema") | |
| class DependencyAudit: | |
| """Expected values score agreement; public boards and feedback establish exposure.""" | |
| def __init__(self, world): | |
| side = ("west", "east")[world.polarity] | |
| self.expected = dict(bus=side, valve=side, order=list(world.restoration_order), | |
| witness=world.current_witness, seal=world.current_seal, | |
| track=world.correct_track) | |
| current = {(record.id, record.issued_at) for record in world.archive.authoritative()} | |
| self.current_revisions = current | |
| self.stale_values = {} | |
| for record in world.archive.revisions(): | |
| facts = _record_facts(wire(record)) | |
| if (record.id,record.issued_at) in current: | |
| self.expected.update(facts) | |
| else: | |
| for key,value in facts.items(): | |
| self.stale_values.setdefault(key,[]).append(value) | |
| self.discovered = {} | |
| self.evidence_provenance = {} | |
| self._public_values = {} | |
| self._order_candidates = set(permutations(("alpha", "beta", "gamma"))) | |
| self._seal_candidates = {"amber", "cobalt", "silver"} | |
| self.reached_phases = set() | |
| self.decisions = [] | |
| self.notebook_samples = [] | |
| self._last_sample = None | |
| def _acquire(self, facts, step, phase, channel): | |
| for key, value in facts.items(): | |
| if not _valid_fact(key, value): | |
| continue | |
| self._public_values[key] = deepcopy(value) | |
| if key in self.expected and value == self.expected[key]: | |
| self.discovered.setdefault(key, step) | |
| acquisitions = self.evidence_provenance.setdefault(key, []) | |
| if not any(item.channel == channel for item in acquisitions): | |
| acquisitions.append(EvidenceAcquisition(step, phase, channel)) | |
| def _provenance(self, keys): | |
| return {key: [asdict(item) for item in self.evidence_provenance.get(key, [])] for key in keys} | |
| def _observe_board(self, observation, step): | |
| phase, board = int(observation["phase"]), observation["board"] | |
| if phase == 1 and type(board.get("live_sensor")) is int and board["live_sensor"] in (0, 1) and type(board.get("latch")) is int: | |
| side = ("west", "east")[board["live_sensor"] ^ (board["latch"] & 1)] | |
| self._acquire(dict(bus=side, valve=side), step, phase, "rule_sensor") | |
| if phase == 1 and board.get("fixture") == "gate" and board.get("gate_open") is True and board.get("latch") in (2, 3): | |
| side = ("west", "east")[board["latch"] - 2] | |
| self._acquire(dict(bus=side, valve=side), step, phase, "rule_gate_feedback") | |
| if phase == 3: | |
| for record in board.get("records", []): | |
| if (record.get("id"),record.get("issued_at")) in self.current_revisions: | |
| self._acquire(_record_facts(record), step, phase, "archive_current_record") | |
| if phase == 5: | |
| seal = board.get("checkpoint_test") | |
| if seal in {"amber", "cobalt", "silver"}: | |
| self._acquire(dict(seal=seal), step, phase, "checkpoint_feedback") | |
| claim = board.get("claim") | |
| if isinstance(claim, dict): | |
| for check in board.get("checks", []): | |
| if check.get("status") == "verified" and check.get("claim") == claim.get("id"): | |
| witness, seal = claim.get("witness"), claim.get("seal") | |
| if _valid_fact("witness", witness) and _valid_fact("seal", seal) and claim.get("statement") == witness + "/" + seal: | |
| self._acquire(dict(witness=witness, seal=seal, record_authorization=check.get("record")), | |
| step, phase, "witness_verification") | |
| if phase == 6: | |
| live = [circuit for circuit in board.get("circuits", []) if circuit.get("status") == "live"] | |
| for circuit in live: | |
| side = circuit.get("bus") | |
| if _valid_fact("bus", side): | |
| self._acquire(dict(bus=side, valve=side), step, phase, "circuit_feedback") | |
| def _observe_feedback(self, public, command, result, step): | |
| if result.get("error") or public["observation"]["phase"] != result["observation"]["phase"]: | |
| return | |
| observation, after = public["observation"], result["observation"] | |
| phase, board = int(observation["phase"]), observation["board"] | |
| tag, args = command.get("tag"), command.get("arguments", {}) | |
| self._observe_board(after, step) | |
| if phase == 5 and tag == "test_checkpoint": | |
| if after["board"].get("checkpoint_test") == "denied": | |
| self._seal_candidates.discard(args.get("seal_id")) | |
| if len(self._seal_candidates) == 1: | |
| self._acquire(dict(seal=next(iter(self._seal_candidates))), step, phase, "checkpoint_elimination") | |
| if phase == 6 and tag == "energize": | |
| live = {circuit["id"] for circuit in board.get("circuits", []) if circuit.get("status") == "live"} | |
| circuit = next((item for item in after["board"].get("circuits", []) if item.get("id") == args.get("circuit")), {}) | |
| status, side = circuit.get("status"), circuit.get("bus") | |
| if len(live) < 3 and (status == "live" or status == "tripped" and side == self._public_values.get("bus")): | |
| self._order_candidates = {order for order in self._order_candidates | |
| if set(order[:len(live)]) == live | |
| and (order[len(live)] == args.get("circuit")) == (status == "live")} | |
| if len(self._order_candidates) == 1: | |
| self._acquire(dict(order=list(next(iter(self._order_candidates)))), step, phase, "circuit_order_feedback") | |
| if phase == 7 and tag == "open_valve" and args.get("side") in ("west", "east"): | |
| side = args["side"] | |
| if after["board"].get("pressure_vented") is True: | |
| side = "east" if side == "west" else "west" | |
| self._acquire(dict(bus=side, valve=side), step, phase, "valve_feedback") | |
| def observe(self, transition, step, context=None, *, exposure=True, dispatched=False): | |
| public = wire(transition) | |
| observation = public["observation"] | |
| phase = int(observation["phase"]) | |
| self.reached_phases.add(phase) | |
| if exposure: | |
| self._observe_board(observation, step) | |
| if dispatched and context is not None: | |
| previous, calls = None, {} | |
| for message in context: | |
| if message["role"] == "assistant": | |
| calls = {call["id"]: json.loads(call["function"]["arguments"]) | |
| for call in message["tool_calls"] | |
| if call["function"]["name"] == "execute_action"} | |
| elif message["role"] in ("user", "tool"): | |
| value = json.loads(message["content"]) | |
| if isinstance(value, dict) and "observation" in value: | |
| call = calls.get(message.get("tool_call_id")) | |
| if call is not None and previous is not None: | |
| self._observe_feedback(previous, call, value, step) | |
| self._observe_board(value["observation"], step) | |
| previous = value | |
| text = observation["notebook"]["text"] | |
| signature = (text, tuple(sorted(self.discovered))) | |
| if signature != self._last_sample or public.get("context_reset"): | |
| sample = self.notebook_quality(text) | |
| sample.update(step=step, phase=phase, context_reset=bool(public.get("context_reset"))) | |
| self.notebook_samples.append(sample) | |
| self._last_sample = signature | |
| def notebook_quality(self, text): | |
| note = extract_notebook(text) | |
| discovered = sorted(self.discovered) | |
| retained = [key for key in discovered if note.retains(key, self.expected[key])] if note.scorable else [] | |
| stale, incorrect, duplicate = [], [], [] | |
| if note.scorable: | |
| for key, values in note.values.items(): | |
| if any(value in values[:index] for index, value in enumerate(values)): | |
| duplicate.append(key) | |
| for value in values: | |
| if key in self.expected and value != self.expected[key]: | |
| destination = stale if value in self.stale_values.get(key, []) else incorrect | |
| if key not in destination: | |
| destination.append(key) | |
| return dict(scorable=note.scorable, reason=note.reason, discovered_facts=discovered, | |
| discovered_denominator=len(discovered), retained_facts=retained, | |
| retained_count=len(retained) if note.scorable else None, | |
| retention_rate=len(retained) / len(discovered) if note.scorable and discovered else None, | |
| stale_fact_keys=stale if note.scorable else None, | |
| incorrect_fact_keys=incorrect if note.scorable else None, | |
| redundant_representation_keys=duplicate if note.scorable else None) | |
| def record_decision(self, before, action, after, step, *, expose_result=True): | |
| public, result, command = wire(before), wire(after), wire(action) | |
| observation = public["observation"] | |
| phase, board = int(observation["phase"]), observation["board"] | |
| tag = command.get("tag") if isinstance(command, dict) else None | |
| args = command.get("arguments", {}) if isinstance(command, dict) else {} | |
| if not isinstance(args, dict): | |
| tag, args = None, {} | |
| note = extract_notebook(observation["notebook"]["text"]) | |
| evaluations = [] | |
| for dependency in DEPENDENCIES: | |
| if phase != dependency.target_phase or tag not in dependency.decisions: | |
| continue | |
| if tag == "escort" and args.get("neighbor") != 4: | |
| continue | |
| correct, unit = self._agreement(dependency.name, tag, args, board) | |
| if result.get("error"): | |
| correct = False | |
| discovered = all(key in self.discovered for key in dependency.facts) | |
| recorded = note.scorable and all(note.retains(key, self.expected[key]) for key in dependency.facts) | |
| category = "correct" if correct else ( | |
| "evidence_undiscovered" if not discovered else | |
| "unscorable" if not note.scorable else | |
| "discovered_but_not_recorded" if not recorded else "recorded_but_action_incorrect") | |
| evaluations.append(dict(dependency=dependency.name, correct=correct, unit=unit, | |
| evidence_mode="state_agreement" if tag == "launch" else "action_agreement", | |
| evidence_discovered=discovered, notebook_scorable=note.scorable, | |
| designated_source_exposed=all(any(item.channel in DESIGNATED_CHANNELS | |
| for item in self.evidence_provenance.get(key, [])) for key in dependency.facts), | |
| evidence_provenance=self._provenance(dependency.facts), | |
| evidence_recorded=recorded if note.scorable else None, category=category)) | |
| row = dict(step=step, phase=phase, action=deepcopy(command), | |
| decision_tags=[item["dependency"] for item in evaluations], evaluations=evaluations) | |
| self.decisions.append(row) | |
| if tag is not None and expose_result: | |
| self._observe_feedback(public, command, result, step) | |
| return row | |
| def _agreement(self, name, tag, args, board): | |
| if name == "wiring_to_bus": | |
| return args.get("bus") == self.expected["bus"], args.get("circuit") | |
| if name == "wiring_to_valve": | |
| return args.get("side") == self.expected["valve"], "valve" | |
| if name == "restoration_to_order": | |
| live = {c["id"] for c in board.get("circuits", []) if c["status"] == "live"} | |
| next_circuit = next((cid for cid in self.expected["order"] if cid not in live), None) | |
| return args.get("circuit") == next_circuit, args.get("circuit") | |
| if name == "authorization_to_witness": | |
| return args.get("witness") == self.expected["witness"], "escort" | |
| if name == "authorization_to_cross_check": | |
| return args.get("record_id") == self.expected["record_authorization"], "check" | |
| if name in {"authorization_to_checkpoint", "authorization_to_exit_seal"}: | |
| return args.get("seal_id") == self.expected["seal"], "checkpoint" if tag == "test_checkpoint" else "seal" | |
| return (args.get("position") if tag == "align_track" else board.get("track")) == self.expected["track"], "launch" if tag == "launch" else "alignment" | |
| def report(self): | |
| rows = [] | |
| for dependency in DEPENDENCIES: | |
| attempts = [evaluation for row in self.decisions for evaluation in row["evaluations"] | |
| if evaluation["dependency"] == dependency.name] | |
| successful_units = {item["unit"] for item in attempts if item["correct"]} | |
| complete = set(dependency.required) <= successful_units | |
| correct = complete and all(item["correct"] for item in attempts) | |
| reached = dependency.target_phase in self.reached_phases | |
| failures = Counter(item["category"] for item in attempts if not item["correct"]) | |
| if correct: | |
| attribution = "correct" | |
| elif not reached: | |
| attribution = "target_unreached" | |
| elif not attempts: | |
| attribution = "no_decision_opportunity" | |
| elif failures: | |
| attribution = next(item["category"] for item in attempts if not item["correct"]) | |
| else: | |
| attribution = "required_units_incomplete" | |
| rows.append(dict(dependency=dependency.name, source_phase=dependency.source_phase, | |
| target_phase=dependency.target_phase, facts=list(dependency.facts), | |
| target_reached=reached, opportunities=len(attempts), | |
| successful_opportunities=sum(item["correct"] for item in attempts), | |
| required_units=list(dependency.required), completed=complete, | |
| correct=correct, unreached=not reached, | |
| attribution=attribution, failure_categories=dict(failures))) | |
| opportunities = sum(row["opportunities"] for row in rows) | |
| successful = sum(row["successful_opportunities"] for row in rows) | |
| state_checks = [item for row in self.decisions for item in row["evaluations"] | |
| if item["evidence_mode"] == "state_agreement"] | |
| return dict(metric_kind="action_state_agreement", causal_reliance="not_identified", | |
| attribution_semantics="Legacy discovered fields mean public evidence availability, not internal discovery. Designated source exposure and alternative feedback are recorded separately.", | |
| scope="Eight declared Rule chamber and Archive cross-room dependencies only; observed agreement does not identify causal use of a fact or its source.", | |
| coverage_rule="All declared required units must be observed correct, with no observed mismatch for that dependency.", | |
| predefined_denominator=len(rows), dependencies_matched=sum(row["correct"] for row in rows), | |
| dependency_agreement_rate=sum(row["correct"] for row in rows) / len(rows), | |
| opportunities=opportunities, successful_opportunities=successful, | |
| opportunity_correctness=successful / opportunities if opportunities else None, | |
| state_agreement_checks=len(state_checks), | |
| successful_state_agreement_checks=sum(item["correct"] for item in state_checks), | |
| unreached_dependencies=[row["dependency"] for row in rows if row["unreached"]], | |
| dependencies=rows, decisions=deepcopy(self.decisions), | |
| evidence_provenance=self._provenance(sorted(self.evidence_provenance)), | |
| notebook_quality=dict(samples=deepcopy(self.notebook_samples), | |
| latest=deepcopy(self.notebook_samples[-1]) if self.notebook_samples else None), | |
| limitations=["Correct choices can occur without evidence discovery or notebook retention.", | |
| "Evidence provenance is a conservative extraction of declared public proof channels, not a complete inference model or a claim about internal knowledge.", | |
| "Recorded-but-action-incorrect labels observable disagreement despite correct retained facts, not internal intent.", | |
| "Facts can be reacquired through later public feedback; agreement does not establish reliance on their earlier source.", | |
| "No Maze equipment, Osmosis strategy or full-campaign causal-necessity claim.", | |
| "Track agreement at launch can be satisfied by the initial alignment without an evidence-dependent action.", | |
| "Unrecognized notebook prose is unscorable; supported JSON fields can omit unrecognized content.", | |
| "Redundancy counts multiple supported representations of the same fact only."]) | |
| def dependency_audit(world): | |
| if world.generation_version == 3: | |
| from .causal_dependencies import CausalDependencyAudit | |
| return CausalDependencyAudit(world) | |
| return DependencyAudit(world) | |
| def checkpoint_policy_response(certificate, policy_factory): | |
| """Evaluate fresh policies on certified matched public inputs without training.""" | |
| if not certificate.get("pass") or len(certificate.get("decision_views", [])) != 2: | |
| return dict(status="unavailable", reason="matched_checkpoint_not_certified") | |
| from borderland.domain import Action, ActionError, Observation, Phase, Transition, canonical | |
| actions, values, errors = [], [], [] | |
| for view, table in zip(certificate["decision_views"], certificate["action_values"]): | |
| observation = deepcopy(view["observation"]) | |
| observation["phase"] = Phase(observation["phase"]) | |
| fields = deepcopy(view) | |
| fields["observation"] = Observation(**observation) | |
| fields["legal_actions"] = tuple(Action(**a) for a in view["legal_actions"]) | |
| if fields.get("error") is not None: | |
| fields["error"] = ActionError(**fields["error"]) | |
| transition = Transition(**fields) | |
| try: | |
| command = wire(policy_factory().choose(transition)) | |
| actions.append(command) | |
| values.append(table.get(canonical(command), 0.0)) | |
| errors.append(None) | |
| except Exception as error: | |
| actions.append(None) | |
| values.append(0.0) | |
| errors.append(type(error).__name__) | |
| return dict(status="measured", scope="Fresh scripted policy responses at the certified empty-notebook final-valve checkpoint only. Values assume the enumerated optimal suffix after the measured response, not a policy rollout.", | |
| actions=actions, errors=errors, action_values=values, paired_mean_optimal_suffix_return=sum(values) / 2, | |
| different_actions=actions[0] != actions[1], policy_updates=0) | |