Spaces:
Running
Running
Download python/borderland_eval/causal_dependencies.py from FatUnc6969/borderland-rl: direct link, hf CLI and curl.
- Browser
- Download file 25.6 kB
-
https://huggingface.co/spaces/FatUnc6969/borderland-rl/resolve/main/python/borderland_eval/causal_dependencies.py
- Command line
-
hf download hf://spaces/FatUnc6969/borderland-rl/python/borderland_eval/causal_dependencies.py
-
curl -L -o causal_dependencies.py https://huggingface.co/spaces/FatUnc6969/borderland-rl/resolve/main/python/borderland_eval/causal_dependencies.py
25.6 kB
| from copy import deepcopy | |
| from dataclasses import asdict, dataclass, field | |
| import json | |
| from borderland.domain import primitive | |
| from borderland.plant import (LANES, SEALS, WITNESSES, causal_uses, circuit_bus, | |
| circuit_ready, corridor_open, delivery_lane, | |
| delivery_ready, exit_decision, permit_decision) | |
| from .context import wire | |
| from .json_codec import strict_json | |
| class DependencyUnit: | |
| id: str | |
| service: str | |
| consumer: str | |
| target_phase: int | |
| sources: tuple[str, ...] | |
| note_field: str | None | |
| expected: object | |
| test_index: int | None = None | |
| note_expected: dict | None = None | |
| class Opportunity: | |
| first_eligible_step: int | None = None | |
| exposure_at_eligibility: dict = field(default_factory=dict) | |
| notebook_at_eligibility: dict | None = None | |
| attempts: list = field(default_factory=list) | |
| completed_step: int | None = None | |
| class NotebookFacts: | |
| scorable: bool | |
| values: dict = field(default_factory=dict) | |
| reason: str | None = None | |
| def extract_notebook(text): | |
| """Accept only the public reference's v3 codec, including partial plans.""" | |
| if text == "": | |
| return NotebookFacts(True) | |
| try: | |
| note = strict_json(text) | |
| except (ValueError, TypeError, RecursionError): | |
| return NotebookFacts(False, reason="unsupported_prose_or_invalid_json") | |
| if (not isinstance(note, dict) or set(note) != {"v", "p"} | |
| or type(note["v"]) is not int or note["v"] != 3 | |
| or not isinstance(note["p"], dict) or set(note["p"]) - set("cwsaged")): | |
| return NotebookFacts(False, reason="unsupported_json_schema") | |
| plan, values = note["p"], {} | |
| valid = True | |
| for key, value in plan.items(): | |
| if key == "c": | |
| valid &= isinstance(value, list) | |
| seen = set() | |
| for circuit in value if isinstance(value, list) else []: | |
| if (not isinstance(circuit, list) or len(circuit) not in (3, 4) | |
| or not isinstance(circuit[0], str) or not circuit[0] | |
| or circuit[0] in seen or circuit[1] not in LANES | |
| or not isinstance(circuit[2], list) | |
| or any(not isinstance(p, str) or not p for p in circuit[2]) | |
| or len(set(circuit[2])) != len(circuit[2]) or circuit[0] in circuit[2] | |
| or len(circuit) == 4 and (not isinstance(circuit[3], str) | |
| or len(circuit[3]) != 8 or any(t not in '0123' for t in circuit[3]))): | |
| valid = False | |
| continue | |
| seen.add(circuit[0]) | |
| values["c." + circuit[0] + ".bus"] = circuit[1] | |
| values["c." + circuit[0] + ".predecessors"] = sorted(circuit[2]) | |
| if len(circuit) == 4: | |
| values["c." + circuit[0] + ".ratings"] = circuit[3] | |
| elif key in ("w", "s"): | |
| valid &= value in (WITNESSES if key == "w" else SEALS) | |
| values[key] = value | |
| elif key in ("a", "g"): | |
| valid &= isinstance(value, str) and bool(value) | |
| values[key] = value | |
| elif key in ("e", "d"): | |
| limit = 3 if key == "e" else 4 | |
| valid_pair = isinstance(value, list) and len(value) == 2 | |
| if key == "e": | |
| valid_pair &= (isinstance(value, list) and len(value) == 2 | |
| and type(value[0]) is int and value[0] in range(limit) | |
| and value[1] in LANES) | |
| else: | |
| valid_pair &= (isinstance(value, list) and len(value) == 2 | |
| and value[0] in LANES and type(value[1]) is int | |
| and value[1] in range(limit)) | |
| valid &= valid_pair | |
| if valid_pair: | |
| values[key + ".0"], values[key + ".1"] = value | |
| return NotebookFacts(bool(valid), values if valid else {}, | |
| None if valid else "malformed_reference_schema") | |
| class CausalDependencyAudit: | |
| def __init__(self, world): | |
| self.plant = world.infrastructure | |
| self.load_schedule = getattr(self.plant, "load_schedule", None) | |
| self.current_records = {s: primitive(self.plant.current(s).to_dict()) for s in self.plant.services} | |
| if self.load_schedule is not None: | |
| for service, record in self.current_records.items(): | |
| if record["body"]["kind"] == "circuit": | |
| record["ratings"] = list(self.load_schedule.curve(record["id"])) | |
| self.units = [] | |
| self.expected_note = {} | |
| uses = {u["service"]: u["sources"] for u in causal_uses(self.plant)} | |
| witness, seal = permit_decision(self.plant) | |
| track, valve = exit_decision(self.plant) | |
| for service in self.plant.services: | |
| body = self.plant.current(service).body | |
| if body.kind == "circuit": | |
| bus_key = "c." + service + ".bus" | |
| base_bus = circuit_bus(self.plant, service) | |
| specs = (("bus", 6, bus_key, base_bus), | |
| ("predecessors", 6, "c." + service + ".predecessors", sorted(body.predecessors))) | |
| if self.load_schedule is not None: | |
| from borderland.plant import compensated_bus | |
| thresholds = self.load_schedule.curve(self.plant.current(service).id) | |
| ratings_key = "c." + service + ".ratings" | |
| retained = {bus_key: base_bus, ratings_key: ''.join(map(str, thresholds))} | |
| self.expected_note.update(retained) | |
| for index in range(3): | |
| meter = self.load_schedule.test(service, index) | |
| expected = compensated_bus(base_bus, thresholds, meter.band, meter.load) | |
| self.units.append(DependencyUnit(service + ":bus:" + str(index + 1), service, | |
| "bus", 6, uses[service] + ("ratings." + service,), bus_key, expected, | |
| index, retained)) | |
| specs = specs[1:] | |
| elif body.kind == "delivery": | |
| residue = next(t for t in range(4) if delivery_ready(self.plant, t)) | |
| specs = (("delivery_lane", 4, "d.0", delivery_lane(self.plant)), | |
| ("delivery_timing", 4, "d.1", residue)) | |
| elif body.kind == "permit": | |
| specs = (("witness", 5, "w", witness), ("checkpoint_seal", 5, "s", seal), | |
| ("permit_record", 5, "a", service), ("corridor", 5, None, None), | |
| ("exit_seal", 7, "s", seal)) | |
| self.expected_note["a"] = service | |
| else: | |
| specs = (("track", 7, "e.0", track), ("valve", 7, "e.1", valve)) | |
| self.expected_note["g"] = service | |
| for consumer, phase, key, expected in specs: | |
| unit = DependencyUnit(service + ":" + consumer, service, consumer, phase, | |
| uses[service], key, expected) | |
| self.units.append(unit) | |
| if key is not None: | |
| self.expected_note[key] = expected | |
| self.opportunities = {u.id: Opportunity() for u in self.units} | |
| self.provenance = {} | |
| self.calibrations = {} | |
| self.reached_phases = set() | |
| self.decisions = [] | |
| self.notebook_samples = [] | |
| self._sample_signature = None | |
| self._selected_notes = {} | |
| self._selected_quality = {} | |
| def _acquire(self, key, step, phase, channel): | |
| acquisition = dict(step=step, phase=phase, channel=channel) | |
| rows = self.provenance.setdefault(key, []) | |
| if not any(row["channel"] == channel for row in rows): | |
| rows.append(acquisition) | |
| def _exposure(self, unit): | |
| return {key: deepcopy(self.provenance.get(key, [])) for key in unit.sources} | |
| def _observe_evidence(self, observation, step): | |
| phase, board = observation["phase"], observation["board"] | |
| if phase == 1 and board.get("channel"): | |
| cid = board["channel"] | |
| readings = self.calibrations.setdefault(cid, {}) | |
| latch = board["latch"] | |
| if type(board.get("live_sensor")) is int: | |
| readings["sensor_xor"] = board["live_sensor"] ^ (latch & 1) | |
| if type(board.get("phase_mark")) is int: | |
| readings["phase_offset"] = (board["phase_mark"] - latch) % 4 | |
| expected = asdict(self.plant.channel(cid)) | |
| if all(readings.get(k) == expected[k] for k in ("sensor_xor", "phase_offset")): | |
| self._acquire("calibration." + cid, step, phase, "source_calibration") | |
| if phase == 2 and board.get("transit_plate") == asdict(self.plant.transit): | |
| self._transit(step, phase, "maze_transit_plate") | |
| if phase == 3: | |
| for record in board.get("records", []): | |
| if record in self.current_records.values(): | |
| self._acquire("charter." + record["subject"], step, phase, "applicable_archive_record") | |
| if "ratings" in record: | |
| self._acquire("ratings." + record["subject"], step, phase, "applicable_archive_record") | |
| feedback = board.get("feedback") | |
| if isinstance(feedback, dict) and isinstance(feedback.get("audit"), dict): | |
| record = feedback["audit"] | |
| service = record.get("subject") | |
| if service in self.current_records and record == self.current_records[service]: | |
| channel = "early_audit_evidence" if phase < 6 else "late_audit_recovery" | |
| self._acquire("charter." + service, step, phase, channel) | |
| if "ratings" in record: | |
| self._acquire("ratings." + service, step, phase, channel) | |
| for key in ("calibration", "applicability_calibration"): | |
| calibration = feedback.get(key) | |
| if isinstance(calibration, dict) and any(calibration == asdict(c) for c in self.plant.calibrations): | |
| self._acquire("calibration." + calibration["id"], step, phase, channel) | |
| if feedback.get("transit") == asdict(self.plant.transit): | |
| self._transit(step, phase, channel) | |
| def _transit(self, step, phase, channel): | |
| for key in asdict(self.plant.transit): | |
| self._acquire("transit." + key, step, phase, channel) | |
| def notebook_quality(self, text): | |
| note = extract_notebook(text) | |
| keys = sorted(self.expected_note) | |
| correct = [k for k in keys if k in note.values and note.values[k] == self.expected_note[k]] | |
| wrong = [k for k in keys if k in note.values and note.values[k] != self.expected_note[k]] | |
| omitted = [k for k in keys if k not in note.values] | |
| return dict(scorable=note.scorable, reason=note.reason, supported_fields=keys, | |
| retained_correct=correct if note.scorable else None, | |
| retained_stale_or_wrong=wrong if note.scorable else None, | |
| omitted_supported_fields=omitted if note.scorable else None, | |
| unexpected_supported_fields=sorted(set(note.values) - set(keys)) if note.scorable else None, | |
| information_coverage=len(correct) / len(keys) if note.scorable else None) | |
| def observe(self, transition, step, context=None, *, exposure=True, dispatched=False): | |
| public = wire(transition) | |
| observation = public["observation"] | |
| self.reached_phases.add(observation["phase"]) | |
| if exposure: | |
| self._observe_evidence(observation, step) | |
| notebook = observation["notebook"] | |
| quality = self.notebook_quality(notebook["text"]) | |
| quality.update(capacity_bytes=notebook["capacity_bytes"], | |
| used_bytes=len(notebook["text"].encode("utf-8"))) | |
| if context is not None: | |
| users = [m for m in context if m.get("role") == "user"] | |
| if dispatched: | |
| users = [] | |
| for message in context: | |
| if message.get("role") in ("user", "tool"): | |
| candidate = json.loads(message["content"]) | |
| if isinstance(candidate, dict) and "observation" in candidate: | |
| users.append(message) | |
| if users: | |
| selected = json.loads(users[-1]["content"])["observation"]["notebook"] | |
| self._selected_notes[step] = selected["text"] | |
| self._selected_quality[step] = dict(self.notebook_quality(selected["text"]), | |
| capacity_bytes=selected["capacity_bytes"], | |
| used_bytes=len(selected["text"].encode("utf-8"))) | |
| if step in self._selected_quality: | |
| quality["selected_notebook"] = self._selected_quality[step] | |
| signature = json.dumps([observation["phase"], bool(public.get("context_reset")), quality, self.provenance], sort_keys=True) | |
| if signature != self._sample_signature: | |
| self.notebook_samples.append(dict(step=step, phase=observation["phase"], | |
| context_reset=bool(public.get("context_reset")), **quality)) | |
| self._sample_signature = signature | |
| for unit in self.units: | |
| opportunity = self.opportunities[unit.id] | |
| if exposure and opportunity.first_eligible_step is None and self._eligible(unit, public): | |
| opportunity.first_eligible_step = step | |
| opportunity.exposure_at_eligibility = self._exposure(unit) | |
| opportunity.notebook_at_eligibility = deepcopy(quality) | |
| elif exposure and opportunity.first_eligible_step == step: | |
| opportunity.notebook_at_eligibility = deepcopy(quality) | |
| if opportunity.completed_step is None and self._completed(unit, observation): | |
| opportunity.completed_step = step | |
| def _eligible(self, unit, public): | |
| if public.get("terminated") or public.get("truncated") or public["observation"]["phase"] != unit.target_phase: | |
| return False | |
| return any(self._relevant(unit, a, public["observation"]["board"]) | |
| for a in public["legal_actions"]) | |
| def _relevant(self, unit, action, board): | |
| tag, args = action.get("tag"), action.get("arguments", {}) | |
| consumer = unit.consumer | |
| if consumer == "delivery_lane": | |
| actor = next((a for a in board.get("allies", []) if a["actor"] == args.get("actor")), {}) | |
| return tag == "bank" or tag == "deploy" and args.get("neighbor") in LANES and bool(actor.get("flag")) | |
| if consumer == "delivery_timing": | |
| return tag == "bank" | |
| if consumer == "witness": | |
| return tag == "interview" or tag == "escort" and args.get("neighbor") == 4 | |
| if consumer == "checkpoint_seal": | |
| return tag == "test_checkpoint" | |
| if consumer == "permit_record": | |
| return tag == "cross_check" | |
| if consumer == "corridor": | |
| return tag == "escort" and args.get("neighbor") == 4 | |
| if consumer in ("bus", "predecessors"): | |
| if consumer == "bus" and unit.test_index is not None: | |
| circuit = next((c for c in board.get("circuits", []) if c["id"] == unit.service), {}) | |
| meter = circuit.get("meter") | |
| if not isinstance(meter, dict) or meter.get("test") != unit.test_index + 1: | |
| return False | |
| return tag == ("connect" if consumer == "bus" else "energize") and args.get("circuit") == unit.service | |
| return tag == {"track": "align_track", "exit_seal": "arm_exit", "valve": "open_valve"}[consumer] or tag == "launch" | |
| def _agreement(self, unit, action, board): | |
| args, tag, p = action.get("arguments", {}), action.get("tag"), self.plant | |
| consumer = unit.consumer | |
| if consumer == "delivery_lane": | |
| if tag == "bank": | |
| actor = next(a for a in board["allies"] if a["actor"] == args.get("actor")) | |
| return actor.get("route_lane") == delivery_lane(p) | |
| return args.get("neighbor") == delivery_lane(p) | |
| if consumer == "delivery_timing": | |
| return delivery_ready(p, board["turn"]) | |
| if consumer == "witness": | |
| return args.get("witness") == permit_decision(p)[0] | |
| if consumer == "checkpoint_seal": | |
| return args.get("seal_id") == permit_decision(p)[1] | |
| if consumer == "permit_record": | |
| return args.get("record_id") == unit.service | |
| if consumer == "corridor": | |
| return corridor_open(p, board["clock"]) | |
| if consumer == "bus": | |
| expected = circuit_bus(p, unit.service) | |
| if unit.test_index is not None: | |
| from borderland.plant import compensated_bus | |
| circuit = next(c for c in board["circuits"] if c["id"] == unit.service) | |
| meter = circuit["meter"] | |
| expected = compensated_bus(expected, self.load_schedule.curve(p.current(unit.service).id), | |
| meter["band"], meter["load"]) | |
| return args.get("bus") == expected | |
| if consumer == "predecessors": | |
| live = [c["id"] for c in board["circuits"] if c["status"] == "live"] | |
| return circuit_ready(p, unit.service, live) | |
| if consumer == "track": | |
| return (board["track"] if tag == "launch" else args.get("position")) == exit_decision(p)[0] | |
| if consumer == "exit_seal": | |
| return (board["armed_with"] if tag == "launch" else args.get("seal_id")) == permit_decision(p)[1] | |
| return (board["valve"] if tag == "launch" else args.get("side")) == exit_decision(p)[1] | |
| def _completed(self, unit, observation): | |
| phase, board = observation["phase"], observation["board"] | |
| if unit.consumer in ("witness", "corridor"): | |
| return phase > 5 and 5 in self.reached_phases | |
| if phase != unit.target_phase: | |
| return False | |
| if unit.consumer in ("bus", "predecessors"): | |
| if unit.test_index is not None: | |
| return any(c["id"] == unit.service and c["completed_tests"] > unit.test_index | |
| for c in board["circuits"]) | |
| return any(c["id"] == unit.service and c["status"] == "live" for c in board["circuits"]) | |
| if unit.consumer == "checkpoint_seal": | |
| return board["checkpoint_test"] == permit_decision(self.plant)[1] | |
| if unit.consumer == "permit_record": | |
| return any(c["record"] == unit.service and c["status"] == "verified" for c in board["checks"]) | |
| return False | |
| def record_decision(self, before, action, after, step, *, expose_result=True): | |
| public, command, result = wire(before), wire(action), wire(after) | |
| observation = public["observation"] | |
| self.observe(before, step, exposure=expose_result) | |
| evaluations = [] | |
| note = extract_notebook(self._selected_notes.get(step, observation["notebook"]["text"])) | |
| if isinstance(command, dict) and isinstance(command.get("arguments", {}), dict): | |
| for unit in self.units: | |
| if unit.target_phase != observation["phase"] or not self._relevant(unit, command, observation["board"]): | |
| continue | |
| opportunity = self.opportunities[unit.id] | |
| exposure = self._exposure(unit) | |
| exposed = all(exposure.values()) | |
| required = unit.note_expected if unit.note_expected is not None else {unit.note_field: unit.expected} | |
| retained = (all(key in note.values and note.values[key] == value for key, value in required.items()) | |
| if unit.note_field is not None else None) | |
| correct = not result.get("error") and self._agreement(unit, command, observation["board"]) | |
| category = ("correct_agreement" if correct else "source_not_exposed" if not exposed | |
| else "notebook_unscored" if not note.scorable else "no_supported_note_field" | |
| if unit.note_field is None else "supported_field_omitted" | |
| if any(key not in note.values for key in required) else "retained_stale_or_wrong" | |
| if not retained else "retained_correct_action_disagrees") | |
| evaluation = dict(unit=unit.id, step=step, correct=correct, evidence_exposed=exposed, | |
| opportunity_opened=opportunity.first_eligible_step is not None, | |
| evidence_provenance=exposure, notebook_scorable=note.scorable, | |
| evidence_recorded=retained if note.scorable else None, category=category, | |
| agreement_mode="state_agreement" if command.get("tag") in ("bank", "launch") else "action_agreement", | |
| charge_spent=observation["charge"] - result["observation"]["charge"]) | |
| opportunity.attempts.append(deepcopy(evaluation)) | |
| evaluations.append(evaluation) | |
| if correct and unit.consumer in ("delivery_timing", "track", "exit_seal", "valve"): | |
| if unit.consumer != "delivery_timing" or result["observation"]["board"].get("allied_points", 0) > observation["board"]["allied_points"]: | |
| opportunity.completed_step = opportunity.completed_step if opportunity.completed_step is not None else step | |
| if observation["phase"] == 4 and command.get("tag") == "bank" and not result.get("error"): | |
| board = observation["board"] | |
| actor = next(a for a in board["allies"] if a["actor"] == command["arguments"].get("actor")) | |
| if delivery_ready(self.plant, board["turn"]) and actor.get("route_lane") == delivery_lane(self.plant): | |
| unit = next(u for u in self.units if u.consumer == "delivery_lane") | |
| opportunity = self.opportunities[unit.id] | |
| opportunity.completed_step = opportunity.completed_step if opportunity.completed_step is not None else step | |
| row = dict(step=step, phase=observation["phase"], action=deepcopy(command), | |
| decision_tags=[e["unit"] for e in evaluations], evaluations=evaluations) | |
| self.decisions.append(row) | |
| if expose_result: | |
| self._observe_evidence(result["observation"], step + 1) | |
| for unit in self.units: | |
| opportunity = self.opportunities[unit.id] | |
| if opportunity.completed_step is None and self._completed(unit, result["observation"]): | |
| opportunity.completed_step = step | |
| return row | |
| def report(self): | |
| rows = [] | |
| for unit in self.units: | |
| opportunity = self.opportunities[unit.id] | |
| first = next((a for a in opportunity.attempts if a["opportunity_opened"]), None) | |
| reached = unit.target_phase in self.reached_phases | |
| rows.append(dict(**asdict(unit), **asdict(opportunity), target_reached=reached, | |
| eligible=opportunity.first_eligible_step is not None, | |
| first_choice=deepcopy(first), first_choice_correct=first["correct"] if first else None, | |
| completed=opportunity.completed_step is not None, | |
| recovered_after_wrong_first=bool(first and not first["correct"] and opportunity.completed_step is not None), | |
| censoring="target_unreached" if not reached else "not_eligible" if opportunity.first_eligible_step is None | |
| else "no_relevant_choice" if first is None else "incomplete_at_observation_end" | |
| if opportunity.completed_step is None else None)) | |
| denominator = len(rows) | |
| eligible = sum(r["eligible"] for r in rows) | |
| first_correct = sum(r["first_choice_correct"] is True for r in rows) | |
| return primitive(dict(generation_version=3, metric_kind="preaction_fixed_unit_agreement", causal_reliance="not_identified", | |
| predefined_denominator=denominator, eligible_units=eligible, opportunities=eligible, | |
| eligible_reach=eligible / denominator, dependencies_matched=first_correct, | |
| dependency_agreement_rate=first_correct / denominator, | |
| opportunity_correctness=first_correct / eligible if eligible else None, | |
| completed_units=sum(r["completed"] for r in rows), | |
| dependencies=rows, decisions=deepcopy(self.decisions), evidence_provenance=deepcopy(self.provenance), | |
| notebook_quality=dict(latest=deepcopy(self.notebook_samples[-1]) if self.notebook_samples else None, | |
| samples=deepcopy(self.notebook_samples)), | |
| limits=["Literal public-reference codec control; arbitrary native prose is unscored.", | |
| "Exposure and agreement do not identify internal use or causal reliance.", | |
| "Delivery lane compares loaded lane crossings or the route state at bank; bridge paths may not expose a lane choice.", | |
| "Corridor timing is measured only when a public escort-to-checkpoint action is available; the reference codec has no corridor timing field."])) | |