import json import random import sys from collections import Counter from pathlib import Path from .context import actor_transition, wire from .causal_dependencies import CausalDependencyAudit from .dependencies import DEPENDENCIES, DependencyAudit, checkpoint_policy_response from .manifests import digest, manifests, source_hash COSTS = {"pulse": 2, "scan": 2, "turn_rotor": 2, "compare_seal": 2, "bind_docket": 2, "challenge": 2, "support": 2, "cross_check": 2, "test_checkpoint": 2, "escort": 2, "connect": 2, "energize": 2, "open_valve": 4} PRIORITY = {"launch": 0, "leave_maze": 1, "end_osmosis": 1, "restore": 1, "collect": 2, "lock_chamber": 3, "bank": 3, "energize": 3, "board": 3, "arm_exit": 3, "bind_docket": 4, "escort": 4, "install": 4, "observe_sensor": 5, "cross_check": 5, "challenge": 5, "walk": 6, "deploy": 6, "interview": 6, "align_track": 6, "open_valve": 7, "pulse": 7, "connect": 7, "open_scroll": 7, "visit": 8, "set_latch": 8, "scan": 9, "rest": 9} class PublicBaseline: def __init__(self, kind, seed=0): self.kind = kind self.rng = random.Random(seed) self.rule_stage = 0 self.learned_bit = None def choose(self, transition): public = wire(transition) actions = public["legal_actions"] if not actions: raise ValueError("no_legal_actions") if self.kind == "random": return self.rng.choice(actions) ordered = sorted(actions, key=lambda action: json.dumps(action, sort_keys=True)) if self.kind == "heuristic": board = public["observation"]["board"] phase = public["observation"]["phase"] if phase == 1: if board.get("live_sensor") is not None: self.learned_bit = board["live_sensor"] ^ (board["latch"] & 1) sequence = [("set_latch", {"value": 0}), ("pulse", {"fixture": "sensor"}), ("observe_sensor", {}), ("set_latch", {"value": 2 + (self.learned_bit or 0)}), ("pulse", {"fixture": "relay"}), ("pulse", {"fixture": "gate"}), ("lock_chamber", {})] tag, arguments = sequence[min(self.rule_stage, len(sequence) - 1)] self.rule_stage += 1 return {"tag": tag, "arguments": arguments} if phase == 2: for tag in ("leave_maze", "collect", "scan"): matching = [action for action in ordered if action["tag"] == tag] if matching and (tag != "scan" or not board.get("scanned")): return matching[0] for direction in ("south", "east", "north", "west"): matching = [action for action in ordered if action["tag"] == "walk" and action["arguments"]["direction"] == direction] if matching: return matching[0] if self.kind == "memoryless": return ordered[0] if self.kind == "cheapest_legal": return min(ordered, key=lambda action: COSTS.get(action["tag"], 1)) if self.kind == "always_scan": scans = [action for action in ordered if action["tag"] in {"scan", "observe_sensor"}] return scans[0] if scans else ordered[0] return min(ordered, key=lambda action: PRIORITY.get(action["tag"], 10)) class EvidenceAudit: def __init__(self): self.records = {} self.events = [] @staticmethod def contains(text, value, field=None): try: parsed = json.loads(text) except (ValueError, TypeError): return isinstance(value, (dict, list, str)) and str(value) in text def search(candidate): if field is None and candidate == value: return True if isinstance(candidate, dict): if field is not None and field in candidate and candidate[field] == value: return True return any(search(item) for item in candidate.values()) if isinstance(candidate, list): return any(search(item) for item in candidate) return False return search(parsed) def observe(self, transition, step, context=None): public = wire(transition) observation = public["observation"] board = observation["board"] notebook = observation["notebook"]["text"] evidence = [(field,board.get(field)) for field in ("live_sensor", "current_record", "current_claim", "record", "claim")] evidence.extend(("record_revision",record) for record in board.get("records",[])) for field,value in evidence: if value is None: continue encoded = json.dumps(value, sort_keys=True) key = field + ":" + encoded if key not in self.records: self.records[key] = {"field": field, "value": value, "exposed_step": step} self.events.append({"category": "evidence_exposed", "step": step, "field": field}) if isinstance(value, dict) and value.get("supersedes"): self.events.append({"category": "superseding_record_exposed", "step": step, "record": value.get("id"), "supersedes": value["supersedes"]}) def revisions(text): try: pending=[json.loads(text)] except (ValueError,TypeError): return set() found=set() while pending: value=pending.pop() if isinstance(value,dict): if isinstance(value.get('id'),str) and type(value.get('issued_at')) is int: found.add(json.dumps(value,sort_keys=True)) pending.extend(value.values()) elif isinstance(value,list): pending.extend(value) return found notebook_revisions=revisions(notebook) selected_revisions=set() if context is not None: for message in context: if message['role']!='system': selected_revisions.update(revisions(message.get('content') or '')) for record in self.records.values(): record["last_checked_step"] = step if record['field']=='record_revision': encoded=json.dumps(record['value'],sort_keys=True) record['value_present_in_notebook']=encoded in notebook_revisions if context is not None: record['value_present_in_selected_context']=encoded in selected_revisions continue scalar_field = record["field"] if not isinstance(record["value"], (dict, list)) else None record["value_present_in_notebook"] = self.contains(notebook, record["value"], scalar_field) if context is not None: record["value_present_in_selected_context"] = any( self.contains(message.get("content") or "", record["value"], scalar_field) for message in context if message["role"] != "system") if public.get("error"): self.events.append({"category": "engine_action_rejected", "step": step, "error": public["error"]}) if public.get("context_reset"): self.events.append({"category": "context_reset", "step": step, "notebook_chars": len(notebook)}) def run_episode(seed, policy=None, max_steps=256, output_path=None, *, generation_version=2, archive_workload="standard", notebook_bytes=768, memory_profile="static", policy_factory=None): from borderland import Campaign from borderland_verify.reference import public_reference campaign = Campaign.generated(seed, generation_version=generation_version, archive_workload=archive_workload, notebook_bytes=notebook_bytes, memory_profile=memory_profile) transition = campaign.reset() if policy is None: policy = public_reference(actor_transition(transition)) if policy_factory is None: if isinstance(policy, PublicBaseline): policy_factory = lambda: PublicBaseline(policy.kind, seed ^ 0xBA5E) else: policy_factory = type(policy) audit = EvidenceAudit() dependencies = (CausalDependencyAudit if generation_version == 3 else DependencyAudit)(campaign._world) trace = [] category = "engine_terminal" steps = 0 decisions = 0 controller_resets = 0 for steps in range(max_steps): audit.observe(transition, steps) dependencies.observe(transition, steps) if transition.done: break try: if hasattr(policy, "notebook"): note = policy.notebook(actor_transition(transition)) if note is not None and note != wire(transition)["observation"]["notebook"]["text"]: transition = campaign.replace_notebook(note) dependencies.observe(transition, steps) decisions += 1 action = policy.choose(actor_transition(transition)) before = transition transition = campaign.step(action) dependencies.record_decision(before, action, transition, steps) trace.append({"action": wire(action), "transition": wire(transition)}) if generation_version == 3 and transition.context_reset: policy = policy_factory() controller_resets += 1 except Exception as error: category = "policy_" + type(error).__name__ break else: category = "evaluator_step_budget" if transition.done and category == "engine_terminal": category = "engine_" + str(transition.outcome) dependencies.observe(transition, decisions) if output_path: campaign.save_replay(output_path, visibility="public") return {"seed": seed, "outcome": transition.outcome, "verified_escape": transition.outcome == "escaped", "generation_version": generation_version, "archive_workload": archive_workload, "memory_profile": memory_profile, "notebook_bytes": notebook_bytes, "controller_resets": controller_resets, "boarded_teammates": list(transition.observation.board.get("boarded", [])), "accepted_actions": sum(row["transition"].get("error") is None for row in trace), "allocated_decisions": decisions, "gameplay_reward": sum(row["transition"]["reward"] for row in trace), "charge": wire(transition)["observation"]["charge"], "termination_category": category, "simulator_truncated": transition.truncated, "evaluator_truncated": not transition.done, "evidence": list(audit.records.values()), "diagnostics": audit.events, "dependency_metrics": dependencies.report(), "trace": trace} def run_suite(seeds, output_dir=None, *, split_manifests=None, generation_version=2, archive_workload="standard", notebook_bytes=768, memory_profile="static"): from borderland_verify.certificates import certify_twins from borderland_verify.reference import run_privileged_reference from borderland.worlds import causal_candidate if type(generation_version) is not int or generation_version not in (2, 3): raise ValueError("generation_version must be 2 or 3") if archive_workload not in ("compact", "standard", "extended"): raise ValueError("archive_workload must be compact, standard, or extended") if type(notebook_bytes) is not int or notebook_bytes < 0: raise ValueError("notebook_bytes must be a nonnegative integer") if memory_profile not in ("static", "commissioning"): raise ValueError("memory_profile must be static or commissioning") if generation_version == 2 and notebook_bytes != 768: raise ValueError("custom byte capacity requires generation three") if generation_version == 2 and memory_profile != "static": raise ValueError("commissioning requires generation three") seeds = list(seeds) output = Path(output_dir) if output_dir else None if output: output.mkdir(parents=True, exist_ok=True) frozen = manifests(generation_version) if split_manifests is None else split_manifests if any(value["generation_version"] != generation_version for value in frozen.values()): raise ValueError("split manifest generation mismatch") configuration = dict(generation_version=generation_version, archive_workload=archive_workload, notebook_bytes=notebook_bytes, memory_profile=memory_profile) unit_count = (len(CausalDependencyAudit(causal_candidate(seeds[0], archive_workload=archive_workload, notebook_bytes=notebook_bytes, memory_profile=memory_profile)).units) if generation_version == 3 and seeds else len(DEPENDENCIES)) policies = [] failures = [] reference_name = "public_reference" if generation_version == 3 else "legal_history" for kind in (reference_name, "memoryless", "heuristic", "random", "cheapest_legal", "always_scan"): rows = [] for seed in seeds: policy = None if kind == reference_name else PublicBaseline(kind, seed ^ 0xBA5E) try: path = output / f"{kind}-{seed}-replay.json" if output else None row = run_episode(seed, policy, output_path=path, **configuration) if not row["verified_escape"]: failures.append({"policy": kind, "seed": seed, "category": row["termination_category"], "outcome": row["outcome"]}) except Exception as error: row = {"seed": seed, "verified_escape": False, "outcome": "not_started", "termination_category": "generation_or_runner_" + type(error).__name__} failures.append({"policy": kind, **row}) rows.append(row) dependency_reports = [row["dependency_metrics"] for row in rows if "dependency_metrics" in row] denominator = len(rows) * unit_count matched = sum(report["dependencies_matched"] for report in dependency_reports) opportunities = sum(report["opportunities"] for report in dependency_reports) successful_opportunities = sum(report["dependencies_matched"] if generation_version == 3 else report["successful_opportunities"] for report in dependency_reports) qualities = [report["notebook_quality"]["latest"] for report in dependency_reports if report["notebook_quality"]["latest"] is not None] scored_qualities = [quality for quality in qualities if quality["scorable"]] discovered = sum(len(quality["supported_fields"]) if generation_version == 3 else quality["discovered_denominator"] for quality in scored_qualities) retained = sum(len(quality["retained_correct"]) if generation_version == 3 else quality["retained_count"] for quality in scored_qualities) policies.append({"name": kind, "student_type": "scripted non-LLM baseline", "episodes": rows, "scope": ("Literal lexicographic current-view baseline; no memoryless ceiling or optimality claim." if kind == "memoryless" else "Uniform random legal action baseline." if kind == "random" else "Scripted public-observation controller; not an optimality bound."), "denominator": len(seeds), "escapes": sum(row["verified_escape"] for row in rows), "escape_rate": sum(row["verified_escape"] for row in rows) / len(rows) if rows else None, "dependency_metrics": {"metric_kind": "preaction_fixed_unit_agreement" if generation_version == 3 else "action_state_agreement", "causal_reliance": "not_identified", "predefined_denominator": denominator, "dependencies_matched": matched, "dependency_agreement_rate": matched / denominator if denominator else None, "opportunities": opportunities, ("first_choices_correct" if generation_version == 3 else "successful_opportunities"): successful_opportunities, "opportunity_correctness": successful_opportunities / opportunities if opportunities else None, **({"eligible_units": opportunities, "eligible_reach": opportunities / denominator if denominator else None, "completed_units": sum(report["completed_units"] for report in dependency_reports)} if generation_version == 3 else {}), "audit_unavailable_episodes": len(rows) - len(dependency_reports)}, "notebook_quality": {"scorable_episodes": len(scored_qualities), "unscorable_episodes": len(qualities) - len(scored_qualities), "audit_unavailable_episodes": len(rows) - len(qualities), ("supported_field_denominator_scored" if generation_version == 3 else "discovered_fact_denominator_scored"): discovered, ("retained_correct_fields" if generation_version == 3 else "retained_facts"): retained, ("information_coverage" if generation_version == 3 else "retention_rate"): retained / discovered if discovered else None}, "categories": dict(Counter(row["termination_category"] for row in rows))}) certificates = [] for seed in seeds if generation_version == 2 else (): try: certificates.append(certify_twins(seed)) except Exception as error: certificates.append({"pass": False, "failures": ["certificate_" + type(error).__name__]}) privileged = [] for seed in seeds if generation_version == 2 else (): try: privileged.append({"seed": seed, **run_privileged_reference(seed)}) except Exception as error: privileged.append({"seed": seed, "outcome": "not_started", "failure": type(error).__name__}) split_names = [name for name, manifest in frozen.items() if set(seeds).issubset(manifest["root_seeds"])] checkpoint_behavior = { kind: [checkpoint_policy_response(certificate, lambda kind=kind: PublicBaseline(kind)) for certificate in certificates] for kind in (("memoryless", "heuristic", "random", "cheapest_legal", "always_scan") if generation_version == 2 else ())} reference_pass = bool(seeds) and policies[0]["escapes"] == len(seeds) result = {"core_pass": (reference_pass and all(certificate.get("pass", False) for certificate in certificates)) if generation_version == 2 else None, "configuration": configuration, "public_reference_pass": reference_pass, "policies": policies, "certificates": certificates, "failures": failures, "certificate_status": "measured" if generation_version == 2 else "unsupported_current_generation", "matched_checkpoint_behavior": checkpoint_behavior, "matched_checkpoint_status": "measured" if generation_version == 2 else "unsupported_current_generation", "privileged_reference": {"status": "measured" if generation_version == 2 else "unsupported_current_generation", "episodes": privileged, "escapes": sum(row["outcome"] == "escaped" for row in privileged), "denominator": len(privileged), "scope": ("Perfect-information diagnostic; does not establish fair solvability or optimality." if generation_version == 2 else "The historical generation-two oracle and twin certificate are not run for generation three. No current oracle bound is available.")}, "comparison": {row["name"]: {"escape_rate": row["escape_rate"], ("escape_gap_to_public_reference" if generation_version == 3 else "escape_gap_to_legal_history"): policies[0]["escape_rate"] - row["escape_rate"] if seeds else None} for row in policies}, "manifests": frozen, "manifest_hashes": {key: digest(value) for key, value in frozen.items()}, "receipt": {"source_hash": source_hash(), "python": sys.version, "seeds": list(seeds), "unique_twin_root_groups": len({seed // 2 for seed in seeds}), "configuration_hash": digest({"max_steps": 256, "policies": [row["name"] for row in policies], **(configuration if generation_version == 3 else {})}), "evaluation_split": split_names[0] if len(split_names) == 1 else "ad_hoc"}, "limitations": (["Frozen manifests hold out raid opponents and guard rotation 3; ad hoc seeds are separately declared in the receipt.", "Full-campaign memoryless results allow physical memory side channels."] if generation_version == 2 else ["Generation-three manifests are disjoint root fixtures. They do not hold out opponent families or prove held-out transfer.", "Memoryless selects the literal first lexicographic legal action. Random selects uniformly. Neither is a memoryless ceiling or an oracle.", "Current-view policies can read physical state. Failure of these literal policies does not bound stronger physical-memory strategies."]) + ["Paired seeds share nuisance structure; episode counts are not independent family counts.", "Evidence diagnostics describe observations, not model mental states."]} if output: (output / "evaluation.json").write_text(json.dumps(result, indent=2)) return result