Spaces:
Running
Running
Download python/borderland_eval/suite.py from FatUnc6969/borderland-rl: direct link, hf CLI and curl.
- Browser
- Download file 22.8 kB
-
https://huggingface.co/spaces/FatUnc6969/borderland-rl/resolve/main/python/borderland_eval/suite.py
- Command line
-
hf download hf://spaces/FatUnc6969/borderland-rl/python/borderland_eval/suite.py
-
curl -L -o suite.py https://huggingface.co/spaces/FatUnc6969/borderland-rl/resolve/main/python/borderland_eval/suite.py
22.8 kB
| import json | |
| import random | |
| import sys | |
| from collections import Counter | |
| from pathlib import Path | |
| from .context import actor_transition, wire | |
| from .causal_dependencies import CausalDependencyAudit | |
| from .dependencies import DEPENDENCIES, DependencyAudit, checkpoint_policy_response | |
| from .manifests import digest, manifests, source_hash | |
| COSTS = {"pulse": 2, "scan": 2, "turn_rotor": 2, "compare_seal": 2, | |
| "bind_docket": 2, "challenge": 2, "support": 2, "cross_check": 2, | |
| "test_checkpoint": 2, "escort": 2, "connect": 2, "energize": 2, "open_valve": 4} | |
| PRIORITY = {"launch": 0, "leave_maze": 1, "end_osmosis": 1, "restore": 1, | |
| "collect": 2, "lock_chamber": 3, "bank": 3, "energize": 3, | |
| "board": 3, "arm_exit": 3, "bind_docket": 4, "escort": 4, | |
| "install": 4, "observe_sensor": 5, "cross_check": 5, "challenge": 5, | |
| "walk": 6, "deploy": 6, "interview": 6, "align_track": 6, | |
| "open_valve": 7, "pulse": 7, "connect": 7, "open_scroll": 7, | |
| "visit": 8, "set_latch": 8, "scan": 9, "rest": 9} | |
| class PublicBaseline: | |
| def __init__(self, kind, seed=0): | |
| self.kind = kind | |
| self.rng = random.Random(seed) | |
| self.rule_stage = 0 | |
| self.learned_bit = None | |
| def choose(self, transition): | |
| public = wire(transition) | |
| actions = public["legal_actions"] | |
| if not actions: | |
| raise ValueError("no_legal_actions") | |
| if self.kind == "random": | |
| return self.rng.choice(actions) | |
| ordered = sorted(actions, key=lambda action: json.dumps(action, sort_keys=True)) | |
| if self.kind == "heuristic": | |
| board = public["observation"]["board"] | |
| phase = public["observation"]["phase"] | |
| if phase == 1: | |
| if board.get("live_sensor") is not None: | |
| self.learned_bit = board["live_sensor"] ^ (board["latch"] & 1) | |
| sequence = [("set_latch", {"value": 0}), ("pulse", {"fixture": "sensor"}), | |
| ("observe_sensor", {}), ("set_latch", {"value": 2 + (self.learned_bit or 0)}), | |
| ("pulse", {"fixture": "relay"}), ("pulse", {"fixture": "gate"}), ("lock_chamber", {})] | |
| tag, arguments = sequence[min(self.rule_stage, len(sequence) - 1)] | |
| self.rule_stage += 1 | |
| return {"tag": tag, "arguments": arguments} | |
| if phase == 2: | |
| for tag in ("leave_maze", "collect", "scan"): | |
| matching = [action for action in ordered if action["tag"] == tag] | |
| if matching and (tag != "scan" or not board.get("scanned")): | |
| return matching[0] | |
| for direction in ("south", "east", "north", "west"): | |
| matching = [action for action in ordered if action["tag"] == "walk" and action["arguments"]["direction"] == direction] | |
| if matching: | |
| return matching[0] | |
| if self.kind == "memoryless": | |
| return ordered[0] | |
| if self.kind == "cheapest_legal": | |
| return min(ordered, key=lambda action: COSTS.get(action["tag"], 1)) | |
| if self.kind == "always_scan": | |
| scans = [action for action in ordered if action["tag"] in {"scan", "observe_sensor"}] | |
| return scans[0] if scans else ordered[0] | |
| return min(ordered, key=lambda action: PRIORITY.get(action["tag"], 10)) | |
| class EvidenceAudit: | |
| def __init__(self): | |
| self.records = {} | |
| self.events = [] | |
| def contains(text, value, field=None): | |
| try: | |
| parsed = json.loads(text) | |
| except (ValueError, TypeError): | |
| return isinstance(value, (dict, list, str)) and str(value) in text | |
| def search(candidate): | |
| if field is None and candidate == value: | |
| return True | |
| if isinstance(candidate, dict): | |
| if field is not None and field in candidate and candidate[field] == value: | |
| return True | |
| return any(search(item) for item in candidate.values()) | |
| if isinstance(candidate, list): | |
| return any(search(item) for item in candidate) | |
| return False | |
| return search(parsed) | |
| def observe(self, transition, step, context=None): | |
| public = wire(transition) | |
| observation = public["observation"] | |
| board = observation["board"] | |
| notebook = observation["notebook"]["text"] | |
| evidence = [(field,board.get(field)) for field in ("live_sensor", "current_record", "current_claim", "record", "claim")] | |
| evidence.extend(("record_revision",record) for record in board.get("records",[])) | |
| for field,value in evidence: | |
| if value is None: | |
| continue | |
| encoded = json.dumps(value, sort_keys=True) | |
| key = field + ":" + encoded | |
| if key not in self.records: | |
| self.records[key] = {"field": field, "value": value, "exposed_step": step} | |
| self.events.append({"category": "evidence_exposed", "step": step, "field": field}) | |
| if isinstance(value, dict) and value.get("supersedes"): | |
| self.events.append({"category": "superseding_record_exposed", "step": step, | |
| "record": value.get("id"), "supersedes": value["supersedes"]}) | |
| def revisions(text): | |
| try: pending=[json.loads(text)] | |
| except (ValueError,TypeError): return set() | |
| found=set() | |
| while pending: | |
| value=pending.pop() | |
| if isinstance(value,dict): | |
| if isinstance(value.get('id'),str) and type(value.get('issued_at')) is int: | |
| found.add(json.dumps(value,sort_keys=True)) | |
| pending.extend(value.values()) | |
| elif isinstance(value,list): pending.extend(value) | |
| return found | |
| notebook_revisions=revisions(notebook) | |
| selected_revisions=set() | |
| if context is not None: | |
| for message in context: | |
| if message['role']!='system': selected_revisions.update(revisions(message.get('content') or '')) | |
| for record in self.records.values(): | |
| record["last_checked_step"] = step | |
| if record['field']=='record_revision': | |
| encoded=json.dumps(record['value'],sort_keys=True) | |
| record['value_present_in_notebook']=encoded in notebook_revisions | |
| if context is not None: record['value_present_in_selected_context']=encoded in selected_revisions | |
| continue | |
| scalar_field = record["field"] if not isinstance(record["value"], (dict, list)) else None | |
| record["value_present_in_notebook"] = self.contains(notebook, record["value"], scalar_field) | |
| if context is not None: | |
| record["value_present_in_selected_context"] = any( | |
| self.contains(message.get("content") or "", record["value"], scalar_field) | |
| for message in context if message["role"] != "system") | |
| if public.get("error"): | |
| self.events.append({"category": "engine_action_rejected", "step": step, | |
| "error": public["error"]}) | |
| if public.get("context_reset"): | |
| self.events.append({"category": "context_reset", "step": step, | |
| "notebook_chars": len(notebook)}) | |
| def run_episode(seed, policy=None, max_steps=256, output_path=None, *, generation_version=2, | |
| archive_workload="standard", notebook_bytes=768, memory_profile="static", policy_factory=None): | |
| from borderland import Campaign | |
| from borderland_verify.reference import public_reference | |
| campaign = Campaign.generated(seed, generation_version=generation_version, archive_workload=archive_workload, | |
| notebook_bytes=notebook_bytes, memory_profile=memory_profile) | |
| transition = campaign.reset() | |
| if policy is None: | |
| policy = public_reference(actor_transition(transition)) | |
| if policy_factory is None: | |
| if isinstance(policy, PublicBaseline): | |
| policy_factory = lambda: PublicBaseline(policy.kind, seed ^ 0xBA5E) | |
| else: | |
| policy_factory = type(policy) | |
| audit = EvidenceAudit() | |
| dependencies = (CausalDependencyAudit if generation_version == 3 else DependencyAudit)(campaign._world) | |
| trace = [] | |
| category = "engine_terminal" | |
| steps = 0 | |
| decisions = 0 | |
| controller_resets = 0 | |
| for steps in range(max_steps): | |
| audit.observe(transition, steps) | |
| dependencies.observe(transition, steps) | |
| if transition.done: | |
| break | |
| try: | |
| if hasattr(policy, "notebook"): | |
| note = policy.notebook(actor_transition(transition)) | |
| if note is not None and note != wire(transition)["observation"]["notebook"]["text"]: | |
| transition = campaign.replace_notebook(note) | |
| dependencies.observe(transition, steps) | |
| decisions += 1 | |
| action = policy.choose(actor_transition(transition)) | |
| before = transition | |
| transition = campaign.step(action) | |
| dependencies.record_decision(before, action, transition, steps) | |
| trace.append({"action": wire(action), "transition": wire(transition)}) | |
| if generation_version == 3 and transition.context_reset: | |
| policy = policy_factory() | |
| controller_resets += 1 | |
| except Exception as error: | |
| category = "policy_" + type(error).__name__ | |
| break | |
| else: | |
| category = "evaluator_step_budget" | |
| if transition.done and category == "engine_terminal": | |
| category = "engine_" + str(transition.outcome) | |
| dependencies.observe(transition, decisions) | |
| if output_path: | |
| campaign.save_replay(output_path, visibility="public") | |
| return {"seed": seed, "outcome": transition.outcome, "verified_escape": transition.outcome == "escaped", | |
| "generation_version": generation_version, "archive_workload": archive_workload, | |
| "memory_profile": memory_profile, "notebook_bytes": notebook_bytes, | |
| "controller_resets": controller_resets, | |
| "boarded_teammates": list(transition.observation.board.get("boarded", [])), | |
| "accepted_actions": sum(row["transition"].get("error") is None for row in trace), | |
| "allocated_decisions": decisions, "gameplay_reward": sum(row["transition"]["reward"] for row in trace), | |
| "charge": wire(transition)["observation"]["charge"], "termination_category": category, | |
| "simulator_truncated": transition.truncated, "evaluator_truncated": not transition.done, | |
| "evidence": list(audit.records.values()), "diagnostics": audit.events, | |
| "dependency_metrics": dependencies.report(), "trace": trace} | |
| def run_suite(seeds, output_dir=None, *, split_manifests=None, generation_version=2, | |
| archive_workload="standard", notebook_bytes=768, memory_profile="static"): | |
| from borderland_verify.certificates import certify_twins | |
| from borderland_verify.reference import run_privileged_reference | |
| from borderland.worlds import causal_candidate | |
| if type(generation_version) is not int or generation_version not in (2, 3): | |
| raise ValueError("generation_version must be 2 or 3") | |
| if archive_workload not in ("compact", "standard", "extended"): | |
| raise ValueError("archive_workload must be compact, standard, or extended") | |
| if type(notebook_bytes) is not int or notebook_bytes < 0: | |
| raise ValueError("notebook_bytes must be a nonnegative integer") | |
| if memory_profile not in ("static", "commissioning"): | |
| raise ValueError("memory_profile must be static or commissioning") | |
| if generation_version == 2 and notebook_bytes != 768: | |
| raise ValueError("custom byte capacity requires generation three") | |
| if generation_version == 2 and memory_profile != "static": | |
| raise ValueError("commissioning requires generation three") | |
| seeds = list(seeds) | |
| output = Path(output_dir) if output_dir else None | |
| if output: | |
| output.mkdir(parents=True, exist_ok=True) | |
| frozen = manifests(generation_version) if split_manifests is None else split_manifests | |
| if any(value["generation_version"] != generation_version for value in frozen.values()): | |
| raise ValueError("split manifest generation mismatch") | |
| configuration = dict(generation_version=generation_version, archive_workload=archive_workload, | |
| notebook_bytes=notebook_bytes, memory_profile=memory_profile) | |
| unit_count = (len(CausalDependencyAudit(causal_candidate(seeds[0], archive_workload=archive_workload, | |
| notebook_bytes=notebook_bytes, memory_profile=memory_profile)).units) | |
| if generation_version == 3 and seeds else len(DEPENDENCIES)) | |
| policies = [] | |
| failures = [] | |
| reference_name = "public_reference" if generation_version == 3 else "legal_history" | |
| for kind in (reference_name, "memoryless", "heuristic", "random", "cheapest_legal", "always_scan"): | |
| rows = [] | |
| for seed in seeds: | |
| policy = None if kind == reference_name else PublicBaseline(kind, seed ^ 0xBA5E) | |
| try: | |
| path = output / f"{kind}-{seed}-replay.json" if output else None | |
| row = run_episode(seed, policy, output_path=path, **configuration) | |
| if not row["verified_escape"]: | |
| failures.append({"policy": kind, "seed": seed, "category": row["termination_category"], | |
| "outcome": row["outcome"]}) | |
| except Exception as error: | |
| row = {"seed": seed, "verified_escape": False, "outcome": "not_started", | |
| "termination_category": "generation_or_runner_" + type(error).__name__} | |
| failures.append({"policy": kind, **row}) | |
| rows.append(row) | |
| dependency_reports = [row["dependency_metrics"] for row in rows if "dependency_metrics" in row] | |
| denominator = len(rows) * unit_count | |
| matched = sum(report["dependencies_matched"] for report in dependency_reports) | |
| opportunities = sum(report["opportunities"] for report in dependency_reports) | |
| successful_opportunities = sum(report["dependencies_matched"] if generation_version == 3 | |
| else report["successful_opportunities"] for report in dependency_reports) | |
| qualities = [report["notebook_quality"]["latest"] for report in dependency_reports | |
| if report["notebook_quality"]["latest"] is not None] | |
| scored_qualities = [quality for quality in qualities if quality["scorable"]] | |
| discovered = sum(len(quality["supported_fields"]) if generation_version == 3 | |
| else quality["discovered_denominator"] for quality in scored_qualities) | |
| retained = sum(len(quality["retained_correct"]) if generation_version == 3 | |
| else quality["retained_count"] for quality in scored_qualities) | |
| policies.append({"name": kind, "student_type": "scripted non-LLM baseline", "episodes": rows, | |
| "scope": ("Literal lexicographic current-view baseline; no memoryless ceiling or optimality claim." | |
| if kind == "memoryless" else "Uniform random legal action baseline." | |
| if kind == "random" else "Scripted public-observation controller; not an optimality bound."), | |
| "denominator": len(seeds), "escapes": sum(row["verified_escape"] for row in rows), | |
| "escape_rate": sum(row["verified_escape"] for row in rows) / len(rows) if rows else None, | |
| "dependency_metrics": {"metric_kind": "preaction_fixed_unit_agreement" if generation_version == 3 else "action_state_agreement", | |
| "causal_reliance": "not_identified", | |
| "predefined_denominator": denominator, | |
| "dependencies_matched": matched, | |
| "dependency_agreement_rate": matched / denominator if denominator else None, | |
| "opportunities": opportunities, | |
| ("first_choices_correct" if generation_version == 3 else "successful_opportunities"): successful_opportunities, | |
| "opportunity_correctness": successful_opportunities / opportunities if opportunities else None, | |
| **({"eligible_units": opportunities, | |
| "eligible_reach": opportunities / denominator if denominator else None, | |
| "completed_units": sum(report["completed_units"] for report in dependency_reports)} | |
| if generation_version == 3 else {}), | |
| "audit_unavailable_episodes": len(rows) - len(dependency_reports)}, | |
| "notebook_quality": {"scorable_episodes": len(scored_qualities), | |
| "unscorable_episodes": len(qualities) - len(scored_qualities), | |
| "audit_unavailable_episodes": len(rows) - len(qualities), | |
| ("supported_field_denominator_scored" if generation_version == 3 else "discovered_fact_denominator_scored"): discovered, | |
| ("retained_correct_fields" if generation_version == 3 else "retained_facts"): retained, | |
| ("information_coverage" if generation_version == 3 else "retention_rate"): retained / discovered if discovered else None}, | |
| "categories": dict(Counter(row["termination_category"] for row in rows))}) | |
| certificates = [] | |
| for seed in seeds if generation_version == 2 else (): | |
| try: | |
| certificates.append(certify_twins(seed)) | |
| except Exception as error: | |
| certificates.append({"pass": False, "failures": ["certificate_" + type(error).__name__]}) | |
| privileged = [] | |
| for seed in seeds if generation_version == 2 else (): | |
| try: | |
| privileged.append({"seed": seed, **run_privileged_reference(seed)}) | |
| except Exception as error: | |
| privileged.append({"seed": seed, "outcome": "not_started", "failure": type(error).__name__}) | |
| split_names = [name for name, manifest in frozen.items() if set(seeds).issubset(manifest["root_seeds"])] | |
| checkpoint_behavior = { | |
| kind: [checkpoint_policy_response(certificate, lambda kind=kind: PublicBaseline(kind)) | |
| for certificate in certificates] | |
| for kind in (("memoryless", "heuristic", "random", "cheapest_legal", "always_scan") | |
| if generation_version == 2 else ())} | |
| reference_pass = bool(seeds) and policies[0]["escapes"] == len(seeds) | |
| result = {"core_pass": (reference_pass and all(certificate.get("pass", False) for certificate in certificates)) | |
| if generation_version == 2 else None, | |
| "configuration": configuration, "public_reference_pass": reference_pass, | |
| "policies": policies, "certificates": certificates, "failures": failures, | |
| "certificate_status": "measured" if generation_version == 2 else "unsupported_current_generation", | |
| "matched_checkpoint_behavior": checkpoint_behavior, | |
| "matched_checkpoint_status": "measured" if generation_version == 2 else "unsupported_current_generation", | |
| "privileged_reference": {"status": "measured" if generation_version == 2 else "unsupported_current_generation", "episodes": privileged, | |
| "escapes": sum(row["outcome"] == "escaped" for row in privileged), | |
| "denominator": len(privileged), | |
| "scope": ("Perfect-information diagnostic; does not establish fair solvability or optimality." | |
| if generation_version == 2 else "The historical generation-two oracle and twin certificate are not run for generation three. No current oracle bound is available.")}, | |
| "comparison": {row["name"]: {"escape_rate": row["escape_rate"], | |
| ("escape_gap_to_public_reference" if generation_version == 3 else "escape_gap_to_legal_history"): policies[0]["escape_rate"] - row["escape_rate"] if seeds else None} | |
| for row in policies}, | |
| "manifests": frozen, "manifest_hashes": {key: digest(value) for key, value in frozen.items()}, | |
| "receipt": {"source_hash": source_hash(), "python": sys.version, "seeds": list(seeds), | |
| "unique_twin_root_groups": len({seed // 2 for seed in seeds}), | |
| "configuration_hash": digest({"max_steps": 256, "policies": [row["name"] for row in policies], | |
| **(configuration if generation_version == 3 else {})}), | |
| "evaluation_split": split_names[0] if len(split_names) == 1 else "ad_hoc"}, | |
| "limitations": (["Frozen manifests hold out raid opponents and guard rotation 3; ad hoc seeds are separately declared in the receipt.", | |
| "Full-campaign memoryless results allow physical memory side channels."] | |
| if generation_version == 2 else | |
| ["Generation-three manifests are disjoint root fixtures. They do not hold out opponent families or prove held-out transfer.", | |
| "Memoryless selects the literal first lexicographic legal action. Random selects uniformly. Neither is a memoryless ceiling or an oracle.", | |
| "Current-view policies can read physical state. Failure of these literal policies does not bound stronger physical-memory strategies."]) | |
| + ["Paired seeds share nuisance structure; episode counts are not independent family counts.", | |
| "Evidence diagnostics describe observations, not model mental states."]} | |
| if output: | |
| (output / "evaluation.json").write_text(json.dumps(result, indent=2)) | |
| return result | |