Download scripts/rescue_spurious.py from AProver/aprover: direct link, hf CLI and curl.
- Browser
- Download file 7.43 kB
-
https://huggingface.co/spaces/AProver/aprover/resolve/main/scripts/rescue_spurious.py
- Command line
-
hf download hf://spaces/AProver/aprover/scripts/rescue_spurious.py
-
curl -L -o rescue_spurious.py https://huggingface.co/spaces/AProver/aprover/resolve/main/scripts/rescue_spurious.py
7.43 kB
| """ | |
| Run the realism check on every classification.json with outcome=spurious. | |
| The pipeline currently DROPS these findings before they reach the realism | |
| stage. This script asks the realism LLM whether the classifier was right. | |
| A REALISTIC verdict here indicates a missed real bug — the classifier | |
| heuristic (path-divergent-unwind filter OR "no caller can produce | |
| state") false-rejected a finding that the LLM, with full source + | |
| witness + call-chain context, identifies as plausible. | |
| Output: <fn_dir>/rescue_realism.json with the realism verdict + reasoning. | |
| Original classification.json and bug_report.json are NEVER modified. | |
| Usage: | |
| .venv/bin/python scripts/rescue_spurious.py \ | |
| --sweep /tmp/libarchive_n3_full_out/seedhunt_n3 \ | |
| --corpus /tmp/libarchive_seedhunt_full \ | |
| --log /tmp/rescue_spurious.log | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import logging | |
| import sys | |
| import traceback | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from bmc_agent.cbmc import Counterexample | |
| from bmc_agent.cex_validator import CExOutcome, ValidationResult | |
| from bmc_agent.config import Config | |
| from bmc_agent.llm import LLMClient | |
| from bmc_agent.parser import parse_c_file | |
| from bmc_agent import realism_checker as _rc_mod | |
| from bmc_agent.realism_checker import RealismChecker | |
| from bmc_agent.spec import Spec | |
| # The rescue script's whole point is to bypass the pre-LLM heuristic | |
| # detectors that already rejected these findings — otherwise we just | |
| # get the same "unrealistic" verdict from the same logic. Stub each | |
| # detector to return None so realism.check() always reaches the LLM. | |
| _WITNESS_DETECTORS = ( | |
| "_witness_indicates_uninitialized_library", | |
| "_witness_indicates_jv_stub_disconnect", | |
| "_witness_indicates_null_guard_violation", | |
| "_witness_indicates_usb_serial_framework_invariant", | |
| "_witness_indicates_phy_framework_invariant", | |
| "_witness_indicates_netdev_private_framework_invariant", | |
| "_witness_indicates_intentional_truncation", | |
| "_witness_indicates_path_divergent_unwind", | |
| ) | |
| for _det_name in _WITNESS_DETECTORS: | |
| if hasattr(_rc_mod, _det_name): | |
| setattr(_rc_mod, _det_name, lambda *a, **kw: None) | |
| def _load_cex(cls: dict) -> Counterexample: | |
| cex_data = cls.get("counterexample") or {} | |
| return Counterexample( | |
| failing_property=cex_data.get("failing_property", "<unknown>"), | |
| variable_assignments=cex_data.get("variable_assignments") or {}, | |
| trace=cex_data.get("trace") or [], | |
| description=cex_data.get("description", "") or "", | |
| failure_location=cex_data.get("failure_location") or {}, | |
| ) | |
| def _process_one( | |
| classification_path: Path, | |
| corpus: Path, | |
| config: Config, | |
| llm: LLMClient, | |
| log: logging.Logger, | |
| ) -> tuple[str, str]: | |
| """Returns (status, message).""" | |
| try: | |
| with open(classification_path) as f: | |
| doc = json.load(f) | |
| except Exception as e: | |
| return "ERROR", f"load: {e}" | |
| cls = doc.get("classification") or {} | |
| if cls.get("outcome") != "spurious": | |
| return "SKIP", f"outcome={cls.get('outcome')}" | |
| fn_dir = classification_path.parent | |
| rescue_path = fn_dir / "rescue_realism.json" | |
| if rescue_path.exists(): | |
| return "SKIP", "rescue_realism.json already exists" | |
| fn_name = cls.get("function_name") or "" | |
| file_stem = fn_dir.parent.name | |
| source_path = corpus / f"{file_stem}.c" | |
| if not source_path.exists(): | |
| return "ERROR", f"source missing: {source_path}" | |
| parsed = parse_c_file(source_path) | |
| func = parsed.get_function_info(fn_name) | |
| if func is None: | |
| return "SKIP", f"function '{fn_name}' not parsed" | |
| cex = _load_cex(cls) | |
| # Load spec next to classification.json if present | |
| spec_path = fn_dir / "spec.json" | |
| if spec_path.exists(): | |
| try: | |
| spec = Spec.from_dict(json.load(open(spec_path))) | |
| except Exception: | |
| spec = Spec(function_name=fn_name, precondition="", postcondition="") | |
| else: | |
| spec = Spec(function_name=fn_name, precondition="", postcondition="") | |
| # Force REAL_BUG so the realism checker actually runs (it skips on | |
| # other outcomes). The rescue interpretation: we're asking the LLM | |
| # "if you treat this as a candidate real bug, is it plausible?" | |
| vr = ValidationResult( | |
| function_name=fn_name, | |
| counterexample=cex, | |
| caller_path=list(cls.get("caller_path") or []), | |
| system_entry_input=None, | |
| refinement_history=list(cls.get("refinement_history") or []), | |
| final_precondition=spec.precondition, | |
| reasoning=cls.get("reasoning", "") or "", | |
| outcome=CExOutcome.REAL_BUG, | |
| system_entry_reached=bool(cls.get("system_entry_reached", False)), | |
| ) | |
| all_funcs = {} | |
| for name in (parsed.functions or {}): | |
| info = parsed.get_function_info(name) | |
| if info is not None: | |
| all_funcs[name] = info | |
| checker = RealismChecker(config, llm) | |
| try: | |
| result = checker.check( | |
| func=func, | |
| counterexample=cex, | |
| validation_result=vr, | |
| parsed_file=parsed, | |
| all_funcs=all_funcs, | |
| spec=spec, | |
| ) | |
| except Exception: | |
| return "ERROR", f"realism crashed:\n{traceback.format_exc()[:500]}" | |
| out = { | |
| "source": "rescue_spurious_2026-05-24", | |
| "function_name": fn_name, | |
| "failing_property": cex.failing_property, | |
| "classifier_reason": (cls.get("reasoning") or "")[:300], | |
| "rescue_verdict": result.verdict.value, | |
| "rescue_reasoning": result.reasoning, | |
| "rescue_key_concern": result.key_concern, | |
| "rescue_llm_confidence": result.llm_confidence, | |
| } | |
| with open(rescue_path, "w") as f: | |
| json.dump(out, f, indent=2) | |
| return "OK", result.verdict.value | |
| def main() -> int: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--sweep", required=True) | |
| ap.add_argument("--corpus", required=True) | |
| ap.add_argument("--log", default="/tmp/rescue_spurious.log") | |
| args = ap.parse_args() | |
| logging.basicConfig( | |
| filename=args.log, | |
| level=logging.INFO, | |
| format="%(asctime)s %(levelname)s %(message)s", | |
| ) | |
| log = logging.getLogger("rescue_spurious") | |
| sweep = Path(args.sweep).resolve() | |
| corpus = Path(args.corpus).resolve() | |
| if not sweep.is_dir() or not corpus.is_dir(): | |
| print(f"missing dir: {sweep} / {corpus}", file=sys.stderr) | |
| return 2 | |
| config = Config.from_env() | |
| config.enable_realism_check = True | |
| llm = LLMClient(config) | |
| cls_files = sorted(sweep.rglob("classification.json")) | |
| log.info("found %d classification.json files", len(cls_files)) | |
| counts = {"OK": 0, "SKIP": 0, "ERROR": 0} | |
| verdict_counts: dict[str, int] = {} | |
| for p in cls_files: | |
| status, msg = _process_one(p, corpus, config, llm, log) | |
| counts[status] += 1 | |
| log.info("%s %s :: %s", status, p.relative_to(sweep), msg) | |
| if status == "OK": | |
| verdict_counts[msg] = verdict_counts.get(msg, 0) + 1 | |
| for h in log.handlers: | |
| try: | |
| h.flush() | |
| except Exception: | |
| pass | |
| log.info("DONE counts=%s verdicts=%s", counts, verdict_counts) | |
| print(f"DONE counts={counts} verdicts={verdict_counts}") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |