| """File-reference entry point that runs the PFB-MAX stack under the OFFICIAL
|
| astabench harness (so submissions and live gpt-4o grading use the real path).
|
|
|
| inspect eval astabench/paper_finder_validation \
|
| --solver "<abs>/pfbmax/inspect_entry.py@pfbmax_solver" \
|
| --model openai/gpt-4o-mini
|
|
|
| Why a file shim (same reasoning as iris_asta/inspect_solver.py): inspect
|
| discovers solvers by AST-scanning for a LITERAL top-level @solver function,
|
| and its loader execs the file without registering it in sys.modules — so this
|
| module stays dataclass-free at top level and pulls the implementation in
|
| through ordinary imports.
|
|
|
| Corpus access: the harness hands the solver its own date-wrapped MCP tools,
|
| but those are async and PFB is search-heavy (the IRIS project measured the
|
| async tool bridge deadlocking on this task). We therefore use a direct
|
| AstaClient like IRIS does, and enforce the sample's snapshot date ourselves
|
| from state.metadata["insertion_date"] — the same contract the task tools
|
| enforce, so results stay date-legal.
|
| """
|
|
|
| import json
|
| import os
|
| import sys
|
|
|
| from inspect_ai.model import ChatMessageAssistant
|
| from inspect_ai.solver import solver
|
|
|
| _HERE = os.path.dirname(os.path.abspath(__file__))
|
| _BUNDLE = os.path.dirname(_HERE)
|
| for _p in (_HERE, os.path.join(_BUNDLE, "iris_asta")):
|
| if _p not in sys.path:
|
| sys.path.insert(0, _p)
|
|
|
|
|
| def _load_env() -> None:
|
| env_path = os.path.join(_BUNDLE, "iris_asta", ".env")
|
| if not os.path.exists(env_path):
|
| return
|
| for line in open(env_path, encoding="utf-8"):
|
| line = line.strip()
|
| if "=" in line and not line.startswith("#"):
|
| k, v = line.split("=", 1)
|
| os.environ.setdefault(k, v.strip().strip('"').strip("'"))
|
|
|
|
|
| def _extract_query(state) -> str:
|
| meta = getattr(state, "metadata", None) or {}
|
| raw = meta.get("raw_query")
|
| if isinstance(raw, str) and raw.strip():
|
| return raw.strip()
|
| text = str(getattr(state, "input_text", "") or "")
|
| marker = "find papers relevant to the following query"
|
| low = text.lower()
|
| idx = low.find(marker)
|
| if idx >= 0:
|
| text = text[idx + len(marker):]
|
| return text.strip().strip(":").strip()
|
|
|
|
|
| @solver
|
| def pfbmax_solver(**kwargs):
|
| """PFB-MAX: compile -> route -> (metadata | specific | semantic) -> emit."""
|
| _load_env()
|
|
|
|
|
|
|
| if not (os.environ.get("OPENAI_API_KEY") or "").strip():
|
| raise RuntimeError(
|
| "OPENAI_API_KEY is not set. Export it, or copy "
|
| "iris_asta/.env.example to iris_asta/.env and fill it in.")
|
| if not (os.environ.get("ASTA_TOOL_KEY") or "").strip():
|
| raise RuntimeError(
|
| "ASTA_TOOL_KEY is not set: corpus search would run against the "
|
| "anonymous rate-limit pool and score near zero. Request a free key "
|
| "and put it in iris_asta/.env (see iris_asta/.env.example).")
|
| os.environ.setdefault("IRIS_ASTA_MCP_DEADLINE_S", "15")
|
| os.environ.setdefault("IRIS_ASTA_TIMEOUT_S", "45")
|
| os.environ.setdefault("IRIS_ASTA_RATE_LIMIT_RPS", "2.0")
|
|
|
| async def solve(state, generate):
|
| import asyncio
|
|
|
| def _run():
|
|
|
|
|
| here = os.environ.get("PFBMAX_DIR") or _HERE
|
| bundle = os.path.dirname(here)
|
| for p in (here, os.path.join(bundle, "iris_asta")):
|
| if p not in sys.path:
|
| sys.path.insert(0, p)
|
|
|
| from iris_asta.asta_client import AstaClient
|
| from iris_asta.config import load_config
|
| import router
|
| from costmeter import CostMeter
|
| from llm import LLM
|
| try:
|
| from corpus_cache import CachedClient
|
| except Exception:
|
| CachedClient = None
|
|
|
| meta = getattr(state, "metadata", None) or {}
|
| inserted_before = meta.get("insertion_date") or "2025-06-01"
|
| query = _extract_query(state)
|
|
|
|
|
|
|
|
|
| _mcp = bool((os.environ.get("PFBMAX_USE_MCP") or "").strip())
|
| if _mcp:
|
|
|
|
|
|
|
|
|
|
|
| try:
|
| import iris_asta.solvers.pfb as _pfb
|
| _pfb._CITATION_LIMIT = min(
|
| getattr(_pfb, "_CITATION_LIMIT", 1000), 200)
|
| except Exception:
|
| pass
|
| client = AstaClient(load_config(),
|
| use_mcp=True if _mcp else None)
|
| if CachedClient is not None:
|
| client = CachedClient(client)
|
| meter = CostMeter()
|
| llm = LLM(meter=meter)
|
| trace = {}
|
| try:
|
| results = router.solve(query, client, llm, inserted_before,
|
| trace=trace)
|
| except Exception as exc:
|
| trace["fatal"] = repr(exc)[:300]
|
| results = []
|
| return results, meter.total_usd(), trace
|
|
|
| results, usd, trace = await asyncio.to_thread(_run)
|
|
|
| payload = {"output": {"results": [
|
| {"paper_id": str(pid), "markdown_evidence": ev}
|
| for pid, ev in results]}}
|
| completion = json.dumps(payload, ensure_ascii=True)
|
|
|
| state.output.completion = completion
|
| state.messages.append(ChatMessageAssistant(content=completion))
|
| try:
|
| state.metadata["pfbmax_usd"] = round(usd, 6)
|
| state.metadata["pfbmax_route"] = trace.get("route")
|
| state.metadata["pfbmax_n"] = len(results)
|
| except Exception:
|
| pass
|
| return state
|
|
|
| return solve
|
|
|
