"""Score the real Analyst against a content-checked private snapshot, never the serving store.""" import builtins import copy import datetime as dt import hashlib import json import os import re import tempfile from pathlib import Path from types import ModuleType, SimpleNamespace import yaml GOLDEN = Path(__file__).resolve().parents[1] / "evals" / "analyst_golden.yml" SNAPSHOT = GOLDEN.with_name("analyst_snapshot.json") class SnapshotError(RuntimeError): """The pinned evaluation data is absent, corrupt or incompatible with the golden set.""" class _SnapshotTools: """Run the shipped tools, semantic compiler and transforms with private module state.""" READS = frozenset(("list_topics", "describe_topic", "get_field_values", "run_semantic_query", "transform_result", "make_chart", "make_table", "make_kpi_card", "report_gap")) def __init__(self, manifest_path, golden, offline=False): self._tmp = None self._modules = {} self.events = [] try: manifest = json.loads(Path(manifest_path).read_text(encoding="utf-8")) ids = [item["id"] for item in golden["items"]] if (manifest.get("version") != 1 or manifest.get("format") != "duckdb" or len(set(ids)) != len(ids) or set(ids) != set(manifest["items"]) or len(ids) != len(manifest["items"])): raise ValueError("manifest must bind every golden item exactly once") seed = manifest["seed"] if any(seed[key] != golden["seed"][key] for key in ("dataset", "revision", "filename", "retainedFilename", "contentSha", "bytes", "cutDate")): raise ValueError("golden seed differs from the snapshot manifest") if (not re.fullmatch(r"[0-9a-f]{40}", seed["revision"]) or not re.fullmatch(r"[0-9a-f]{64}", seed["contentSha"]) or not seed["filename"] or Path(seed["filename"]).is_absolute() or ".." in Path(seed["filename"]).parts or seed["retainedFilename"] != f"analyst_snapshots/{seed['contentSha']}.duckdb" or type(seed["bytes"]) is not int or seed["bytes"] <= 0): raise ValueError("an immutable revision, checksum and relative filename are required") dt.date.fromisoformat(seed["cutDate"]) self.today = dt.date.fromisoformat(manifest["clock"]).isoformat() clock = dt.date.fromisoformat(self.today) class Date(dt.date): @classmethod def today(cls): return cls(clock.year, clock.month, clock.day) class DateTime(dt.datetime): @classmethod def now(cls, tz=None): value = cls(clock.year, clock.month, clock.day, tzinfo=dt.timezone.utc) return value.astimezone(tz) if tz else value.replace(tzinfo=None) @classmethod def utcnow(cls): return cls.now() self._datetime = SimpleNamespace(**{**vars(dt), "date": Date, "datetime": DateTime}) self.identity = {"seed": dict(seed), "clock": self.today, "items": ids} override = os.environ.get("ANALYST_EVAL_SNAPSHOT_FILE") if override: source = Path(override) self.identity["resolved_from"] = "exact-file-override" else: from huggingface_hub import hf_hub_download from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError, RevisionNotFoundError kwargs = dict(repo_type="dataset", token=os.environ.get("HF_TOKEN") or None, local_files_only=offline) try: source = Path(hf_hub_download(seed["dataset"], seed["filename"], revision=seed["revision"], **kwargs)) self.identity["resolved_from"] = "pinned-revision" except (EntryNotFoundError, RevisionNotFoundError) as exc: if offline or isinstance(exc, LocalEntryNotFoundError): raise # Squashed history may recover only the content-addressed object at HEAD. source = Path(hf_hub_download(seed["dataset"], seed["retainedFilename"], revision="main", **kwargs)) self.identity["resolved_from"] = "retained-content-at-head" self._tmp = tempfile.TemporaryDirectory(prefix="analyst-eval-") self.path = Path(self._tmp.name) / "snapshot.duckdb" digest = hashlib.sha256() with source.open("rb") as src, self.path.open("wb") as dst: while chunk := src.read(8 * 1024 * 1024): digest.update(chunk) dst.write(chunk) if digest.hexdigest() != seed["contentSha"] or self.path.stat().st_size != seed["bytes"]: raise ValueError("snapshot checksum mismatch") self.path.chmod(0o400) import harness.datastore as DS self._datastore = SimpleNamespace( DB_PATH=self.path, ENTITIES=copy.deepcopy(DS.ENTITIES), ready=lambda: True, ro_cursor=self._cursor, entity_live=self._entity_live, _columns_backfilled=DS._columns_backfilled) cur = self._cursor() try: for table in manifest["required_tables"]: if not re.fullmatch(r"[a-z_]+", table): raise ValueError("invalid required table") cur.execute(f'SELECT 1 FROM "{table}" LIMIT 0') if not self._entity_live(table): raise ValueError(f"snapshot source {table} has not completed its backfill") finally: cur.close() self._package = ModuleType("harness") self._package.datastore = self._datastore self._package.telemetry = SimpleNamespace(log=self.log) self._core = ModuleType("core") self._core.odoo = SimpleNamespace(_ctx=lambda: None, sale_line_domain=self._refuse) self._core.store = SimpleNamespace(available=lambda: False) self._semantic = self._load("semantic") tenant = yaml.safe_load((self._semantic.MODEL_DIR / "tenant.yml").read_text(encoding="utf-8")) self._semantic._tenant = lambda: copy.deepcopy(tenant) self._semantic._store_con = self._cursor query = self._semantic.store_query def pinned_query(*args, **kwargs): if any(kwargs.get(key) is not None for key in ("runtime", "user", "cur")): raise SnapshotError("evaluation queries cannot bind another data source") kwargs["today"] = self.today return query(*args, **kwargs) self._semantic.store_query = pinned_query self._semantic._model() # Freeze source metadata before the first model turn. self._tools = self._load("tools") self._RESULTS = self._tools._RESULTS except Exception as exc: self.close() # Hub exceptions can contain signed URLs. Report only the dependency identity. if isinstance(exc, (ValueError, KeyError)): why = str(exc) else: why = type(exc).__name__ raise SnapshotError(f"pinned Analyst snapshot unavailable or invalid: {why}; " "supply ANALYST_EVAL_SNAPSHOT_FILE with the manifest's exact bytes; " "no serving-store fallback") from None @staticmethod def _refuse(*args, **kwargs): raise SnapshotError("live source access is forbidden during a pinned evaluation") def _entity_live(self, table): cur = self._cursor() try: row = cur.execute("SELECT phase FROM _sync_state WHERE entity = ?", [table]).fetchone() spec = self._datastore.ENTITIES.get(table) or {} columns_ready = (not spec.get("backfill_fields") or self._datastore._columns_backfilled(cur, table, spec)) return bool(row and row[0] == "live" and columns_ready) finally: cur.close() def _cursor(self): import duckdb connection = duckdb.connect(str(self.path), read_only=True, config={"enable_external_access": False, "threads": 1}) clock = self.today class Cursor: def execute(self, sql, params=None): # Pin SQL clock expressions without rewriting quoted data. parts = re.split(r"('(?:''|[^'])*')", sql) for i in range(0, len(parts), 2): parts[i] = re.sub(r"\bCURRENT_DATE\b(?:\s*\(\s*\))?|\btoday\s*\(\s*\)", f"DATE '{clock}'", parts[i], flags=re.I) parts[i] = re.sub(r"\bCURRENT_TIMESTAMP\b|\bnow\s*\(\s*\)", f"TIMESTAMP '{clock} 00:00:00'", parts[i], flags=re.I) connection.execute("".join(parts), params or []) return self def __getattr__(self, name): return getattr(connection, name) return Cursor() def _import(self, name, globals=None, locals=None, fromlist=(), level=0): if name == "datetime": return self._datetime if name in ("harness.semantic", "harness.tools", "harness.transforms"): module = self._load(name.split(".")[-1]) return module if fromlist else self._package if name == "harness.datastore": return self._datastore if fromlist else self._package if name == "harness.telemetry": return self._package.telemetry if fromlist else self._package if name == "harness": for child in fromlist: if child in ("semantic", "tools", "transforms"): self._load(child) elif child not in ("datastore", "telemetry"): self._refuse() return self._package if name == "core.odoo": return self._core.odoo if fromlist else self._core if name == "core" and tuple(fromlist) == ("store",): return self._core if name in ("core", "modules", "connectors", "odoo_client"): self._refuse() if name.startswith(("harness.", "core.", "modules.", "connectors.")) and name not in ( "harness.filter_sql", "core.filter_contract", "modules.finance_spine", "modules.amazon_profit"): self._refuse() return builtins.__import__(name, globals, locals, fromlist, level) def _load(self, name): if name not in self._modules: path = Path(__file__).with_name(name + ".py") module = ModuleType("harness." + name) module.__file__ = str(path) module.__builtins__ = {**vars(builtins), "__import__": self._import} self._modules[name] = module setattr(self._package, name, module) # Private globals preserve the shipped implementation without swapping sys.modules. exec(compile(path.read_text(encoding="utf-8"), str(path), "exec"), module.__dict__) if name == "tools": import time import uuid epoch = dt.datetime.fromisoformat(self.today).replace(tzinfo=dt.timezone.utc).timestamp() module.time = SimpleNamespace( strftime=lambda fmt: time.strftime(fmt, time.gmtime(epoch))) counter = iter(range(1, 10**12)) module.uuid = SimpleNamespace(uuid4=lambda: uuid.UUID(int=next(counter) << 88)) return self._modules[name] def log(self, kind, **fields): self.events.append({"kind": kind, **fields}) def reset(self): self._RESULTS.clear() self._tools._RESULT_CALLERS.clear() self.events.clear() def list_topics(self): return self._tools.list_topics() def openai_tools(self): return [tool for tool in self._tools.openai_tools() if tool["function"]["name"] in self.READS] def dispatch(self, name, args): if name not in self.READS: return {"ok": False, "error": "tool unavailable in a read-only pinned evaluation"} return self._tools.dispatch(name, args) def close(self): if self._tmp is not None: if self.path.exists(): self.path.chmod(0o600) self._tmp.cleanup() self._tmp = None def _numbers(text): """Every number in the answer, normalized: $1,234.56 -> 1234.56; '59%'/'59.0 %' -> 0.59 too.""" out = [] for m in re.finditer(r"\$?([\d,]+(?:\.\d+)?)\s*(%?)", text): try: v = float(m.group(1).replace(",", "")) except ValueError: continue out.append(v) if m.group(2): # percent form: also offer the fraction out.append(v / 100.0) return out def _norm_ws(s): """Collapse ALL unicode whitespace to single spaces. Models emit U+202F (narrow no-break space) and friends inside names — 'Poppy\\u202fFlowers' failed a plain substring check on two different models (the 'top_customer flake', diagnosed 2026-07-12).""" return re.sub(r"\s+", " ", s) def _check(item, result): exp = item.get("expect") or {} answer = _norm_ws(result.get("answer") or "") reasons = [] if result.get("exhausted"): reasons.append("tool budget exhausted") if "value" in exp: want = float(exp["value"]) tol = exp.get("abs", exp.get("rel", 0.001) * abs(want) or 0.01) nums = _numbers(answer) if exp.get("allow_pct_form"): nums += [n for n in list(nums)] if not any(abs(n - want) <= tol for n in nums): reasons.append(f"expected {want:,.2f} (±{tol:,.2f}) not in answer") for s in exp.get("contains", []): if _norm_ws(s).lower() not in answer.lower(): reasons.append(f"missing substring {s!r}") # WRONG-ANSWER guard (2026-07-28). `contains` can only prove an answer said the right thing; # some failures are about saying a WRONG thing that a right answer never says — e.g. naming an # internal salesperson as an agent ([[invoice-line-agent-commission]]). Those need a negative # assertion or the eval passes on an answer that is confidently incorrect. for s in exp.get("not_contains", []): if _norm_ws(s).lower() in answer.lower(): reasons.append(f"forbidden substring {s!r} present") used = [t["tool"] for t in result.get("tool_trace", [])] for t in exp.get("tools", []): if t not in used: reasons.append(f"tool {t!r} not used (trace: {used})") for t in exp.get("not_tools", []): # over-refusal guard: these must NOT appear if t in used: reasons.append(f"tool {t!r} used but forbidden here (trace: {used})") if exp.get("artifact") == "chart": if not any("chart" in a for a in result.get("artifacts", [])): reasons.append("no chart artifact produced") return (not reasons), reasons def _cost_since(A, start): """Ledger calls and USD cost written since `start`, or None when ANALYST_CALL_LOG is unset.""" import os as _os if not _os.environ.get("ANALYST_CALL_LOG"): return None now = A.CALL_TOTALS return {"calls": now["calls"] - start["calls"], "cost_usd": round(now["cost"] - start["cost"], 6), "cost_missing": now["cost_missing"] - start["cost_missing"]} def _print_cost(cost): if cost is not None: print(f"COST: {cost['calls']} model call(s), ${cost['cost_usd']:.4f}" + (f" ({cost['cost_missing']} call(s) returned no cost)" if cost["cost_missing"] else "")) def run(chat_fn=None, model=None, only=None, bar=0.8, verbose=True, snapshot_path=SNAPSHOT): """Score golden questions on their pinned snapshot, including when chat_fn is supplied.""" import harness.analyst as A golden = yaml.safe_load(GOLDEN.read_text(encoding="utf-8")) items = golden["items"] if only: unknown = set(only) - {item["id"] for item in items} if unknown: raise ValueError(f"unknown golden items: {sorted(unknown)}") items = [item for item in items if item["id"] in only] source = _SnapshotTools(snapshot_path, golden, offline=chat_fn is not None) patient, call_item = os.environ.get("ANALYST_PATIENT"), A.CALL_ITEM try: return _run(items, source, chat_fn, model, bar, verbose) finally: A.CALL_ITEM = call_item if patient is None: os.environ.pop("ANALYST_PATIENT", None) else: os.environ["ANALYST_PATIENT"] = patient source.close() def _run(items, source, chat_fn, model, bar, verbose): """Run the golden set. Returns {passed, total, pass_rate, gate_ok, results, cost}. ANALYST_CALL_LOG names a JSONL cost ledger; `cost` is None when it is unset. ANALYST_EVAL_PACE_S sets the sleep before each item (default 6 s). """ import harness.analyst as A results, passed = [], 0 import os as _os import time as _time _os.environ["ANALYST_PATIENT"] = "1" # batch mode: wait out rate windows, don't fail fast infra_fails = 0 aborted = False pace = 0 if chat_fn is not None else float(_os.environ.get("ANALYST_EVAL_PACE_S") or 6) start = dict(A.CALL_TOTALS) for item in items: _time.sleep(pace) # pace the free-tier per-minute windows (Groq/Cerebras) A.CALL_ITEM = item["id"] source.reset() kw = {"chat_fn": chat_fn} if chat_fn is not None else {} if model: kw["model"] = model try: res = A.ask(item["ask"], telemetry_kind="analyst_eval", data_source=source, **kw) ok, reasons = _check(item, res) infra_fails = 0 except Exception as e: res, ok, reasons = {"answer": "", "tool_trace": []}, False, [f"RUN ERROR: {e}"] # Provider exhaustion is NOT a model failure: two in a row means the ladder is out # of quota — abort and report INCONCLUSIVE instead of burning the remaining items # into a misleading FAIL (learned 2026-07-16: 14 wasted items, 4/18 "22%"). if "all LLM providers failed" in str(e): infra_fails += 1 if infra_fails >= 2: aborted = True finally: A.CALL_ITEM = None passed += ok results.append({"id": item["id"], "ok": ok, "reasons": reasons, "usage": res.get("usage"), "iterations": res.get("iterations"), "snapshot_sha": source.identity["seed"]["contentSha"]}) if verbose: mark = "OK " if ok else "XX " print(f" {mark}{item['id']:18s}" + ("" if ok else f" — {'; '.join(reasons)[:90]}")) if aborted: break cost = _cost_since(A, start) if verbose: _print_cost(cost) if aborted: ran = [r for r in results if not str(r["reasons"])[:60].startswith("['RUN ERROR")] if verbose: print(f"\nEVAL ABORTED; LLM providers exhausted after {len(results)} item(s); " f"{passed}/{len(ran) or 1} of the items that actually ran passed. " "Gate is INCONCLUSIVE (not a model failure); re-run when quota recovers.") return {"passed": passed, "total": len(items), "pass_rate": None, "gate_ok": False, "inconclusive": True, "results": results, "cost": cost, "snapshot": source.identity} rate = passed / len(items) if items else 0.0 gate = rate >= bar if verbose: print(f"\nEVAL: {passed}/{len(items)} = {rate:.0%} GATE({bar:.0%}): " f"{'PASS' if gate else 'FAIL'}") return {"passed": passed, "total": len(items), "pass_rate": rate, "gate_ok": gate, "results": results, "cost": cost, "snapshot": source.identity}