fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
8873406 verified
Raw History Blame Contribute Delete
20.4 kB
"""Score the real Analyst against a content-checked private snapshot, never the serving store."""
import builtins
import copy
import datetime as dt
import hashlib
import json
import os
import re
import tempfile
from pathlib import Path
from types import ModuleType, SimpleNamespace
import yaml
GOLDEN = Path(__file__).resolve().parents[1] / "evals" / "analyst_golden.yml"
SNAPSHOT = GOLDEN.with_name("analyst_snapshot.json")
class SnapshotError(RuntimeError):
"""The pinned evaluation data is absent, corrupt or incompatible with the golden set."""
class _SnapshotTools:
"""Run the shipped tools, semantic compiler and transforms with private module state."""
READS = frozenset(("list_topics", "describe_topic", "get_field_values",
"run_semantic_query", "transform_result", "make_chart", "make_table",
"make_kpi_card", "report_gap"))
def __init__(self, manifest_path, golden, offline=False):
self._tmp = None
self._modules = {}
self.events = []
try:
manifest = json.loads(Path(manifest_path).read_text(encoding="utf-8"))
ids = [item["id"] for item in golden["items"]]
if (manifest.get("version") != 1 or manifest.get("format") != "duckdb"
or len(set(ids)) != len(ids) or set(ids) != set(manifest["items"])
or len(ids) != len(manifest["items"])):
raise ValueError("manifest must bind every golden item exactly once")
seed = manifest["seed"]
if any(seed[key] != golden["seed"][key] for key in
("dataset", "revision", "filename", "retainedFilename", "contentSha", "bytes", "cutDate")):
raise ValueError("golden seed differs from the snapshot manifest")
if (not re.fullmatch(r"[0-9a-f]{40}", seed["revision"])
or not re.fullmatch(r"[0-9a-f]{64}", seed["contentSha"])
or not seed["filename"] or Path(seed["filename"]).is_absolute()
or ".." in Path(seed["filename"]).parts
or seed["retainedFilename"] != f"analyst_snapshots/{seed['contentSha']}.duckdb"
or type(seed["bytes"]) is not int or seed["bytes"] <= 0):
raise ValueError("an immutable revision, checksum and relative filename are required")
dt.date.fromisoformat(seed["cutDate"])
self.today = dt.date.fromisoformat(manifest["clock"]).isoformat()
clock = dt.date.fromisoformat(self.today)
class Date(dt.date):
@classmethod
def today(cls):
return cls(clock.year, clock.month, clock.day)
class DateTime(dt.datetime):
@classmethod
def now(cls, tz=None):
value = cls(clock.year, clock.month, clock.day, tzinfo=dt.timezone.utc)
return value.astimezone(tz) if tz else value.replace(tzinfo=None)
@classmethod
def utcnow(cls):
return cls.now()
self._datetime = SimpleNamespace(**{**vars(dt), "date": Date, "datetime": DateTime})
self.identity = {"seed": dict(seed), "clock": self.today, "items": ids}
override = os.environ.get("ANALYST_EVAL_SNAPSHOT_FILE")
if override:
source = Path(override)
self.identity["resolved_from"] = "exact-file-override"
else:
from huggingface_hub import hf_hub_download
from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError, RevisionNotFoundError
kwargs = dict(repo_type="dataset", token=os.environ.get("HF_TOKEN") or None,
local_files_only=offline)
try:
source = Path(hf_hub_download(seed["dataset"], seed["filename"],
revision=seed["revision"], **kwargs))
self.identity["resolved_from"] = "pinned-revision"
except (EntryNotFoundError, RevisionNotFoundError) as exc:
if offline or isinstance(exc, LocalEntryNotFoundError):
raise
# Squashed history may recover only the content-addressed object at HEAD.
source = Path(hf_hub_download(seed["dataset"], seed["retainedFilename"],
revision="main", **kwargs))
self.identity["resolved_from"] = "retained-content-at-head"
self._tmp = tempfile.TemporaryDirectory(prefix="analyst-eval-")
self.path = Path(self._tmp.name) / "snapshot.duckdb"
digest = hashlib.sha256()
with source.open("rb") as src, self.path.open("wb") as dst:
while chunk := src.read(8 * 1024 * 1024):
digest.update(chunk)
dst.write(chunk)
if digest.hexdigest() != seed["contentSha"] or self.path.stat().st_size != seed["bytes"]:
raise ValueError("snapshot checksum mismatch")
self.path.chmod(0o400)
import harness.datastore as DS
self._datastore = SimpleNamespace(
DB_PATH=self.path, ENTITIES=copy.deepcopy(DS.ENTITIES),
ready=lambda: True, ro_cursor=self._cursor, entity_live=self._entity_live,
_columns_backfilled=DS._columns_backfilled)
cur = self._cursor()
try:
for table in manifest["required_tables"]:
if not re.fullmatch(r"[a-z_]+", table):
raise ValueError("invalid required table")
cur.execute(f'SELECT 1 FROM "{table}" LIMIT 0')
if not self._entity_live(table):
raise ValueError(f"snapshot source {table} has not completed its backfill")
finally:
cur.close()
self._package = ModuleType("harness")
self._package.datastore = self._datastore
self._package.telemetry = SimpleNamespace(log=self.log)
self._core = ModuleType("core")
self._core.odoo = SimpleNamespace(_ctx=lambda: None, sale_line_domain=self._refuse)
self._core.store = SimpleNamespace(available=lambda: False)
self._semantic = self._load("semantic")
tenant = yaml.safe_load((self._semantic.MODEL_DIR / "tenant.yml").read_text(encoding="utf-8"))
self._semantic._tenant = lambda: copy.deepcopy(tenant)
self._semantic._store_con = self._cursor
query = self._semantic.store_query
def pinned_query(*args, **kwargs):
if any(kwargs.get(key) is not None for key in ("runtime", "user", "cur")):
raise SnapshotError("evaluation queries cannot bind another data source")
kwargs["today"] = self.today
return query(*args, **kwargs)
self._semantic.store_query = pinned_query
self._semantic._model() # Freeze source metadata before the first model turn.
self._tools = self._load("tools")
self._RESULTS = self._tools._RESULTS
except Exception as exc:
self.close()
# Hub exceptions can contain signed URLs. Report only the dependency identity.
if isinstance(exc, (ValueError, KeyError)):
why = str(exc)
else:
why = type(exc).__name__
raise SnapshotError(f"pinned Analyst snapshot unavailable or invalid: {why}; "
"supply ANALYST_EVAL_SNAPSHOT_FILE with the manifest's exact bytes; "
"no serving-store fallback") from None
@staticmethod
def _refuse(*args, **kwargs):
raise SnapshotError("live source access is forbidden during a pinned evaluation")
def _entity_live(self, table):
cur = self._cursor()
try:
row = cur.execute("SELECT phase FROM _sync_state WHERE entity = ?", [table]).fetchone()
spec = self._datastore.ENTITIES.get(table) or {}
columns_ready = (not spec.get("backfill_fields")
or self._datastore._columns_backfilled(cur, table, spec))
return bool(row and row[0] == "live" and columns_ready)
finally:
cur.close()
def _cursor(self):
import duckdb
connection = duckdb.connect(str(self.path), read_only=True,
config={"enable_external_access": False, "threads": 1})
clock = self.today
class Cursor:
def execute(self, sql, params=None):
# Pin SQL clock expressions without rewriting quoted data.
parts = re.split(r"('(?:''|[^'])*')", sql)
for i in range(0, len(parts), 2):
parts[i] = re.sub(r"\bCURRENT_DATE\b(?:\s*\(\s*\))?|\btoday\s*\(\s*\)",
f"DATE '{clock}'", parts[i], flags=re.I)
parts[i] = re.sub(r"\bCURRENT_TIMESTAMP\b|\bnow\s*\(\s*\)",
f"TIMESTAMP '{clock} 00:00:00'", parts[i], flags=re.I)
connection.execute("".join(parts), params or [])
return self
def __getattr__(self, name):
return getattr(connection, name)
return Cursor()
def _import(self, name, globals=None, locals=None, fromlist=(), level=0):
if name == "datetime":
return self._datetime
if name in ("harness.semantic", "harness.tools", "harness.transforms"):
module = self._load(name.split(".")[-1])
return module if fromlist else self._package
if name == "harness.datastore":
return self._datastore if fromlist else self._package
if name == "harness.telemetry":
return self._package.telemetry if fromlist else self._package
if name == "harness":
for child in fromlist:
if child in ("semantic", "tools", "transforms"):
self._load(child)
elif child not in ("datastore", "telemetry"):
self._refuse()
return self._package
if name == "core.odoo":
return self._core.odoo if fromlist else self._core
if name == "core" and tuple(fromlist) == ("store",):
return self._core
if name in ("core", "modules", "connectors", "odoo_client"):
self._refuse()
if name.startswith(("harness.", "core.", "modules.", "connectors.")) and name not in (
"harness.filter_sql", "core.filter_contract", "modules.finance_spine", "modules.amazon_profit"):
self._refuse()
return builtins.__import__(name, globals, locals, fromlist, level)
def _load(self, name):
if name not in self._modules:
path = Path(__file__).with_name(name + ".py")
module = ModuleType("harness." + name)
module.__file__ = str(path)
module.__builtins__ = {**vars(builtins), "__import__": self._import}
self._modules[name] = module
setattr(self._package, name, module)
# Private globals preserve the shipped implementation without swapping sys.modules.
exec(compile(path.read_text(encoding="utf-8"), str(path), "exec"), module.__dict__)
if name == "tools":
import time
import uuid
epoch = dt.datetime.fromisoformat(self.today).replace(tzinfo=dt.timezone.utc).timestamp()
module.time = SimpleNamespace(
strftime=lambda fmt: time.strftime(fmt, time.gmtime(epoch)))
counter = iter(range(1, 10**12))
module.uuid = SimpleNamespace(uuid4=lambda: uuid.UUID(int=next(counter) << 88))
return self._modules[name]
def log(self, kind, **fields):
self.events.append({"kind": kind, **fields})
def reset(self):
self._RESULTS.clear()
self._tools._RESULT_CALLERS.clear()
self.events.clear()
def list_topics(self):
return self._tools.list_topics()
def openai_tools(self):
return [tool for tool in self._tools.openai_tools() if tool["function"]["name"] in self.READS]
def dispatch(self, name, args):
if name not in self.READS:
return {"ok": False, "error": "tool unavailable in a read-only pinned evaluation"}
return self._tools.dispatch(name, args)
def close(self):
if self._tmp is not None:
if self.path.exists():
self.path.chmod(0o600)
self._tmp.cleanup()
self._tmp = None
def _numbers(text):
"""Every number in the answer, normalized: $1,234.56 -> 1234.56; '59%'/'59.0 %' -> 0.59 too."""
out = []
for m in re.finditer(r"\$?([\d,]+(?:\.\d+)?)\s*(%?)", text):
try:
v = float(m.group(1).replace(",", ""))
except ValueError:
continue
out.append(v)
if m.group(2): # percent form: also offer the fraction
out.append(v / 100.0)
return out
def _norm_ws(s):
"""Collapse ALL unicode whitespace to single spaces. Models emit U+202F (narrow no-break
space) and friends inside names — 'Poppy\\u202fFlowers' failed a plain substring check on
two different models (the 'top_customer flake', diagnosed 2026-07-12)."""
return re.sub(r"\s+", " ", s)
def _check(item, result):
exp = item.get("expect") or {}
answer = _norm_ws(result.get("answer") or "")
reasons = []
if result.get("exhausted"):
reasons.append("tool budget exhausted")
if "value" in exp:
want = float(exp["value"])
tol = exp.get("abs", exp.get("rel", 0.001) * abs(want) or 0.01)
nums = _numbers(answer)
if exp.get("allow_pct_form"):
nums += [n for n in list(nums)]
if not any(abs(n - want) <= tol for n in nums):
reasons.append(f"expected {want:,.2f} (±{tol:,.2f}) not in answer")
for s in exp.get("contains", []):
if _norm_ws(s).lower() not in answer.lower():
reasons.append(f"missing substring {s!r}")
# WRONG-ANSWER guard (2026-07-28). `contains` can only prove an answer said the right thing;
# some failures are about saying a WRONG thing that a right answer never says — e.g. naming an
# internal salesperson as an agent ([[invoice-line-agent-commission]]). Those need a negative
# assertion or the eval passes on an answer that is confidently incorrect.
for s in exp.get("not_contains", []):
if _norm_ws(s).lower() in answer.lower():
reasons.append(f"forbidden substring {s!r} present")
used = [t["tool"] for t in result.get("tool_trace", [])]
for t in exp.get("tools", []):
if t not in used:
reasons.append(f"tool {t!r} not used (trace: {used})")
for t in exp.get("not_tools", []): # over-refusal guard: these must NOT appear
if t in used:
reasons.append(f"tool {t!r} used but forbidden here (trace: {used})")
if exp.get("artifact") == "chart":
if not any("chart" in a for a in result.get("artifacts", [])):
reasons.append("no chart artifact produced")
return (not reasons), reasons
def _cost_since(A, start):
"""Ledger calls and USD cost written since `start`, or None when ANALYST_CALL_LOG is unset."""
import os as _os
if not _os.environ.get("ANALYST_CALL_LOG"):
return None
now = A.CALL_TOTALS
return {"calls": now["calls"] - start["calls"],
"cost_usd": round(now["cost"] - start["cost"], 6),
"cost_missing": now["cost_missing"] - start["cost_missing"]}
def _print_cost(cost):
if cost is not None:
print(f"COST: {cost['calls']} model call(s), ${cost['cost_usd']:.4f}"
+ (f" ({cost['cost_missing']} call(s) returned no cost)"
if cost["cost_missing"] else ""))
def run(chat_fn=None, model=None, only=None, bar=0.8, verbose=True, snapshot_path=SNAPSHOT):
"""Score golden questions on their pinned snapshot, including when chat_fn is supplied."""
import harness.analyst as A
golden = yaml.safe_load(GOLDEN.read_text(encoding="utf-8"))
items = golden["items"]
if only:
unknown = set(only) - {item["id"] for item in items}
if unknown:
raise ValueError(f"unknown golden items: {sorted(unknown)}")
items = [item for item in items if item["id"] in only]
source = _SnapshotTools(snapshot_path, golden, offline=chat_fn is not None)
patient, call_item = os.environ.get("ANALYST_PATIENT"), A.CALL_ITEM
try:
return _run(items, source, chat_fn, model, bar, verbose)
finally:
A.CALL_ITEM = call_item
if patient is None:
os.environ.pop("ANALYST_PATIENT", None)
else:
os.environ["ANALYST_PATIENT"] = patient
source.close()
def _run(items, source, chat_fn, model, bar, verbose):
"""Run the golden set. Returns {passed, total, pass_rate, gate_ok, results, cost}.
ANALYST_CALL_LOG names a JSONL cost ledger; `cost` is None when it is unset.
ANALYST_EVAL_PACE_S sets the sleep before each item (default 6 s).
"""
import harness.analyst as A
results, passed = [], 0
import os as _os
import time as _time
_os.environ["ANALYST_PATIENT"] = "1" # batch mode: wait out rate windows, don't fail fast
infra_fails = 0
aborted = False
pace = 0 if chat_fn is not None else float(_os.environ.get("ANALYST_EVAL_PACE_S") or 6)
start = dict(A.CALL_TOTALS)
for item in items:
_time.sleep(pace) # pace the free-tier per-minute windows (Groq/Cerebras)
A.CALL_ITEM = item["id"]
source.reset()
kw = {"chat_fn": chat_fn} if chat_fn is not None else {}
if model:
kw["model"] = model
try:
res = A.ask(item["ask"], telemetry_kind="analyst_eval", data_source=source, **kw)
ok, reasons = _check(item, res)
infra_fails = 0
except Exception as e:
res, ok, reasons = {"answer": "", "tool_trace": []}, False, [f"RUN ERROR: {e}"]
# Provider exhaustion is NOT a model failure: two in a row means the ladder is out
# of quota — abort and report INCONCLUSIVE instead of burning the remaining items
# into a misleading FAIL (learned 2026-07-16: 14 wasted items, 4/18 "22%").
if "all LLM providers failed" in str(e):
infra_fails += 1
if infra_fails >= 2:
aborted = True
finally:
A.CALL_ITEM = None
passed += ok
results.append({"id": item["id"], "ok": ok, "reasons": reasons,
"usage": res.get("usage"), "iterations": res.get("iterations"),
"snapshot_sha": source.identity["seed"]["contentSha"]})
if verbose:
mark = "OK " if ok else "XX "
print(f" {mark}{item['id']:18s}" + ("" if ok else f" — {'; '.join(reasons)[:90]}"))
if aborted:
break
cost = _cost_since(A, start)
if verbose:
_print_cost(cost)
if aborted:
ran = [r for r in results if not str(r["reasons"])[:60].startswith("['RUN ERROR")]
if verbose:
print(f"\nEVAL ABORTED; LLM providers exhausted after {len(results)} item(s); "
f"{passed}/{len(ran) or 1} of the items that actually ran passed. "
"Gate is INCONCLUSIVE (not a model failure); re-run when quota recovers.")
return {"passed": passed, "total": len(items), "pass_rate": None, "gate_ok": False,
"inconclusive": True, "results": results, "cost": cost,
"snapshot": source.identity}
rate = passed / len(items) if items else 0.0
gate = rate >= bar
if verbose:
print(f"\nEVAL: {passed}/{len(items)} = {rate:.0%} GATE({bar:.0%}): "
f"{'PASS' if gate else 'FAIL'}")
return {"passed": passed, "total": len(items), "pass_rate": rate, "gate_ok": gate,
"results": results, "cost": cost, "snapshot": source.identity}