Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Download platform/harness/evals.py from fsanyoto/runloopable: direct link, hf CLI and curl.
- Browser
- Download file 20.4 kB
-
https://huggingface.co/spaces/fsanyoto/runloopable/resolve/main/platform/harness/evals.py
- Command line
-
hf download hf://spaces/fsanyoto/runloopable/platform/harness/evals.py
-
curl -L -o evals.py https://huggingface.co/spaces/fsanyoto/runloopable/resolve/main/platform/harness/evals.py
20.4 kB
| """Score the real Analyst against a content-checked private snapshot, never the serving store.""" | |
| import builtins | |
| import copy | |
| import datetime as dt | |
| import hashlib | |
| import json | |
| import os | |
| import re | |
| import tempfile | |
| from pathlib import Path | |
| from types import ModuleType, SimpleNamespace | |
| import yaml | |
| GOLDEN = Path(__file__).resolve().parents[1] / "evals" / "analyst_golden.yml" | |
| SNAPSHOT = GOLDEN.with_name("analyst_snapshot.json") | |
| class SnapshotError(RuntimeError): | |
| """The pinned evaluation data is absent, corrupt or incompatible with the golden set.""" | |
| class _SnapshotTools: | |
| """Run the shipped tools, semantic compiler and transforms with private module state.""" | |
| READS = frozenset(("list_topics", "describe_topic", "get_field_values", | |
| "run_semantic_query", "transform_result", "make_chart", "make_table", | |
| "make_kpi_card", "report_gap")) | |
| def __init__(self, manifest_path, golden, offline=False): | |
| self._tmp = None | |
| self._modules = {} | |
| self.events = [] | |
| try: | |
| manifest = json.loads(Path(manifest_path).read_text(encoding="utf-8")) | |
| ids = [item["id"] for item in golden["items"]] | |
| if (manifest.get("version") != 1 or manifest.get("format") != "duckdb" | |
| or len(set(ids)) != len(ids) or set(ids) != set(manifest["items"]) | |
| or len(ids) != len(manifest["items"])): | |
| raise ValueError("manifest must bind every golden item exactly once") | |
| seed = manifest["seed"] | |
| if any(seed[key] != golden["seed"][key] for key in | |
| ("dataset", "revision", "filename", "retainedFilename", "contentSha", "bytes", "cutDate")): | |
| raise ValueError("golden seed differs from the snapshot manifest") | |
| if (not re.fullmatch(r"[0-9a-f]{40}", seed["revision"]) | |
| or not re.fullmatch(r"[0-9a-f]{64}", seed["contentSha"]) | |
| or not seed["filename"] or Path(seed["filename"]).is_absolute() | |
| or ".." in Path(seed["filename"]).parts | |
| or seed["retainedFilename"] != f"analyst_snapshots/{seed['contentSha']}.duckdb" | |
| or type(seed["bytes"]) is not int or seed["bytes"] <= 0): | |
| raise ValueError("an immutable revision, checksum and relative filename are required") | |
| dt.date.fromisoformat(seed["cutDate"]) | |
| self.today = dt.date.fromisoformat(manifest["clock"]).isoformat() | |
| clock = dt.date.fromisoformat(self.today) | |
| class Date(dt.date): | |
| def today(cls): | |
| return cls(clock.year, clock.month, clock.day) | |
| class DateTime(dt.datetime): | |
| def now(cls, tz=None): | |
| value = cls(clock.year, clock.month, clock.day, tzinfo=dt.timezone.utc) | |
| return value.astimezone(tz) if tz else value.replace(tzinfo=None) | |
| def utcnow(cls): | |
| return cls.now() | |
| self._datetime = SimpleNamespace(**{**vars(dt), "date": Date, "datetime": DateTime}) | |
| self.identity = {"seed": dict(seed), "clock": self.today, "items": ids} | |
| override = os.environ.get("ANALYST_EVAL_SNAPSHOT_FILE") | |
| if override: | |
| source = Path(override) | |
| self.identity["resolved_from"] = "exact-file-override" | |
| else: | |
| from huggingface_hub import hf_hub_download | |
| from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError, RevisionNotFoundError | |
| kwargs = dict(repo_type="dataset", token=os.environ.get("HF_TOKEN") or None, | |
| local_files_only=offline) | |
| try: | |
| source = Path(hf_hub_download(seed["dataset"], seed["filename"], | |
| revision=seed["revision"], **kwargs)) | |
| self.identity["resolved_from"] = "pinned-revision" | |
| except (EntryNotFoundError, RevisionNotFoundError) as exc: | |
| if offline or isinstance(exc, LocalEntryNotFoundError): | |
| raise | |
| # Squashed history may recover only the content-addressed object at HEAD. | |
| source = Path(hf_hub_download(seed["dataset"], seed["retainedFilename"], | |
| revision="main", **kwargs)) | |
| self.identity["resolved_from"] = "retained-content-at-head" | |
| self._tmp = tempfile.TemporaryDirectory(prefix="analyst-eval-") | |
| self.path = Path(self._tmp.name) / "snapshot.duckdb" | |
| digest = hashlib.sha256() | |
| with source.open("rb") as src, self.path.open("wb") as dst: | |
| while chunk := src.read(8 * 1024 * 1024): | |
| digest.update(chunk) | |
| dst.write(chunk) | |
| if digest.hexdigest() != seed["contentSha"] or self.path.stat().st_size != seed["bytes"]: | |
| raise ValueError("snapshot checksum mismatch") | |
| self.path.chmod(0o400) | |
| import harness.datastore as DS | |
| self._datastore = SimpleNamespace( | |
| DB_PATH=self.path, ENTITIES=copy.deepcopy(DS.ENTITIES), | |
| ready=lambda: True, ro_cursor=self._cursor, entity_live=self._entity_live, | |
| _columns_backfilled=DS._columns_backfilled) | |
| cur = self._cursor() | |
| try: | |
| for table in manifest["required_tables"]: | |
| if not re.fullmatch(r"[a-z_]+", table): | |
| raise ValueError("invalid required table") | |
| cur.execute(f'SELECT 1 FROM "{table}" LIMIT 0') | |
| if not self._entity_live(table): | |
| raise ValueError(f"snapshot source {table} has not completed its backfill") | |
| finally: | |
| cur.close() | |
| self._package = ModuleType("harness") | |
| self._package.datastore = self._datastore | |
| self._package.telemetry = SimpleNamespace(log=self.log) | |
| self._core = ModuleType("core") | |
| self._core.odoo = SimpleNamespace(_ctx=lambda: None, sale_line_domain=self._refuse) | |
| self._core.store = SimpleNamespace(available=lambda: False) | |
| self._semantic = self._load("semantic") | |
| tenant = yaml.safe_load((self._semantic.MODEL_DIR / "tenant.yml").read_text(encoding="utf-8")) | |
| self._semantic._tenant = lambda: copy.deepcopy(tenant) | |
| self._semantic._store_con = self._cursor | |
| query = self._semantic.store_query | |
| def pinned_query(*args, **kwargs): | |
| if any(kwargs.get(key) is not None for key in ("runtime", "user", "cur")): | |
| raise SnapshotError("evaluation queries cannot bind another data source") | |
| kwargs["today"] = self.today | |
| return query(*args, **kwargs) | |
| self._semantic.store_query = pinned_query | |
| self._semantic._model() # Freeze source metadata before the first model turn. | |
| self._tools = self._load("tools") | |
| self._RESULTS = self._tools._RESULTS | |
| except Exception as exc: | |
| self.close() | |
| # Hub exceptions can contain signed URLs. Report only the dependency identity. | |
| if isinstance(exc, (ValueError, KeyError)): | |
| why = str(exc) | |
| else: | |
| why = type(exc).__name__ | |
| raise SnapshotError(f"pinned Analyst snapshot unavailable or invalid: {why}; " | |
| "supply ANALYST_EVAL_SNAPSHOT_FILE with the manifest's exact bytes; " | |
| "no serving-store fallback") from None | |
| def _refuse(*args, **kwargs): | |
| raise SnapshotError("live source access is forbidden during a pinned evaluation") | |
| def _entity_live(self, table): | |
| cur = self._cursor() | |
| try: | |
| row = cur.execute("SELECT phase FROM _sync_state WHERE entity = ?", [table]).fetchone() | |
| spec = self._datastore.ENTITIES.get(table) or {} | |
| columns_ready = (not spec.get("backfill_fields") | |
| or self._datastore._columns_backfilled(cur, table, spec)) | |
| return bool(row and row[0] == "live" and columns_ready) | |
| finally: | |
| cur.close() | |
| def _cursor(self): | |
| import duckdb | |
| connection = duckdb.connect(str(self.path), read_only=True, | |
| config={"enable_external_access": False, "threads": 1}) | |
| clock = self.today | |
| class Cursor: | |
| def execute(self, sql, params=None): | |
| # Pin SQL clock expressions without rewriting quoted data. | |
| parts = re.split(r"('(?:''|[^'])*')", sql) | |
| for i in range(0, len(parts), 2): | |
| parts[i] = re.sub(r"\bCURRENT_DATE\b(?:\s*\(\s*\))?|\btoday\s*\(\s*\)", | |
| f"DATE '{clock}'", parts[i], flags=re.I) | |
| parts[i] = re.sub(r"\bCURRENT_TIMESTAMP\b|\bnow\s*\(\s*\)", | |
| f"TIMESTAMP '{clock} 00:00:00'", parts[i], flags=re.I) | |
| connection.execute("".join(parts), params or []) | |
| return self | |
| def __getattr__(self, name): | |
| return getattr(connection, name) | |
| return Cursor() | |
| def _import(self, name, globals=None, locals=None, fromlist=(), level=0): | |
| if name == "datetime": | |
| return self._datetime | |
| if name in ("harness.semantic", "harness.tools", "harness.transforms"): | |
| module = self._load(name.split(".")[-1]) | |
| return module if fromlist else self._package | |
| if name == "harness.datastore": | |
| return self._datastore if fromlist else self._package | |
| if name == "harness.telemetry": | |
| return self._package.telemetry if fromlist else self._package | |
| if name == "harness": | |
| for child in fromlist: | |
| if child in ("semantic", "tools", "transforms"): | |
| self._load(child) | |
| elif child not in ("datastore", "telemetry"): | |
| self._refuse() | |
| return self._package | |
| if name == "core.odoo": | |
| return self._core.odoo if fromlist else self._core | |
| if name == "core" and tuple(fromlist) == ("store",): | |
| return self._core | |
| if name in ("core", "modules", "connectors", "odoo_client"): | |
| self._refuse() | |
| if name.startswith(("harness.", "core.", "modules.", "connectors.")) and name not in ( | |
| "harness.filter_sql", "core.filter_contract", "modules.finance_spine", "modules.amazon_profit"): | |
| self._refuse() | |
| return builtins.__import__(name, globals, locals, fromlist, level) | |
| def _load(self, name): | |
| if name not in self._modules: | |
| path = Path(__file__).with_name(name + ".py") | |
| module = ModuleType("harness." + name) | |
| module.__file__ = str(path) | |
| module.__builtins__ = {**vars(builtins), "__import__": self._import} | |
| self._modules[name] = module | |
| setattr(self._package, name, module) | |
| # Private globals preserve the shipped implementation without swapping sys.modules. | |
| exec(compile(path.read_text(encoding="utf-8"), str(path), "exec"), module.__dict__) | |
| if name == "tools": | |
| import time | |
| import uuid | |
| epoch = dt.datetime.fromisoformat(self.today).replace(tzinfo=dt.timezone.utc).timestamp() | |
| module.time = SimpleNamespace( | |
| strftime=lambda fmt: time.strftime(fmt, time.gmtime(epoch))) | |
| counter = iter(range(1, 10**12)) | |
| module.uuid = SimpleNamespace(uuid4=lambda: uuid.UUID(int=next(counter) << 88)) | |
| return self._modules[name] | |
| def log(self, kind, **fields): | |
| self.events.append({"kind": kind, **fields}) | |
| def reset(self): | |
| self._RESULTS.clear() | |
| self._tools._RESULT_CALLERS.clear() | |
| self.events.clear() | |
| def list_topics(self): | |
| return self._tools.list_topics() | |
| def openai_tools(self): | |
| return [tool for tool in self._tools.openai_tools() if tool["function"]["name"] in self.READS] | |
| def dispatch(self, name, args): | |
| if name not in self.READS: | |
| return {"ok": False, "error": "tool unavailable in a read-only pinned evaluation"} | |
| return self._tools.dispatch(name, args) | |
| def close(self): | |
| if self._tmp is not None: | |
| if self.path.exists(): | |
| self.path.chmod(0o600) | |
| self._tmp.cleanup() | |
| self._tmp = None | |
| def _numbers(text): | |
| """Every number in the answer, normalized: $1,234.56 -> 1234.56; '59%'/'59.0 %' -> 0.59 too.""" | |
| out = [] | |
| for m in re.finditer(r"\$?([\d,]+(?:\.\d+)?)\s*(%?)", text): | |
| try: | |
| v = float(m.group(1).replace(",", "")) | |
| except ValueError: | |
| continue | |
| out.append(v) | |
| if m.group(2): # percent form: also offer the fraction | |
| out.append(v / 100.0) | |
| return out | |
| def _norm_ws(s): | |
| """Collapse ALL unicode whitespace to single spaces. Models emit U+202F (narrow no-break | |
| space) and friends inside names — 'Poppy\\u202fFlowers' failed a plain substring check on | |
| two different models (the 'top_customer flake', diagnosed 2026-07-12).""" | |
| return re.sub(r"\s+", " ", s) | |
| def _check(item, result): | |
| exp = item.get("expect") or {} | |
| answer = _norm_ws(result.get("answer") or "") | |
| reasons = [] | |
| if result.get("exhausted"): | |
| reasons.append("tool budget exhausted") | |
| if "value" in exp: | |
| want = float(exp["value"]) | |
| tol = exp.get("abs", exp.get("rel", 0.001) * abs(want) or 0.01) | |
| nums = _numbers(answer) | |
| if exp.get("allow_pct_form"): | |
| nums += [n for n in list(nums)] | |
| if not any(abs(n - want) <= tol for n in nums): | |
| reasons.append(f"expected {want:,.2f} (±{tol:,.2f}) not in answer") | |
| for s in exp.get("contains", []): | |
| if _norm_ws(s).lower() not in answer.lower(): | |
| reasons.append(f"missing substring {s!r}") | |
| # WRONG-ANSWER guard (2026-07-28). `contains` can only prove an answer said the right thing; | |
| # some failures are about saying a WRONG thing that a right answer never says — e.g. naming an | |
| # internal salesperson as an agent ([[invoice-line-agent-commission]]). Those need a negative | |
| # assertion or the eval passes on an answer that is confidently incorrect. | |
| for s in exp.get("not_contains", []): | |
| if _norm_ws(s).lower() in answer.lower(): | |
| reasons.append(f"forbidden substring {s!r} present") | |
| used = [t["tool"] for t in result.get("tool_trace", [])] | |
| for t in exp.get("tools", []): | |
| if t not in used: | |
| reasons.append(f"tool {t!r} not used (trace: {used})") | |
| for t in exp.get("not_tools", []): # over-refusal guard: these must NOT appear | |
| if t in used: | |
| reasons.append(f"tool {t!r} used but forbidden here (trace: {used})") | |
| if exp.get("artifact") == "chart": | |
| if not any("chart" in a for a in result.get("artifacts", [])): | |
| reasons.append("no chart artifact produced") | |
| return (not reasons), reasons | |
| def _cost_since(A, start): | |
| """Ledger calls and USD cost written since `start`, or None when ANALYST_CALL_LOG is unset.""" | |
| import os as _os | |
| if not _os.environ.get("ANALYST_CALL_LOG"): | |
| return None | |
| now = A.CALL_TOTALS | |
| return {"calls": now["calls"] - start["calls"], | |
| "cost_usd": round(now["cost"] - start["cost"], 6), | |
| "cost_missing": now["cost_missing"] - start["cost_missing"]} | |
| def _print_cost(cost): | |
| if cost is not None: | |
| print(f"COST: {cost['calls']} model call(s), ${cost['cost_usd']:.4f}" | |
| + (f" ({cost['cost_missing']} call(s) returned no cost)" | |
| if cost["cost_missing"] else "")) | |
| def run(chat_fn=None, model=None, only=None, bar=0.8, verbose=True, snapshot_path=SNAPSHOT): | |
| """Score golden questions on their pinned snapshot, including when chat_fn is supplied.""" | |
| import harness.analyst as A | |
| golden = yaml.safe_load(GOLDEN.read_text(encoding="utf-8")) | |
| items = golden["items"] | |
| if only: | |
| unknown = set(only) - {item["id"] for item in items} | |
| if unknown: | |
| raise ValueError(f"unknown golden items: {sorted(unknown)}") | |
| items = [item for item in items if item["id"] in only] | |
| source = _SnapshotTools(snapshot_path, golden, offline=chat_fn is not None) | |
| patient, call_item = os.environ.get("ANALYST_PATIENT"), A.CALL_ITEM | |
| try: | |
| return _run(items, source, chat_fn, model, bar, verbose) | |
| finally: | |
| A.CALL_ITEM = call_item | |
| if patient is None: | |
| os.environ.pop("ANALYST_PATIENT", None) | |
| else: | |
| os.environ["ANALYST_PATIENT"] = patient | |
| source.close() | |
| def _run(items, source, chat_fn, model, bar, verbose): | |
| """Run the golden set. Returns {passed, total, pass_rate, gate_ok, results, cost}. | |
| ANALYST_CALL_LOG names a JSONL cost ledger; `cost` is None when it is unset. | |
| ANALYST_EVAL_PACE_S sets the sleep before each item (default 6 s). | |
| """ | |
| import harness.analyst as A | |
| results, passed = [], 0 | |
| import os as _os | |
| import time as _time | |
| _os.environ["ANALYST_PATIENT"] = "1" # batch mode: wait out rate windows, don't fail fast | |
| infra_fails = 0 | |
| aborted = False | |
| pace = 0 if chat_fn is not None else float(_os.environ.get("ANALYST_EVAL_PACE_S") or 6) | |
| start = dict(A.CALL_TOTALS) | |
| for item in items: | |
| _time.sleep(pace) # pace the free-tier per-minute windows (Groq/Cerebras) | |
| A.CALL_ITEM = item["id"] | |
| source.reset() | |
| kw = {"chat_fn": chat_fn} if chat_fn is not None else {} | |
| if model: | |
| kw["model"] = model | |
| try: | |
| res = A.ask(item["ask"], telemetry_kind="analyst_eval", data_source=source, **kw) | |
| ok, reasons = _check(item, res) | |
| infra_fails = 0 | |
| except Exception as e: | |
| res, ok, reasons = {"answer": "", "tool_trace": []}, False, [f"RUN ERROR: {e}"] | |
| # Provider exhaustion is NOT a model failure: two in a row means the ladder is out | |
| # of quota — abort and report INCONCLUSIVE instead of burning the remaining items | |
| # into a misleading FAIL (learned 2026-07-16: 14 wasted items, 4/18 "22%"). | |
| if "all LLM providers failed" in str(e): | |
| infra_fails += 1 | |
| if infra_fails >= 2: | |
| aborted = True | |
| finally: | |
| A.CALL_ITEM = None | |
| passed += ok | |
| results.append({"id": item["id"], "ok": ok, "reasons": reasons, | |
| "usage": res.get("usage"), "iterations": res.get("iterations"), | |
| "snapshot_sha": source.identity["seed"]["contentSha"]}) | |
| if verbose: | |
| mark = "OK " if ok else "XX " | |
| print(f" {mark}{item['id']:18s}" + ("" if ok else f" — {'; '.join(reasons)[:90]}")) | |
| if aborted: | |
| break | |
| cost = _cost_since(A, start) | |
| if verbose: | |
| _print_cost(cost) | |
| if aborted: | |
| ran = [r for r in results if not str(r["reasons"])[:60].startswith("['RUN ERROR")] | |
| if verbose: | |
| print(f"\nEVAL ABORTED; LLM providers exhausted after {len(results)} item(s); " | |
| f"{passed}/{len(ran) or 1} of the items that actually ran passed. " | |
| "Gate is INCONCLUSIVE (not a model failure); re-run when quota recovers.") | |
| return {"passed": passed, "total": len(items), "pass_rate": None, "gate_ok": False, | |
| "inconclusive": True, "results": results, "cost": cost, | |
| "snapshot": source.identity} | |
| rate = passed / len(items) if items else 0.0 | |
| gate = rate >= bar | |
| if verbose: | |
| print(f"\nEVAL: {passed}/{len(items)} = {rate:.0%} GATE({bar:.0%}): " | |
| f"{'PASS' if gate else 'FAIL'}") | |
| return {"passed": passed, "total": len(items), "pass_rate": rate, "gate_ok": gate, | |
| "results": results, "cost": cost, "snapshot": source.identity} | |