bidbrain / backend /tests /test_workflow_eval.py
atharvgaur18's picture
Upload folder using huggingface_hub
ca83ef2 verified
Raw History Blame Contribute Delete
1.62 kB
"""Fast subset of the deterministic workflow evaluation (eval/workflow.py): temporal + task completion.
Throwaway graphs only (session copies, a private company graph); no LLM. The full run is
`python -m eval.workflow`.
"""
from __future__ import annotations
import pytest
from bidbrain.graph.client import Graph, db
# cases that fail because of a product behaviour diagnosed in the workflow report (not a harness problem)
KNOWN_PRODUCT_FAILURES = {"tc.renewal_actions_match_lapsing"}
@pytest.fixture(scope="module")
def result():
assert Graph().exists(), "load the base graph first: python -m bidbrain.ingest.load_graph"
from eval import workflow
return workflow.run(["temporal", "task_completion"], write=False)
def test_rows_are_well_formed(result):
rows = result["rows"]
assert len(rows) >= 15
for r in rows:
assert set(r) == {"id", "dimension", "description", "passed", "expected", "observed", "ms"}
assert isinstance(r["passed"], bool) and r["ms"] >= 0
assert len({r["id"] for r in rows}) == len(rows)
assert not [r for r in rows if r["id"].endswith(".crashed")]
def test_every_case_passes_except_diagnosed_product_failures(result):
failed = {r["id"] for r in result["rows"] if not r["passed"]}
assert failed <= KNOWN_PRODUCT_FAILURES, failed
def test_cleans_up_and_never_touches_shared_graphs(result):
assert not [n for n in db().list_graphs() if n.startswith("wfeval_")]
guard = next(r for r in result["rows"] if r["id"] == "is.base_and_org_untouched")
assert guard["passed"], guard
assert result["summary"]["llm_calls"] == 0