| """Smoke-test the Decision Index engine on a few rows of every normalised benchmark file.""" |
| import glob |
| import json |
| import os |
| import sys |
| import time |
|
|
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) |
| from decider_engine import DeciderEngine |
| from decision_index.engines import Unsupported, validate |
|
|
| engine = DeciderEngine(**dict(arg.split("=", 1) for arg in sys.argv[1:])) |
| engine.warmup() |
| for path in sorted(glob.glob("work/artifacts/benchmark-suite/normalized/*.jsonl")): |
| rows = [] |
| with open(path) as file: |
| for line in file: |
| rows.append(json.loads(line)) |
| if len(rows) == 4: |
| break |
| hits = total = 0 |
| started = time.time() |
| status = "ok" |
| for row in rows: |
| try: |
| response, _ = engine(row["state"], row["questions"]) |
| validate(row["questions"], response) |
| for key, answer in response["answers"].items(): |
| expected = (row.get("expected") or {}).get(key) |
| if expected is not None and answer["type"] == "choice": |
| total += 1 |
| hits += answer["choice"] == str(expected) |
| except Unsupported as error: |
| status = f"unsupported: {error}" |
| except Exception as error: |
| status = f"ERROR {type(error).__name__}: {str(error)[:150]}" |
| questions = sum(len(row["questions"]) for row in rows) |
| options = max(len(q.get("criteria") or {}) for row in rows for q in row["questions"].values()) |
| print(f"{os.path.basename(path)[:38]:38} {status[:60]:60} {hits}/{total} right, {questions} questions, " |
| f"max {options} options, {(time.time() - started) / max(len(rows), 1) * 1000:.0f} ms/request", flush=True) |
| print("TEST DONE", flush=True) |
|
|
