"""Smoke-test the Decision Index engine on a few rows of every normalised benchmark file."""
import glob
import json
import os
import sys
import time

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from decider_engine import DeciderEngine  # noqa: E402
from decision_index.engines import Unsupported, validate  # noqa: E402

engine = DeciderEngine(**dict(arg.split("=", 1) for arg in sys.argv[1:]))
engine.warmup()
for path in sorted(glob.glob("work/artifacts/benchmark-suite/normalized/*.jsonl")):
    rows = []
    with open(path) as file:
        for line in file:
            rows.append(json.loads(line))
            if len(rows) == 4:
                break
    hits = total = 0
    started = time.time()
    status = "ok"
    for row in rows:
        try:
            response, _ = engine(row["state"], row["questions"])
            validate(row["questions"], response)
            for key, answer in response["answers"].items():
                expected = (row.get("expected") or {}).get(key)
                if expected is not None and answer["type"] == "choice":
                    total += 1
                    hits += answer["choice"] == str(expected)
        except Unsupported as error:
            status = f"unsupported: {error}"
        except Exception as error:
            status = f"ERROR {type(error).__name__}: {str(error)[:150]}"
    questions = sum(len(row["questions"]) for row in rows)
    options = max(len(q.get("criteria") or {}) for row in rows for q in row["questions"].values())
    print(f"{os.path.basename(path)[:38]:38} {status[:60]:60} {hits}/{total} right, {questions} questions, "
          f"max {options} options, {(time.time() - started) / max(len(rows), 1) * 1000:.0f} ms/request", flush=True)
print("TEST DONE", flush=True)