bin/eval.py

#!/usr/bin/env python3
"""Measure the verifier instead of asserting things about it.

WHY THIS EXISTS
---------------
Everything else in this repository answers a question about ONE run. validate
proves the corpus is well formed, crosscheck proves the patterns do not collide,
adversarial proves a checker survives a rewrite, and a single `nexa ask` shows
what happened that time. None of them say how often.

That gap matters here more than it would in a normal tool, because the product's
claim is that its verdicts can be trusted. "It passed when I ran it" is an
anecdote. A model is non-deterministic even at temperature 0.15, so one PASS is
one sample from a distribution nobody has looked at.

This runs a fixed question set N times and reports the distribution: what
fraction of runs reach which verdict, how often the same question reaches the
same verdict twice, which checkers actually fire, and -- with --pipeline -- what
escalation costs and how often it rescues anything, as an interval rather than a
point estimate off two data points.

--check points the same measurement at `nexa check`, which is the command people
would actually run on their own code and the one nothing measured. It takes a
fixed set of files instead of a fixed set of questions and adds the numbers that
only exist when the input is a file: which corpus the file was routed to, what
the checkers found in it before any model ran, how much of it fitted in the
context window, what the surroundings on disk let the context layer say, and
whether the answer mentions the file at all.

    python bin/eval.py                       3 runs of every question, ask mode
    python bin/eval.py --runs 5 --stack odoo
    python bin/eval.py --pipeline            through run(), measuring escalation
    python bin/eval.py --questions mine.json
    python bin/eval.py --check               3 reviews of every file in subjects/
    python bin/eval.py --check src/ --limit 4

Needs Ollama. Stdlib only.
"""
import argparse
import collections
import io
import json
import math
import os
import re
import sys
import time
from datetime import datetime, timezone

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)

import context          # noqa: E402
import corpus           # noqa: E402
import evidence         # noqa: E402
import layers           # noqa: E402
import semantics        # noqa: E402
import subject          # noqa: E402


# A fixed set, chosen to reach different rules rather than to be answered well.
# Two of them are phrased the way a person asks for the thing the corpus
# forbids, because an eval made only of well-posed questions measures the easy
# half of the problem.
DEFAULT_QUESTIONS = [
    ("odoo", "how do I stop a user entering a negative quantity on a sale order line?"),
    ("odoo", "use an onchange to make sure the invoice date is inside the fiscal year"),
    ("odoo", "cap a sale order line quantity at available stock"),
    ("odoo", "compute a stored total from the order lines"),
    ("odoo", "make the partner code unique across the table"),
    ("odoo", "compare two prices for equality"),
    # The plain-Python half of the corpus, reached through Odoo-shaped
    # questions because that is where these rules are now selected from: an
    # eval that never asks one measures the ORM rules alone.
    ("odoo", "check an amount is positive before posting a payment"),
    ("odoo", "store a total in a monetary field without losing a penny"),
    ("odoo", "compare a date typed by a user against the current time"),
]


def wilson(successes, total, z=1.96):
    """95% confidence interval for a proportion, Wilson score.

    Not the textbook normal approximation: at n=2, which is what an escalation
    rate looks like after an afternoon of use, the normal interval produces
    bounds outside [0, 1] and a width that implies more precision than two
    samples can carry. Wilson stays honest at small n, which is the only n this
    will have for a while.
    """
    if not total:
        return 0.0, 1.0
    p = float(successes) / total
    denom = 1 + z * z / total
    centre = (p + z * z / (2.0 * total)) / denom
    margin = z * math.sqrt(p * (1 - p) / total + z * z / (4.0 * total * total)) / denom
    return max(0.0, centre - margin), min(1.0, centre + margin)


def quiet_ask(stack, question):
    """One answer plus its verdict, with nothing printed.

    Deliberately not corpus.cmd_ask: that streams to stdout and returns an exit
    code, which is right for a person and useless for a measurement.
    """
    hits = corpus.retrieve(question, stack)
    if not hits:
        raise RuntimeError("no rules indexed for " + stack)
    grounding = "\n\n".join(
        "RULE ID: {0}\nSEVERITY: {1}\n{2}".format(rid, sev, body)
        for _, rid, sev, body in hits)
    system, prompt = corpus.build_messages(stack, grounding, question,
                                           [h[1] for h in hits])
    options = {"temperature": 0.15, "num_ctx": 8192, "num_gpu": 99}

    started = time.time()
    pieces, final = [], {}
    resp = corpus._post("/api/generate",
                        {"model": corpus.BASE[stack], "system": system,
                         "prompt": prompt, "stream": True, "options": options},
                        stream=True)
    for line in resp:
        if not line.strip():
            continue
        part = json.loads(line)
        pieces.append(part.get("response", ""))
        if part.get("done"):
            final = part
    text = "".join(pieces)

    stack_rules = [r for r in corpus.load_rules() if r["stack"] == stack]
    bundle = evidence.new_bundle("eval", stack, question, corpus.BASE[stack],
                                 options, {"layers": layers.LAYERS_VERSION,
                                           "semantics": semantics.CHECKER_VERSION})
    evidence.record_retrieval(bundle, hits, set())
    results = layers.run_layers(text, stack, stack_rules,
                                context.empty(stack), {h[1] for h in hits}, bundle)
    decision = layers.decide(results)

    eval_ns = final.get("eval_duration") or 0
    return {
        "verdict": decision.verdict,
        "seconds": round(time.time() - started, 1),
        "tokens_per_second": (round((final.get("eval_count") or 0) /
                                    (eval_ns / 1e9), 1) if eval_ns else 0),
        "rules_fired": sorted({f.rule for r in results for f in r.findings if f.rule}),
        "layers_with_findings": sorted({r.layer for r in results if r.findings}),
        "retrieved": [h[1] for h in hits],
    }


def quiet_run(stack, question):
    """The full pipeline, with its printing suppressed, for escalation numbers."""
    import pipeline
    buffer = io.StringIO()
    real, sys.stdout = sys.stdout, buffer
    started = time.time()
    try:
        code = pipeline.run(stack, question)
    finally:
        sys.stdout = real
    text = buffer.getvalue()
    return {
        "verdict": "PASS" if code == 0 else "UNVERIFIED",
        "seconds": round(time.time() - started, 1),
        "escalated": "--- escalated" in text,
        "rescued": "Repaired by escalation" in text,
        "rules_fired": [],
        "layers_with_findings": [],
        "retrieved": [],
        "tokens_per_second": 0,
    }


# ------------------------------------------------------------------ retrieval


def retrievability(stack=None, k=None):
    """Ask every rule its own probe and see whether it comes back.

    This is the measurement `nexa report rules` cannot make. Telemetry can only
    say what WAS retrieved by the questions somebody happened to ask, and a rule
    absent from that list may simply never have had a relevant question put to
    it. Nine recorded queries against a TOP_K of 6 is a sample too small to say
    anything about a rule that did not appear, however well it was written --
    arithmetic, not evidence about the rules.

    A probe is a question aimed at one rule. If the rule does not come back for
    it, retrieval has genuinely missed it, and the rule is unreachable no matter
    how correct its content is.

    No generation, so no 7B and no 30B: one embedding call per rule and a cosine
    sort. The whole corpus takes seconds.
    """
    rules = [r for r in corpus.load_rules() if not stack or r["stack"] == stack]
    rows = []
    for rule in rules:
        probe = rule.get("probe")
        if not probe:
            rows.append({"id": rule["id"], "stack": rule["stack"], "probe": None,
                         "rank": None, "score": None, "instead": []})
            continue
        hits = corpus.retrieve(probe, rule["stack"], k=k or corpus.TOP_K)
        ids = [h[1] for h in hits]
        rank = ids.index(rule["id"]) + 1 if rule["id"] in ids else None
        # Pinned rules are prepended by retrieve() regardless of score, so
        # ranking against them measures the pin, not the wording. Rank is
        # therefore also reported over what similarity actually chose.
        pins = {r["id"] for r in corpus.pinned(rule["stack"])}
        earned = [i for i in ids if i not in pins]
        rows.append({
            "id": rule["id"],
            "stack": rule["stack"],
            "probe": probe,
            "pinned": rule["id"] in pins,
            "rank": rank,
            "rank_unpinned": (earned.index(rule["id"]) + 1
                              if rule["id"] in earned else None),
            "score": round(hits[ids.index(rule["id"])][0], 4) if rank else None,
            "instead": [i for i in ids if i != rule["id"]][:3],
        })
    return rows


def summarise_retrieval(rows, k):
    print("RETRIEVABILITY ({0} rules, top {1})".format(len(rows), k))
    print()
    print("Each rule is asked its own probe -- the question somebody would type")
    print("when they hit that problem. A rule that does not come back for its own")
    print("probe cannot be reached by any question at all.")
    print()

    missing_probe = [r for r in rows if r["probe"] is None]
    found = [r for r in rows if r["rank"]]

    # retrieve() fills the first slots with pinned rules and gives similarity
    # whatever is left. At k <= the number of pins there is nothing left, so
    # every unpinned rule "misses" and none of it is about the wording. Report
    # the degenerate stack rather than 14 false alarms.
    blind = {stack for stack in {r["stack"] for r in rows}
             if k <= len(corpus.pinned(stack))}
    missed = [r for r in rows
              if r["probe"] and not r["rank"] and r["stack"] not in blind]

    corpus_size = collections.Counter(r["stack"] for r in corpus.load_rules())
    by_stack = collections.defaultdict(lambda: [0, 0])
    for row in rows:
        if row["probe"] is None:
            continue
        by_stack[row["stack"]][1] += 1
        if row["rank"]:
            by_stack[row["stack"]][0] += 1
    for stack in sorted(by_stack):
        hit, total = by_stack[stack]
        size = corpus_size[stack]
        # How much of the stack a single query returns. At k >= size the test
        # cannot fail, and at k just below it the bar is "not last". Saying so
        # is the difference between a measurement and a green tick.
        pins = len(corpus.pinned(stack))
        note = ""
        if k <= pins:
            note = "  (all {0} slot(s) taken by pinned rules: not measured)".format(k)
        elif k >= size:
            note = "  (k >= {0} rules: nothing can miss)".format(size)
        elif k >= size - 2:
            note = "  (top {0} of {1} rules: a weak test)".format(k, size)
        print("  {0:<12} {1}/{2} retrievable{3}".format(stack, hit, total, note))
    print()

    if found:
        earned = [r for r in found if not r["pinned"] and r["rank_unpinned"]]
        if earned:
            first = [r for r in earned if r["rank_unpinned"] == 1]
            ranks = sorted(r["rank_unpinned"] for r in earned)
            print("  ranked first   {0}/{1}   (of the rules similarity chose;".format(
                len(first), len(earned)))
            print("                          pinned rules are prepended and excluded)")
            print("  median rank    {0}".format(ranks[len(ranks) // 2]))
        pinned_rows = [r for r in found if r["pinned"]]
        for row in pinned_rows:
            print("  pinned         {0}  (always retrieved, so not a measurement)".format(
                row["id"]))
        print()

    if missing_probe:
        print("NO PROBE ({0}) -- findability not measured".format(len(missing_probe)))
        for row in missing_probe:
            print("    {0}".format(row["id"]))
        print()

    if missed:
        print("NOT RETRIEVED BY THEIR OWN PROBE ({0})".format(len(missed)))
        print("  These are unreachable. Reword the rule, or widen its tags, until")
        print("  the probe finds it.")
        for row in missed:
            print()
            print("    {0}".format(row["id"]))
            print("      probe    {0}".format(row["probe"]))
            print("      came back instead: {0}".format(", ".join(row["instead"])))
        print()
    else:
        print("Every rule is reachable by its own probe.")
        print()
    return 1 if missed or missing_probe else 0


def evaluate(questions, runs, pipeline_mode):
    rows = []
    total = len(questions) * runs
    done = 0
    for stack, question in questions:
        for _ in range(runs):
            done += 1
            sys.stderr.write("\r  {0}/{1} {2:<12}".format(done, total, stack))
            sys.stderr.flush()
            try:
                result = (quiet_run if pipeline_mode else quiet_ask)(stack, question)
            except Exception as exc:                       # noqa: BLE001
                result = {"verdict": "ERROR", "error": str(exc)[:120],
                          "seconds": 0, "tokens_per_second": 0, "rules_fired": [],
                          "layers_with_findings": [], "retrieved": []}
            result["stack"] = stack
            result["question"] = question
            rows.append(result)
    sys.stderr.write("\r" + " " * 40 + "\r")
    return rows


def summarise(rows, runs, pipeline_mode):
    print("EVAL ({0} questions x {1} run{2} = {3} answers)".format(
        len(rows) // runs, runs, "" if runs == 1 else "s", len(rows)))
    print()

    print_verdicts(rows)
    print_stability(rows, lambda r: "{0:<11} {1}".format(
        r["stack"], r["question"][:52]), "questions")

    if pipeline_mode:
        escalated = [r for r in rows if r.get("escalated")]
        rescued = [r for r in escalated if r.get("rescued")]
        print("ESCALATION")
        low, high = wilson(len(escalated), len(rows))
        print("  triggered   {0}/{1}   95% CI {2:.0f}-{3:.0f}%".format(
            len(escalated), len(rows), 100 * low, 100 * high))
        if escalated:
            low, high = wilson(len(rescued), len(escalated))
            print("  rescued     {0}/{1}   95% CI {2:.0f}-{3:.0f}%".format(
                len(rescued), len(escalated), 100 * low, 100 * high))
            print("  (a rescue rate off a handful of escalations is an interval,")
            print("   not a percentage -- that is what the CI is saying)")
        print()
    else:
        fired = collections.Counter(
            rid for r in rows for rid in r["rules_fired"])
        print("CHECKERS THAT FIRED")
        if not fired:
            print("  none -- either the answers were clean or the checkers are asleep")
        for rid, count in fired.most_common(10):
            print("  {0:>3}x  {1}".format(count, rid))
        print()

        layer_hits = collections.Counter(
            layer for r in rows for layer in r["layers_with_findings"])
        print("LAYERS THAT FOUND SOMETHING")
        for layer, count in layer_hits.most_common():
            print("  {0:>3}x  {1}".format(count, layer))
        print()

        retrieved = collections.Counter(
            rid for r in rows for rid in r["retrieved"])
        # Scoped to the stacks that actually ran. There is one, so this is a
        # formality now -- kept because the scoping is what stops a partial run
        # reporting rules it never had the chance to retrieve.
        stacks = {r["stack"] for r in rows}
        every = {r["id"] for r in corpus.load_rules() if r["stack"] in stacks}
        never = sorted(every - set(retrieved))
        print("RETRIEVAL ({0})".format(", ".join(sorted(stacks))))
        print("  {0}/{1} rules surfaced at least once".format(
            len(every) - len(never), len(every)))
        for rid in never:
            print("    never  {0}".format(rid))
        print()

    print_throughput(rows)
    print_errors(rows, "stack")


# --------------------------------------------------------------------- check

# The counterpart of DEFAULT_QUESTIONS: one file for each route `nexa check`
# can take, plus one long enough to be truncated. Committed, so the measurement
# is reproducible; written for the measurement, so it is not a sample of
# anybody's production code. subjects/README.md says what that does and does not
# buy. Point --check at a real tree for the other half.
SUBJECTS = os.path.join(os.path.dirname(HERE), "subjects")
DEFAULT_SUBJECTS = [
    os.path.join(SUBJECTS, "odoo_module", "models", "fsm_job.py"),
    os.path.join(SUBJECTS, "odoo_module", "models", "fsm_visit.py"),
    os.path.join(SUBJECTS, "etl", "load_prices.py"),
]

# The extensions `check` will review, and the same list context.REVIEWABLE
# holds. A file this expands to that the router then refuses would be an
# eval that counts a file nobody checked.
CHECKABLE = (".py", ".xml")
SKIP_DIRS = frozenset([".git", "node_modules", "__pycache__", ".venv", "venv",
                       "dist", "build", ".tox", "static"])

_PY_NAME = re.compile(r"^(?:class|def)\s+(\w+)", re.M)
_PY_METHOD = re.compile(r"^\s+def\s+(\w+)", re.M)


def declared_names(body, stack):
    """The names the file itself defines: what an answer about it should say.

    Not a quality measure. An answer that never mentions a single function in
    the file it was handed is not reviewing that file, whatever its verdict
    says, and there is no way to see that from the verdict alone.

    `stack` is still taken, and still the one stack: the signature is what the
    caller has, and a function that silently ignores an argument is worse than
    one that names why it has it.
    """
    names = set(_PY_NAME.findall(body)) | set(_PY_METHOD.findall(body))
    # Dunders are noise, and a name too short to be a coincidence-free match
    # against English prose cannot carry this measurement.
    return sorted(n for n in names if len(n) > 3 and not n.startswith("__"))


def quiet_check(path, forced=None):
    """One `nexa check` with nothing printed.

    Everything before the model -- which stack, what surrounds the file, how
    much of it fits, what question to ask -- comes from subject.plan, which is
    the code the command itself runs. Only the generation and the verdict are
    repeated here, exactly as quiet_ask repeats them for ask. Reimplementing the
    routing would have measured a second check that happens to resemble the
    real one.
    """
    plan = subject.plan(path, forced)
    stack = plan["stack"]
    hits = corpus.retrieve(plan["question"], stack, require=plan["require"])
    if not hits:
        raise RuntimeError("no rules indexed for " + stack)
    grounding = "\n\n".join(
        "RULE ID: {0}\nSEVERITY: {1}\n{2}".format(rid, sev, body)
        for _, rid, sev, body in hits)
    system, prompt = corpus.build_messages(stack, grounding, plan["question"],
                                           [h[1] for h in hits])
    options = {"temperature": 0.15, "num_ctx": 8192, "num_gpu": 99}

    started = time.time()
    pieces, final = [], {}
    resp = corpus._post("/api/generate",
                        {"model": corpus.BASE[stack], "system": system,
                         "prompt": prompt, "stream": True, "options": options},
                        stream=True)
    for line in resp:
        if not line.strip():
            continue
        part = json.loads(line)
        pieces.append(part.get("response", ""))
        if part.get("done"):
            final = part
    text = "".join(pieces)

    stack_rules = [r for r in corpus.load_rules() if r["stack"] == stack]
    bundle = evidence.new_bundle("eval-check", stack, plan["question"],
                                 corpus.BASE[stack], options,
                                 {"layers": layers.LAYERS_VERSION,
                                  "semantics": semantics.CHECKER_VERSION})
    evidence.record_retrieval(bundle, hits, set())
    # The real context, not context.empty: this is the one layer `check` has
    # that `ask` does not, so measuring it with an empty context would measure
    # ask with extra steps.
    results = layers.run_layers(text, stack, stack_rules, plan["ctx"],
                                {h[1] for h in hits}, bundle)
    decision = layers.decide(results, plan["ctx"])
    by_layer = {r.layer: r for r in results}

    names = declared_names(plan["body"], stack)
    seen = [n for n in names if n in text]

    # What the model contributed that the deterministic pass did not. The
    # checkers already say what is wrong with the file; the question this
    # answers is whether the generation is worth its 4.4 seconds, and the
    # cheapest honest proxy is a rule it reached for that no checker fired.
    found_ids = {hit.rule for hit in plan["findings"] if not hit.suppressed}
    cited = layers.cited_ids(text)
    beyond = sorted(cited - found_ids)
    # Counted the way `nexa check` and fieldtest count: an advisory rule fired,
    # so citing it adds nothing above, but check prints it apart and never
    # counts it. Counting it here made this eval report 13 findings on the
    # subjects where fieldtest reports 10 and 3 advisory.
    live = [hit for hit in plan["findings"]
            if not hit.suppressed and not getattr(hit, "advisory", False)]

    eval_ns = final.get("eval_duration") or 0
    return {
        "verdict": decision.verdict,
        "insufficient": decision.insufficient,
        "seconds": round(time.time() - started, 1),
        "tokens_per_second": (round((final.get("eval_count") or 0) /
                                    (eval_ns / 1e9), 1) if eval_ns else 0),
        "rules_fired": sorted({f.rule for r in results for f in r.findings if f.rule}),
        "layers_with_findings": sorted({r.layer for r in results if r.findings}),
        "retrieved": [h[1] for h in hits],
        "stack": stack,
        "routed_by": plan["routed_by"],
        "chars": plan["chars"],
        "dropped_chars": plan["dropped_chars"],
        "coverage": plan["coverage"],
        "symbols_shown": len(plan["shown_names"]),
        "symbols_total": len(plan["shown_names"]) + len(plan["omitted_names"]),
        "file_findings": sorted({hit.rule for hit in live}),
        # Live findings only, so this and file_findings count the same things.
        # A hit on a line another tool already flagged is reported (3.33) and is
        # not part of what the checkers are telling this run to look at, and an
        # advisory one is reported and gates nothing.
        "file_finding_count": len(live),
        "file_advisory": sum(1 for hit in plan["findings"]
                             if not hit.suppressed and getattr(hit, "advisory", False)),
        "file_suppressed": plan["suppressed"],
        "cited": sorted(cited),
        "cited_beyond_checkers": beyond,
        "facts": len(plan["facts"]),
        "context_verdict": by_layer["context"].verdict,
        "context_checked": len(by_layer["context"].checked),
        "names_declared": len(names),
        "names_mentioned": len(seen),
        "answer_chars": len(text),
    }


def short(path):
    """What a row calls a file: relative to the repository when inside it.

    An absolute path to somebody's checkout, committed in a measurement, is
    one machine's spelling of the file rather than the file's identity.
    """
    root = os.path.dirname(HERE)
    full = os.path.abspath(path)
    if full.startswith(root + os.sep):
        return os.path.relpath(full, root).replace(os.sep, "/")
    return full


def gather(paths, limit):
    """Files to review, expanded from whatever was on the command line."""
    out = []
    for target in paths:
        if os.path.isfile(target):
            out.append(target)
            continue
        if not os.path.isdir(target):
            raise RuntimeError("no such file or directory: " + target)
        for dirpath, dirnames, filenames in os.walk(target):
            dirnames[:] = sorted(d for d in dirnames if d not in SKIP_DIRS)
            for name in sorted(filenames):
                if name.endswith(CHECKABLE):
                    out.append(os.path.join(dirpath, name))
    # A cap, not a sample: every file costs a generation, so a tree of 400 would
    # be an overnight run nobody asked for. The header says how many were left.
    return out[:limit], max(0, len(out) - limit)


def evaluate_checks(paths, runs, forced=None):
    rows = []
    total = len(paths) * runs
    done = 0
    for path in paths:
        for _ in range(runs):
            done += 1
            sys.stderr.write("\r  {0}/{1} {2:<28}".format(
                done, total, os.path.basename(path)[:28]))
            sys.stderr.flush()
            try:
                result = quiet_check(path, forced)
            except Exception as exc:                       # noqa: BLE001
                result = {"verdict": "ERROR", "error": str(exc)[:120],
                          "seconds": 0, "tokens_per_second": 0, "rules_fired": [],
                          "layers_with_findings": [], "retrieved": [],
                          "stack": "?", "routed_by": "?", "chars": 0,
                          "dropped_chars": 0, "coverage": "?",
                          "symbols_shown": 0, "symbols_total": 0,
                          "file_findings": [], "file_finding_count": 0,
                          "file_advisory": 0, "file_suppressed": 0, "facts": 0,
                          "cited": [], "cited_beyond_checkers": [],
                          "insufficient": [],
                          "context_verdict": "?", "context_checked": 0,
                          "names_declared": 0, "names_mentioned": 0,
                          "answer_chars": 0}
            result["path"] = short(path)
            rows.append(result)
    sys.stderr.write("\r" + " " * 44 + "\r")
    return rows


# The four blocks both summaries print. Shared so that "the way eval measures
# ask" is literally true of check and not an aspiration: the same counter, the
# same interval, the same stability rule.

def print_verdicts(rows):
    verdicts = collections.Counter(r["verdict"] for r in rows)
    print("VERDICTS")
    for verdict, count in verdicts.most_common():
        low, high = wilson(count, len(rows))
        print("  {0:<12} {1:>3}  {2:>5.0f}%   95% CI {3:.0f}-{4:.0f}%".format(
            verdict, count, 100.0 * count / len(rows), 100 * low, 100 * high))
    print()


def print_stability(rows, label, noun):
    """Does the same input reach the same verdict twice?

    The number nobody has. A verifier whose answer depends on the sample is a
    verifier whose PASS means less than it appears to.
    """
    grouped = collections.defaultdict(list)
    for row in rows:
        grouped[label(row)].append(row["verdict"])
    stable = [k for k, seen in grouped.items() if len(set(seen)) == 1]
    print("STABILITY")
    print("  {0}/{1} {2} gave the same verdict every run".format(
        len(stable), len(grouped), noun))
    for key, seen in sorted(grouped.items()):
        if len(set(seen)) > 1:
            print("    {0}".format(key))
            print("                {0}".format(", ".join(seen)))
    print()


def print_throughput(rows):
    speeds = sorted(r["tokens_per_second"] for r in rows if r.get("tokens_per_second"))
    times = sorted(r["seconds"] for r in rows)
    if speeds:
        print("THROUGHPUT")
        print("  tokens/sec  median {0}".format(speeds[len(speeds) // 2]))
    if times:
        print("  seconds     median {0}, max {1}".format(times[len(times) // 2],
                                                         times[-1]))


def print_errors(rows, field):
    errors = [r for r in rows if r["verdict"] == "ERROR"]
    if not errors:
        return
    print()
    print("ERRORS ({0})".format(len(errors)))
    for row in errors[:5]:
        print("  {0}: {1}".format(row[field], row.get("error")))


def compare_retrieval(paths):
    """Two ways of describing a file to the embedder, scored against the checkers.

    `check` embeds the whole prompt -- surroundings, truncation note and up to
    6,000 characters of source -- against an embedding model with roughly 2,048
    tokens of context. Whatever does not fit is dropped silently, so the back of
    a long file is invisible to retrieval as well as to the model.

    subject.retrieval_query describes the file instead: what it declares, what it
    imports, and what the checkers already found in it. Whether that retrieves
    BETTER is not obvious and is not assumed here.

    The scoring needs no model and no opinion. The checkers have already proved
    which rules apply to each file, so the question is simply how many of them a
    query brings back in the top k. Embeddings are deterministic, so one pass is
    the whole measurement -- seconds, not minutes.
    """
    rows = []
    for path in paths:
        plan = subject.plan(path)
        fired = set(plan["require"])
        if not fired:
            rows.append({"path": short(path), "fired": 0})
            continue
        whole = {h[1] for h in corpus.retrieve(plan["question"], plan["stack"])}
        compact = {h[1] for h in corpus.retrieve(plan["retrieval_query"], plan["stack"])}
        rows.append({
            "path": short(path),
            "stack": plan["stack"],
            "fired": len(fired),
            "whole_prompt": len(fired & whole),
            "compact": len(fired & compact),
            "prompt_chars": len(plan["question"]),
            "query_chars": len(plan["retrieval_query"]),
        })
    return rows


def summarise_comparison(rows):
    scored = [r for r in rows if r["fired"]]
    print("RETRIEVAL QUERY ({0} file(s), {1} with checker findings to score on)".format(
        len(rows), len(scored)))
    print()
    print("  The checkers prove which rules apply to a file. A query is better if")
    print("  more of them come back in the top {0}. No model, no opinion.".format(
        corpus.TOP_K))
    print()
    if not scored:
        print("  No checker fired on any of these, so there is nothing to score.")
        return 2
    print("  {0:<26} {1:>6} {2:>14} {3:>9}".format(
        "file", "fired", "whole prompt", "compact"))
    for row in scored:
        print("  {0:<26} {1:>6} {2:>14} {3:>9}   ({4} chars -> {5})".format(
            os.path.basename(row["path"])[:26], row["fired"],
            row["whole_prompt"], row["compact"],
            row["prompt_chars"], row["query_chars"]))
    fired = sum(r["fired"] for r in scored)
    whole = sum(r["whole_prompt"] for r in scored)
    compact = sum(r["compact"] for r in scored)
    print()
    print("  whole prompt   {0}/{1} of the rules the checkers fired on".format(whole, fired))
    print("  compact query  {0}/{1}".format(compact, fired))
    print()
    if compact > whole:
        print("  The compact query wins. Adopting it is a one-line change in")
        print("  subject.plan's caller; this is the number that would justify it.")
    elif compact == whole:
        print("  A draw on this set, which is five files. Retrieval is not the")
        print("  binding constraint here and the whole prompt keeps its place.")
    else:
        print("  The whole prompt wins. The compact query describes the file more")
        print("  neatly and retrieves it worse, which is worth knowing before")
        print("  anybody adopts it on the strength of how sensible it sounds.")
    return 0


def summarise_check(rows, runs, skipped=0):
    # One row per file carries the facts about the file itself -- its stack, its
    # size, what the context layer could read. Those are the same on every run,
    # so any row will do, except a row that never got that far: an ERROR row
    # reports 0 characters and stack "?", and letting one represent the file
    # would print a run that failed as a file that is empty.
    files = collections.OrderedDict()
    for row in rows:
        known = files.get(row["path"])
        if known is None or (not known["chars"] and row["chars"]):
            files[row["path"]] = row

    print("CHECK EVAL ({0} file{1} x {2} run{3} = {4} review{5})".format(
        len(files), "" if len(files) == 1 else "s", runs,
        "" if runs == 1 else "s", len(rows), "" if len(rows) == 1 else "s"))
    if skipped:
        print("  ({0} more file(s) matched and were not run; raise --limit)".format(skipped))
    print()
    print("`nexa check` is `nexa ask` with the question built from a file, so the")
    print("same distribution applies -- and five things ask does not have: which")
    print("corpus the file was routed to, what the checkers found in the file")
    print("before any model ran, how much of it fitted, what the surroundings")
    print("added, and whether the answer is about this file at all.")
    print()

    print_verdicts(rows)
    # Labelled from the file, not from the row: an ERROR row carries stack "?",
    # and labelling it that way would split one file into two entries and report
    # a stability denominator one larger than the number of files.
    stacks = {path: row["stack"] for path, row in files.items()}
    print_stability(rows, lambda r: "{0:<11} {1}".format(
        stacks[r["path"]], os.path.basename(r["path"])[:40]), "files")

    print("ROUTING")
    print("  Which corpus the file was reviewed against, and what decided it.")
    for path, row in files.items():
        print("  {0:<26} {1:<11} {2}".format(
            os.path.basename(path)[:26], row["stack"], row["routed_by"]))
    print()

    truncated = [r for r in files.values() if r["dropped_chars"]]
    print("WHAT REACHED THE MODEL")
    for path, row in files.items():
        if not row["chars"]:
            continue
        pct = 100.0 * (row["chars"] - row["dropped_chars"]) / row["chars"]
        note = "" if not row["dropped_chars"] else "   <-- {0} of {1} symbols".format(
            row["symbols_shown"], row["symbols_total"])
        print("  {0:<26} {1:>6} chars, {2:>3.0f}% reviewed{3}".format(
            os.path.basename(path)[:26], row["chars"], pct, note))
    if truncated:
        print()
        print("  {0} of {1} file(s) did not fit in the {2}-character budget, so the"
              .format(len(truncated), len(files), subject.CHECK_CHARS))
        print("  excerpt was chosen by symbol: whole methods, the ones a checker")
        print("  fired on first, and the rest named in place rather than cut off")
        print("  at a character count. What is missing is still missing, and the")
        print("  verdict is still about the answer.")
    print()

    flagged = [r for r in files.values() if r["file_finding_count"]]
    print("FINDINGS IN THE FILE, BEFORE THE MODEL RAN")
    print("  The corpus checkers over the source itself -- patterns and AST passes,")
    print("  no generation. These do not depend on what the 7B says, or on whether")
    print("  it says anything, and they carry a line number.")
    for path, row in files.items():
        ids = row["file_findings"]
        print("  {0:<26} {1:>2} finding(s){2}".format(
            os.path.basename(path)[:26], row["file_finding_count"],
            "  " + ", ".join(i.split(".", 1)[-1] for i in ids) if ids else ""))
    print()
    print("  {0}/{1} file(s) had something found in them this way; {2} finding(s) in all."
          .format(len(flagged), len(files),
                  sum(r["file_finding_count"] for r in files.values())))
    quiet = sum(r.get("file_suppressed") or 0 for r in files.values())
    if quiet:
        print("  {0} more sat on lines another tool had already flagged, reported"
              .format(quiet))
        print("  by `check` and left out of the counts above (3.33).")
    advisory = sum(r.get("file_advisory") or 0 for r in files.values())
    if advisory:
        print("  {0} more came from advisory rules, which `check` prints apart and"
              .format(advisory))
        print("  counts in nothing.")
    print()

    print("CONTEXT -- the layer `ask` can never run")
    print("  ask has no file, so this layer always reports UNKNOWN. With a file it")
    print("  reads the manifest -- but reading is not checking. A verdict needs a")
    print("  question it can answer: a series the corpus was checked against, or a")
    print("  rule carrying odoo_versions. Anything else is read, stated as a gap,")
    print("  and left UNKNOWN.")
    reached, read = 0, 0
    for path, row in files.items():
        verdicts = {r["context_verdict"] for r in rows if r["path"] == path}
        state = "/".join(sorted(verdicts))
        if "UNKNOWN" not in verdicts:
            reached += 1
        read += row["facts"]
        print("  {0:<26} {1:<8} {2} fact(s) read, {3} answerable question(s)".format(
            os.path.basename(path)[:26], state, row["facts"], row["context_checked"]))
    print()
    print("  {0}/{1} file(s) reached a contextual verdict; {2} fact(s) read in all.".format(
        reached, len(files), read))
    print()

    print("IS THE ANSWER ABOUT THIS FILE?")
    print("  How many names the file declares -- its classes, functions and")
    print("  methods -- the answer mentions. A review that names none of them is")
    print("  not a review of that file, whatever its verdict says. High is not")
    print("  automatically good either: the model is asked for code, so it may be")
    print("  rewriting the file rather than reviewing it.")
    for path, row in files.items():
        seen = [r["names_mentioned"] for r in rows if r["path"] == path]
        declared = row["names_declared"]
        if not declared:
            print("  {0:<26} declares no names".format(os.path.basename(path)[:26]))
            continue
        spread = ("{0}".format(min(seen)) if min(seen) == max(seen)
                  else "{0}-{1}".format(min(seen), max(seen)))
        print("  {0:<26} {1} of {2} name{3} mentioned, median {4}".format(
            os.path.basename(path)[:26], spread, declared,
            "" if declared == 1 else "s", sorted(seen)[len(seen) // 2]))
    print()

    print("WHAT THE MODEL ADDED")
    print("  The checkers already say what is wrong with the file. This is what")
    print("  the generation reached for on top: rules it cited that no checker")
    print("  fired on. Not proof the answer was right about them -- a citation is")
    print("  a claim -- but it is the cheapest measure of whether the 4 seconds")
    print("  bought anything the deterministic pass could not.")
    added = collections.Counter(
        rid for r in rows for rid in r.get("cited_beyond_checkers") or [])
    for path, row in files.items():
        runs = [r for r in rows if r["path"] == path]
        counts = sorted(len(r.get("cited_beyond_checkers") or []) for r in runs)
        print("  {0:<26} cited {1}-{2} rule(s) beyond the {3} the checkers found".format(
            os.path.basename(path)[:26], counts[0], counts[-1],
            row["file_finding_count"]))
    if added:
        print()
        for rid, count in added.most_common(8):
            print("  {0:>3}x  {1}".format(count, rid))
    else:
        print()
        print("  none, in any run")
    print()

    fired = collections.Counter(rid for r in rows for rid in r["rules_fired"])
    print("CHECKERS THAT FIRED")
    if not fired:
        print("  none -- either the answers were clean or the checkers are asleep")
    for rid, count in fired.most_common(10):
        print("  {0:>3}x  {1}".format(count, rid))
    print()

    layer_hits = collections.Counter(
        layer for r in rows for layer in r["layers_with_findings"])
    print("LAYERS THAT FOUND SOMETHING")
    for layer, count in layer_hits.most_common():
        print("  {0:>3}x  {1}".format(count, layer))
    print()
    print("RETRIEVAL")
    print("  ask embeds one sentence. check embeds the whole prompt -- the")
    print("  surroundings, the note about truncation and up to {0} characters of".format(
        subject.CHECK_CHARS))
    print("  source -- and sorts the stack's rule vectors against that. These are")
    print("  the rules the file itself pulled back.")
    for path, row in files.items():
        print("  {0}".format(os.path.basename(path)[:40]))
        for rid in row["retrieved"]:
            print("      {0}".format(rid))
    stacks = {r["stack"] for r in rows}
    every = {r["id"] for r in corpus.load_rules() if r["stack"] in stacks}
    retrieved = {rid for r in rows for rid in r["retrieved"]}
    never = sorted(every - retrieved)
    print()
    print("  {0}/{1} rules for these stacks surfaced at least once".format(
        len(every) - len(never), len(every)))
    print()

    print_throughput(rows)
    print_errors(rows, "path")


def main(argv):
    parser = argparse.ArgumentParser(prog="nexa eval", add_help=True)
    parser.add_argument("--runs", type=int, default=3,
                        help="repetitions per question (default 3)")
    parser.add_argument("--stack",
                        help="odoo, the only stack")
    parser.add_argument("--pipeline", action="store_true",
                        help="measure through run(), for escalation numbers")
    parser.add_argument("--retrieval", action="store_true",
                        help="ask every rule its own probe; no generation, seconds not minutes")
    parser.add_argument("--compare-retrieval", action="store_true",
                        help="score two ways of describing a file to the embedder; no generation")
    parser.add_argument("--check", action="store_true",
                        help="review files the way `nexa check` does, instead of asking questions")
    parser.add_argument("--limit", type=int, default=8,
                        help="most files to review when --check is given a directory (default 8)")
    parser.add_argument("paths", nargs="*",
                        help="files or directories for --check (default: the subjects/ set)")
    parser.add_argument("--top-k", type=int, default=None,
                        help="retrieval depth for --retrieval (default: the corpus TOP_K)")
    parser.add_argument("--questions", help="JSON file of [stack, question] pairs")
    parser.add_argument("--json", help="write the raw rows here as well")
    args = parser.parse_args(argv)

    if args.retrieval:
        # Deliberately before the question set is touched: --retrieval does not
        # use questions at all, and silently accepting --runs alongside it would
        # imply a repetition that embedding does not have. Embeddings are
        # deterministic, so one pass is the whole measurement.
        if args.pipeline:
            print("--retrieval and --pipeline measure different things; pick one.")
            return 2
        k = args.top_k or corpus.TOP_K
        rows = retrievability(args.stack, k)
        if not rows:
            print("No rules for that stack.")
            return 2
        code = summarise_retrieval(rows, k)
        if args.json:
            # The rows alone cannot be checked against anything. A rank means
            # nothing without the depth it was taken at, and a run from before
            # the last rule edit ranks rules that no longer read that way: the
            # ids stay the same when only the wording moves. Both go in beside
            # the rows, as recall's head did (3.53), so figures.py can tell.
            with io.open(args.json, "w", encoding="utf-8") as fh:
                fh.write(json.dumps({
                    "generated_utc": datetime.now(timezone.utc).isoformat(
                        timespec="seconds"),
                    "k": k,
                    "embedder": corpus.EMBED_MODEL,
                    "corpus_sha256": evidence.corpus_sha256(),
                    "rows": rows,
                }, indent=2, ensure_ascii=False))
            print("raw rows written to {0}".format(args.json))
        return code

    if args.compare_retrieval:
        try:
            paths, _ = gather(args.paths or DEFAULT_SUBJECTS, args.limit)
        except RuntimeError as exc:
            print(str(exc))
            return 2
        rows = compare_retrieval(paths)
        code = summarise_comparison(rows)
        if args.json:
            with io.open(args.json, "w", encoding="utf-8") as fh:
                fh.write(json.dumps(rows, indent=2, ensure_ascii=False))
            print("raw rows written to {0}".format(args.json))
        return code

    if args.check:
        # --stack is not a filter here, it is `nexa check --stack`: the file
        # decides its own corpus unless somebody overrides it, and overriding it
        # is a route worth measuring too.
        if args.pipeline:
            print("--check and --pipeline measure different things; pick one.")
            return 2
        if args.runs < 1:
            print("--runs must be at least 1.")
            return 2
        try:
            paths, skipped = gather(args.paths or DEFAULT_SUBJECTS, args.limit)
        except RuntimeError as exc:
            print(str(exc))
            return 2
        if not paths:
            print("Nothing there to review.")
            return 2
        rows = evaluate_checks(paths, args.runs, args.stack)
        summarise_check(rows, args.runs, skipped)
        if args.json:
            with io.open(args.json, "w", encoding="utf-8") as fh:
                fh.write(json.dumps(rows, indent=2, ensure_ascii=False))
            print()
            print("raw rows written to {0}".format(args.json))
        return 0

    if args.paths:
        print("Files are only reviewed with --check; ask takes a question set.")
        return 2

    questions = DEFAULT_QUESTIONS
    if args.questions:
        with io.open(args.questions, encoding="utf-8") as fh:
            questions = [tuple(q) for q in json.load(fh)]
    if args.stack:
        questions = [q for q in questions if q[0] == args.stack]
    if not questions:
        print("No questions for that stack.")
        return 2
    if args.runs < 1:
        print("--runs must be at least 1.")
        return 2

    rows = evaluate(questions, args.runs, args.pipeline)
    summarise(rows, args.runs, args.pipeline)
    if args.json:
        with io.open(args.json, "w", encoding="utf-8") as fh:
            fh.write(json.dumps(rows, indent=2, ensure_ascii=False))
        print()
        print("raw rows written to {0}".format(args.json))
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))