File size: 11,525 Bytes
e6f80f3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
"""Paired room-level comparison of two evaluate outputs on the same rows (A minus B).

    python compare.py --a runs/test-X --b runs/test-X-argmax --label-a spread --label-b argmax --out C.json \
        [--a-outcomes outcomes.jsonl] [--b-outcomes outcomes.jsonl] [--rows ROWS] [--where rule=frozen_prep] \
        [--max-objects 128] [--boot 10000] [--cache DIR]

Refuses unless both runs scored the same rows: equal report ``data_sha256`` and projection, and the same outcome rows
with the same ``provenance.scene_id`` in the same order; and unless both were decoded and scored by the same code:
equal report ``implementation_sha256`` and ``baseline`` (online / supplied_head_outputs / ...) and, for head outputs,
equal forward-pass code (the head-output manifests' ``implementation_sha256``). Per-object values come from
``stratify.collect`` (evaluate's own metric code, reproduction-checked). ``--where KEY=V1[,V2]`` (repeatable; rule /
place on objects, family / objects on rooms) keeps a subset; ``--max-objects N`` keeps rooms of at most N requested
objects (rooms both models can answer); room-level validator metrics are reported only without an object filter.

Per metric, over the rooms where both runs have a layout and the metric has a value: PRIMARY the mean of per-room
differences with its paired room-bootstrap 95% CI (``--boot`` resamples of rooms); significant when the CI excludes 0,
direction from its sign. ROBUSTNESS an exact two-sided sign test on the per-room differences (ties dropped), Holm's
correction across the metrics of this comparison; ``verdict.agree`` false flags a disagreement of the two. The pooled
(object-weighted) means of A and B (``a`` / ``b``) and their difference with CI, and the per-room means
(``a_room_mean`` / ``b_room_mean``, whose difference is ``room_mean_diff``) are descriptive. Lower is better except
bev_iou and room_clean_rate.
"""
from __future__ import annotations

import argparse
import json
import math
from pathlib import Path

import numpy as np

import stratify as S

ERROR_METRICS = S.E.REFERENCE_METRICS + ("object_collision_rate", "object_hard_violation_rate")
ROOM_METRICS = ("room_clean_rate", "room_collision_rate")
HIGHER_IS_BETTER = {"bev_iou", "room_clean_rate", "clean_room_rate_incl_failures", "interface_pass_rate"}


def sign_test(d):
    pos, neg = int((d > 0).sum()), int((d < 0).sum())
    n, k = pos + neg, min(pos, neg)
    p = min(1., 2 * sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n) if n else 1.
    return {"a_greater": pos, "b_greater": neg, "ties": int(len(d) - n), "p": p}


def holm(ps):
    order, adjusted, running = sorted(range(len(ps)), key=lambda i: ps[i]), [None] * len(ps), 0.
    for rank, i in enumerate(order):
        running = max(running, min(1., (len(ps) - rank) * ps[i]))
        adjusted[i] = running
    return adjusted


def keep_fn(where, max_objects=None):
    """Object filter and room filter from ``--where`` pairs and ``--max-objects``."""
    obj = {k: v for k, v in where.items() if k in S.OBJECT_KINDS}
    room = {k: v for k, v in where.items() if k in S.ROOM_KINDS}
    room_key = {"family": "family", "objects": "objects_bin"}
    return (lambda o: all(o[k] in v for k, v in obj.items())), (lambda r: all(r[room_key[k]] in v for k, v in room.items())
                                                               and (max_objects is None or r["n_objects"] <= max_objects)), bool(obj)


def paired(rooms_a, rooms_b, metrics, *, boot, seed, keep=lambda o: True, fallback=False):
    """Paired statistics of aligned room lists; a metric counts a room only when both runs give it a value."""
    out = {}
    for m, metric in enumerate(metrics):
        cells = []
        for ra, rb in zip(rooms_a, rooms_b):
            va = S.room_values(ra, metric, [o for o in ra["objects"].values() if keep(o)], fallback=fallback)
            vb = S.room_values(rb, metric, [o for o in rb["objects"].values() if keep(o)], fallback=fallback)
            if va and vb and va[1] and vb[1]:
                cells.append((*va, *vb))
        if not cells:
            continue
        c = np.array(cells)
        d = c[:, 0] / c[:, 1] - c[:, 2] / c[:, 3]
        # one set of room weights for pooled A, pooled B and the per-room differences: the resamples stay paired
        point, samples = S.bootstrap(np.column_stack((c[:, 0], c[:, 2], d)),
                                     np.column_stack((c[:, 1], c[:, 3], np.ones(len(d)))), boot, seed + m)
        out[metric] = {"rooms": len(d), "objects_a": int(c[:, 1].sum()), "objects_b": int(c[:, 3].sum()),
                       "a": float(point[0]), "b": float(point[1]), "pooled_diff": float(point[0] - point[1]),
                       "a_room_mean": float(np.mean(c[:, 0] / c[:, 1])), "b_room_mean": float(np.mean(c[:, 2] / c[:, 3])),
                       "pooled_diff_ci95": [float(x) for x in S.ci(samples[:, 0] - samples[:, 1])],
                       "room_mean_diff": float(point[2]), "room_mean_diff_ci95": [float(x) for x in S.ci(samples[:, 2])],
                       "sign_test": sign_test(d), "higher_is_better": metric in HIGHER_IS_BETTER}
    adjusted = holm([v["sign_test"]["p"] for v in out.values()])
    for v, p in zip(out.values(), adjusted):
        v["sign_test"]["p_holm"] = p
    return verdicts(out)


def verdicts(out):
    """primary: room-mean-difference CI excludes 0; sign_test: Holm p < .05; each a_better / a_worse / None."""
    for v in out.values():
        lo, hi = v["room_mean_diff_ci95"]
        st = v["sign_test"]
        side = lambda higher: None if higher is None else "a_better" if higher == v["higher_is_better"] else "a_worse"
        v["verdict"] = {"primary": side(True if lo > 0 else False if hi < 0 else None),
                        "sign_test": side(st["a_greater"] > st["b_greater"] if st["p_holm"] < .05 else None)}
        v["verdict"]["agree"] = v["verdict"]["primary"] == v["verdict"]["sign_test"]
    return out


def same_code(runs):
    """Refuse unless every (eval dir, outcomes file) was decoded and scored by the same code from the same kind of
    source, and head outputs were produced by the same forward-pass code."""
    codes = []
    for d, o in runs:
        report = S.run_info(d, o)[0]
        manifest = report.get("head_outputs_manifest") or {}
        codes.append((report.get("implementation_sha256"), report.get("baseline"), manifest.get("implementation_sha256")))
    if len(set(codes)) > 1:
        raise SystemExit("refused: the runs were not decoded by the same code (implementation_sha256, source, forward code): "
                         + "; ".join(f"{d}/{o}: {(k[0] or '-')[:12]} {k[1]} {(k[2] or '-')[:12]}" for (d, o), k in zip(runs, codes)))


def same_inputs(runs):
    """Refuse (before any work) unless every (eval dir, outcomes file) scored the same data in the same projection."""
    info = [S.run_info(d, o) for d, o in runs]
    keys = [(r["data_sha256"], projection, r.get("subset_limit")) for r, projection, _ in info]
    if len(set(keys)) > 1:
        raise SystemExit("refused: the runs did not score the same rows (data_sha256, projection, subset_limit): "
                         + "; ".join(f"{d}/{o}: {k[0][:12]} {k[1]} {k[2]}" for (d, o), k in zip(runs, keys)))


def aligned(*room_lists):
    """Refuse unless every run holds the same outcome rows with the same scene ids, in the same order."""
    if len({tuple((r["row"], r["scene_id"]) for r in rooms) for rooms in room_lists}) > 1:
        raise SystemExit("refused: the runs did not score the same rows (outcome rows / scene ids differ)")


def main(argv=None):
    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    p.add_argument("--a", required=True, type=Path)
    p.add_argument("--b", required=True, type=Path)
    p.add_argument("--a-outcomes", default="outcomes.jsonl")
    p.add_argument("--b-outcomes", default="outcomes.jsonl")
    p.add_argument("--label-a", default="A")
    p.add_argument("--label-b", default="B")
    p.add_argument("--rows", type=Path, help="the scored rows file (default: report.json data_path)")
    p.add_argument("--where", action="append", default=[], metavar="KEY=V1[,V2]")
    p.add_argument("--max-objects", type=int, help="keep rooms of at most N requested objects")
    p.add_argument("--cache", type=Path, help="stratify.collect cache directory")
    p.add_argument("--out", required=True, type=Path)
    p.add_argument("--boot", type=int, default=10000)
    p.add_argument("--seed", type=int, default=0)
    p.add_argument("--workers", type=int, default=6)
    a = p.parse_args(argv)
    where = {}
    for item in a.where:
        key, _, values = item.partition("=")
        if key not in S.OBJECT_KINDS + S.ROOM_KINDS or not values:
            raise SystemExit(f"--where {item}: KEY must be one of {S.OBJECT_KINDS + S.ROOM_KINDS}")
        where[key] = set(values.split(","))
    same_inputs([(a.a, a.a_outcomes), (a.b, a.b_outcomes)])
    same_code([(a.a, a.a_outcomes), (a.b, a.b_outcomes)])
    meta_a, rooms_a, checks_a = S.collect(a.a, a.a_outcomes, a.rows, a.workers, a.cache)
    meta_b, rooms_b, checks_b = S.collect(a.b, a.b_outcomes, a.rows, a.workers, a.cache)
    aligned(rooms_a, rooms_b)
    keep, keep_room, object_filter = keep_fn(where, a.max_objects)
    excluded = sum(not keep_room(r) for r in rooms_a)
    pairs = [(ra, rb) for ra, rb in zip(rooms_a, rooms_b) if keep_room(ra)]
    rooms_a, rooms_b = [ra for ra, _ in pairs], [rb for _, rb in pairs]
    metrics = ERROR_METRICS + (() if object_filter else ROOM_METRICS)
    result = {"meta": {"a": {**meta_a, "label": a.label_a}, "b": {**meta_b, "label": a.label_b},
                       "where": {k: sorted(v) for k, v in where.items()}, "max_objects": a.max_objects, "boot": a.boot,
                       "seed": a.seed, "difference": "A minus B",
                       "primary": "paired room-bootstrap 95% CI of the mean per-room difference (direction from it)",
                       "robustness": "exact two-sided sign test on per-room differences, Holm across metrics",
                       "pooled": "object-weighted means and their difference: descriptive only"},
              "checks": {"a_reproduces_report": checks_a["reproduces_report_exactly"],
                         "b_reproduces_report": checks_b["reproduces_report_exactly"]},
              "rooms": {"compared": len(pairs), "excluded_by_room_filter": excluded,
                        "layout_both": sum(ra["layout"] and rb["layout"] for ra, rb in pairs),
                        "layout_only_a": sum(ra["layout"] and not rb["layout"] for ra, rb in pairs),
                        "layout_only_b": sum(rb["layout"] and not ra["layout"] for ra, rb in pairs),
                        "layout_neither": sum(not ra["layout"] and not rb["layout"] for ra, rb in pairs)},
              "metrics": paired(rooms_a, rooms_b, metrics, boot=a.boot, seed=a.seed, keep=keep)}
    a.out.parent.mkdir(parents=True, exist_ok=True)
    a.out.write_text(json.dumps(result, indent=1, allow_nan=False) + "\n")
    print(json.dumps(result["rooms"]))
    for metric, v in result["metrics"].items():
        print(f"{metric:34s} room_mean_diff={v['room_mean_diff']:+.4f} [{v['room_mean_diff_ci95'][0]:+.4f},"
              f"{v['room_mean_diff_ci95'][1]:+.4f}] rooms={v['rooms']} sign {v['sign_test']['a_greater']}/"
              f"{v['sign_test']['b_greater']} p_holm={v['sign_test']['p_holm']:.3g} {v['verdict']}")


if __name__ == "__main__":
    main()