Instructions to use XXXpilar/Fastfill-test with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use XXXpilar/Fastfill-test with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
File size: 11,525 Bytes
e6f80f3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 | """Paired room-level comparison of two evaluate outputs on the same rows (A minus B).
python compare.py --a runs/test-X --b runs/test-X-argmax --label-a spread --label-b argmax --out C.json \
[--a-outcomes outcomes.jsonl] [--b-outcomes outcomes.jsonl] [--rows ROWS] [--where rule=frozen_prep] \
[--max-objects 128] [--boot 10000] [--cache DIR]
Refuses unless both runs scored the same rows: equal report ``data_sha256`` and projection, and the same outcome rows
with the same ``provenance.scene_id`` in the same order; and unless both were decoded and scored by the same code:
equal report ``implementation_sha256`` and ``baseline`` (online / supplied_head_outputs / ...) and, for head outputs,
equal forward-pass code (the head-output manifests' ``implementation_sha256``). Per-object values come from
``stratify.collect`` (evaluate's own metric code, reproduction-checked). ``--where KEY=V1[,V2]`` (repeatable; rule /
place on objects, family / objects on rooms) keeps a subset; ``--max-objects N`` keeps rooms of at most N requested
objects (rooms both models can answer); room-level validator metrics are reported only without an object filter.
Per metric, over the rooms where both runs have a layout and the metric has a value: PRIMARY the mean of per-room
differences with its paired room-bootstrap 95% CI (``--boot`` resamples of rooms); significant when the CI excludes 0,
direction from its sign. ROBUSTNESS an exact two-sided sign test on the per-room differences (ties dropped), Holm's
correction across the metrics of this comparison; ``verdict.agree`` false flags a disagreement of the two. The pooled
(object-weighted) means of A and B (``a`` / ``b``) and their difference with CI, and the per-room means
(``a_room_mean`` / ``b_room_mean``, whose difference is ``room_mean_diff``) are descriptive. Lower is better except
bev_iou and room_clean_rate.
"""
from __future__ import annotations
import argparse
import json
import math
from pathlib import Path
import numpy as np
import stratify as S
ERROR_METRICS = S.E.REFERENCE_METRICS + ("object_collision_rate", "object_hard_violation_rate")
ROOM_METRICS = ("room_clean_rate", "room_collision_rate")
HIGHER_IS_BETTER = {"bev_iou", "room_clean_rate", "clean_room_rate_incl_failures", "interface_pass_rate"}
def sign_test(d):
pos, neg = int((d > 0).sum()), int((d < 0).sum())
n, k = pos + neg, min(pos, neg)
p = min(1., 2 * sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n) if n else 1.
return {"a_greater": pos, "b_greater": neg, "ties": int(len(d) - n), "p": p}
def holm(ps):
order, adjusted, running = sorted(range(len(ps)), key=lambda i: ps[i]), [None] * len(ps), 0.
for rank, i in enumerate(order):
running = max(running, min(1., (len(ps) - rank) * ps[i]))
adjusted[i] = running
return adjusted
def keep_fn(where, max_objects=None):
"""Object filter and room filter from ``--where`` pairs and ``--max-objects``."""
obj = {k: v for k, v in where.items() if k in S.OBJECT_KINDS}
room = {k: v for k, v in where.items() if k in S.ROOM_KINDS}
room_key = {"family": "family", "objects": "objects_bin"}
return (lambda o: all(o[k] in v for k, v in obj.items())), (lambda r: all(r[room_key[k]] in v for k, v in room.items())
and (max_objects is None or r["n_objects"] <= max_objects)), bool(obj)
def paired(rooms_a, rooms_b, metrics, *, boot, seed, keep=lambda o: True, fallback=False):
"""Paired statistics of aligned room lists; a metric counts a room only when both runs give it a value."""
out = {}
for m, metric in enumerate(metrics):
cells = []
for ra, rb in zip(rooms_a, rooms_b):
va = S.room_values(ra, metric, [o for o in ra["objects"].values() if keep(o)], fallback=fallback)
vb = S.room_values(rb, metric, [o for o in rb["objects"].values() if keep(o)], fallback=fallback)
if va and vb and va[1] and vb[1]:
cells.append((*va, *vb))
if not cells:
continue
c = np.array(cells)
d = c[:, 0] / c[:, 1] - c[:, 2] / c[:, 3]
# one set of room weights for pooled A, pooled B and the per-room differences: the resamples stay paired
point, samples = S.bootstrap(np.column_stack((c[:, 0], c[:, 2], d)),
np.column_stack((c[:, 1], c[:, 3], np.ones(len(d)))), boot, seed + m)
out[metric] = {"rooms": len(d), "objects_a": int(c[:, 1].sum()), "objects_b": int(c[:, 3].sum()),
"a": float(point[0]), "b": float(point[1]), "pooled_diff": float(point[0] - point[1]),
"a_room_mean": float(np.mean(c[:, 0] / c[:, 1])), "b_room_mean": float(np.mean(c[:, 2] / c[:, 3])),
"pooled_diff_ci95": [float(x) for x in S.ci(samples[:, 0] - samples[:, 1])],
"room_mean_diff": float(point[2]), "room_mean_diff_ci95": [float(x) for x in S.ci(samples[:, 2])],
"sign_test": sign_test(d), "higher_is_better": metric in HIGHER_IS_BETTER}
adjusted = holm([v["sign_test"]["p"] for v in out.values()])
for v, p in zip(out.values(), adjusted):
v["sign_test"]["p_holm"] = p
return verdicts(out)
def verdicts(out):
"""primary: room-mean-difference CI excludes 0; sign_test: Holm p < .05; each a_better / a_worse / None."""
for v in out.values():
lo, hi = v["room_mean_diff_ci95"]
st = v["sign_test"]
side = lambda higher: None if higher is None else "a_better" if higher == v["higher_is_better"] else "a_worse"
v["verdict"] = {"primary": side(True if lo > 0 else False if hi < 0 else None),
"sign_test": side(st["a_greater"] > st["b_greater"] if st["p_holm"] < .05 else None)}
v["verdict"]["agree"] = v["verdict"]["primary"] == v["verdict"]["sign_test"]
return out
def same_code(runs):
"""Refuse unless every (eval dir, outcomes file) was decoded and scored by the same code from the same kind of
source, and head outputs were produced by the same forward-pass code."""
codes = []
for d, o in runs:
report = S.run_info(d, o)[0]
manifest = report.get("head_outputs_manifest") or {}
codes.append((report.get("implementation_sha256"), report.get("baseline"), manifest.get("implementation_sha256")))
if len(set(codes)) > 1:
raise SystemExit("refused: the runs were not decoded by the same code (implementation_sha256, source, forward code): "
+ "; ".join(f"{d}/{o}: {(k[0] or '-')[:12]} {k[1]} {(k[2] or '-')[:12]}" for (d, o), k in zip(runs, codes)))
def same_inputs(runs):
"""Refuse (before any work) unless every (eval dir, outcomes file) scored the same data in the same projection."""
info = [S.run_info(d, o) for d, o in runs]
keys = [(r["data_sha256"], projection, r.get("subset_limit")) for r, projection, _ in info]
if len(set(keys)) > 1:
raise SystemExit("refused: the runs did not score the same rows (data_sha256, projection, subset_limit): "
+ "; ".join(f"{d}/{o}: {k[0][:12]} {k[1]} {k[2]}" for (d, o), k in zip(runs, keys)))
def aligned(*room_lists):
"""Refuse unless every run holds the same outcome rows with the same scene ids, in the same order."""
if len({tuple((r["row"], r["scene_id"]) for r in rooms) for rooms in room_lists}) > 1:
raise SystemExit("refused: the runs did not score the same rows (outcome rows / scene ids differ)")
def main(argv=None):
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--a", required=True, type=Path)
p.add_argument("--b", required=True, type=Path)
p.add_argument("--a-outcomes", default="outcomes.jsonl")
p.add_argument("--b-outcomes", default="outcomes.jsonl")
p.add_argument("--label-a", default="A")
p.add_argument("--label-b", default="B")
p.add_argument("--rows", type=Path, help="the scored rows file (default: report.json data_path)")
p.add_argument("--where", action="append", default=[], metavar="KEY=V1[,V2]")
p.add_argument("--max-objects", type=int, help="keep rooms of at most N requested objects")
p.add_argument("--cache", type=Path, help="stratify.collect cache directory")
p.add_argument("--out", required=True, type=Path)
p.add_argument("--boot", type=int, default=10000)
p.add_argument("--seed", type=int, default=0)
p.add_argument("--workers", type=int, default=6)
a = p.parse_args(argv)
where = {}
for item in a.where:
key, _, values = item.partition("=")
if key not in S.OBJECT_KINDS + S.ROOM_KINDS or not values:
raise SystemExit(f"--where {item}: KEY must be one of {S.OBJECT_KINDS + S.ROOM_KINDS}")
where[key] = set(values.split(","))
same_inputs([(a.a, a.a_outcomes), (a.b, a.b_outcomes)])
same_code([(a.a, a.a_outcomes), (a.b, a.b_outcomes)])
meta_a, rooms_a, checks_a = S.collect(a.a, a.a_outcomes, a.rows, a.workers, a.cache)
meta_b, rooms_b, checks_b = S.collect(a.b, a.b_outcomes, a.rows, a.workers, a.cache)
aligned(rooms_a, rooms_b)
keep, keep_room, object_filter = keep_fn(where, a.max_objects)
excluded = sum(not keep_room(r) for r in rooms_a)
pairs = [(ra, rb) for ra, rb in zip(rooms_a, rooms_b) if keep_room(ra)]
rooms_a, rooms_b = [ra for ra, _ in pairs], [rb for _, rb in pairs]
metrics = ERROR_METRICS + (() if object_filter else ROOM_METRICS)
result = {"meta": {"a": {**meta_a, "label": a.label_a}, "b": {**meta_b, "label": a.label_b},
"where": {k: sorted(v) for k, v in where.items()}, "max_objects": a.max_objects, "boot": a.boot,
"seed": a.seed, "difference": "A minus B",
"primary": "paired room-bootstrap 95% CI of the mean per-room difference (direction from it)",
"robustness": "exact two-sided sign test on per-room differences, Holm across metrics",
"pooled": "object-weighted means and their difference: descriptive only"},
"checks": {"a_reproduces_report": checks_a["reproduces_report_exactly"],
"b_reproduces_report": checks_b["reproduces_report_exactly"]},
"rooms": {"compared": len(pairs), "excluded_by_room_filter": excluded,
"layout_both": sum(ra["layout"] and rb["layout"] for ra, rb in pairs),
"layout_only_a": sum(ra["layout"] and not rb["layout"] for ra, rb in pairs),
"layout_only_b": sum(rb["layout"] and not ra["layout"] for ra, rb in pairs),
"layout_neither": sum(not ra["layout"] and not rb["layout"] for ra, rb in pairs)},
"metrics": paired(rooms_a, rooms_b, metrics, boot=a.boot, seed=a.seed, keep=keep)}
a.out.parent.mkdir(parents=True, exist_ok=True)
a.out.write_text(json.dumps(result, indent=1, allow_nan=False) + "\n")
print(json.dumps(result["rooms"]))
for metric, v in result["metrics"].items():
print(f"{metric:34s} room_mean_diff={v['room_mean_diff']:+.4f} [{v['room_mean_diff_ci95'][0]:+.4f},"
f"{v['room_mean_diff_ci95'][1]:+.4f}] rooms={v['rooms']} sign {v['sign_test']['a_greater']}/"
f"{v['sign_test']['b_greater']} p_holm={v['sign_test']['p_holm']:.3g} {v['verdict']}")
if __name__ == "__main__":
main()
|