Spaces:
Sleeping
Sleeping
File size: 6,256 Bytes
c96d602 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 | """Evaluation harness: metric properties demanded by the design review."""
import json
import math
import os
import numpy as np
import pytest
from src.bench import baselines, build
from src.bench import evaluate as E
DOCS = {"train": 1, "val": 1, "test-dev": 1, "test": 1}
@pytest.fixture(scope="module")
def pages(tmp_path_factory):
rel = str(tmp_path_factory.mktemp("rel"))
build.write_release(rel, ["easy", "hard"], DOCS, ["val"])
out = []
for tier in ("easy", "hard"):
for pid, ans in E.split_answers(rel, tier, "val").items():
pdir = os.path.join(rel, "puzzles", tier, "val", pid)
with open(os.path.join(pdir, "puzzle.json")) as fh:
puzzle = json.load(fh)
out.append((E.load_page(pdir, ans), ans, puzzle))
return out
def _moved(sol, g):
return {"fragments": {f: {"affine": (g @ E.to3(v["affine"]))[:2].tolist()}
for f, v in sol["fragments"].items()}}
def _rot(theta, tx, ty):
c, s = math.cos(theta), math.sin(theta)
return np.array([[c, -s, tx], [s, c, ty], [0, 0, 1.0]])
def test_oracle_scores_one(pages):
for page, ans, _ in pages:
s = E.score_page(page, baselines.oracle_solution(ans))
assert s.direct_acc == 1.0
assert s.neighbor_acc == 1.0
assert s.perfect == 1.0
assert s.adj_recall == 1.0
assert s.adj_precision >= 0.95
def test_invariant_to_global_rigid_transform(pages):
for page, ans, _ in pages:
s = E.score_page(page, _moved(baselines.oracle_solution(ans), _rot(2.1, -400, 900)))
assert s.direct_acc == 1.0
assert s.neighbor_acc == 1.0
def test_random_and_single_fragment_score_exactly_zero(pages):
for page, ans, puzzle in pages:
r = E.score_page(page, baselines.random_solution(puzzle, 3))
assert r.direct_acc == 0.0
assert r.neighbor_acc == 0.0
assert r.adj_f1 == 0.0
orc = baselines.oracle_solution(ans)
one = {"fragments": dict(list(orc["fragments"].items())[:1])}
assert E.score_page(page, one).direct_acc == 0.0
def test_non_rigid_poses_are_invalid_not_crashes(pages):
page, ans, _ = pages[0]
orc = baselines.oracle_solution(ans)
mirrored = _moved(orc, np.diag([-1.0, 1.0, 1.0]))
sheared = _moved(orc, np.array([[1.0, 0.4, 0], [0, 1.0, 0], [0, 0, 1.0]]))
for sol in (mirrored, sheared):
s = E.score_page(page, sol)
assert s.direct_acc == 0.0
assert s.invalid == len(page.gt)
broken = json.loads(json.dumps(orc))
fids = sorted(broken["fragments"])
broken["fragments"][fids[0]]["affine"] = [[0, 0, 0], [0, 0, 0]] # singular
broken["fragments"][fids[1]]["affine"] = "nonsense"
broken["fragments"]["not-a-fragment"] = {"affine": [[1, 0, 0], [0, 1, 0]]}
s = E.score_page(page, broken)
assert s.invalid == 3
assert s.placed == len(fids) - 2
def test_missing_solution_scores_zero(pages):
page, _, _ = pages[0]
s = E.score_page(page, None)
assert (s.direct_acc, s.neighbor_acc, s.perfect) == (0.0, 0.0, 0.0)
def test_one_misplaced_fragment_gives_partial_credit(pages):
page, ans, _ = pages[0]
orc = baselines.oracle_solution(ans)
fid = sorted(orc["fragments"])[0]
orc["fragments"][fid]["affine"] = (_rot(0, 250, 0) @ E.to3(orc["fragments"][fid]["affine"]))[:2].tolist()
s = E.score_page(page, orc)
n = len(page.gt)
assert s.direct_acc == pytest.approx((n - 1) / n)
assert 0 < s.neighbor_acc < 1
assert s.perfect == 0.0
def test_neighbor_test_is_symmetric_and_perfect_implies_one(pages):
page, ans, _ = pages[-1]
rng = np.random.default_rng(0)
noisy = {"fragments": {}}
for f, v in ans["fragments"].items(): # sub-tolerance per-fragment jitter
jitter = _rot(float(rng.normal(0, 0.0015)), float(rng.normal(0, 1.0)), float(rng.normal(0, 1.0)))
noisy["fragments"][f] = {"affine": (E.to3(v["affine"]) @ jitter)[:2].tolist()}
pred = {f: E.to3(v["affine"]) for f, v in noisy["fragments"].items()}
tau = E.TAU_FRAC * page.canvas[0]
g, _ = E.best_alignment(pred, page, tau)
for a, b, _ in page.adjacency:
assert E._pair_ok(pred, page, a, b, g, tau) == E._pair_ok(pred, page, b, a, g, tau)
s = E.score_page(page, noisy)
if s.perfect == 1.0:
assert s.neighbor_acc == 1.0
def test_candidates_hit_at_k(pages):
page, ans, _ = pages[0]
min_shared = E.min_shared_px(page.canvas[0])
cands = {}
for a, b, shared in page.adjacency:
if shared >= min_shared:
cands.setdefault(a, []).append(b)
cands.setdefault(b, []).append(a)
sol = dict(baselines.oracle_solution(ans), candidates=cands)
s = E.score_page(page, sol)
assert (s.hit1, s.hit5, s.mrr) == (1.0, 1.0, 1.0)
def test_aggregate_is_order_independent(pages):
scores = [E.score_page(p, baselines.random_solution(z, 1)) for p, _, z in pages]
scores += [E.score_page(p, baselines.oracle_solution(a)) for p, a, _ in pages]
assert E.aggregate(scores) == E.aggregate(list(reversed(scores)))
def test_geometry_helpers():
g = _rot(0.8, 12.0, -3.0)
src = np.random.default_rng(1).uniform(0, 100, (20, 2))
fit = E.kabsch2d(src, src @ g[:2, :2].T + g[:2, 2])
assert np.allclose(fit, g)
assert np.allclose(E.rigid_inverse(g) @ g, np.eye(3))
assert E.is_rigid(g[:2])
assert not E.is_rigid([[1, 0, 0], [0, -1, 0]]) # mirror
assert not E.is_rigid([[float("nan"), 0, 0], [0, 1, 0]])
assert not E.is_rigid("x")
@pytest.mark.parametrize("bad", ["abc", ["f000"], 7, None])
def test_malformed_fragments_field_scores_zero(pages, bad):
page, _, _ = pages[0]
s = E.score_page(page, {"fragments": bad})
assert s.placed == 0
assert s.direct_acc == 0.0
@pytest.mark.parametrize("bad", [7, "f001", [{"x": 1}], [[]], [None, 3.5]])
def test_malformed_candidates_score_zero(pages, bad):
page, ans, _ = pages[0]
cands = {fid: bad for fid in ans["fragments"]}
s = E.score_page(page, dict(baselines.oracle_solution(ans), candidates=cands))
assert (s.hit1, s.hit5, s.mrr) == (0.0, 0.0, 0.0)
assert s.direct_acc == 1.0 # placements still scored
|