Buckets:
| """Synthetic sanity tests for aide/eval/metrics.py and protocol constants. | |
| python -m pytest aide/tests/test_metrics.py -q (or: python aide/tests/test_metrics.py) | |
| """ | |
| from __future__ import annotations | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[2])) | |
| from aide.data import protocol as P # noqa: E402 | |
| from aide.eval import metrics as M # noqa: E402 | |
| rng = np.random.default_rng(0) | |
| def _gt(n=7): | |
| g = np.zeros((n, P.T_PRED, P.J, 3)) | |
| g[..., 0] = rng.uniform(0, P.W, (n, P.T_PRED, P.J)) | |
| g[..., 1] = rng.uniform(0, P.H, (n, P.T_PRED, P.J)) | |
| g[..., 2] = rng.uniform(0, 1, (n, P.T_PRED, P.J)) | |
| return g | |
| def test_zero_error(): | |
| g = _gt() | |
| err = M.joint_errors(g[..., :2], g) | |
| assert M.mpjpe(err) == 0.0 and M.dn_mpjpe(err) == 0.0 | |
| assert M.pck(err, 0.05) == 100.0 and M.pck(err, 0.10) == 100.0 | |
| assert np.all(M.per_horizon(err) == 0) and all(v == 0 for v in M.per_part(err).values()) | |
| def test_known_displacement(): | |
| g = _gt() | |
| pred = g[..., :2] + np.array([3.0, 4.0]) # 5 px everywhere | |
| err = M.joint_errors(pred, g) | |
| assert np.allclose(err, 5.0) | |
| assert abs(M.mpjpe(err) - 5.0) < 1e-9 | |
| assert abs(M.dn_mpjpe(err) - 100 * 5.0 / P.DIAG) < 1e-9 | |
| assert M.pck(err, 0.05) == 100.0 | |
| assert M.pck(err, 0.001) == 0.0 # 0.001*1920 = 1.92 px < 5 px | |
| assert M.pck(err, 0.05, "diag") == 100.0 | |
| def test_pck_threshold_locked_to_max_side(): | |
| assert P.pck_threshold_px(0.05) == 96.0 | |
| assert abs(P.pck_threshold_px(0.05, "diag") - 0.05 * P.DIAG) < 1e-9 | |
| g = _gt() | |
| pred = g[..., :2] + np.array([100.0, 0.0]) # 100 px: > 96 (max side) but < 110.1 (diag) | |
| err = M.joint_errors(pred, g) | |
| assert M.pck(err, 0.05) == 0.0 and M.pck(err, 0.05, "diag") == 100.0 | |
| def test_per_horizon_and_part(): | |
| g = _gt() | |
| pred = g[..., :2].copy() | |
| for h in range(P.T_PRED): | |
| pred[:, h, :, 0] += h + 1 # h+1 px at horizon h | |
| err = M.joint_errors(pred, g) | |
| assert np.allclose(M.per_horizon(err), [1, 2, 3, 4, 5]) | |
| pred = g[..., :2].copy() | |
| pred[:, :, P.PARTS["face"], 1] += 7.0 | |
| pp = M.per_part(M.joint_errors(pred, g)) | |
| assert abs(pp["face"] - 7.0) < 1e-9 and pp["body"] == 0 and pp["lhand"] == 0 and pp["rhand"] == 0 | |
| def test_hm_mask_and_threshold(): | |
| n = 20 | |
| g = np.zeros((n, P.T_PRED, P.J, 3)) | |
| for k in range(n): # clip k moves (k+1) px per transition (x only) | |
| for h in range(P.T_PRED): | |
| g[k, h, :, 0] = (k + 1) * h | |
| score = M.motion_score(g) | |
| assert np.allclose(score, np.arange(1, n + 1)) | |
| m, thr, _ = M.hm_mask(g, n_top=4) | |
| assert m.sum() == 4 and set(np.flatnonzero(m)) == {16, 17, 18, 19} and thr == 17.0 | |
| assert P.hm_top(609) == 60 and P.hm_top(405) == 40 | |
| # displacement inside the observed block must not matter (only future transitions) | |
| g2 = g.copy() | |
| m2, thr2, _ = M.hm_mask(g2, n_top=4) | |
| assert (m2 == m).all() and thr2 == thr | |
| def test_bootstrap_and_paired(): | |
| v = rng.normal(10, 1, 500) | |
| lo, hi = M.bootstrap_ci(v, n_boot=2000) | |
| assert lo < v.mean() < hi and hi - lo < 0.5 | |
| d = M.paired_diff(v, v, n_boot=500) | |
| assert d["mean"] == 0 and d["ci_lo"] == 0 and d["ci_hi"] == 0 | |
| def test_evaluate_end_to_end(): | |
| g = _gt(30) | |
| pred = g[..., :2] + np.array([3.0, 4.0]) | |
| out, meta = M.evaluate(pred, g, n_boot=200, ref_pred=g[..., :2]) | |
| assert abs(out["mpjpe_all"]["value"] - 5) < 1e-9 and out["pck05"]["value"] == 100 | |
| assert out["diff_vs_ref_all"]["value"] == 5.0 | |
| assert meta["hm_n"] == 3 and out["mpjpe_hm"]["n"] == 3 | |
| def test_classification(): | |
| y = np.array([0, 1, 2, 0, 1, 2]) | |
| assert M.macro_f1(y, y) == 100.0 and M.accuracy(y, y) == 100.0 | |
| yh = np.array([0, 1, 2, 0, 1, 0]) | |
| assert abs(M.accuracy(y, yh) - 100 * 5 / 6) < 1e-9 | |
| def test_protocol_sampling(): | |
| assert P.sample_idx(45, "floor").tolist() == [0, 4, 9, 14, 19, 24, 29, 34, 39, 44] | |
| assert P.sample_idx(45, "round").tolist() == [0, 5, 10, 15, 20, 24, 29, 34, 39, 44] | |
| assert P.sample_idx(45).tolist() == P.sample_idx(45, P.SAMPLE_GRID).tolist() | |
| assert np.array_equal(P.sample_idx(45, "floor"), np.linspace(0, 44, 10).astype(int)) | |
| for n in (46, 60, 90): | |
| i = P.sample_idx(n) | |
| assert i[0] == 0 and i[-1] == n - 1 and np.all(np.diff(i) > 0) | |
| def test_driver_selection_rules(): | |
| def det(x, w, score): | |
| k = np.zeros((P.J, 3)); k[:, 0] = x | |
| return {"keypoints": k.ravel().tolist(), "bbox": [x, 0, w, 100], "proposal_score": score, "idx": [0]} | |
| frames = [{"imgname": "0.jpg", "result": [det(1, 10, 0.5), det(2, 50, 0.9)]}, | |
| {"imgname": "1.jpg", "result": []}, | |
| {"imgname": "2.jpg", "result": [det(3, 10, 0.1)]}] | |
| kp, bb, nd, filled = P.select_driver(frames, "first") | |
| assert kp[0, 0, 0] == 1 and kp[1, 0, 0] == 1 and kp[2, 0, 0] == 3 and filled.tolist() == [False, True, False] | |
| kp, _, _, _ = P.select_driver(frames, "largest_bbox") | |
| assert kp[0, 0, 0] == 2 and kp[1, 0, 0] == 2 | |
| kp, _, _, _ = P.select_driver(frames, "max_score") | |
| assert kp[0, 0, 0] == 2 | |
| assert nd.tolist() == [2, 0, 1] | |
| def test_flip_perm_is_involution(): | |
| p = P.flip_perm() | |
| assert np.array_equal(p[p], np.arange(P.J)) | |
| assert len(P.EDGES_BODY_HANDS) == 24 + 40 and max(max(e) for e in P.EDGES_ALL) < P.J | |
| def test_dn_mpjpe_hm_is_100_hm_over_diag(): | |
| """Roadmap B A0: the HM d-nMPJPE column equals 100 * HM-MPJPE / image diagonal (paper ZV 139.19 -> 6.32).""" | |
| g = _gt(30) | |
| pred = g[..., :2] + rng.normal(0, 15, g[..., :2].shape) | |
| out, meta = M.evaluate(pred, g, n_boot=100) | |
| assert abs(out["dn_mpjpe_hm"]["value"] - 100.0 * out["mpjpe_hm"]["value"] / P.DIAG) < 1e-9 | |
| assert out["dn_mpjpe_hm"]["n"] == meta["hm_n"] == P.hm_top(30) | |
| assert round(100.0 * P.PAPER["zero_velocity"]["mpjpe_hm"] / P.DIAG, 2) == 6.32 # paper Table 1 ZV row | |
| if __name__ == "__main__": | |
| for k, f in list(globals().items()): | |
| if k.startswith("test_") and callable(f): | |
| f() | |
| print("ok", k) | |
Xet Storage Details
- Size:
- 6.12 kB
- Xet hash:
- b637d4ab89dccd5e4e6eace3e3aceadacea046dbd2afaddc808afda934c7ed09
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.