File size: 2,925 Bytes
e6f80f3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
"""Self-check of stratify's selection-rule attribution on new-data rows (CPU only).

    python selftest_shift.py --rows runs/analysis/ablation-sample-500/new.jsonl --work runs/analysis/test/shift

Writes the labels as predictions with +0.5 m along x on every target whose selection_rule is not frozen_prep, scores
them with ``evaluate --predictions`` (full condition) and checks via stratify: report reproduced exactly; position error
0 on frozen_prep and 0.5 on each added rule; size and yaw error 0 everywhere.
"""
import argparse
import json
from pathlib import Path
import subprocess
import sys

import stratify as S

SHIFT = .5


def main():
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument("--rows", required=True, type=Path)
    p.add_argument("--work", required=True, type=Path)
    a = p.parse_args()
    a.work.mkdir(parents=True, exist_ok=False)
    with open(a.rows) as src, open(a.work / "predictions.jsonl", "w") as out:
        for line in src:
            row = json.loads(line)
            rules = [(e or {}).get("selection_rule") or "frozen_prep" for e in row["provenance"]["field_evidence"]]
            objects = []
            for t, rule in zip(row["target"]["objects"], rules):
                position = [v if v is not None else 0. for v in t["bottom_center_m"]]
                size = [v if v else 1. for v in t["target_size_local_m"]]
                objects.append({"id": t["id"], "bottom_center_m": [position[0] + (rule != "frozen_prep") * SHIFT, *position[1:]],
                                "target_size_local_m": size, "yaw_rad": t["yaw_rad"] if t["yaw_rad"] is not None else 0.})
            out.write(json.dumps({"layout": {"schema_version": "fastfill.v2", "objects": objects}}) + "\n")
    subprocess.run([sys.executable, "-m", "fastfill.v2.evaluate", "--data", str(a.rows), "--predictions",
                    str(a.work / "predictions.jsonl"), "--output", str(a.work / "eval")], check=True, stdout=subprocess.DEVNULL)
    meta, rooms, checks = S.collect(a.work / "eval", "outcomes.jsonl", a.rows, 6)
    strata = S.aggregate(rooms, boot=200)
    assert checks["reproduces_report_exactly"], checks
    seen = {}
    for rule, entry in strata["rule"].items():
        m = entry["metrics"]
        seen[rule] = (m["bottom_center_error_m"]["mean"], m["bottom_center_error_m"]["n"])
        assert abs(m["bottom_center_error_m"]["mean"] - (0. if rule == "frozen_prep" else SHIFT)) < 1e-9, (rule, m)
        assert m["yaw_error_rad"]["mean"] < 1e-9 and m.get("log_size_error", {"mean": 0.})["mean"] < 1e-9, (rule, m)
    assert set(seen) == set(S.RULES), seen
    print(json.dumps({"ok": True, "rooms": len(rooms), "layouts": checks["requests_with_layout"],
                      "position_error_mean_and_objects_by_rule": seen,
                      "places": {k: v["objects"] for k, v in strata["place"].items()}}, indent=1))


if __name__ == "__main__":
    main()