Buckets:
| #!/usr/bin/env python3 | |
| """tools/review_tool/apply_review.py — turn the reviewer's export (cwm3_review_labels.json or .csv from results/review_tool/index.html) into | |
| (1) results/targets_kin_causal_routes_v2/driver_box_verified.json — per suspect route: driver / passenger / unsure / none (+ note, rater, ts); | |
| the verified mask for the masked protocol = routes labelled passenger or none | |
| (unsure routes are listed separately and reported both ways); | |
| (2) results/tables/review_tool_summary.md — Task A counts and the affected test-anchor shares; Task B precision of the | |
| automatic hand_leave / hand_return / head_turn_onset labels per regime with | |
| Wilson 95 % intervals. | |
| Append-only for the table; the json is (re)written from the export only. python tools/review_tool/apply_review.py <export.json|export.csv> | |
| """ | |
| from __future__ import annotations | |
| import csv, json, math, sys, time | |
| from pathlib import Path | |
| CWM3 = Path(__file__).resolve().parents[2] | |
| ITEMS = CWM3 / "results/review_tool/items.json" | |
| SUSPECTS = CWM3 / "results/targets_kin_causal_routes_v2/driver_box_suspect_routes.json" | |
| OUT_JSON = CWM3 / "results/targets_kin_causal_routes_v2/driver_box_verified.json" | |
| OUT_MD = CWM3 / "results/tables/review_tool_summary.md" | |
| def wilson(k: int, n: int, z: float = 1.96): | |
| if n == 0: | |
| return (float("nan"), float("nan")) | |
| p = k / n | |
| d = 1 + z * z / n | |
| c = (p + z * z / (2 * n)) / d | |
| h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d | |
| return (c - h, c + h) | |
| def load_export(p: Path) -> dict: | |
| if p.suffix == ".json": | |
| o = json.load(open(p)) | |
| if isinstance(o, dict) and "items" in o: # browser export | |
| rows = o["items"] | |
| else: # server labels_latest.json: {id: {label, note, rater, ts}} | |
| rows = [dict(id=k, **v) for k, v in o.items() if not k.startswith("_")] | |
| else: | |
| rows = list(csv.DictReader(open(p, encoding="utf-8"))) | |
| return {r["id"]: r for r in rows if r.get("label")} | |
| def main(argv=None) -> int: | |
| argv = argv or sys.argv[1:] | |
| if not argv: | |
| print(__doc__); return 2 | |
| exp = load_export(Path(argv[0])) | |
| items = {i["id"]: i for i in json.load(open(ITEMS))["items"]} | |
| sus = json.load(open(SUSPECTS)) | |
| ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) | |
| # ---- Task A | |
| A = {} | |
| for iid, it in items.items(): | |
| if it["task"] != "A": | |
| continue | |
| r = exp.get(iid) | |
| A[it["meta"]["uid"]] = {"label": r["label"] if r else "", "note": (r or {}).get("note", ""), "rater": (r or {}).get("rater", ""), "ts": (r or {}).get("ts", ""), | |
| "suspect_meta": it["meta"]} | |
| lab_counts = {} | |
| for v in A.values(): | |
| lab_counts[v["label"] or "unlabelled"] = lab_counts.get(v["label"] or "unlabelled", 0) + 1 | |
| mask_routes = sorted(u for u, v in A.items() if v["label"] in ("passenger", "none")) | |
| unsure_routes = sorted(u for u, v in A.items() if v["label"] == "unsure") | |
| driver_routes = sorted(u for u, v in A.items() if v["label"] == "driver") | |
| # affected anchors per (regime, split) from the suspect list | |
| anchors = {} | |
| for e in sus["flagged"]: | |
| key = f"{e['reg']}_{e['sp']}" | |
| anchors.setdefault(key, {"total_flagged": 0, "verified_wrong": 0, "unsure": 0, "verified_driver": 0}) | |
| anchors[key]["total_flagged"] += e["n_anchors"] | |
| lab = A.get(e["uid"], {}).get("label", "") | |
| if lab in ("passenger", "none"): | |
| anchors[key]["verified_wrong"] += e["n_anchors"] | |
| elif lab == "unsure": | |
| anchors[key]["unsure"] += e["n_anchors"] | |
| elif lab == "driver": | |
| anchors[key]["verified_driver"] += e["n_anchors"] | |
| totals = {k: v["anchors_total"] for k, v in sus["summary"].items()} if isinstance(sus.get("summary"), dict) and all(isinstance(v, dict) and "anchors_total" in v for v in sus["summary"].values()) else {} | |
| json.dump({"written_utc": ts, "source_export": str(argv[0]), "definition": "human review of the causal driver box on the 64 suspect routes (results/review_tool); " | |
| "mask = routes labelled passenger or none; unsure listed separately", "routes": A, "mask_routes": mask_routes, "unsure_routes": unsure_routes, | |
| "driver_routes": driver_routes, "anchors_by_split": anchors}, open(OUT_JSON, "w"), indent=1) | |
| # ---- Task B | |
| B = {} | |
| for iid, it in items.items(): | |
| if it["task"] != "B": | |
| continue | |
| r = exp.get(iid) | |
| if not r: | |
| continue | |
| key = (it["meta"]["regime"], it["meta"]["event"]) | |
| B.setdefault(key, {"n": 0, "correct": 0, "wrong": 0, "timing": 0, "unsure": 0}) | |
| B[key]["n"] += 1 | |
| B[key][r["label"]] = B[key].get(r["label"], 0) + 1 | |
| lines = [f"\n## Review-tool summary {ts} (export {Path(argv[0]).name}; rater(s) {sorted({v['rater'] for v in A.values() if v['rater']})})\n", | |
| "### Task A — driver association on the 64 suspect routes\n", "| label | routes |", "|---|---|"] | |
| lines += [f"| {k} | {v} |" for k, v in sorted(lab_counts.items())] | |
| lines += ["", "| regime_split | flagged anchors | verified wrong occupant | unsure | verified driver |", "|---|---|---|---|---|"] | |
| lines += [f"| {k} | {v['total_flagged']:,} | {v['verified_wrong']:,} | {v['unsure']:,} | {v['verified_driver']:,} |" for k, v in sorted(anchors.items())] | |
| lines += ["", f"Verified mask ({len(mask_routes)} routes: passenger or none) written to `{OUT_JSON.relative_to(CWM3)}`; unsure {len(unsure_routes)}; driver {len(driver_routes)}.", | |
| "", "### Task B — precision of the automatic switch labels (human 'correct' among labelled items; Wilson 95 % CI)\n", | |
| "| regime | event | n labelled | correct | wrong | timing >0.5 s | unsure | precision (correct / labelled excl. unsure) [CI] |", "|---|---|---|---|---|---|---|---|"] | |
| for (reg, ev), v in sorted(B.items()): | |
| n_eff = v["n"] - v.get("unsure", 0) | |
| k = v.get("correct", 0) | |
| lo, hi = wilson(k, n_eff) | |
| prec = f"{k / n_eff:.3f} [{lo:.3f}, {hi:.3f}]" if n_eff else "--" | |
| lines.append(f"| {reg} | {ev} | {v['n']} | {k} | {v.get('wrong', 0)} | {v.get('timing', 0)} | {v.get('unsure', 0)} | {prec} |") | |
| with open(OUT_MD, "a", encoding="utf-8") as f: | |
| f.write("\n".join(lines) + "\n") | |
| print("\n".join(lines)) | |
| print(f"\nwrote {OUT_JSON} and appended {OUT_MD}") | |
| return 0 | |
| # --------------------------------------------------------------------------------------------------------------------------------------- | |
| # Second labelling pass (owner request 2026-09-16): hand_return items re-labelled independently on relabel_hr.html (ids <id>__r2). | |
| # python tools/review_tool/apply_review.py results/review_tool/labels_latest.json --pass2 results/review_tool/items_relabel_hr.json | |
| # Appends a section to review_tool_summary.md: pass-1 vs pass-2 precision per regime, cross-tab, agreement + Cohen's kappa, phone notes; | |
| # the reported hand_return precision is the pass-2 value (pass 1 shown alongside). Does NOT touch driver_box_verified.json. | |
| LABELS4 = ("correct", "wrong", "timing", "unsure") | |
| def cohen_kappa(pairs): | |
| n = len(pairs) | |
| if n == 0: | |
| return float("nan") | |
| po = sum(a == b for a, b in pairs) / n | |
| pe = sum((sum(a == l for a, _ in pairs) / n) * (sum(b == l for _, b in pairs) / n) for l in LABELS4) | |
| return (po - pe) / (1 - pe) if pe < 1 else float("nan") | |
| def _prec_row(reg, ev, tag, v): | |
| n_eff = v["n"] - v.get("unsure", 0); k = v.get("correct", 0); lo, hi = wilson(k, n_eff) | |
| prec = f"{k / n_eff:.3f} [{lo:.3f}, {hi:.3f}]" if n_eff else "--" | |
| return f"| {reg} | {ev} | {tag} | {v['n']} | {k} | {v.get('wrong', 0)} | {v.get('timing', 0)} | {v.get('unsure', 0)} | {prec} |" | |
| def main_pass2(export_path: Path, items_p2_path: Path) -> int: | |
| exp = load_export(export_path) | |
| items1 = {i["id"]: i for i in json.load(open(ITEMS))["items"]} | |
| P2 = json.load(open(items_p2_path, encoding="utf-8")) | |
| items2 = P2["items"]; ev = P2.get("event", "hand_return") | |
| ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) | |
| regs = sorted({i["meta"]["regime"] for i in items2}) | |
| tab = {r: {"pass1": {"n": 0}, "pass2": {"n": 0}} for r in regs} | |
| pairs = {r: [] for r in regs}; cross = {r: {} for r in regs}; phone2 = {r: {} for r in regs} | |
| n_unlab = 0 | |
| for it in items2: | |
| reg = it["meta"]["regime"]; r2 = exp.get(it["id"]); r1 = exp.get(it["meta"]["orig_id"]) | |
| if r1: | |
| tab[reg]["pass1"]["n"] += 1; tab[reg]["pass1"][r1["label"]] = tab[reg]["pass1"].get(r1["label"], 0) + 1 | |
| if not r2: | |
| n_unlab += 1; continue | |
| tab[reg]["pass2"]["n"] += 1; tab[reg]["pass2"][r2["label"]] = tab[reg]["pass2"].get(r2["label"], 0) + 1 | |
| if "phone" in (r2.get("note") or "").lower(): | |
| phone2[reg][r2["label"]] = phone2[reg].get(r2["label"], 0) + 1 | |
| if r1: | |
| pairs[reg].append((r1["label"], r2["label"])); cross[reg][(r1["label"], r2["label"])] = cross[reg].get((r1["label"], r2["label"]), 0) + 1 | |
| raters = sorted({(exp[i["id"]].get("rater") or "") for i in items2 if i["id"] in exp}) | |
| lines = [f"\n## Review-tool summary — pass 2 ({ev} re-labelled independently) {ts} (export {export_path.name}; items {items_p2_path.name}; rater(s) {raters}; unlabelled {n_unlab})\n", | |
| f"Pass 2 = the owner's second, blind labelling of the same {len(items2)} {ev} items on `relabel_hr.html` (first-pass answers hidden; explicit definition shown). " | |
| f"The precision to report for {ev} is the pass-2 value; pass 1 is kept for the record.\n", | |
| "| regime | event | pass | n labelled | correct | wrong | timing >0.5 s | unsure | precision (correct / labelled excl. unsure) [CI] |", "|---|---|---|---|---|---|---|---|---|"] | |
| for reg in regs: | |
| lines.append(_prec_row(reg, ev, "1", tab[reg]["pass1"])); lines.append(_prec_row(reg, ev, "**2 (reported)**", tab[reg]["pass2"])) | |
| lines += ["", "| regime | n both passes | agree | agreement | Cohen's κ | cross-tab pass1→pass2 (n) |", "|---|---|---|---|---|---|"] | |
| for reg in regs: | |
| pr = pairs[reg]; ag = sum(a == b for a, b in pr) | |
| ct = "; ".join(f"{a}→{b}: {n}" for (a, b), n in sorted(cross[reg].items())) | |
| lines.append(f"| {reg} | {len(pr)} | {ag} | {ag / len(pr):.3f} | {cohen_kappa(pr):.3f} | {ct} |" if pr else f"| {reg} | 0 | -- | -- | -- | -- |") | |
| allp = [x for r in regs for x in pairs[r]] | |
| if allp: | |
| lines.append(f"| all | {len(allp)} | {sum(a == b for a, b in allp)} | {sum(a == b for a, b in allp) / len(allp):.3f} | {cohen_kappa(allp):.3f} | |") | |
| ph = "; ".join(f"{reg} " + ", ".join(f"{l}: {n}" for l, n in sorted(v.items())) for reg, v in phone2.items() if v) | |
| lines += ["", ("Phone-use notes in pass 2 (label: n): " + ph) if ph else "Phone-use notes in pass 2: none."] | |
| with open(OUT_MD, "a", encoding="utf-8") as f: | |
| f.write("\n".join(lines) + "\n") | |
| print("\n".join(lines)); print(f"\nappended {OUT_MD} (driver_box_verified.json untouched)") | |
| return 0 | |
| def main_pass3(export_path: Path, items_p3_path: Path, items_p2_path: Path) -> int: | |
| """Third pass (engaged hand_return at 0.1 s/frame): reported engaged value = pass 3; manual stays pass 2; agreements 1<->3 and 2<->3.""" | |
| exp = load_export(export_path) | |
| P3 = json.load(open(items_p3_path, encoding="utf-8")); P2 = json.load(open(items_p2_path, encoding="utf-8")) | |
| ev = P3.get("event", "hand_return"); reg3 = P3.get("regime", "engaged") | |
| ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) | |
| cnt = {"1": {"n": 0}, "2": {"n": 0}, "3": {"n": 0}}; pairs13, pairs23 = [], []; cross23 = {}; phone3 = {}; n_unlab = 0 | |
| for it in P3["items"]: | |
| o = it["meta"]["orig_id"]; r1 = exp.get(o); r2 = exp.get(o + "__r2"); r3 = exp.get(it["id"]) | |
| for tag, r in (("1", r1), ("2", r2)): | |
| if r: cnt[tag]["n"] += 1; cnt[tag][r["label"]] = cnt[tag].get(r["label"], 0) + 1 | |
| if not r3: n_unlab += 1; continue | |
| cnt["3"]["n"] += 1; cnt["3"][r3["label"]] = cnt["3"].get(r3["label"], 0) + 1 | |
| if "phone" in (r3.get("note") or "").lower(): phone3[r3["label"]] = phone3.get(r3["label"], 0) + 1 | |
| if r1: pairs13.append((r1["label"], r3["label"])) | |
| if r2: pairs23.append((r2["label"], r3["label"])); cross23[(r2["label"], r3["label"])] = cross23.get((r2["label"], r3["label"]), 0) + 1 | |
| # manual: pass 2 stays the reported value | |
| man = {"n": 0} | |
| for it in P2["items"]: | |
| if it["meta"]["regime"] == reg3: continue | |
| r2 = exp.get(it["id"]) | |
| if r2: man["n"] += 1; man[r2["label"]] = man.get(r2["label"], 0) + 1 | |
| lines = [f"\n## Review-tool summary — pass 3 ({reg3} {ev} at 0.1 s/frame, 21 frames) {ts} (export {export_path.name}; items {items_p3_path.name}; unlabelled {n_unlab})\n", | |
| f"Pass 3 = the owner's third, blind labelling of the same {len(P3['items'])} {reg3} {ev} events with 21 frames at 0.1 s (−1.0 … +1.0 s) and a frame player; passes 1–2 hidden. " | |
| f"Pre-registered rule: REPORTED {reg3} {ev} precision = pass 3; manual {ev} stays pass 2; other events stay pass 1.\n", | |
| "| regime | event | pass | n labelled | correct | wrong | timing >0.5 s | unsure | precision (correct / labelled excl. unsure) [CI] |", "|---|---|---|---|---|---|---|---|---|"] | |
| for tag in ("1", "2"): lines.append(_prec_row(reg3, ev, tag, cnt[tag])) | |
| lines.append(_prec_row(reg3, ev, "**3 (reported; 0.1 s frames)**", cnt["3"])) | |
| lines.append(_prec_row("manual", ev, "**2 (reported)**", man)) | |
| lines += ["", "| comparison | n | agree | agreement | Cohen's κ | cross-tab (n) |", "|---|---|---|---|---|---|"] | |
| for name, pr, ct in (("pass 1 → pass 3", pairs13, None), ("pass 2 → pass 3", pairs23, cross23)): | |
| cts = "; ".join(f"{a}→{b}: {n}" for (a, b), n in sorted(ct.items())) if ct else "" | |
| lines.append(f"| {name} | {len(pr)} | {sum(a == b for a, b in pr)} | {sum(a == b for a, b in pr) / len(pr):.3f} | {cohen_kappa(pr):.3f} | {cts} |" if pr else f"| {name} | 0 | -- | -- | -- | |") | |
| ph = ", ".join(f"{l}: {n}" for l, n in sorted(phone3.items())) | |
| lines += ["", ("Phone-use notes in pass 3 (label: n): " + ph) if ph else "Phone-use notes in pass 3: none."] | |
| with open(OUT_MD, "a", encoding="utf-8") as f: | |
| f.write("\n".join(lines) + "\n") | |
| print("\n".join(lines)); print(f"\nappended {OUT_MD} (driver_box_verified.json untouched)") | |
| return 0 | |
| def main_pass4(export_path: Path, items_p4_path: Path) -> int: | |
| """Frame-click diagnostic (engaged hand_return): first frame with a hand on the wheel, or never_on / always_on / unsure.""" | |
| exp = load_export(export_path); P4 = json.load(open(items_p4_path, encoding="utf-8")) | |
| ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) | |
| tally = {"return_frame": 0, "never_on": 0, "always_on": 0, "unsure": 0, "unlabelled": 0}; dts = []; per = [] | |
| for it in P4["items"]: | |
| r = exp.get(it["id"]); l = (r or {}).get("label", "") | |
| o = it["meta"]["orig_id"] | |
| if not l: tally["unlabelled"] += 1 | |
| elif l.startswith("ret_dt="): tally["return_frame"] += 1; dts.append(float(l.split("=")[1])) | |
| else: tally[l] = tally.get(l, 0) + 1 | |
| per.append((o, it["meta"]["car"], exp.get(o, {}).get("label", ""), exp.get(o + "__r2", {}).get("label", ""), exp.get(o + "__r3", {}).get("label", ""), l or "--", (r or {}).get("note", "") or "")) | |
| n = len(P4["items"]); n_lab = n - tally["unlabelled"]; k = tally["return_frame"] | |
| lo, hi = wilson(k, n_lab); lo2, hi2 = wilson(k, n_lab - tally["unsure"]) | |
| lines = [f"\n## Review-tool summary — pass 4 (engaged hand_return frame-click diagnostic) {ts} (export {export_path.name}; items {items_p4_path.name})\n", | |
| "Question: the FIRST of the 21 frames (dt = −1.0 … +1.0 s, 0.1 s) with a hand on the wheel; or 'never on the wheel in the window' / 'on the wheel in every frame (never left)' / 'unsure'. " | |
| "Owner's definition of 'unsure' (2026-09-16): hand on the wheel at the start and then taken away, or a return tendency mid-way that is not completed.\n", | |
| "| answer | n |", "|---|---|"] + [f"| {a} | {v} |" for a, v in tally.items()] | |
| lines += ["", f"Real hand return within ±1 s of the automatic switch: **{k} / {n_lab}** labelled (Wilson 95 % [{lo:.3f}, {hi:.3f}]); excluding 'unsure': {k} / {n_lab - tally['unsure']} [{lo2:.3f}, {hi2:.3f}]." | |
| + (f" Return-frame offsets (s): {sorted(dts)}." if dts else " No item received a return frame."), | |
| "", "| item | car | pass 1 | pass 2 | pass 3 | pass 4 | note |", "|---|---|---|---|---|---|---|"] | |
| lines += [f"| {o} | {c} | {a} | {b} | {d} | {e} | {f_} |" for o, c, a, b, d, e, f_ in per] | |
| with open(OUT_MD, "a", encoding="utf-8") as f: | |
| f.write("\n".join(lines) + "\n") | |
| print("\n".join(lines[:12])); print(f"\nappended {OUT_MD}") | |
| return 0 | |
| _main_pass1 = main | |
| def main(argv=None) -> int: # noqa: F811 — dispatcher: --pass2 <items_relabel.json> selects the second-pass summary | |
| argv = list(argv or sys.argv[1:]) | |
| if "--pass4" in argv: | |
| i = argv.index("--pass4"); p4 = argv[i + 1]; rest = argv[:i] + argv[i + 2:] | |
| if not rest: | |
| print(__doc__); return 2 | |
| return main_pass4(Path(rest[0]), Path(p4)) | |
| if "--pass3" in argv: | |
| i = argv.index("--pass3"); p3 = argv[i + 1]; rest = argv[:i] + argv[i + 2:] | |
| p2 = str(CWM3 / "results/review_tool/items_relabel_hr.json") | |
| if "--pass2" in rest: | |
| j = rest.index("--pass2"); p2 = rest[j + 1]; rest = rest[:j] + rest[j + 2:] | |
| if not rest: | |
| print(__doc__); return 2 | |
| return main_pass3(Path(rest[0]), Path(p3), Path(p2)) | |
| if "--pass2" in argv: | |
| i = argv.index("--pass2"); p2 = argv[i + 1]; rest = argv[:i] + argv[i + 2:] | |
| if not rest: | |
| print(__doc__); return 2 | |
| return main_pass2(Path(rest[0]), Path(p2)) | |
| return _main_pass1(argv) | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |
Xet Storage Details
- Size:
- 18.4 kB
- Xet hash:
- 464d7da54219f17efacd43f0f1c82633594b7307e5b5462e31b9d69a4c1a651a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.