Buckets:
| #!/usr/bin/env python3 | |
| """tools/review_tool/apply_gold_engaged.py — turn the owner's gold-event labels (relabel_gold_engaged.html, ids G_*; pre-registered | |
| docs/decisions.md 2026-09-17T00:28:00Z) into the GOLD EVENT SET for the engaged hand cells. | |
| From results/review_tool/labels_latest.json and items_gold_engaged.json: | |
| * gold events = answers 1/2: at_event -> t* = t_event (the automatic candidate time); ret_dt=<dt> / off_dt=<dt> -> t* = t_event + dt (the clicked frame); | |
| * non-events = answers 3/4 (never / always) = v3 candidate false positives; unsure and unlabelled are excluded from the precision denominators; | |
| * per cell (engaged hand_return, engaged hand_leave, manual hand_return = block MR): counts of every answer, v3 candidate precision = | |
| (at_event + shifted) / (labelled - unsure) with the Wilson 95 % interval, and the strict variant at_event / (labelled - unsure) (event within | |
| +-0.5 s of the candidate); the clicked-dt distribution of the shifted events; | |
| * block W (60 engaged 10 s recall windows without a v3 hand event; yes | no | unsure): miss share q = yes / (yes + no) with Wilson CI and the | |
| apply_review_v3.py recall estimate TP / (TP + FN), TP = precision_event(engaged hand_return) x N_v3 returns, FN = q x (v3-negative 10 s windows); | |
| * milestone flags (>= 300 labelled = minimum set; engaged blocks complete; + W; + MR = full) and per-block counts. | |
| Writes results/review_tool/gold_engaged_events.json (--json-out) and appends one section to results/tables/review_tool_summary.md (--out). | |
| Dry run: --out /tmp/x.md --json-out /tmp/x.json. Read-only on everything else. | |
| python tools/review_tool/apply_gold_engaged.py [results/review_tool/labels_latest.json] [--items results/review_tool/items_gold_engaged.json] | |
| [--out results/tables/review_tool_summary.md] [--json-out results/review_tool/gold_engaged_events.json] [--tol 0.5] | |
| """ | |
| from __future__ import annotations | |
| import argparse, json, sys, time | |
| from pathlib import Path | |
| CWM3 = Path(__file__).resolve().parents[2] | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| import apply_review as AR # noqa: E402 (wilson, load_export) | |
| LABELS_DEFAULT = CWM3 / "results/review_tool/labels_latest.json" | |
| ITEMS_DEFAULT = CWM3 / "results/review_tool/items_gold_engaged.json" | |
| OUT_MD_DEFAULT = CWM3 / "results/tables/review_tool_summary.md" | |
| JSON_DEFAULT = CWM3 / "results/review_tool/gold_engaged_events.json" | |
| MIN_SET, TOL = 300, 0.5 | |
| BLOCKS = ("R200", "L100", "R100", "L50", "W", "MR") | |
| CELLS = ("engaged_hand_return", "engaged_hand_leave", "manual_hand_return") | |
| def parse(label: str): | |
| """-> (kind, dt) with kind in at_event | shifted | never | always | unsure | unlabelled | other.""" | |
| if not label: return "unlabelled", None | |
| if label == "at_event": return "at_event", 0.0 | |
| if label.startswith("ret_dt=") or label.startswith("off_dt="): | |
| try: return "shifted", float(label.split("=")[1]) | |
| except ValueError: return "other", None | |
| if label in ("never", "always", "unsure"): return label, None | |
| return "other", None | |
| def fmt_ci(k, n): | |
| if not n: return "--" | |
| lo, hi = AR.wilson(k, n); return f"{k / n:.3f} [{lo:.3f}, {hi:.3f}] (n={n})" | |
| def dt_stats(dts): | |
| if not dts: return "--" | |
| s = sorted(dts); n = len(s); med = s[n // 2] if n % 2 else 0.5 * (s[n // 2 - 1] + s[n // 2]) | |
| return f"n {n}, median {med:+.1f}, mean {sum(s) / n:+.2f}, min {s[0]:+.1f}, max {s[-1]:+.1f}" | |
| def main(argv=None) -> int: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("labels", nargs="?", default=str(LABELS_DEFAULT)) | |
| ap.add_argument("--items", default=str(ITEMS_DEFAULT)); ap.add_argument("--out", default=str(OUT_MD_DEFAULT)) | |
| ap.add_argument("--json-out", default=str(JSON_DEFAULT)); ap.add_argument("--tol", type=float, default=TOL) | |
| a = ap.parse_args(argv) | |
| exp = AR.load_export(Path(a.labels)); P = json.load(open(a.items, encoding="utf-8")); items = P["items"] | |
| ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) | |
| gold, rows = [], [] | |
| C = {cell: {"n": 0, "labelled": 0, "at_event": 0, "shifted": 0, "never": 0, "always": 0, "unsure": 0, "unlabelled": 0, "other": 0, "dts": []} for cell in CELLS} | |
| W = {"n": 0, "yes": 0, "no": 0, "unsure": 0, "unlabelled": 0, "other": 0} | |
| blk = {b: {"n": 0, "labelled": 0} for b in BLOCKS} | |
| for it in items: | |
| m = it["meta"]; r = exp.get(it["id"]) or {}; lab = r.get("label", "") or "" | |
| b = blk.setdefault(m["block"], {"n": 0, "labelled": 0}); b["n"] += 1 | |
| if m["event"] == "recall_window": | |
| kind = "yes" if lab == "yes" else "no" if lab == "no" else "unsure" if lab == "unsure" else "unlabelled" if not lab else "other" | |
| W["n"] += 1; W[kind] += 1 | |
| if kind != "unlabelled": b["labelled"] += 1 | |
| rows.append({"id": it["id"], "block": m["block"], "event": m["event"], "regime": m["regime"], "car": m["car"], "uid": m["uid"], "t_event": m["t0"], "label": lab, "kind": kind, "note": r.get("note") or ""}) | |
| continue | |
| kind, dt = parse(lab); c = C[f"{m['regime']}_{m['event']}"]; c["n"] += 1; c[kind] += 1 | |
| if kind != "unlabelled": c["labelled"] += 1; b["labelled"] += 1 | |
| if kind == "shifted": c["dts"].append(dt) | |
| if kind in ("at_event", "shifted"): | |
| gold.append({"id": it["id"], "uid": m["uid"], "car": m["car"], "regime": m["regime"], "event": m["event"], "t_anchor": m["t_anchor"], "t_event_candidate": m["t_event"], | |
| "t_star": round(m["t_event"] + dt, 3), "dt_star": dt, "label": lab, "note": r.get("note") or "", "block": m["block"], "ts": r.get("ts")}) | |
| rows.append({"id": it["id"], "block": m["block"], "event": m["event"], "regime": m["regime"], "car": m["car"], "uid": m["uid"], "t_event": m["t_event"], "label": lab, "kind": kind, "note": r.get("note") or ""}) | |
| for ev, c in C.items(): | |
| c["n_eff"] = c["labelled"] - c["unsure"] - c["other"] | |
| c["n_gold"] = c["at_event"] + c["shifted"]; c["n_false_positive"] = c["never"] + c["always"] | |
| c["precision_event"] = c["n_gold"] / c["n_eff"] if c["n_eff"] else None; c["precision_event_ci"] = list(AR.wilson(c["n_gold"], c["n_eff"])) if c["n_eff"] else None | |
| c["precision_at_event"] = c["at_event"] / c["n_eff"] if c["n_eff"] else None; c["precision_at_event_ci"] = list(AR.wilson(c["at_event"], c["n_eff"])) if c["n_eff"] else None | |
| c["n_gold_routes"] = len({g["uid"] for g in gold if f"{g['regime']}_{g['event']}" == ev}) | |
| # ---- block W: miss share and recall estimate (apply_review_v3.py logic) | |
| W["n_eff"] = W["yes"] + W["no"]; W["q"] = W["yes"] / W["n_eff"] if W["n_eff"] else None; W["q_ci"] = list(AR.wilson(W["yes"], W["n_eff"])) if W["n_eff"] else None | |
| st = P.get("sampling_stats", {}).get("recall_window", {}) | |
| n_ret = st.get("n_v3_hand_return_engaged_test"); n_starts = st.get("n_valid_window_starts"); step = st.get("start_step_s", 0.2); win_s = st.get("window_s", 10.0) | |
| R = {"n_v3_returns_engaged_test": n_ret, "n_neg_windows": (n_starts * step / win_s) if n_starts else None} | |
| pe = C["engaged_hand_return"] | |
| if R["n_neg_windows"] and pe["n_eff"] and W["n_eff"]: | |
| def rec(p, q): tp = p * n_ret; fn = q * R["n_neg_windows"]; return tp / (tp + fn) if tp + fn > 0 else None | |
| R.update(tp_est=pe["precision_event"] * n_ret, fn_est=W["q"] * R["n_neg_windows"], recall=rec(pe["precision_event"], W["q"]), | |
| recall_lo=rec(pe["precision_event_ci"][0], W["q_ci"][1]), recall_hi=rec(pe["precision_event_ci"][1], W["q_ci"][0])) | |
| n_lab = sum(c["labelled"] for c in C.values()) + W["n"] - W["unlabelled"]; n_all = len(items) | |
| n_eng = sum(v["n"] for b, v in blk.items() if b in ("R200", "L100", "R100", "L50")); n_engw = n_eng + blk["W"]["n"] | |
| tot = {"n_items": n_all, "n_labelled": n_lab, "n_gold": len(gold), "n_gold_engaged": sum(1 for g in gold if g["regime"] == "engaged"), "n_gold_manual": sum(1 for g in gold if g["regime"] == "manual"), | |
| "n_unsure": sum(c["unsure"] for c in C.values()) + W["unsure"], "n_unlabelled": sum(c["unlabelled"] for c in C.values()) + W["unlabelled"], | |
| "milestones": {"minimum_set": MIN_SET, "engaged_events": n_eng, "engaged_events_plus_windows": n_engw, "full": n_all}, "milestones_planned": [300, 450, 510, 610], | |
| "minimum_set_reached": n_lab >= MIN_SET, "engaged_events_complete": all(v["labelled"] == v["n"] for b, v in blk.items() if b in ("R200", "L100", "R100", "L50")), | |
| "windows_complete": blk["W"]["labelled"] == blk["W"]["n"], "full_set_reached": n_lab >= n_all} | |
| # ---- gold json | |
| summ = {"written_utc": ts, "labels": str(a.labels), "items": str(a.items), "items_built_utc": P.get("built_utc"), "label_version_candidates": P.get("label_version"), | |
| "split": "engaged test", "tol_s": a.tol, "definition": {"gold_event": "answer 1 (at_event: t* = candidate t_event) or 2 (ret_dt/off_dt: t* = t_event + clicked dt)", | |
| "false_positive": "answer 3 (never) or 4 (always): no such event in the -2 ... +1 s window", "excluded": "unsure, unlabelled"}, | |
| "totals": tot, "per_cell": {ev: {k: v for k, v in c.items() if k != "dts"} | {"clicked_dts": c["dts"]} for ev, c in C.items()}, "recall_windows": W, "recall_estimate": R, | |
| "per_block": blk, "gold_events": gold, "all_items": rows} | |
| jo = Path(a.json_out); jo.parent.mkdir(parents=True, exist_ok=True); jo.write_text(json.dumps(summ, indent=1, ensure_ascii=False), encoding="utf-8") | |
| # ---- markdown section | |
| L = [f"\n## Review-tool summary — human GOLD EVENT SET, engaged hand cells (+ recall windows, + manual hand_return) {ts} (labels {Path(a.labels).name}; items {Path(a.items).name} built {P.get('built_utc')}; labelled {n_lab}/{n_all}; " | |
| f"minimum set {MIN_SET}: {'reached' if tot['minimum_set_reached'] else 'NOT reached'}; engaged event blocks ({n_eng}): {'complete' if tot['engaged_events_complete'] else 'incomplete'}; windows: {'complete' if tot['windows_complete'] else 'incomplete'}; full {n_all}: {'reached' if tot['full_set_reached'] else 'NOT reached'})\n", | |
| f"Protocol (pre-registered 2026-09-17T00:28:00Z): v3 first-step candidates on the ENGAGED TEST split, <= {P.get('per_route')} per route per event type, seed {P.get('seed')}, none within +-5 s of any " | |
| f"previously reviewed/validation item; 31 frames at 0.1 s (-2.0 ... +1.0 s), EVENT = dt 0 = candidate time; real-time playback. Gold event = answer 1 (t* = candidate time) or 2 " | |
| f"(t* = clicked frame); answers 3/4 = candidate false positives; unsure excluded. precision_event = (1 + 2) / (labelled - unsure); precision_at_event = 1 / (labelled - unsure) " | |
| f"(event within +-{a.tol:.1f} s of the candidate). Evaluation only: training targets stay v2.\n", | |
| "| cell | n | labelled | 1 at_event | 2 shifted | 3 never | 4 always | 5 unsure | unlabelled | gold events (routes) | v3 precision_event [Wilson 95 %] | precision_at_event [Wilson 95 %] |", | |
| "|---|---|---|---|---|---|---|---|---|---|---|---|"] | |
| for ev, c in C.items(): | |
| L.append(f"| {ev} | {c['n']} | {c['labelled']} | {c['at_event']} | {c['shifted']} | {c['never']} | {c['always']} | {c['unsure']} | {c['unlabelled']} | {c['n_gold']} ({c['n_gold_routes']}) | " | |
| f"{fmt_ci(c['n_gold'], c['n_eff'])} | {fmt_ci(c['at_event'], c['n_eff'])} |") | |
| L += ["", "Clicked offsets of the shifted (answer 2) events relative to the candidate (s): " + "; ".join(f"{ev}: {dt_stats(c['dts'])}" for ev, c in C.items()), | |
| "Blocks: " + " · ".join(f"{b} {v['labelled']}/{v['n']}" for b, v in blk.items()), f"Gold file: `{jo.relative_to(CWM3) if jo.is_relative_to(CWM3) else jo}` ({len(gold)} gold events: {tot['n_gold_engaged']} engaged, {tot['n_gold_manual']} manual).", "", | |
| "### Block W — engaged TEST 10 s windows without a v3 hand event (1 Hz frames): 'did a hand return to the wheel and stay >= 1 s?'", "", | |
| "| windows | yes (v3 missed a return) | no | unsure | unlabelled | miss share q = yes / (yes + no) [Wilson 95 %] |", "|---|---|---|---|---|---|", | |
| f"| {W['n']} | {W['yes']} | {W['no']} | {W['unsure']} | {W['unlabelled']} | {fmt_ci(W['yes'], W['n_eff'])} |", ""] | |
| if n_starts: | |
| L.append(f"Window population: {n_starts:,} valid start positions x {step} s = {n_starts * step / 3600:.2f} h of engaged TEST time with no v3 hand_return / hand_leave in [t0 - 1 s, t0 + 10 s], " | |
| f"not overlapping any excluded zone (= {R['n_neg_windows']:.0f} disjoint 10 s windows); v3 engaged TEST hand_return candidates (row_ok & EM, first step): {n_ret}.") | |
| if R.get("recall") is not None: | |
| L.append(f"Recall estimate of the v3 candidate set (engaged hand_return) = TP / (TP + FN), TP = precision_event x {n_ret} = {R['tp_est']:.1f}, FN = q x {R['n_neg_windows']:.0f} = {R['fn_est']:.1f}: " | |
| f"**{R['recall']:.3f}** (rough interval [{R['recall_lo']:.3f}, {R['recall_hi']:.3f}] from the Wilson bounds crosswise; assumes <= 1 real return per 10 s negative window).") | |
| else: | |
| L.append("Recall estimate: not available yet (needs labelled engaged hand_return items AND labelled windows).") | |
| L.append("") | |
| if n_lab: | |
| L += ["| item | block | regime | event | car | t_event / t0 | label | note |", "|---|---|---|---|---|---|---|---|"] | |
| L += [f"| {r['id']} | {r['block']} | {r['regime']} | {r['event']} | {r['car']} | {r['t_event']:.1f} | {r['label'] or '--'} | {r['note'].replace('|', '/')} |" for r in rows if r["label"]] | |
| text = "\n".join(L) + "\n" | |
| out = Path(a.out); out.parent.mkdir(parents=True, exist_ok=True) | |
| with open(out, "a", encoding="utf-8") as f: f.write(text) | |
| print("\n".join(L[:12])); print(f"\nappended {out}; wrote {jo} ({len(gold)} gold events; labelled {n_lab}/{n_all})") | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |
Xet Storage Details
- Size:
- 13.9 kB
- Xet hash:
- 282d72357154fdd9bf96a6ac906208097ebd99f20cb00e81b8f4487ce48a7062
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.