Buckets:
| """cwm3/aide/features/build_aide_kin11.py — the 11 BATON driver-kinematics channels from the AIDE in-cabin RTMW keypoints (claim C1, E10 (a)). | |
| Source: /home/USER/Drive/explore/opendriver/out/aide/motion/<clip>.npz (opendriver adapter_aide.py: RTMW-x whole-body 133-slot | |
| COCO-WholeBody keypoints of the AIDE `incar` view, ONE person per frame = the detection with the largest body-17 bounding extent, x / 1920, | |
| y / 1080, score; already re-gridded to 10 Hz by nearest frame: t = 0.0, 0.1, ... (3 s clips -> 30 steps; 13 longer clips -> more), t_actual = | |
| the sampled frame's time). All 2,898 protocol clips (aide/results/cache/aide_kp.npz clip_id / split) are present. | |
| Channel definitions are IMPORTED from data/build_kin_causal.py (CHANNELS, raw_from_kpts, clip_series, hysteresis, CONF_TH, WHEEL_TH, XSCALE, | |
| REFQ) so order, units and thresholds are the BATON pack contract by construction: | |
| head_cx, head_cy, Lwri_x, Lwri_y, Rwri_x, Rwri_y, head_turn, head_turn_dev, Lhand_rest_dist, Rhand_rest_dist, min_hand_rest_dist | |
| units "crop-normalised-equivalent": x = x_norm * 1.6, y = y_norm; distances / angles dimensionless (build_kin_causal.UNITS). | |
| Body indices 0-16 of COCO-WholeBody-133 (RTMW) and HALPE-136 are the same COCO-17 joints, so the BATON code path applies unchanged. | |
| DEVIATIONS from the BATON build (documented, every one is forced by the AIDE data): | |
| 1. Rest / head-turn references: BATON uses expanding per-route medians over MOVING frames <= t seeded from the device's earlier routes. | |
| AIDE has no route history and no vehicle speed -> REF_q(k) = median{ q(j) : j <= k, conf > 0.3 } over the clip's OWN frames only | |
| (causal, per clip), defined once >= N_MIN_AIDE = 5 valid samples (BATON N_MIN = 25 would leave a 3 s clip almost entirely undefined). | |
| Consequence: `head_turn_dev` and the `*_rest_dist` channels measure deviation from the clip's own running median, not from a long-run | |
| hands-on-wheel pose. The absolute channels (head_cx/cy, wrist x/y, head_turn) are unaffected. | |
| 2. Camera: AIDE incar (1920x1080, driver-facing dash camera) vs the BATON comma device camera; absolute x/y offsets differ, as they | |
| already differ per route inside BATON (docs/k4_handoff.md) -> standardise per channel on the consumer side (the pretraining uses | |
| AIDE-train statistics, saved here as kmu / ksd). | |
| 3. Grid: nearest-frame 10 Hz samples of a 15 fps video (no interpolation); BATON routes are native 10 Hz. | |
| 4. States hands_on (min_hand_rest_dist < 0.10) and head_turned (hysteresis on |head_turn_dev| with the BATON TRAIN thresholds) are | |
| computed for description only; with deviation 1 they are not the BATON events. | |
| Validity per step: frame present AND score >= 0.3 for nose, both shoulders, both wrists AND finite head_turn AND reference defined | |
| (= build_kin_causal's V rule) -> every valid step has all 11 channels finite. | |
| Output aide/results/cache/aide_kin11.npz (N = 2,898 clips, T = max steps, NaN-padded): | |
| K f32[N,T,11], valid bool[N,T], frame_ok bool[N,T], n_steps i32[N], t_grid f32[T] (0.1 s), ref f32[N,T,5] (REFQ order), hands_on f32[N,T], | |
| head_turned f32[N,T], clip_id, split, channels, units, ref_quantities, kmu / ksd f32[11] (AIDE TRAIN split, valid steps, float64), | |
| source, deviations (json string); + aide_kin11_summary.json and a run json. No BATON file is read; no DMS field exists in AIDE. | |
| python aide/features/build_aide_kin11.py [--motion-dir DIR] [--limit K] | |
| """ | |
| from __future__ import annotations | |
| import os | |
| for _v in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS", "NUMEXPR_NUM_THREADS"): | |
| os.environ.setdefault(_v, "4") | |
| import argparse # noqa: E402 | |
| import bisect # noqa: E402 | |
| import json # noqa: E402 | |
| import sys # noqa: E402 | |
| import time # noqa: E402 | |
| from pathlib import Path # noqa: E402 | |
| import numpy as np # noqa: E402 | |
| CWM3 = Path(__file__).resolve().parents[2] | |
| for p in (CWM3, CWM3 / "data", CWM3 / "common"): | |
| sys.path.insert(0, str(p)) | |
| import build_kin_causal as BK # noqa: E402 (channel definitions: the BATON pack contract) | |
| from runmeta import write_run_json # noqa: E402 | |
| MOTION_DIR = Path("/home/USER/Drive/explore/opendriver/out/aide/motion") | |
| CACHE_KP = CWM3 / "aide" / "results" / "cache" / "aide_kp.npz" | |
| OUT = CWM3 / "aide" / "results" / "cache" / "aide_kin11.npz" | |
| SUMMARY = CWM3 / "aide" / "results" / "cache" / "aide_kin11_summary.json" | |
| N_MIN_AIDE = 5 | |
| GRID_DT = 0.1 | |
| DEVIATIONS = { | |
| "references": f"per-clip causal expanding median over the clip's own valid frames (j <= k), N_MIN={N_MIN_AIDE} (BATON: per-route over moving frames, " | |
| "device pool from earlier routes, N_MIN=25); head_turn_dev / *_rest_dist are deviations from the clip's running median", | |
| "camera": "AIDE incar 1920x1080 driver-facing camera; absolute x/y offsets differ from the BATON device camera (standardise per channel)", | |
| "grid": "nearest-frame 10 Hz resampling of 15 fps video (opendriver adapter), no interpolation", | |
| "states": "hands_on / head_turned computed with the BATON thresholds for description only (not BATON events)", | |
| "moving_gate": "none (AIDE has no vehicle speed)", | |
| } | |
| def expanding_median_clip(vals, n_min=N_MIN_AIDE): | |
| """vals[T] with NaN -> causal expanding median ref[T] (NaN while fewer than n_min valid samples so far).""" | |
| arr = [] | |
| out = np.full(len(vals), np.nan, np.float32) | |
| last = np.nan | |
| for k, v in enumerate(vals): | |
| if np.isfinite(v): | |
| bisect.insort(arr, float(v)) | |
| n = len(arr) | |
| if n >= n_min: | |
| last = arr[n // 2] if n % 2 else 0.5 * (arr[n // 2 - 1] + arr[n // 2]) | |
| out[k] = last | |
| return out | |
| def clip_channels(kpts, head_on, head_off): | |
| """kpts f32[T,133,3] (x_norm, y_norm, score) -> dict(K [T,11], valid [T], frame_ok [T], ref [T,5], hands_on [T], head_turned [T]).""" | |
| frame_ok = np.isfinite(kpts[:, 0, 0]) & np.isfinite(kpts[:, 0, 2]) | |
| raw = BK.raw_from_kpts(kpts.astype(np.float16), frame_ok) # BATON code path: x * XSCALE, conf mask, head, head_turn, V rule | |
| q = raw["q"] | |
| ref = np.stack([expanding_median_clip(q[:, j]) for j in range(len(BK.REFQ))], 1).astype(np.float32) | |
| G = BK.clip_series(raw, ref) | |
| V = raw["V"] & np.isfinite(G).all(1) # BATON checks {0,6,7,10}; here BOTH wrist refs must be defined (they can appear at different steps) | |
| G = G.astype(np.float16).astype(np.float32) # stored precision of the BATON packs (states from stored values) | |
| hands_on = np.where(np.isnan(G[:, 10]), np.nan, (G[:, 10] < BK.WHEEL_TH).astype(np.float32)) | |
| head_turned = BK.hysteresis(np.abs(G[:, 7]), head_on, head_off) | |
| G[~V] = np.nan # the pack contract: a valid step has all 11 channels; invalid -> NaN | |
| return dict(K=G, valid=V, frame_ok=frame_ok, ref=ref, hands_on=hands_on, head_turned=head_turned) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--motion-dir", default=str(MOTION_DIR)) | |
| ap.add_argument("--limit", type=int, default=0) | |
| ap.add_argument("--out", default=str(OUT)) | |
| a = ap.parse_args() | |
| t0 = time.time() | |
| mdir = Path(a.motion_dir) | |
| kp = np.load(CACHE_KP, allow_pickle=True) | |
| clip_ids, splits = [str(c) for c in kp["clip_id"]], [str(s) for s in kp["split"]] | |
| if a.limit: | |
| clip_ids, splits = clip_ids[:a.limit], splits[:a.limit] | |
| old = json.load(open(BK.OLD_STATS)) | |
| head_on, head_off = float(old["head_turn_on"]), float(old["head_turn_off"]) | |
| missing = [c for c in clip_ids if not (mdir / f"{c}.npz").exists()] | |
| assert not missing, f"{len(missing)} clips without RTMW motion npz, e.g. {missing[:5]}" | |
| per = [] | |
| for i, cid in enumerate(clip_ids): | |
| z = np.load(mdir / f"{cid}.npz") | |
| kpts = z["kpts"].astype(np.float32) | |
| assert kpts.shape[1:] == (133, 3), kpts.shape | |
| assert np.allclose(np.diff(z["t"]), GRID_DT, atol=1e-4), "motion grid is not 10 Hz" | |
| per.append(clip_channels(kpts, head_on, head_off)) | |
| if (i + 1) % 500 == 0: | |
| print(f" {i + 1}/{len(clip_ids)} clips ({time.time() - t0:.0f}s)", flush=True) | |
| N, T = len(per), max(len(p["valid"]) for p in per) | |
| D = len(BK.CHANNELS) | |
| K = np.full((N, T, D), np.nan, np.float32) | |
| valid = np.zeros((N, T), bool) | |
| frame_ok = np.zeros((N, T), bool) | |
| ref = np.full((N, T, len(BK.REFQ)), np.nan, np.float32) | |
| hands_on = np.full((N, T), np.nan, np.float32) | |
| head_turned = np.full((N, T), np.nan, np.float32) | |
| n_steps = np.zeros(N, np.int32) | |
| for i, p in enumerate(per): | |
| n = len(p["valid"]) | |
| n_steps[i] = n | |
| K[i, :n], valid[i, :n], frame_ok[i, :n], ref[i, :n] = p["K"], p["valid"], p["frame_ok"], p["ref"] | |
| hands_on[i, :n], head_turned[i, :n] = p["hands_on"], p["head_turned"] | |
| split_arr = np.array(splits) | |
| tr = (split_arr == "train")[:, None] & valid | |
| Ktr = K[tr].astype(np.float64) # [n_valid_train_steps, 11], all finite by the V rule | |
| assert np.isfinite(Ktr).all() | |
| kmu = Ktr.mean(0) | |
| ksd = Ktr.std(0) + 1e-6 | |
| t_grid = (np.arange(T) * GRID_DT).astype(np.float32) | |
| np.savez_compressed(a.out, K=K, valid=valid, frame_ok=frame_ok, n_steps=n_steps, t_grid=t_grid, ref=ref, hands_on=hands_on, head_turned=head_turned, | |
| clip_id=np.array(clip_ids), split=split_arr, channels=np.array(BK.CHANNELS), units=np.array(BK.UNITS), ref_quantities=np.array(BK.REFQ), | |
| kmu=kmu.astype(np.float32), ksd=ksd.astype(np.float32), stats_split=np.array("train"), n_min_ref=np.array(N_MIN_AIDE), | |
| head_turn_on=np.array(head_on), head_turn_off=np.array(head_off), source=np.array(f"rtmw133 incar 10 Hz: {mdir}"), | |
| deviations=np.array(json.dumps(DEVIATIONS))) | |
| inside = np.arange(T)[None, :] < n_steps[:, None] # [N,T] real (non-padded) steps | |
| summary = { | |
| "n_clips": int(N), "T_max": int(T), "n_steps_hist": {str(k): int(v) for k, v in zip(*np.unique(n_steps, return_counts=True))}, | |
| "frame_ok_frac": float(frame_ok[inside].mean()), | |
| "valid_frac_of_steps": float(valid.sum() / n_steps.sum()), "valid_frac_first_second": float(valid[:, :10].mean()), | |
| "valid_frac_after_1s": float(valid[:, 10:].sum() / np.maximum(n_steps - 10, 0).sum()), | |
| "clips_with_no_valid_step": int((~valid.any(1)).sum()), | |
| "per_split_valid_frac": {s: float(valid[split_arr == s].sum() / n_steps[split_arr == s].sum()) for s in ("train", "val", "test")}, | |
| "hands_on_prev_valid": float(np.nanmean(hands_on[valid])), "head_turned_prev_valid": float(np.nanmean(head_turned[valid])), | |
| "channel_stats_train": {c: {"mean": float(kmu[j]), "sd": float(ksd[j]), "p01": float(np.percentile(Ktr[:, j], 1)), "p99": float(np.percentile(Ktr[:, j], 99))} | |
| for j, c in enumerate(BK.CHANNELS)}, | |
| "head_turn_thresholds": {"on": head_on, "off": head_off, "source": str(BK.OLD_STATS)}, "n_min_ref": N_MIN_AIDE, "deviations": DEVIATIONS, | |
| "units": BK.UNITS, "channels": BK.CHANNELS, "wall_s": round(time.time() - t0, 1), | |
| } | |
| Path(SUMMARY).write_text(json.dumps(summary, indent=1)) | |
| write_run_json(Path(a.out).parent, "build_aide_kin11", [Path(__file__), CWM3 / "data" / "build_kin_causal.py"], | |
| [CACHE_KP, Path(BK.OLD_STATS)] + [mdir / f"{c}.npz" for c in clip_ids[:50]], seed=0, t_start=t0, input_listing=False, | |
| extra={"n_clips": N, "T_max": T, "motion_dir": str(mdir), "out": str(a.out), "summary": {k: v for k, v in summary.items() if k not in ("channel_stats_train",)}, | |
| "dms_fields_used": [], "note_inputs": "input listing truncated to 50 motion npz (2,898 files); all are enumerated in aide_kp.npz clip_id"}) | |
| print(json.dumps({k: v for k, v in summary.items() if k not in ("channel_stats_train", "deviations")}, indent=1)) | |
| print("AIDE_KIN11_DONE", flush=True) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 12.1 kB
- Xet hash:
- c9ec6fb53902f2b8e3d5aac1ac40fedc90997033ba7892460e80b2e3e9a3116e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.