Buckets:
| """cwm3/data/device_identity_audit.py — dongle <-> VIN <-> car-model audit of the canonical BATON corpus (WP5), for the §4.1 wording | |
| 'device-disjoint and vehicle-disjoint for all but N devices; driver identity is not recorded'. | |
| Sources: metadata.json of every canonical leaf (results/baton_leaves.json): local copies under /home/USER/BATON/<leaf>/ | |
| (1,349 leaves after the WP5 CSV fetches) and, for leaves with no local directory, the remote file read once over the SSH ControlMaster | |
| (base64 over one batched ssh per 120 leaves; read-only; cached in results/tables/metadata_remote_cache.json). Fields: dongle_id, | |
| car_model (= the car directory), carParams.carVin / carFingerprint / carName, initData.version / deviceType / gitBranch. | |
| VIN placeholder rule: empty, None, or all-zero / shorter than 11 characters -> 'unknown' (excluded from every VIN count). | |
| Splits: benchmark dongle -> split from s12_wm/results/targets_{regime}_{split}.npz (train / val / test; consistency across the six files | |
| asserted); holdout dongles from results/holdout/draw_v1.json; everything else 'unused'. | |
| Counts written (results/tables/device_identity.md + .json + run json): | |
| devices, leaves, leaves with metadata; devices with exactly one VIN / several VINs / no known VIN; VINs seen on several devices (and | |
| whether those devices fall in different benchmark splits or benchmark vs holdout); car models per device; VIN overlap train vs val/test, | |
| benchmark vs holdout; the set of metadata keys (no driver / user / person field exists). | |
| """ | |
| import base64, json, subprocess, sys, time | |
| from collections import Counter, defaultdict | |
| from pathlib import Path | |
| import numpy as np | |
| ROOT = Path(__file__).resolve().parents[1]; EXPLORE = ROOT.parent; sys.path.insert(0, str(ROOT / "common")) | |
| from runmeta import write_run_json # noqa: E402 | |
| LROOT = Path("/home/USER/BATON"); S12R = EXPLORE / "s12_wm" / "results"; TABLES = ROOT / "results" / "tables" | |
| LEAVES = ROOT / "results" / "baton_leaves.json"; DRAW = ROOT / "results" / "holdout" / "draw_v1.json"; CACHE = TABLES / "metadata_remote_cache.json" | |
| REMOTE = "USER@REMOTE_HOST"; RROOT = "/data/backup/BATON"; SSH = ["ssh", "-S", "/tmp/baton_bak_ctl", "-o", "BatchMode=yes", "-o", "ConnectTimeout=20"] | |
| REGIMES, SPLITS = ("manual", "engaged"), ("train", "val", "test") | |
| DRIVER_WORDS = ("driver", "user", "person", "name", "email", "account", "owner", "operator") | |
| VEHICLE_NAME_KEYS = {"carParams.carName", "car_model", "carParams.carFingerprint"} # vehicle make / model fields, not a person | |
| def vin_of(meta): | |
| v = (meta.get("carParams") or {}).get("carVin") | |
| if not isinstance(v, str): return None | |
| v = v.strip() | |
| if len(v) < 11 or set(v) <= {"0"}: return None | |
| return v | |
| def all_keys(obj, prefix="", out=None): | |
| out = set() if out is None else out | |
| if isinstance(obj, dict): | |
| for k, v in obj.items(): out.add(prefix + k); all_keys(v, prefix + k + ".", out) | |
| elif isinstance(obj, list) and obj: all_keys(obj[0], prefix + "[]" + ".", out) | |
| return out | |
| def remote_metadata(leaves): | |
| cache = json.load(open(CACHE)) if CACHE.exists() else {} | |
| todo = [l for l in leaves if l not in cache] | |
| if todo: | |
| assert subprocess.run(SSH + ["-O", "check", REMOTE], capture_output=True).returncode == 0, "ssh ControlMaster /tmp/baton_bak_ctl not alive (bash cwm2/private/ssh_master.sh)" | |
| for i in range(0, len(todo), 120): | |
| chunk = todo[i:i + 120] | |
| cmd = "; ".join(f"printf '%s\\t' '{l}'; base64 -w0 '{RROOT}/{l}/metadata.json' 2>/dev/null; printf '\\n'" for l in chunk) | |
| r = subprocess.run(SSH + [REMOTE, cmd], capture_output=True, text=True, timeout=1800) | |
| for line in r.stdout.splitlines(): | |
| p = line.split("\t", 1) | |
| if len(p) != 2: continue | |
| try: cache[p[0]] = json.loads(base64.b64decode(p[1]).decode()) if p[1] else None | |
| except Exception: cache[p[0]] = None | |
| print(f" remote metadata {min(i + 120, len(todo))}/{len(todo)}", flush=True) | |
| TABLES.mkdir(parents=True, exist_ok=True); CACHE.write_text(json.dumps(cache)) | |
| return cache | |
| def main(): | |
| t0 = time.time(); TABLES.mkdir(parents=True, exist_ok=True); lv = json.load(open(LEAVES))["leaves"]; leaves = sorted(lv) | |
| # splits per dongle | |
| dsplit = {}; incons = [] | |
| for r in REGIMES: | |
| for s in SPLITS: | |
| for d in np.unique(np.load(S12R / f"targets_{r}_{s}.npz", allow_pickle=True)["dongle"]).tolist(): | |
| if dsplit.get(d, s) != s: incons.append((d, dsplit[d], s)) | |
| dsplit[d] = s | |
| hold = {c["dongle"] for c in json.load(open(DRAW))["leaves"]} if DRAW.exists() else set() | |
| for d in hold: dsplit.setdefault(d, "holdout") | |
| # metadata per leaf | |
| local = {l: (LROOT / l / "metadata.json") for l in leaves if (LROOT / l / "metadata.json").exists()} | |
| metas = {} | |
| for l, p in local.items(): | |
| try: metas[l] = json.load(open(p)) | |
| except Exception: metas[l] = None | |
| remote_needed = [l for l in leaves if l not in local] | |
| rem = remote_metadata(remote_needed) if remote_needed else {} | |
| for l in remote_needed: metas[l] = rem.get(l) | |
| n_meta = sum(m is not None for m in metas.values()) | |
| # per device | |
| dev = defaultdict(lambda: dict(leaves=0, meta=0, vins=set(), cars=set(), fps=set(), versions=set(), devtypes=set(), unknown_vin_leaves=0)) | |
| keys = set(); vin_dev = defaultdict(set) | |
| for l in leaves: | |
| d = l.split("/")[1]; car = l.split("/")[0]; D = dev[d]; D["leaves"] += 1; D["cars"].add(car); m = metas.get(l) | |
| if m is None: continue | |
| D["meta"] += 1; keys |= all_keys(m); v = vin_of(m) | |
| if v: D["vins"].add(v); vin_dev[v].add(d) | |
| else: D["unknown_vin_leaves"] += 1 | |
| cp = m.get("carParams") or {}; ini = m.get("initData") or {} | |
| if cp.get("carFingerprint"): D["fps"].add(cp["carFingerprint"]) | |
| if ini.get("version"): D["versions"].add(str(ini["version"])) | |
| if ini.get("deviceType"): D["devtypes"].add(str(ini["deviceType"])) | |
| if m.get("dongle_id") and m["dongle_id"] != d: D.setdefault("dongle_mismatch", 0); D["dongle_mismatch"] += 1 | |
| devices = sorted(dev); split_of = {d: dsplit.get(d, "unused") for d in devices} | |
| one = [d for d in devices if len(dev[d]["vins"]) == 1]; several = [d for d in devices if len(dev[d]["vins"]) > 1]; none = [d for d in devices if not dev[d]["vins"]] | |
| shared = {v: sorted(ds) for v, ds in vin_dev.items() if len(ds) > 1} | |
| def grp(d): return {"train": "train", "val": "valtest", "test": "valtest", "holdout": "holdout", "unused": "unused"}[split_of[d]] | |
| cross_tv = {v: ds for v, ds in shared.items() if {grp(d) for d in ds} >= {"train", "valtest"}} | |
| cross_bh = {v: ds for v, ds in shared.items() if ({grp(d) for d in ds} & {"train", "valtest"}) and ("holdout" in {grp(d) for d in ds})} | |
| cross_any_bench = {v: ds for v, ds in shared.items() if len({split_of[d] for d in ds if split_of[d] in SPLITS}) > 1} | |
| bench_dev = [d for d in devices if split_of[d] in SPLITS]; hold_dev = [d for d in devices if split_of[d] == "holdout"] | |
| n_bench_not_vehicle_disjoint = len({d for v, ds in cross_any_bench.items() for d in ds}) | |
| multi_car = [d for d in devices if len(dev[d]["cars"]) > 1] | |
| driver_keys = sorted(k for k in keys if any(w in k.lower() for w in DRIVER_WORDS) and k not in VEHICLE_NAME_KEYS) | |
| rec = dict(utc=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), leaves=len(leaves), leaves_with_metadata=n_meta, leaves_metadata_local=len(local), leaves_metadata_remote=len(remote_needed), devices=len(devices), | |
| devices_benchmark=len(bench_dev), devices_holdout=len(hold_dev), devices_unused=sum(split_of[d] == "unused" for d in devices), split_inconsistencies=incons, | |
| devices_one_vin=len(one), devices_several_vins=len(several), devices_no_known_vin=len(none), several_vins=[dict(dongle=d, n_vins=len(dev[d]["vins"]), split=split_of[d], cars=sorted(dev[d]["cars"])) for d in several], | |
| vins_known=len(vin_dev), vins_on_several_devices=len(shared), shared_vins=[dict(vin_sha12=__import__("hashlib").sha256(v.encode()).hexdigest()[:12], devices=ds, splits=[split_of[d] for d in ds], cars=sorted({c for d in ds for c in dev[d]["cars"]})) for v, ds in shared.items()], | |
| vin_overlap_train_vs_valtest=len(cross_tv), vin_overlap_benchmark_vs_holdout=len(cross_bh), vin_overlap_across_benchmark_splits=len(cross_any_bench), benchmark_devices_not_vehicle_disjoint=n_bench_not_vehicle_disjoint, | |
| devices_multiple_car_dirs=len(multi_car), multi_car=[dict(dongle=d, cars=sorted(dev[d]["cars"]), split=split_of[d]) for d in multi_car], dongle_id_mismatch_leaves=sum(D.get("dongle_mismatch", 0) for D in dev.values()), | |
| metadata_keys=sorted(keys), driver_like_keys=driver_keys, device_types=dict(Counter(t for D in dev.values() for t in D["devtypes"])), | |
| per_split_devices={s: sum(split_of[d] == s for d in devices) for s in SPLITS + ("holdout", "unused")}, | |
| per_split_vin_known_devices={s: sum(split_of[d] == s and bool(dev[d]["vins"]) for d in devices) for s in SPLITS + ("holdout", "unused")}) | |
| (TABLES / "device_identity.json").write_text(json.dumps(rec, indent=1)) | |
| N = n_bench_not_vehicle_disjoint | |
| L = ["# Device / vehicle identity audit (BATON canonical corpus)", "", f"Generated {rec['utc']} by cwm3/data/device_identity_audit.py (raw values in results/tables/device_identity.json).", "", | |
| f"Sources: metadata.json of {rec['leaves_with_metadata']} / {rec['leaves']} canonical leaves ({rec['leaves_metadata_local']} local, {rec['leaves_metadata_remote']} read once from the remote); VIN = carParams.carVin with the placeholder rule (empty / all zeros / < 11 chars = unknown). Benchmark split per dongle from the s12 anchor files (inconsistencies: {len(incons)}); holdout dongles from results/holdout/draw_v1.json.", "", | |
| "| quantity | value |", "|---|---|", | |
| f"| devices (dongles) in the corpus | {rec['devices']} (benchmark {rec['devices_benchmark']}: train {rec['per_split_devices']['train']} / val {rec['per_split_devices']['val']} / test {rec['per_split_devices']['test']}; holdout {rec['devices_holdout']}; unused {rec['devices_unused']}) |", | |
| f"| devices with exactly one known VIN | {rec['devices_one_vin']} |", f"| devices with several known VINs | {rec['devices_several_vins']} |", f"| devices with no known VIN (placeholder only) | {rec['devices_no_known_vin']} |", | |
| f"| known VINs | {rec['vins_known']} |", f"| VINs seen on several devices | {rec['vins_on_several_devices']} |", | |
| f"| VIN shared between a TRAIN device and a VAL/TEST device | {rec['vin_overlap_train_vs_valtest']} |", f"| VIN shared across any two benchmark splits | {rec['vin_overlap_across_benchmark_splits']} (devices involved: {N}) |", | |
| f"| VIN shared between a benchmark device and a holdout device | {rec['vin_overlap_benchmark_vs_holdout']} (the draw excluded the one candidate device whose VIN matched a benchmark VIN) |", | |
| f"| devices appearing under more than one car directory | {rec['devices_multiple_car_dirs']} |", f"| metadata.json keys that look like a driver / user identity | {len(driver_keys)} ({', '.join(driver_keys) if driver_keys else 'none'}) |", "", | |
| "## Devices with several VINs", "", "| dongle | VINs | split | car dirs |", "|---|---|---|---|"] + [f"| {r['dongle']} | {r['n_vins']} | {r['split']} | {', '.join(r['cars'])} |" for r in rec["several_vins"]] + \ | |
| ["", "## VINs on several devices (VIN shown as sha256 prefix)", "", "| vin (sha12) | devices | splits | car dirs |", "|---|---|---|---|"] + [f"| {r['vin_sha12']} | {', '.join(r['devices'])} | {', '.join(r['splits'])} | {', '.join(r['cars'])} |" for r in rec["shared_vins"]] + \ | |
| ["", "## Wording for §4.1", "", f"\"Splits are by device (dongle id). Vehicle identity (VIN) is known for {sum(rec['per_split_vin_known_devices'][s] for s in SPLITS)} of the {rec['devices_benchmark']} benchmark devices; the benchmark splits are device-disjoint by construction and vehicle-disjoint for all but {N} device{'s' if N != 1 else ''} ({rec['vin_overlap_across_benchmark_splits']} VIN{'s' if rec['vin_overlap_across_benchmark_splits'] != 1 else ''} appear{'s' if rec['vin_overlap_across_benchmark_splits'] == 1 else ''} on devices of two splits; {rec['vin_overlap_train_vs_valtest']} of them between train and val/test). The holdout devices share no VIN with any benchmark device. Driver identity is not recorded anywhere in the metadata (no driver / user / person field; {rec['devices_several_vins']} devices were installed in more than one vehicle, and {rec['vins_on_several_devices']} vehicles carried more than one device), so a device-disjoint split does not guarantee driver-disjointness.\"", "", | |
| f"Metadata keys seen ({len(rec['metadata_keys'])}): {', '.join(rec['metadata_keys'])}"] | |
| (TABLES / "device_identity.md").write_text("\n".join(L) + "\n") | |
| write_run_json(TABLES, "device_identity_audit", [Path(__file__)], [LEAVES] + ([DRAW] if DRAW.exists() else []) + [S12R / f"targets_{r}_{s}.npz" for r in REGIMES for s in SPLITS], None, t0, extra=dict(summary={k: v for k, v in rec.items() if k not in ("metadata_keys", "several_vins", "shared_vins", "multi_car")}), input_listing=False) | |
| print("DEVICE_IDENTITY_DONE", json.dumps({k: v for k, v in rec.items() if k not in ("metadata_keys", "several_vins", "shared_vins", "multi_car", "driver_like_keys")}, indent=1)) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 13.6 kB
- Xet hash:
- 1b048fbf0c54380f12768a2e7fb850dd2d2566be78f9f6eba940a3607fedf532
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.