Buckets:
| """cwm3/data/remote_baton.py — canonical BATON lives on the backup host; the local copy is partial (2026-09-10). | |
| Real remote layout (discovered 2026-09-10 with `ls` two levels + `find`; the first probe used -maxdepth 3 and found 0): | |
| /data/backup/BATON/<CAR_MODEL>/<dongle_16hex>/<NNNNNNNN--routehash>/route_<a>-<b>/{dcamera.mp4,qcamera.mp4,*.csv} | |
| i.e. a LEAF is a directory 4 levels below the root and the files sit at depth 5. Some <NNNNNNNN--routehash> dirs | |
| additionally hold raw openpilot segment files `<seg>--dcamera.hevc` (depth 4 files); those are NOT leaves. | |
| Local convention is identical (cwm2/private/clip_map_full.json `dcam` = /home/USER/BATON/<same 4 levels>/dcamera.mp4). | |
| Writes results/baton_leaves.json: every remote leaf (relative path, dcamera/qcamera sizes), local match status | |
| (identical size / size differs / missing), leaves used by the 26,578-clip manifest, plus a run_<name>_<utc>.json. | |
| Uses the SSH ControlMaster of cwm2/private/ssh_master.sh (no password in code). Never writes to the remote host.""" | |
| import json, os, subprocess, sys, time | |
| from collections import Counter | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(ROOT / "common")) | |
| from runmeta import write_run_json | |
| REMOTE = "USER@REMOTE_HOST"; RROOT = "/data/backup/BATON"; LROOT = Path("/home/USER/BATON") | |
| CTL = "/tmp/baton_bak_ctl"; SSH = ["ssh", "-o", f"ControlPath={CTL}", "-o", "BatchMode=yes", "-o", "ConnectTimeout=20"] | |
| OUT = ROOT / "results"; OUT.mkdir(exist_ok=True) | |
| CLIP_MAP = Path("/home/USER/cwm2/private/clip_map_full.json") | |
| LEAF_DEPTH = 4 # dirs below the root; files are at depth LEAF_DEPTH + 1 | |
| def ensure_master(): | |
| r = subprocess.run(SSH + ["-O", "check", REMOTE], capture_output=True, text=True) | |
| if r.returncode != 0: | |
| subprocess.run(["bash", "/home/USER/cwm2/private/ssh_master.sh"], capture_output=True, check=False) | |
| time.sleep(2) | |
| r = subprocess.run(SSH + ["-O", "check", REMOTE], capture_output=True, text=True) | |
| return r.returncode == 0 | |
| def ssh_out(cmd, timeout=900): | |
| r = subprocess.run(SSH + [REMOTE, cmd], capture_output=True, text=True, timeout=timeout) | |
| if r.returncode != 0: | |
| raise RuntimeError(f"ssh rc={r.returncode}: {r.stderr[:300]}") | |
| return r.stdout | |
| def remote_listing(): | |
| d = LEAF_DEPTH + 1 | |
| cmd = (f"cd {RROOT} && find . -mindepth {d} -maxdepth {d} -type f \\( -name dcamera.mp4 -o -name qcamera.mp4 \\) " | |
| "-printf '%h\\t%f\\t%s\\n' | sed 's#^\\./##'") | |
| leaves = {} | |
| for line in ssh_out(cmd).splitlines(): | |
| leaf, fn, size = line.split("\t") | |
| leaves.setdefault(leaf, {})[fn.split(".")[0]] = int(size) | |
| # raw segment dirs (not leaves) — reported for the layout record only | |
| seg = ssh_out(f"cd {RROOT} && find . -mindepth {LEAF_DEPTH} -maxdepth {LEAF_DEPTH} -type f -name '*--dcamera.hevc' -printf '%h\\n' | sort -u | wc -l").strip() | |
| depth_hist = Counter(l.count("/") + 1 for l in leaves) | |
| return leaves, int(seg), dict(depth_hist) | |
| def local_listing(): | |
| d = LEAF_DEPTH + 1 | |
| local = {} | |
| for fn in ("dcamera.mp4", "qcamera.mp4"): | |
| # glob at exactly depth d (CAR/dongle/route/route_a-b/file) | |
| for p in LROOT.glob("/".join(["*"] * LEAF_DEPTH) + f"/{fn}"): | |
| local.setdefault(str(p.parent.relative_to(LROOT)), {})[fn.split(".")[0]] = p.stat().st_size | |
| return local | |
| def main(): | |
| t_start = time.time() | |
| if not ensure_master(): | |
| print("no SSH master; run cwm2/private/ssh_master.sh"); sys.exit(2) | |
| remote, n_segdirs, depth_hist = remote_listing() | |
| local = local_listing() | |
| cm = json.load(open(CLIP_MAP)) if CLIP_MAP.exists() else {} | |
| used = Counter(str(Path(v["dcam"]).parent.relative_to(LROOT)) for v in cm.values()) | |
| rows = {} | |
| st = Counter() | |
| for leaf, sz in sorted(remote.items()): | |
| loc = local.get(leaf) | |
| if loc is None or "dcamera" not in loc: | |
| status = "missing_local" | |
| elif loc.get("dcamera") == sz.get("dcamera"): | |
| status = "local_identical_size" | |
| else: | |
| status = "local_size_differs" | |
| st[status] += 1 | |
| rows[leaf] = dict(dcamera_bytes=sz.get("dcamera"), qcamera_bytes=sz.get("qcamera"), | |
| local_dcamera_bytes=None if loc is None else loc.get("dcamera"), | |
| local_qcamera_bytes=None if loc is None else loc.get("qcamera"), | |
| local_status=status, n_clips_manifest=used.get(leaf, 0)) | |
| only_local = sorted(set(local) - set(remote)) | |
| used_status = Counter(rows[l]["local_status"] if l in rows else "not_on_remote" for l in used) | |
| rec = {"utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "remote_root": f"{REMOTE}:{RROOT}", "local_root": str(LROOT), | |
| "layout": {"pattern": "<CAR_MODEL>/<dongle_16hex>/<NNNNNNNN--routehash>/route_<a>-<b>/{dcamera.mp4,qcamera.mp4,*.csv}", | |
| "leaf_depth_dirs_below_root": LEAF_DEPTH, "file_depth": LEAF_DEPTH + 1, | |
| "leaf_depth_histogram": depth_hist, "n_remote_raw_segment_dirs_hevc_not_leaves": n_segdirs, | |
| "note": "first probe (2026-09-10 08:23Z) used find -mindepth 2 -maxdepth 3 -> 0 hits; leaves are one level deeper"}, | |
| "n_remote": len(rows), "n_remote_with_dcamera": sum(1 for r in rows.values() if r["dcamera_bytes"] is not None), | |
| "n_remote_with_qcamera": sum(1 for r in rows.values() if r["qcamera_bytes"] is not None), | |
| "remote_dcamera_bytes_total": int(sum(r["dcamera_bytes"] or 0 for r in rows.values())), | |
| "n_local": len(local), "n_local_identical_size": st["local_identical_size"], "n_local_size_differs": st["local_size_differs"], | |
| "n_missing_local": st["missing_local"], "n_only_local": len(only_local), | |
| "manifest": {"n_clips": len(cm), "n_leaves_used": len(used), "leaf_status_hist": dict(used_status), | |
| "n_clips_by_leaf_status": {k: int(sum(n for l, n in used.items() if (rows[l]["local_status"] if l in rows else "not_on_remote") == k)) for k in used_status}}, | |
| "leaves": rows, "only_local": only_local} | |
| (OUT / "baton_leaves.json").write_text(json.dumps(rec, indent=1)) | |
| summ = {k: rec[k] for k in ("n_remote", "n_remote_with_dcamera", "n_remote_with_qcamera", "n_local", "n_local_identical_size", "n_local_size_differs", "n_missing_local", "n_only_local")} | |
| summ["manifest"] = rec["manifest"]; summ["layout"] = rec["layout"]["pattern"]; summ["remote_dcamera_TB"] = round(rec["remote_dcamera_bytes_total"] / 1e12, 3) | |
| write_run_json(OUT, "remote_baton", [__file__], [CLIP_MAP] if CLIP_MAP.exists() else [], None, t_start, extra={"summary": summ}) | |
| print(json.dumps(summ, indent=1)) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 6.76 kB
- Xet hash:
- a07315ce8d45899f2e27f89850765353151853933ba0f7f81707c0a8bdaba167
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.