"""Frozen availability table: which modality of which recording is usable. ``scripts/build_availability.py`` loads every stream of every recording once with the strict loaders and records, per recording and modality, one status: ``ok`` loads and passes the quality rules ``absent`` no file, and alignment_metadata.json does not list it ``missing_file`` alignment_metadata.json lists the stream (``aligned_modalities``) but the file is absent (ERROR) ``ambiguous`` skeleton TSV present both in raw/ and at the root (ERROR) ``unavailable:`` the file does not provide the modality (e.g. v5 IMU, 72 columns -> ``unavailable:imu_columns_72``) ``quality:`` fails a quality rule (``extreme_values``, ``mostly_zero``) The table is committed (``manifests/availability_v1.1.json``). Task manifests are derived from it, so which recordings a task uses is fixed before any training code runs, and a loader that later fails on an expected recording raises instead of shrinking the dataset. """ from __future__ import annotations import hashlib import json import os from concurrent.futures import ProcessPoolExecutor from pathlib import Path from typing import Dict, Iterable, List, Optional from .. import DATA_TAG from ..errors import (AmbiguousFileError, DataQualityError, MissingFileError, ModalityUnavailableError) from ..paths import DataRoot, split_rid from . import modalities as M from .annotations import load_segments_file from .io import load_alignment_metadata, load_stream, modality_from_stream from .recordings import Recording, load_recordings, sort_rids SCAN_MODALITIES = ( "emg", "imu", "imu_wrist", "imu_finger", "eyetrack", "eyetrack_gaze", "eyetrack_ocular", "pressure", "mocap", "mocap_markers", "myo_quat", ) ERROR_STATUSES = ("missing_file", "ambiguous") SCHEMA = "pulse-availability/1" def _rel(dr: DataRoot, p) -> str: return Path(os.path.relpath(str(p), str(dr.root))).as_posix() def scan_recording(dr: DataRoot, rec: Recording) -> dict: """Scan one recording (all streams, all modalities).""" rid = rec.rid streams_info: Dict[str, dict] = {} loaded = {} try: meta = load_alignment_metadata(dr, rid) meta_error = None except MissingFileError: # v4s1 and v4s4 are listed in recordings.csv with no aligned modality and # ship no files at all; any other recording must have its metadata. meta = {} meta_error = None if not rec.aligned_modalities else "alignment_metadata.json missing" aligned = set(meta.get("aligned_modalities") or []) meta_n = meta.get("n_samples") for sname in M.STREAM_FILES: listed = M.STREAM_ALIGNMENT_NAME[sname] in aligned csv_name = M.STREAM_RECORDINGS_CSV_NAME[sname] info = {"listed_in_alignment_metadata": listed, "listed_in_recordings_csv": rec.lists(csv_name) if csv_name else None} try: st = load_stream(dr, rid, sname) except AmbiguousFileError as e: info.update(status="ambiguous", error=str(e).replace(str(dr.root), "")) except MissingFileError: info.update(status="missing_file" if listed else "absent") else: loaded[sname] = st info.update(status="present", file=_rel(dr, st.path), n_rows=st.n_rows, n_columns=len(st.columns)) if sname == "mocap_skeleton": info["location"] = "raw" if "/raw/" in info["file"] else "recording_root" if meta_n: info["rows_minus_metadata_samples"] = st.n_rows - int(meta_n) streams_info[sname] = info mods: Dict[str, dict] = {} for mod in SCAN_MODALITIES: sname = M.base_stream(mod) sinfo = streams_info[sname] if sinfo["status"] != "present": mods[mod] = {"status": sinfo["status"]} continue try: ma = modality_from_stream(loaded[sname], mod) except ModalityUnavailableError as e: mods[mod] = {"status": f"unavailable:{e.reason}"} except DataQualityError as e: mods[mod] = {"status": f"quality:{e.reason}"} else: mods[mod] = {"status": "ok", "n_frames": int(ma.data.shape[0]), "n_features": int(ma.data.shape[1])} ann: dict = {"exists": False} _, scen = split_rid(rid) if scen != "s9": p = dr.annotation_path(rid, must_exist=False) if p is not None: segs = load_segments_file(p, rid) ann = {"exists": True, "file": _rel(dr, p), "n_segments": len(segs)} return { "volunteer": rec.volunteer, "scenario": rec.scenario, "n_samples_100hz": rec.n_samples_100hz, "alignment_n_samples": meta_n, "has_dense_annotations": rec.has_dense_annotations, "no_data": not rec.aligned_modalities and not loaded, "metadata_error": meta_error, "streams": streams_info, "modalities": mods, "annotation": ann, } def _scan_one(args): root, rec = args return rec.rid, scan_recording(DataRoot(root=Path(root)), rec) def build_availability(dr: DataRoot, rids: Optional[Iterable[str]] = None, workers: int = 1) -> dict: recs = load_recordings(dr) todo = sort_rids(rids if rids is not None else recs.keys()) jobs = [(str(dr.root), recs[r]) for r in todo] if workers > 1: with ProcessPoolExecutor(max_workers=workers) as ex: results = dict(ex.map(_scan_one, jobs, chunksize=1)) else: results = dict(map(_scan_one, jobs)) table = {"schema": SCHEMA, "data_tag": DATA_TAG, "scan_modalities": list(SCAN_MODALITIES), "n_recordings": len(results), "recordings": {r: results[r] for r in sort_rids(results)}} table["summary"] = summarize(table) return table def summarize(table: dict) -> dict: counts: Dict[str, Dict[str, int]] = {} errors: List[str] = [] warnings: List[str] = [] notes: List[str] = [] rows_mismatch: List[str] = [] tsv_loc: Dict[str, int] = {} no_data: List[str] = [] for rid, r in table["recordings"].items(): if r.get("metadata_error"): errors.append(f"{rid}: {r['metadata_error']}") if r.get("no_data"): no_data.append(rid) for mod, m in r["modalities"].items(): counts.setdefault(mod, {}).setdefault(m["status"], 0) counts[mod][m["status"]] += 1 for sname, s in r["streams"].items(): if s["status"] in ERROR_STATUSES: errors.append(f"{rid}: {sname} {s['status']}") if s["status"] == "present" and not s.get("listed_in_alignment_metadata"): warnings.append(f"{rid}: {sname} present but not in alignment_metadata.json") if s["status"] == "present" and s.get("listed_in_recordings_csv") is False: notes.append(f"{rid}: {sname} present but not listed in recordings.csv") if s["status"] == "present" and s.get("rows_minus_metadata_samples"): rows_mismatch.append(f"{rid}: {sname} {s['rows_minus_metadata_samples']:+d} rows") if sname == "mocap_skeleton" and s["status"] == "present": tsv_loc[s["location"]] = tsv_loc.get(s["location"], 0) + 1 return {"modality_status_counts": {k: dict(sorted(v.items())) for k, v in sorted(counts.items())}, "mocap_tsv_locations": tsv_loc, "errors": errors, "warnings": warnings, "notes": notes, "row_count_differences": rows_mismatch, "recordings_without_data": no_data} def file_md5(path) -> str: h = hashlib.md5() with open(path, "rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) return h.hexdigest() class Availability: """Read-only view of a frozen availability table.""" def __init__(self, table: dict, path: Optional[str] = None, md5: Optional[str] = None): if table.get("schema") != SCHEMA: raise ValueError(f"not an availability table (schema={table.get('schema')!r})") self.table = table self.path = path self.md5 = md5 self.data_tag = table.get("data_tag") self._recs = table["recordings"] @classmethod def load(cls, path) -> "Availability": with open(path, encoding="utf-8") as f: table = json.load(f) return cls(table, str(path), file_md5(path)) def rids(self, volunteers: Optional[Iterable[str]] = None, scenarios: Optional[Iterable[str]] = None) -> List[str]: vs = set(volunteers) if volunteers is not None else None ss = set(scenarios) if scenarios is not None else None return sort_rids(r for r, d in self._recs.items() if (vs is None or d["volunteer"] in vs) and (ss is None or d["scenario"] in ss)) def status(self, rid: str, modality: str) -> str: try: return self._recs[rid]["modalities"][modality]["status"] except KeyError: raise KeyError(f"{rid}/{modality} not in availability table") from None def ok(self, rid: str, modality: str) -> bool: return self.status(rid, modality) == "ok" def has_annotation(self, rid: str) -> bool: return bool(self._recs[rid]["annotation"].get("exists")) def errors(self) -> List[str]: return list(self.table.get("summary", {}).get("errors", [])) def default_availability_path(repo_root: Optional[Path] = None) -> Path: base = repo_root or Path(__file__).resolve().parents[2] return base / "manifests" / f"availability_{DATA_TAG}.json"