PULSE-code / pulse /data /availability.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
9.79 kB
"""Frozen availability table: which modality of which recording is usable.
``scripts/build_availability.py`` loads every stream of every recording once
with the strict loaders and records, per recording and modality, one status:
``ok`` loads and passes the quality rules
``absent`` no file, and alignment_metadata.json does not list it
``missing_file`` alignment_metadata.json lists the stream (``aligned_modalities``)
but the file is absent (ERROR)
``ambiguous`` skeleton TSV present both in raw/ and at the root (ERROR)
``unavailable:<why>`` the file does not provide the modality (e.g. v5 IMU,
72 columns -> ``unavailable:imu_columns_72``)
``quality:<why>`` fails a quality rule (``extreme_values``, ``mostly_zero``)
The table is committed (``manifests/availability_v1.1.json``). Task manifests
are derived from it, so which recordings a task uses is fixed before any
training code runs, and a loader that later fails on an expected recording
raises instead of shrinking the dataset.
"""
from __future__ import annotations
import hashlib
import json
import os
from concurrent.futures import ProcessPoolExecutor
from pathlib import Path
from typing import Dict, Iterable, List, Optional
from .. import DATA_TAG
from ..errors import (AmbiguousFileError, DataQualityError, MissingFileError,
ModalityUnavailableError)
from ..paths import DataRoot, split_rid
from . import modalities as M
from .annotations import load_segments_file
from .io import load_alignment_metadata, load_stream, modality_from_stream
from .recordings import Recording, load_recordings, sort_rids
SCAN_MODALITIES = (
"emg", "imu", "imu_wrist", "imu_finger", "eyetrack", "eyetrack_gaze", "eyetrack_ocular",
"pressure", "mocap", "mocap_markers", "myo_quat",
)
ERROR_STATUSES = ("missing_file", "ambiguous")
SCHEMA = "pulse-availability/1"
def _rel(dr: DataRoot, p) -> str:
return Path(os.path.relpath(str(p), str(dr.root))).as_posix()
def scan_recording(dr: DataRoot, rec: Recording) -> dict:
"""Scan one recording (all streams, all modalities)."""
rid = rec.rid
streams_info: Dict[str, dict] = {}
loaded = {}
try:
meta = load_alignment_metadata(dr, rid)
meta_error = None
except MissingFileError:
# v4s1 and v4s4 are listed in recordings.csv with no aligned modality and
# ship no files at all; any other recording must have its metadata.
meta = {}
meta_error = None if not rec.aligned_modalities else "alignment_metadata.json missing"
aligned = set(meta.get("aligned_modalities") or [])
meta_n = meta.get("n_samples")
for sname in M.STREAM_FILES:
listed = M.STREAM_ALIGNMENT_NAME[sname] in aligned
csv_name = M.STREAM_RECORDINGS_CSV_NAME[sname]
info = {"listed_in_alignment_metadata": listed,
"listed_in_recordings_csv": rec.lists(csv_name) if csv_name else None}
try:
st = load_stream(dr, rid, sname)
except AmbiguousFileError as e:
info.update(status="ambiguous", error=str(e).replace(str(dr.root), "<root>"))
except MissingFileError:
info.update(status="missing_file" if listed else "absent")
else:
loaded[sname] = st
info.update(status="present", file=_rel(dr, st.path), n_rows=st.n_rows,
n_columns=len(st.columns))
if sname == "mocap_skeleton":
info["location"] = "raw" if "/raw/" in info["file"] else "recording_root"
if meta_n:
info["rows_minus_metadata_samples"] = st.n_rows - int(meta_n)
streams_info[sname] = info
mods: Dict[str, dict] = {}
for mod in SCAN_MODALITIES:
sname = M.base_stream(mod)
sinfo = streams_info[sname]
if sinfo["status"] != "present":
mods[mod] = {"status": sinfo["status"]}
continue
try:
ma = modality_from_stream(loaded[sname], mod)
except ModalityUnavailableError as e:
mods[mod] = {"status": f"unavailable:{e.reason}"}
except DataQualityError as e:
mods[mod] = {"status": f"quality:{e.reason}"}
else:
mods[mod] = {"status": "ok", "n_frames": int(ma.data.shape[0]),
"n_features": int(ma.data.shape[1])}
ann: dict = {"exists": False}
_, scen = split_rid(rid)
if scen != "s9":
p = dr.annotation_path(rid, must_exist=False)
if p is not None:
segs = load_segments_file(p, rid)
ann = {"exists": True, "file": _rel(dr, p), "n_segments": len(segs)}
return {
"volunteer": rec.volunteer, "scenario": rec.scenario,
"n_samples_100hz": rec.n_samples_100hz, "alignment_n_samples": meta_n,
"has_dense_annotations": rec.has_dense_annotations,
"no_data": not rec.aligned_modalities and not loaded,
"metadata_error": meta_error,
"streams": streams_info, "modalities": mods, "annotation": ann,
}
def _scan_one(args):
root, rec = args
return rec.rid, scan_recording(DataRoot(root=Path(root)), rec)
def build_availability(dr: DataRoot, rids: Optional[Iterable[str]] = None,
workers: int = 1) -> dict:
recs = load_recordings(dr)
todo = sort_rids(rids if rids is not None else recs.keys())
jobs = [(str(dr.root), recs[r]) for r in todo]
if workers > 1:
with ProcessPoolExecutor(max_workers=workers) as ex:
results = dict(ex.map(_scan_one, jobs, chunksize=1))
else:
results = dict(map(_scan_one, jobs))
table = {"schema": SCHEMA, "data_tag": DATA_TAG, "scan_modalities": list(SCAN_MODALITIES),
"n_recordings": len(results),
"recordings": {r: results[r] for r in sort_rids(results)}}
table["summary"] = summarize(table)
return table
def summarize(table: dict) -> dict:
counts: Dict[str, Dict[str, int]] = {}
errors: List[str] = []
warnings: List[str] = []
notes: List[str] = []
rows_mismatch: List[str] = []
tsv_loc: Dict[str, int] = {}
no_data: List[str] = []
for rid, r in table["recordings"].items():
if r.get("metadata_error"):
errors.append(f"{rid}: {r['metadata_error']}")
if r.get("no_data"):
no_data.append(rid)
for mod, m in r["modalities"].items():
counts.setdefault(mod, {}).setdefault(m["status"], 0)
counts[mod][m["status"]] += 1
for sname, s in r["streams"].items():
if s["status"] in ERROR_STATUSES:
errors.append(f"{rid}: {sname} {s['status']}")
if s["status"] == "present" and not s.get("listed_in_alignment_metadata"):
warnings.append(f"{rid}: {sname} present but not in alignment_metadata.json")
if s["status"] == "present" and s.get("listed_in_recordings_csv") is False:
notes.append(f"{rid}: {sname} present but not listed in recordings.csv")
if s["status"] == "present" and s.get("rows_minus_metadata_samples"):
rows_mismatch.append(f"{rid}: {sname} {s['rows_minus_metadata_samples']:+d} rows")
if sname == "mocap_skeleton" and s["status"] == "present":
tsv_loc[s["location"]] = tsv_loc.get(s["location"], 0) + 1
return {"modality_status_counts": {k: dict(sorted(v.items())) for k, v in sorted(counts.items())},
"mocap_tsv_locations": tsv_loc, "errors": errors, "warnings": warnings,
"notes": notes, "row_count_differences": rows_mismatch,
"recordings_without_data": no_data}
def file_md5(path) -> str:
h = hashlib.md5()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
class Availability:
"""Read-only view of a frozen availability table."""
def __init__(self, table: dict, path: Optional[str] = None, md5: Optional[str] = None):
if table.get("schema") != SCHEMA:
raise ValueError(f"not an availability table (schema={table.get('schema')!r})")
self.table = table
self.path = path
self.md5 = md5
self.data_tag = table.get("data_tag")
self._recs = table["recordings"]
@classmethod
def load(cls, path) -> "Availability":
with open(path, encoding="utf-8") as f:
table = json.load(f)
return cls(table, str(path), file_md5(path))
def rids(self, volunteers: Optional[Iterable[str]] = None,
scenarios: Optional[Iterable[str]] = None) -> List[str]:
vs = set(volunteers) if volunteers is not None else None
ss = set(scenarios) if scenarios is not None else None
return sort_rids(r for r, d in self._recs.items()
if (vs is None or d["volunteer"] in vs)
and (ss is None or d["scenario"] in ss))
def status(self, rid: str, modality: str) -> str:
try:
return self._recs[rid]["modalities"][modality]["status"]
except KeyError:
raise KeyError(f"{rid}/{modality} not in availability table") from None
def ok(self, rid: str, modality: str) -> bool:
return self.status(rid, modality) == "ok"
def has_annotation(self, rid: str) -> bool:
return bool(self._recs[rid]["annotation"].get("exists"))
def errors(self) -> List[str]:
return list(self.table.get("summary", {}).get("errors", []))
def default_availability_path(repo_root: Optional[Path] = None) -> Path:
base = repo_root or Path(__file__).resolve().parents[2]
return base / "manifests" / f"availability_{DATA_TAG}.json"