Download pulse/data/availability.py from velvet-pine-22/PULSE-code: direct link, hf CLI and curl.
- Browser
- Download file 9.79 kB
-
https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/pulse/data/availability.py
- Command line
-
hf download hf://velvet-pine-22/PULSE-code/pulse/data/availability.py
-
curl -L -o availability.py https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/pulse/data/availability.py
9.79 kB
| """Frozen availability table: which modality of which recording is usable. | |
| ``scripts/build_availability.py`` loads every stream of every recording once | |
| with the strict loaders and records, per recording and modality, one status: | |
| ``ok`` loads and passes the quality rules | |
| ``absent`` no file, and alignment_metadata.json does not list it | |
| ``missing_file`` alignment_metadata.json lists the stream (``aligned_modalities``) | |
| but the file is absent (ERROR) | |
| ``ambiguous`` skeleton TSV present both in raw/ and at the root (ERROR) | |
| ``unavailable:<why>`` the file does not provide the modality (e.g. v5 IMU, | |
| 72 columns -> ``unavailable:imu_columns_72``) | |
| ``quality:<why>`` fails a quality rule (``extreme_values``, ``mostly_zero``) | |
| The table is committed (``manifests/availability_v1.1.json``). Task manifests | |
| are derived from it, so which recordings a task uses is fixed before any | |
| training code runs, and a loader that later fails on an expected recording | |
| raises instead of shrinking the dataset. | |
| """ | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import os | |
| from concurrent.futures import ProcessPoolExecutor | |
| from pathlib import Path | |
| from typing import Dict, Iterable, List, Optional | |
| from .. import DATA_TAG | |
| from ..errors import (AmbiguousFileError, DataQualityError, MissingFileError, | |
| ModalityUnavailableError) | |
| from ..paths import DataRoot, split_rid | |
| from . import modalities as M | |
| from .annotations import load_segments_file | |
| from .io import load_alignment_metadata, load_stream, modality_from_stream | |
| from .recordings import Recording, load_recordings, sort_rids | |
| SCAN_MODALITIES = ( | |
| "emg", "imu", "imu_wrist", "imu_finger", "eyetrack", "eyetrack_gaze", "eyetrack_ocular", | |
| "pressure", "mocap", "mocap_markers", "myo_quat", | |
| ) | |
| ERROR_STATUSES = ("missing_file", "ambiguous") | |
| SCHEMA = "pulse-availability/1" | |
| def _rel(dr: DataRoot, p) -> str: | |
| return Path(os.path.relpath(str(p), str(dr.root))).as_posix() | |
| def scan_recording(dr: DataRoot, rec: Recording) -> dict: | |
| """Scan one recording (all streams, all modalities).""" | |
| rid = rec.rid | |
| streams_info: Dict[str, dict] = {} | |
| loaded = {} | |
| try: | |
| meta = load_alignment_metadata(dr, rid) | |
| meta_error = None | |
| except MissingFileError: | |
| # v4s1 and v4s4 are listed in recordings.csv with no aligned modality and | |
| # ship no files at all; any other recording must have its metadata. | |
| meta = {} | |
| meta_error = None if not rec.aligned_modalities else "alignment_metadata.json missing" | |
| aligned = set(meta.get("aligned_modalities") or []) | |
| meta_n = meta.get("n_samples") | |
| for sname in M.STREAM_FILES: | |
| listed = M.STREAM_ALIGNMENT_NAME[sname] in aligned | |
| csv_name = M.STREAM_RECORDINGS_CSV_NAME[sname] | |
| info = {"listed_in_alignment_metadata": listed, | |
| "listed_in_recordings_csv": rec.lists(csv_name) if csv_name else None} | |
| try: | |
| st = load_stream(dr, rid, sname) | |
| except AmbiguousFileError as e: | |
| info.update(status="ambiguous", error=str(e).replace(str(dr.root), "<root>")) | |
| except MissingFileError: | |
| info.update(status="missing_file" if listed else "absent") | |
| else: | |
| loaded[sname] = st | |
| info.update(status="present", file=_rel(dr, st.path), n_rows=st.n_rows, | |
| n_columns=len(st.columns)) | |
| if sname == "mocap_skeleton": | |
| info["location"] = "raw" if "/raw/" in info["file"] else "recording_root" | |
| if meta_n: | |
| info["rows_minus_metadata_samples"] = st.n_rows - int(meta_n) | |
| streams_info[sname] = info | |
| mods: Dict[str, dict] = {} | |
| for mod in SCAN_MODALITIES: | |
| sname = M.base_stream(mod) | |
| sinfo = streams_info[sname] | |
| if sinfo["status"] != "present": | |
| mods[mod] = {"status": sinfo["status"]} | |
| continue | |
| try: | |
| ma = modality_from_stream(loaded[sname], mod) | |
| except ModalityUnavailableError as e: | |
| mods[mod] = {"status": f"unavailable:{e.reason}"} | |
| except DataQualityError as e: | |
| mods[mod] = {"status": f"quality:{e.reason}"} | |
| else: | |
| mods[mod] = {"status": "ok", "n_frames": int(ma.data.shape[0]), | |
| "n_features": int(ma.data.shape[1])} | |
| ann: dict = {"exists": False} | |
| _, scen = split_rid(rid) | |
| if scen != "s9": | |
| p = dr.annotation_path(rid, must_exist=False) | |
| if p is not None: | |
| segs = load_segments_file(p, rid) | |
| ann = {"exists": True, "file": _rel(dr, p), "n_segments": len(segs)} | |
| return { | |
| "volunteer": rec.volunteer, "scenario": rec.scenario, | |
| "n_samples_100hz": rec.n_samples_100hz, "alignment_n_samples": meta_n, | |
| "has_dense_annotations": rec.has_dense_annotations, | |
| "no_data": not rec.aligned_modalities and not loaded, | |
| "metadata_error": meta_error, | |
| "streams": streams_info, "modalities": mods, "annotation": ann, | |
| } | |
| def _scan_one(args): | |
| root, rec = args | |
| return rec.rid, scan_recording(DataRoot(root=Path(root)), rec) | |
| def build_availability(dr: DataRoot, rids: Optional[Iterable[str]] = None, | |
| workers: int = 1) -> dict: | |
| recs = load_recordings(dr) | |
| todo = sort_rids(rids if rids is not None else recs.keys()) | |
| jobs = [(str(dr.root), recs[r]) for r in todo] | |
| if workers > 1: | |
| with ProcessPoolExecutor(max_workers=workers) as ex: | |
| results = dict(ex.map(_scan_one, jobs, chunksize=1)) | |
| else: | |
| results = dict(map(_scan_one, jobs)) | |
| table = {"schema": SCHEMA, "data_tag": DATA_TAG, "scan_modalities": list(SCAN_MODALITIES), | |
| "n_recordings": len(results), | |
| "recordings": {r: results[r] for r in sort_rids(results)}} | |
| table["summary"] = summarize(table) | |
| return table | |
| def summarize(table: dict) -> dict: | |
| counts: Dict[str, Dict[str, int]] = {} | |
| errors: List[str] = [] | |
| warnings: List[str] = [] | |
| notes: List[str] = [] | |
| rows_mismatch: List[str] = [] | |
| tsv_loc: Dict[str, int] = {} | |
| no_data: List[str] = [] | |
| for rid, r in table["recordings"].items(): | |
| if r.get("metadata_error"): | |
| errors.append(f"{rid}: {r['metadata_error']}") | |
| if r.get("no_data"): | |
| no_data.append(rid) | |
| for mod, m in r["modalities"].items(): | |
| counts.setdefault(mod, {}).setdefault(m["status"], 0) | |
| counts[mod][m["status"]] += 1 | |
| for sname, s in r["streams"].items(): | |
| if s["status"] in ERROR_STATUSES: | |
| errors.append(f"{rid}: {sname} {s['status']}") | |
| if s["status"] == "present" and not s.get("listed_in_alignment_metadata"): | |
| warnings.append(f"{rid}: {sname} present but not in alignment_metadata.json") | |
| if s["status"] == "present" and s.get("listed_in_recordings_csv") is False: | |
| notes.append(f"{rid}: {sname} present but not listed in recordings.csv") | |
| if s["status"] == "present" and s.get("rows_minus_metadata_samples"): | |
| rows_mismatch.append(f"{rid}: {sname} {s['rows_minus_metadata_samples']:+d} rows") | |
| if sname == "mocap_skeleton" and s["status"] == "present": | |
| tsv_loc[s["location"]] = tsv_loc.get(s["location"], 0) + 1 | |
| return {"modality_status_counts": {k: dict(sorted(v.items())) for k, v in sorted(counts.items())}, | |
| "mocap_tsv_locations": tsv_loc, "errors": errors, "warnings": warnings, | |
| "notes": notes, "row_count_differences": rows_mismatch, | |
| "recordings_without_data": no_data} | |
| def file_md5(path) -> str: | |
| h = hashlib.md5() | |
| with open(path, "rb") as f: | |
| for chunk in iter(lambda: f.read(1 << 20), b""): | |
| h.update(chunk) | |
| return h.hexdigest() | |
| class Availability: | |
| """Read-only view of a frozen availability table.""" | |
| def __init__(self, table: dict, path: Optional[str] = None, md5: Optional[str] = None): | |
| if table.get("schema") != SCHEMA: | |
| raise ValueError(f"not an availability table (schema={table.get('schema')!r})") | |
| self.table = table | |
| self.path = path | |
| self.md5 = md5 | |
| self.data_tag = table.get("data_tag") | |
| self._recs = table["recordings"] | |
| def load(cls, path) -> "Availability": | |
| with open(path, encoding="utf-8") as f: | |
| table = json.load(f) | |
| return cls(table, str(path), file_md5(path)) | |
| def rids(self, volunteers: Optional[Iterable[str]] = None, | |
| scenarios: Optional[Iterable[str]] = None) -> List[str]: | |
| vs = set(volunteers) if volunteers is not None else None | |
| ss = set(scenarios) if scenarios is not None else None | |
| return sort_rids(r for r, d in self._recs.items() | |
| if (vs is None or d["volunteer"] in vs) | |
| and (ss is None or d["scenario"] in ss)) | |
| def status(self, rid: str, modality: str) -> str: | |
| try: | |
| return self._recs[rid]["modalities"][modality]["status"] | |
| except KeyError: | |
| raise KeyError(f"{rid}/{modality} not in availability table") from None | |
| def ok(self, rid: str, modality: str) -> bool: | |
| return self.status(rid, modality) == "ok" | |
| def has_annotation(self, rid: str) -> bool: | |
| return bool(self._recs[rid]["annotation"].get("exists")) | |
| def errors(self) -> List[str]: | |
| return list(self.table.get("summary", {}).get("errors", [])) | |
| def default_availability_path(repo_root: Optional[Path] = None) -> Path: | |
| base = repo_root or Path(__file__).resolve().parents[2] | |
| return base / "manifests" / f"availability_{DATA_TAG}.json" | |