File size: 9,791 Bytes
dffbc1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
"""Frozen availability table: which modality of which recording is usable.

``scripts/build_availability.py`` loads every stream of every recording once
with the strict loaders and records, per recording and modality, one status:

``ok``                  loads and passes the quality rules
``absent``              no file, and alignment_metadata.json does not list it
``missing_file``        alignment_metadata.json lists the stream (``aligned_modalities``)
                        but the file is absent (ERROR)
``ambiguous``           skeleton TSV present both in raw/ and at the root (ERROR)
``unavailable:<why>``   the file does not provide the modality (e.g. v5 IMU,
                        72 columns -> ``unavailable:imu_columns_72``)
``quality:<why>``       fails a quality rule (``extreme_values``, ``mostly_zero``)

The table is committed (``manifests/availability_v1.1.json``).  Task manifests
are derived from it, so which recordings a task uses is fixed before any
training code runs, and a loader that later fails on an expected recording
raises instead of shrinking the dataset.
"""
from __future__ import annotations

import hashlib
import json
import os
from concurrent.futures import ProcessPoolExecutor
from pathlib import Path
from typing import Dict, Iterable, List, Optional

from .. import DATA_TAG
from ..errors import (AmbiguousFileError, DataQualityError, MissingFileError,
                      ModalityUnavailableError)
from ..paths import DataRoot, split_rid
from . import modalities as M
from .annotations import load_segments_file
from .io import load_alignment_metadata, load_stream, modality_from_stream
from .recordings import Recording, load_recordings, sort_rids

SCAN_MODALITIES = (
    "emg", "imu", "imu_wrist", "imu_finger", "eyetrack", "eyetrack_gaze", "eyetrack_ocular",
    "pressure", "mocap", "mocap_markers", "myo_quat",
)
ERROR_STATUSES = ("missing_file", "ambiguous")
SCHEMA = "pulse-availability/1"


def _rel(dr: DataRoot, p) -> str:
    return Path(os.path.relpath(str(p), str(dr.root))).as_posix()


def scan_recording(dr: DataRoot, rec: Recording) -> dict:
    """Scan one recording (all streams, all modalities)."""
    rid = rec.rid
    streams_info: Dict[str, dict] = {}
    loaded = {}
    try:
        meta = load_alignment_metadata(dr, rid)
        meta_error = None
    except MissingFileError:
        # v4s1 and v4s4 are listed in recordings.csv with no aligned modality and
        # ship no files at all; any other recording must have its metadata.
        meta = {}
        meta_error = None if not rec.aligned_modalities else "alignment_metadata.json missing"
    aligned = set(meta.get("aligned_modalities") or [])
    meta_n = meta.get("n_samples")
    for sname in M.STREAM_FILES:
        listed = M.STREAM_ALIGNMENT_NAME[sname] in aligned
        csv_name = M.STREAM_RECORDINGS_CSV_NAME[sname]
        info = {"listed_in_alignment_metadata": listed,
                "listed_in_recordings_csv": rec.lists(csv_name) if csv_name else None}
        try:
            st = load_stream(dr, rid, sname)
        except AmbiguousFileError as e:
            info.update(status="ambiguous", error=str(e).replace(str(dr.root), "<root>"))
        except MissingFileError:
            info.update(status="missing_file" if listed else "absent")
        else:
            loaded[sname] = st
            info.update(status="present", file=_rel(dr, st.path), n_rows=st.n_rows,
                        n_columns=len(st.columns))
            if sname == "mocap_skeleton":
                info["location"] = "raw" if "/raw/" in info["file"] else "recording_root"
            if meta_n:
                info["rows_minus_metadata_samples"] = st.n_rows - int(meta_n)
        streams_info[sname] = info

    mods: Dict[str, dict] = {}
    for mod in SCAN_MODALITIES:
        sname = M.base_stream(mod)
        sinfo = streams_info[sname]
        if sinfo["status"] != "present":
            mods[mod] = {"status": sinfo["status"]}
            continue
        try:
            ma = modality_from_stream(loaded[sname], mod)
        except ModalityUnavailableError as e:
            mods[mod] = {"status": f"unavailable:{e.reason}"}
        except DataQualityError as e:
            mods[mod] = {"status": f"quality:{e.reason}"}
        else:
            mods[mod] = {"status": "ok", "n_frames": int(ma.data.shape[0]),
                         "n_features": int(ma.data.shape[1])}

    ann: dict = {"exists": False}
    _, scen = split_rid(rid)
    if scen != "s9":
        p = dr.annotation_path(rid, must_exist=False)
        if p is not None:
            segs = load_segments_file(p, rid)
            ann = {"exists": True, "file": _rel(dr, p), "n_segments": len(segs)}
    return {
        "volunteer": rec.volunteer, "scenario": rec.scenario,
        "n_samples_100hz": rec.n_samples_100hz, "alignment_n_samples": meta_n,
        "has_dense_annotations": rec.has_dense_annotations,
        "no_data": not rec.aligned_modalities and not loaded,
        "metadata_error": meta_error,
        "streams": streams_info, "modalities": mods, "annotation": ann,
    }


def _scan_one(args):
    root, rec = args
    return rec.rid, scan_recording(DataRoot(root=Path(root)), rec)


def build_availability(dr: DataRoot, rids: Optional[Iterable[str]] = None,
                       workers: int = 1) -> dict:
    recs = load_recordings(dr)
    todo = sort_rids(rids if rids is not None else recs.keys())
    jobs = [(str(dr.root), recs[r]) for r in todo]
    if workers > 1:
        with ProcessPoolExecutor(max_workers=workers) as ex:
            results = dict(ex.map(_scan_one, jobs, chunksize=1))
    else:
        results = dict(map(_scan_one, jobs))
    table = {"schema": SCHEMA, "data_tag": DATA_TAG, "scan_modalities": list(SCAN_MODALITIES),
             "n_recordings": len(results),
             "recordings": {r: results[r] for r in sort_rids(results)}}
    table["summary"] = summarize(table)
    return table


def summarize(table: dict) -> dict:
    counts: Dict[str, Dict[str, int]] = {}
    errors: List[str] = []
    warnings: List[str] = []
    notes: List[str] = []
    rows_mismatch: List[str] = []
    tsv_loc: Dict[str, int] = {}
    no_data: List[str] = []
    for rid, r in table["recordings"].items():
        if r.get("metadata_error"):
            errors.append(f"{rid}: {r['metadata_error']}")
        if r.get("no_data"):
            no_data.append(rid)
        for mod, m in r["modalities"].items():
            counts.setdefault(mod, {}).setdefault(m["status"], 0)
            counts[mod][m["status"]] += 1
        for sname, s in r["streams"].items():
            if s["status"] in ERROR_STATUSES:
                errors.append(f"{rid}: {sname} {s['status']}")
            if s["status"] == "present" and not s.get("listed_in_alignment_metadata"):
                warnings.append(f"{rid}: {sname} present but not in alignment_metadata.json")
            if s["status"] == "present" and s.get("listed_in_recordings_csv") is False:
                notes.append(f"{rid}: {sname} present but not listed in recordings.csv")
            if s["status"] == "present" and s.get("rows_minus_metadata_samples"):
                rows_mismatch.append(f"{rid}: {sname} {s['rows_minus_metadata_samples']:+d} rows")
            if sname == "mocap_skeleton" and s["status"] == "present":
                tsv_loc[s["location"]] = tsv_loc.get(s["location"], 0) + 1
    return {"modality_status_counts": {k: dict(sorted(v.items())) for k, v in sorted(counts.items())},
            "mocap_tsv_locations": tsv_loc, "errors": errors, "warnings": warnings,
            "notes": notes, "row_count_differences": rows_mismatch,
            "recordings_without_data": no_data}


def file_md5(path) -> str:
    h = hashlib.md5()
    with open(path, "rb") as f:
        for chunk in iter(lambda: f.read(1 << 20), b""):
            h.update(chunk)
    return h.hexdigest()


class Availability:
    """Read-only view of a frozen availability table."""

    def __init__(self, table: dict, path: Optional[str] = None, md5: Optional[str] = None):
        if table.get("schema") != SCHEMA:
            raise ValueError(f"not an availability table (schema={table.get('schema')!r})")
        self.table = table
        self.path = path
        self.md5 = md5
        self.data_tag = table.get("data_tag")
        self._recs = table["recordings"]

    @classmethod
    def load(cls, path) -> "Availability":
        with open(path, encoding="utf-8") as f:
            table = json.load(f)
        return cls(table, str(path), file_md5(path))

    def rids(self, volunteers: Optional[Iterable[str]] = None,
             scenarios: Optional[Iterable[str]] = None) -> List[str]:
        vs = set(volunteers) if volunteers is not None else None
        ss = set(scenarios) if scenarios is not None else None
        return sort_rids(r for r, d in self._recs.items()
                         if (vs is None or d["volunteer"] in vs)
                         and (ss is None or d["scenario"] in ss))

    def status(self, rid: str, modality: str) -> str:
        try:
            return self._recs[rid]["modalities"][modality]["status"]
        except KeyError:
            raise KeyError(f"{rid}/{modality} not in availability table") from None

    def ok(self, rid: str, modality: str) -> bool:
        return self.status(rid, modality) == "ok"

    def has_annotation(self, rid: str) -> bool:
        return bool(self._recs[rid]["annotation"].get("exists"))

    def errors(self) -> List[str]:
        return list(self.table.get("summary", {}).get("errors", []))


def default_availability_path(repo_root: Optional[Path] = None) -> Path:
    base = repo_root or Path(__file__).resolve().parents[2]
    return base / "manifests" / f"availability_{DATA_TAG}.json"