Safetensors
English
File size: 19,088 Bytes
d52c89c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d90680b
d52c89c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
"""Verified token-release loading. Never pack unrelated recordings into context."""

import hashlib
import itertools
import json
from collections.abc import Sequence
from fractions import Fraction
from pathlib import Path

import numpy as np
import torch

from .network import TEXT_OPTIONS, TEXT_SHIFT, TRACKS


def sha256(path):
    digest = hashlib.sha256()
    with Path(path).open("rb") as f:
        for chunk in iter(lambda: f.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def read_manifest(path):
    with Path(path).open() as stream:
        rows = [json.loads(line) for line in stream if line.strip()]
    if not rows:
        raise ValueError("Empty token manifest")
    groups = {}
    identities = set()
    recordings = {}
    codec = rows[0]["codec"]
    for row in rows:
        if row["schema_version"] not in (1, 2) or row["split"] not in {
            "train",
            "validation",
            "test",
        }:
            raise ValueError("Invalid schema or split")
        parent = {
            key: row.get(key)
            for key in (
                "source_group",
                "split",
                "split_status",
                "source",
                "audio",
                "metadata",
            )
        }
        if recordings.setdefault(row["recording_id"], parent) != parent:
            raise ValueError("Recording metadata differs across encodings")
        for field in ("recording_id", "source_group"):
            key = (field, row[field])
            if not row[field] or groups.setdefault(key, row["split"]) != row["split"]:
                raise ValueError("Recording/source group crosses splits")
        # Supplied group labels cannot hide known duplicate/uploader identities.
        for namespace, fields in (
            ("source", ("sha256", "uploader_id", "duplicate_cluster")),
            ("audio", ("sha256", "pcm_sha256")),
            ("tokens", ("sha256",)),
        ):
            for field in fields:
                value = row.get(namespace, {}).get(field)
                if value is not None:
                    key = (namespace, field, value)
                    if groups.setdefault(key, row["split"]) != row["split"]:
                        raise ValueError("Known source identity crosses splits")
        for asset in [a for a in assets(row) if a] + [t for t in texts(row) if t]:
            if "sha256" not in asset:
                continue
            key = ("tokens", "sha256", asset["sha256"])
            if groups.setdefault(key, row["split"]) != row["split"]:
                raise ValueError("Known source identity crosses splits")
        identity = (
            row["recording_id"],
            row["window"]["start_sample"],
            row["window"]["end_sample"],
        )
        if identity in identities:
            raise ValueError("Duplicate token window")
        identities.add(identity)
        if row["codec"] != codec:
            raise ValueError("Mixed codecs")
    expected = {"codebooks": 8, "cardinality": 2048, "fps": 12.5, "sample_rate": 24000}
    if any(codec[k] != v for k, v in expected.items()) or not codec.get("repo"):
        raise ValueError("Expected frozen eight-codebook Mimi")
    revision = codec.get("revision", "")
    if len(revision) != 40 or any(c not in "0123456789abcdef" for c in revision):
        raise ValueError("Codec needs immutable full revision")
    return rows


def assets(row):
    """The audio token files of a row's two tracks, in track order; None where a track is empty.

    Schema-2 rows list `tokens.tracks`. Schema-1 rows hold one stream, or two in
    `tokens.streams`; a single stream sits on track 1.
    """
    info = row["tokens"]
    if row["schema_version"] == 2:
        tracks = info["tracks"]
    else:
        tracks = info.get("streams") or [info, None]
    if len(tracks) != 2 or not any(tracks):
        raise ValueError("A row has two tracks, at least one with audio")
    return list(tracks)


def texts(row):
    """The text of a row's two tracks: None (absent), {"count": 0} (present, no words)
    or a verified-on-read asset {"path", "sha256", "count"}. Schema-1 rows have none."""
    if row["schema_version"] == 1:
        return [None, None]
    text = row["text"]
    if len(text) != 2:
        raise ValueError("A row has two text tracks")
    return list(text)


def token_array(row, root, asset=None):
    info = row["tokens"]
    asset = asset or info
    root = Path(root).resolve()
    path = (root / asset["path"]).resolve()
    if not path.is_relative_to(root) or Path(asset["path"]).is_absolute():
        raise ValueError("Token path must be relative to release root")
    if sha256(path) != asset["sha256"]:
        raise ValueError("Token hash mismatch")
    data = np.load(path, allow_pickle=False, mmap_mode="r")
    if (
        data.dtype != np.uint16
        or info["dtype"] != "uint16"
        or data.shape != (8, info["frames"])
    ):
        raise ValueError("Expected uint16 [8,frames]")
    n = info["valid_frames"]
    if type(n) is not int or not 0 < n <= data.shape[1]:
        raise ValueError("Invalid valid_frames or code index")
    # Stream validation: touching every mmap page would inflate resident memory.
    # Both C and Fortran layouts contain the same contiguous uint16 payload.
    with path.open("rb") as stream:
        stream.seek(data.offset)
        remaining = data.size
        while remaining:
            values = np.fromfile(stream, dtype=np.uint16, count=min(remaining, 524288))
            if not len(values) or np.any(values >= 2048):
                raise ValueError("Invalid valid_frames or code index")
            remaining -= len(values)
    return data


def text_array(row, root, asset):
    """A track's text as int64 [count, 2] rows of (frame, id), frames nondecreasing."""
    if asset is None:
        return None
    if not asset["count"]:
        return np.zeros((0, 2), dtype=np.int64)
    root = Path(root).resolve()
    path = (root / asset["path"]).resolve()
    if not path.is_relative_to(root) or Path(asset["path"]).is_absolute():
        raise ValueError("Text path must be relative to release root")
    if sha256(path) != asset["sha256"]:
        raise ValueError("Text hash mismatch")
    data = np.load(path, allow_pickle=False)
    if data.dtype != np.int32 or data.shape != (asset["count"], 2):
        raise ValueError("Expected int32 [count, 2] text")
    frames, ids = data[:, 0], data[:, 1]
    if (
        frames.min() < 0
        or frames.max() >= row["tokens"]["valid_frames"]
        or np.any(np.diff(frames) < 0)
        or ids.min() < 0
        or ids.max() >= 32000
    ):
        raise ValueError("Text frames or ids out of range")
    return data.astype(np.int64)


def load_tokens(row, root):
    data = token_array(row, root)
    return torch.from_numpy(data[:, : row["tokens"]["valid_frames"]].T.astype(np.int64))


def extra_positions(option, counts):
    """Upper bound on a window's text positions whatever the track swap: counts[p][k]
    is the tokens of stored track k written in the window when it lands on track p
    (None where absent)."""
    return max(
        sum(counts[p][order[p]] or 0 for p in TEXT_OPTIONS[option])
        for order in ((0, 1), (1, 0))
    )


class Window:
    """One training window: audio [frames, 16] (zeros on an empty track), which
    tracks have audio, and each track's text rows (frame, id) rebased to the window.

    Text sits where its track's shift writes it; `alternate` is the text as the
    swapped window places it, each speaker's words under the other track's shift
    (default: the same rows, for equal shifts)."""

    def __init__(self, audio, present, text, alternate=None):
        self.audio, self.present, self.text = audio, present, text
        self.alternate = text[::-1] if alternate is None else alternate

    def __len__(self):
        return len(self.audio)

    def swapped(self):
        return Window(
            torch.cat((self.audio[:, 8:], self.audio[:, :8]), 1),
            self.present[::-1],
            self.alternate,
            self.text,
        )


class TokenWindows(Sequence):
    """Metadata-only index; each access maps the row's assets and copies one window.

    No resident token bank. Each asset is hashed and range-checked on first
    access, then its memory map is reused while the file's size and modification
    time are unchanged, so per-window cost stays flat as assets grow. Releases
    must stay immutable for the run. Text arrays are small and load at indexing.

    A span is cut from its start into windows of at most `context` + 1 temporal
    positions under the training option `text`: one per frame, one for the
    flush step and one per text token the option could place, whatever the swap.
    """

    def __init__(self, spans, root, context, text="none", shift=TEXT_SHIFT):
        if context < 1:
            raise ValueError("context must be positive")
        self.spans, self.root, self.context, self.option = spans, root, context, text
        text_option = text
        self.arrays = {}
        self.text = []
        self.windows = []
        self.total_frames = 0
        occupied = {}
        for index, (row, start, end) in enumerate(spans):
            if not 0 <= start < end <= row["tokens"]["valid_frames"]:
                raise ValueError("Invalid chunk bounds")
            if "audio" in row:
                rate = row["audio"]["sample_rate"]
                origin, limit = (
                    row["window"]["start_sample"],
                    row["window"]["end_sample"],
                )
                if (
                    type(rate) is not int
                    or rate <= 0
                    or not 0 <= origin < limit <= row["audio"]["num_samples"]
                ):
                    raise ValueError("Invalid source clock bounds")
                step = Fraction(rate * 2, 25)
                if origin + row["tokens"]["valid_frames"] * step > limit:
                    raise ValueError("Complete frames exceed source bounds")
                occupied.setdefault(row["recording_id"], []).append(
                    (origin + start * step, origin + end * step)
                )
            # placed[p][k]: stored track k's rows moved to the frame track p writes
            # them at, start frame - TEXT_SHIFT[p], since the swap may put either
            # speaker on either track; rows outside the span fall outside every window.
            stored = [text_array(row, root, t) for t in texts(row)]
            placed = [
                [None if t is None else t - np.array([value, 0]) for t in stored]
                for value in shift
            ]
            self.text.append(placed)
            # Words' tokens before each frame, per placement.
            cumulative = [
                [
                    None
                    if t is None
                    else np.searchsorted(t[:, 0], np.arange(end + 1), side="left")
                    for t in track
                ]
                for track in placed
            ]

            def bound(at, stop, cumulative=cumulative):
                counts = [
                    [None if c is None else c[stop] - c[at] for c in track]
                    for track in cumulative
                ]
                return stop - at + 1 + extra_positions(text_option, counts)

            at = start
            while at < end:
                # The longest window from `at` that fits; bound grows with stop.
                low, high = at, min(at + context, end)
                while low < high:
                    middle = (low + high + 1) // 2
                    if bound(at, middle) <= context + 1:
                        low = middle
                    else:
                        high = middle - 1
                stop = low
                if stop == at:
                    raise ValueError("One frame's text exceeds the context")
                self.windows.append((index, at, stop, bound(at, stop)))
                self.total_frames += stop - at
                at = stop
        if not self.windows:
            raise ValueError("No selected windows")
        for intervals in occupied.values():
            intervals.sort()
            if any(left[1] > right[0] for left, right in itertools.pairwise(intervals)):
                raise ValueError("Overlapping source exposure; use verified selection")

    def __len__(self):
        return len(self.windows)

    def bounds(self, index):
        if index < 0:
            index += len(self)
        if not 0 <= index < len(self):
            raise IndexError(index)
        span, start, end, _ = self.windows[index]
        return self.spans[span][0], start, end

    def frames(self, index):
        _, start, end, _ = self.windows[index]
        return end - start

    def positions(self, index):
        """Upper bound on the window's temporal positions under any swap."""
        return self.windows[index][3]

    def __getitem__(self, index):
        row, start, end = self.bounds(index)
        span = self.windows[index][0]
        # Hash and range checks read the whole asset, so an asset is verified on
        # first access and again only when its size or modification time changes.
        parts = []
        for asset in assets(row):
            if asset is None:
                parts.append(np.zeros((8, end - start), dtype=np.uint16))
                continue
            key = (asset["path"], asset["sha256"])
            stat = (Path(self.root) / asset["path"]).stat()
            signature = (stat.st_size, stat.st_mtime_ns)
            if self.arrays.get(key, (None,))[0] != signature:
                self.arrays[key] = (signature, token_array(row, self.root, asset))
            parts.append(self.arrays[key][1][:, start:end])

        def inside(t):
            if t is None:
                return None
            kept = t[(t[:, 0] >= start) & (t[:, 0] < end)].copy()
            kept[:, 0] -= start
            return kept

        placed = self.text[span]
        # Tracks sit side by side: [frames, codebooks x tracks].
        return Window(
            torch.from_numpy(np.concatenate(parts).T.astype(np.int64)),
            [a is not None for a in assets(row)],
            [inside(placed[p][p]) for p in range(TRACKS)],
            [inside(placed[p][1 - p]) for p in range(TRACKS)],
        )

    def metadata(self, index):
        row, start, end = self.bounds(index)
        result = {"recording": row, "start_frame": start, "end_frame": end}
        if "audio" in row:
            step = Fraction(row["audio"]["sample_rate"] * 2, 25)
            origin = row["window"]["start_sample"]
            result["source_sample_bounds"] = [
                {"numerator": value.numerator, "denominator": value.denominator}
                for value in (origin + start * step, origin + end * step)
            ]
        return result


def windows(rows, root, split, context, text="none", shift=TEXT_SHIFT):
    return TokenWindows(
        [
            (row, 0, row["tokens"]["valid_frames"])
            for row in rows
            if row["split"] == split
        ],
        root,
        context,
        text,
        shift,
    )


def read_selection(path, manifest, evidence_path):
    """Recompute the data owner's policy rather than trusting claimed qualification."""
    from duplex.data.contiguous import build_selection

    selection = json.loads(Path(path).read_text())
    evidence = [
        json.loads(line)
        for line in Path(evidence_path).read_text().splitlines()
        if line.strip()
    ]
    expected = build_selection(
        read_manifest(manifest), sha256(manifest), evidence, sha256(evidence_path)
    )
    if selection != expected:
        raise ValueError("Selection does not match frozen pool, evidence and policy")
    return selection


def selected_windows(
    rows, root, split, context, selection, text="none", shift=TEXT_SHIFT
):
    """Final chunking of verified slices, without joining gaps or re-encoding."""
    indexed = {
        (
            r["recording_id"],
            r["window"]["start_sample"],
            r["window"]["end_sample"],
            r["tokens"]["sha256"],
        ): r | selection["recordings"][r["recording_id"]]
        for r in rows
    }
    result = []
    for item in selection["selections"]:
        key = (
            item["recording_id"],
            item["window"]["start_sample"],
            item["window"]["end_sample"],
            item["tokens_sha256"],
        )
        row = indexed[key]
        if row["split"] != split:
            continue
        start, end = item["start_frame"], item["end_frame"]
        if (
            type(start) is not int
            or type(end) is not int
            or not 0 <= start < end <= row["tokens"]["valid_frames"]
        ):
            raise ValueError("Selected frame bounds exceed valid frames")
        result.append((row, start, end))
    return TokenWindows(result, root, context, text, shift)


def length_buckets(items, width):
    """Window indices keyed by padded length: the window's position bound less its
    flush step, rounded up to a multiple of width. Without text that is its frames."""
    buckets = {}
    for index in range(len(items)):
        frames = min(-(-(items.positions(index) - 1) // width) * width, items.context)
        buckets.setdefault(frames, []).append(index)
    return buckets


# Bucketed batches pad audio to a multiple of this, keeping few shapes.
AUDIO_PAD = 64


def batch(items, device, text="none", frames=None):
    """Pad windows to the longest, or, when a bucket of `frames` temporal positions
    fixes the shape, to frames + 1 positions and audio to the longest window rounded
    up to AUDIO_PAD frames (a bucket counts text positions, which hold no audio).
    Returns tokens [batch, audio frames, 16], the [batch, audio frames, tracks] mask
    of real frames of present tracks, and the text layout (None without text).

    Under `one` only track 2's text is kept; under `none` no text."""
    from .network import TEXT_OPTIONS, collate_text, text_layout

    lengths = [len(x) for x in items]
    width = max(lengths)
    if frames:
        width = min(frames, -(-width // AUDIO_PAD) * AUDIO_PAD)
    tokens = torch.zeros((len(items), width, 16), dtype=torch.long, device=device)
    valid = torch.zeros((len(items), width, 2), dtype=torch.bool, device=device)
    for i, item in enumerate(items):
        tokens[i, : len(item)] = item.audio.to(device)
        valid[i, : len(item)] = torch.tensor(item.present, device=device)
    if text == "none":
        return tokens, valid, None
    kept = TEXT_OPTIONS[text]
    layouts = [
        text_layout(
            len(item), [t if j in kept else None for j, t in enumerate(item.text)]
        )
        for item in items
    ]
    positions = frames + 1 if frames else max(len(x["source"]) for x in layouts)
    return tokens, valid, collate_text(layouts, width, positions, device)