Download scripts/duplex/model/tokens.py from Trelis/duplex-0: direct link, hf CLI and curl.
- Browser
- Download file 19.1 kB
-
https://huggingface.co/Trelis/duplex-0/resolve/main/scripts/duplex/model/tokens.py
- Command line
-
hf download hf://Trelis/duplex-0/scripts/duplex/model/tokens.py
-
curl -L -o tokens.py https://huggingface.co/Trelis/duplex-0/resolve/main/scripts/duplex/model/tokens.py
19.1 kB
| """Verified token-release loading. Never pack unrelated recordings into context.""" | |
| import hashlib | |
| import itertools | |
| import json | |
| from collections.abc import Sequence | |
| from fractions import Fraction | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| from .network import TEXT_OPTIONS, TEXT_SHIFT, TRACKS | |
| def sha256(path): | |
| digest = hashlib.sha256() | |
| with Path(path).open("rb") as f: | |
| for chunk in iter(lambda: f.read(1024 * 1024), b""): | |
| digest.update(chunk) | |
| return digest.hexdigest() | |
| def read_manifest(path): | |
| with Path(path).open() as stream: | |
| rows = [json.loads(line) for line in stream if line.strip()] | |
| if not rows: | |
| raise ValueError("Empty token manifest") | |
| groups = {} | |
| identities = set() | |
| recordings = {} | |
| codec = rows[0]["codec"] | |
| for row in rows: | |
| if row["schema_version"] not in (1, 2) or row["split"] not in { | |
| "train", | |
| "validation", | |
| "test", | |
| }: | |
| raise ValueError("Invalid schema or split") | |
| parent = { | |
| key: row.get(key) | |
| for key in ( | |
| "source_group", | |
| "split", | |
| "split_status", | |
| "source", | |
| "audio", | |
| "metadata", | |
| ) | |
| } | |
| if recordings.setdefault(row["recording_id"], parent) != parent: | |
| raise ValueError("Recording metadata differs across encodings") | |
| for field in ("recording_id", "source_group"): | |
| key = (field, row[field]) | |
| if not row[field] or groups.setdefault(key, row["split"]) != row["split"]: | |
| raise ValueError("Recording/source group crosses splits") | |
| # Supplied group labels cannot hide known duplicate/uploader identities. | |
| for namespace, fields in ( | |
| ("source", ("sha256", "uploader_id", "duplicate_cluster")), | |
| ("audio", ("sha256", "pcm_sha256")), | |
| ("tokens", ("sha256",)), | |
| ): | |
| for field in fields: | |
| value = row.get(namespace, {}).get(field) | |
| if value is not None: | |
| key = (namespace, field, value) | |
| if groups.setdefault(key, row["split"]) != row["split"]: | |
| raise ValueError("Known source identity crosses splits") | |
| for asset in [a for a in assets(row) if a] + [t for t in texts(row) if t]: | |
| if "sha256" not in asset: | |
| continue | |
| key = ("tokens", "sha256", asset["sha256"]) | |
| if groups.setdefault(key, row["split"]) != row["split"]: | |
| raise ValueError("Known source identity crosses splits") | |
| identity = ( | |
| row["recording_id"], | |
| row["window"]["start_sample"], | |
| row["window"]["end_sample"], | |
| ) | |
| if identity in identities: | |
| raise ValueError("Duplicate token window") | |
| identities.add(identity) | |
| if row["codec"] != codec: | |
| raise ValueError("Mixed codecs") | |
| expected = {"codebooks": 8, "cardinality": 2048, "fps": 12.5, "sample_rate": 24000} | |
| if any(codec[k] != v for k, v in expected.items()) or not codec.get("repo"): | |
| raise ValueError("Expected frozen eight-codebook Mimi") | |
| revision = codec.get("revision", "") | |
| if len(revision) != 40 or any(c not in "0123456789abcdef" for c in revision): | |
| raise ValueError("Codec needs immutable full revision") | |
| return rows | |
| def assets(row): | |
| """The audio token files of a row's two tracks, in track order; None where a track is empty. | |
| Schema-2 rows list `tokens.tracks`. Schema-1 rows hold one stream, or two in | |
| `tokens.streams`; a single stream sits on track 1. | |
| """ | |
| info = row["tokens"] | |
| if row["schema_version"] == 2: | |
| tracks = info["tracks"] | |
| else: | |
| tracks = info.get("streams") or [info, None] | |
| if len(tracks) != 2 or not any(tracks): | |
| raise ValueError("A row has two tracks, at least one with audio") | |
| return list(tracks) | |
| def texts(row): | |
| """The text of a row's two tracks: None (absent), {"count": 0} (present, no words) | |
| or a verified-on-read asset {"path", "sha256", "count"}. Schema-1 rows have none.""" | |
| if row["schema_version"] == 1: | |
| return [None, None] | |
| text = row["text"] | |
| if len(text) != 2: | |
| raise ValueError("A row has two text tracks") | |
| return list(text) | |
| def token_array(row, root, asset=None): | |
| info = row["tokens"] | |
| asset = asset or info | |
| root = Path(root).resolve() | |
| path = (root / asset["path"]).resolve() | |
| if not path.is_relative_to(root) or Path(asset["path"]).is_absolute(): | |
| raise ValueError("Token path must be relative to release root") | |
| if sha256(path) != asset["sha256"]: | |
| raise ValueError("Token hash mismatch") | |
| data = np.load(path, allow_pickle=False, mmap_mode="r") | |
| if ( | |
| data.dtype != np.uint16 | |
| or info["dtype"] != "uint16" | |
| or data.shape != (8, info["frames"]) | |
| ): | |
| raise ValueError("Expected uint16 [8,frames]") | |
| n = info["valid_frames"] | |
| if type(n) is not int or not 0 < n <= data.shape[1]: | |
| raise ValueError("Invalid valid_frames or code index") | |
| # Stream validation: touching every mmap page would inflate resident memory. | |
| # Both C and Fortran layouts contain the same contiguous uint16 payload. | |
| with path.open("rb") as stream: | |
| stream.seek(data.offset) | |
| remaining = data.size | |
| while remaining: | |
| values = np.fromfile(stream, dtype=np.uint16, count=min(remaining, 524288)) | |
| if not len(values) or np.any(values >= 2048): | |
| raise ValueError("Invalid valid_frames or code index") | |
| remaining -= len(values) | |
| return data | |
| def text_array(row, root, asset): | |
| """A track's text as int64 [count, 2] rows of (frame, id), frames nondecreasing.""" | |
| if asset is None: | |
| return None | |
| if not asset["count"]: | |
| return np.zeros((0, 2), dtype=np.int64) | |
| root = Path(root).resolve() | |
| path = (root / asset["path"]).resolve() | |
| if not path.is_relative_to(root) or Path(asset["path"]).is_absolute(): | |
| raise ValueError("Text path must be relative to release root") | |
| if sha256(path) != asset["sha256"]: | |
| raise ValueError("Text hash mismatch") | |
| data = np.load(path, allow_pickle=False) | |
| if data.dtype != np.int32 or data.shape != (asset["count"], 2): | |
| raise ValueError("Expected int32 [count, 2] text") | |
| frames, ids = data[:, 0], data[:, 1] | |
| if ( | |
| frames.min() < 0 | |
| or frames.max() >= row["tokens"]["valid_frames"] | |
| or np.any(np.diff(frames) < 0) | |
| or ids.min() < 0 | |
| or ids.max() >= 32000 | |
| ): | |
| raise ValueError("Text frames or ids out of range") | |
| return data.astype(np.int64) | |
| def load_tokens(row, root): | |
| data = token_array(row, root) | |
| return torch.from_numpy(data[:, : row["tokens"]["valid_frames"]].T.astype(np.int64)) | |
| def extra_positions(option, counts): | |
| """Upper bound on a window's text positions whatever the track swap: counts[p][k] | |
| is the tokens of stored track k written in the window when it lands on track p | |
| (None where absent).""" | |
| return max( | |
| sum(counts[p][order[p]] or 0 for p in TEXT_OPTIONS[option]) | |
| for order in ((0, 1), (1, 0)) | |
| ) | |
| class Window: | |
| """One training window: audio [frames, 16] (zeros on an empty track), which | |
| tracks have audio, and each track's text rows (frame, id) rebased to the window. | |
| Text sits where its track's shift writes it; `alternate` is the text as the | |
| swapped window places it, each speaker's words under the other track's shift | |
| (default: the same rows, for equal shifts).""" | |
| def __init__(self, audio, present, text, alternate=None): | |
| self.audio, self.present, self.text = audio, present, text | |
| self.alternate = text[::-1] if alternate is None else alternate | |
| def __len__(self): | |
| return len(self.audio) | |
| def swapped(self): | |
| return Window( | |
| torch.cat((self.audio[:, 8:], self.audio[:, :8]), 1), | |
| self.present[::-1], | |
| self.alternate, | |
| self.text, | |
| ) | |
| class TokenWindows(Sequence): | |
| """Metadata-only index; each access maps the row's assets and copies one window. | |
| No resident token bank. Each asset is hashed and range-checked on first | |
| access, then its memory map is reused while the file's size and modification | |
| time are unchanged, so per-window cost stays flat as assets grow. Releases | |
| must stay immutable for the run. Text arrays are small and load at indexing. | |
| A span is cut from its start into windows of at most `context` + 1 temporal | |
| positions under the training option `text`: one per frame, one for the | |
| flush step and one per text token the option could place, whatever the swap. | |
| """ | |
| def __init__(self, spans, root, context, text="none", shift=TEXT_SHIFT): | |
| if context < 1: | |
| raise ValueError("context must be positive") | |
| self.spans, self.root, self.context, self.option = spans, root, context, text | |
| text_option = text | |
| self.arrays = {} | |
| self.text = [] | |
| self.windows = [] | |
| self.total_frames = 0 | |
| occupied = {} | |
| for index, (row, start, end) in enumerate(spans): | |
| if not 0 <= start < end <= row["tokens"]["valid_frames"]: | |
| raise ValueError("Invalid chunk bounds") | |
| if "audio" in row: | |
| rate = row["audio"]["sample_rate"] | |
| origin, limit = ( | |
| row["window"]["start_sample"], | |
| row["window"]["end_sample"], | |
| ) | |
| if ( | |
| type(rate) is not int | |
| or rate <= 0 | |
| or not 0 <= origin < limit <= row["audio"]["num_samples"] | |
| ): | |
| raise ValueError("Invalid source clock bounds") | |
| step = Fraction(rate * 2, 25) | |
| if origin + row["tokens"]["valid_frames"] * step > limit: | |
| raise ValueError("Complete frames exceed source bounds") | |
| occupied.setdefault(row["recording_id"], []).append( | |
| (origin + start * step, origin + end * step) | |
| ) | |
| # placed[p][k]: stored track k's rows moved to the frame track p writes | |
| # them at, start frame - TEXT_SHIFT[p], since the swap may put either | |
| # speaker on either track; rows outside the span fall outside every window. | |
| stored = [text_array(row, root, t) for t in texts(row)] | |
| placed = [ | |
| [None if t is None else t - np.array([value, 0]) for t in stored] | |
| for value in shift | |
| ] | |
| self.text.append(placed) | |
| # Words' tokens before each frame, per placement. | |
| cumulative = [ | |
| [ | |
| None | |
| if t is None | |
| else np.searchsorted(t[:, 0], np.arange(end + 1), side="left") | |
| for t in track | |
| ] | |
| for track in placed | |
| ] | |
| def bound(at, stop, cumulative=cumulative): | |
| counts = [ | |
| [None if c is None else c[stop] - c[at] for c in track] | |
| for track in cumulative | |
| ] | |
| return stop - at + 1 + extra_positions(text_option, counts) | |
| at = start | |
| while at < end: | |
| # The longest window from `at` that fits; bound grows with stop. | |
| low, high = at, min(at + context, end) | |
| while low < high: | |
| middle = (low + high + 1) // 2 | |
| if bound(at, middle) <= context + 1: | |
| low = middle | |
| else: | |
| high = middle - 1 | |
| stop = low | |
| if stop == at: | |
| raise ValueError("One frame's text exceeds the context") | |
| self.windows.append((index, at, stop, bound(at, stop))) | |
| self.total_frames += stop - at | |
| at = stop | |
| if not self.windows: | |
| raise ValueError("No selected windows") | |
| for intervals in occupied.values(): | |
| intervals.sort() | |
| if any(left[1] > right[0] for left, right in itertools.pairwise(intervals)): | |
| raise ValueError("Overlapping source exposure; use verified selection") | |
| def __len__(self): | |
| return len(self.windows) | |
| def bounds(self, index): | |
| if index < 0: | |
| index += len(self) | |
| if not 0 <= index < len(self): | |
| raise IndexError(index) | |
| span, start, end, _ = self.windows[index] | |
| return self.spans[span][0], start, end | |
| def frames(self, index): | |
| _, start, end, _ = self.windows[index] | |
| return end - start | |
| def positions(self, index): | |
| """Upper bound on the window's temporal positions under any swap.""" | |
| return self.windows[index][3] | |
| def __getitem__(self, index): | |
| row, start, end = self.bounds(index) | |
| span = self.windows[index][0] | |
| # Hash and range checks read the whole asset, so an asset is verified on | |
| # first access and again only when its size or modification time changes. | |
| parts = [] | |
| for asset in assets(row): | |
| if asset is None: | |
| parts.append(np.zeros((8, end - start), dtype=np.uint16)) | |
| continue | |
| key = (asset["path"], asset["sha256"]) | |
| stat = (Path(self.root) / asset["path"]).stat() | |
| signature = (stat.st_size, stat.st_mtime_ns) | |
| if self.arrays.get(key, (None,))[0] != signature: | |
| self.arrays[key] = (signature, token_array(row, self.root, asset)) | |
| parts.append(self.arrays[key][1][:, start:end]) | |
| def inside(t): | |
| if t is None: | |
| return None | |
| kept = t[(t[:, 0] >= start) & (t[:, 0] < end)].copy() | |
| kept[:, 0] -= start | |
| return kept | |
| placed = self.text[span] | |
| # Tracks sit side by side: [frames, codebooks x tracks]. | |
| return Window( | |
| torch.from_numpy(np.concatenate(parts).T.astype(np.int64)), | |
| [a is not None for a in assets(row)], | |
| [inside(placed[p][p]) for p in range(TRACKS)], | |
| [inside(placed[p][1 - p]) for p in range(TRACKS)], | |
| ) | |
| def metadata(self, index): | |
| row, start, end = self.bounds(index) | |
| result = {"recording": row, "start_frame": start, "end_frame": end} | |
| if "audio" in row: | |
| step = Fraction(row["audio"]["sample_rate"] * 2, 25) | |
| origin = row["window"]["start_sample"] | |
| result["source_sample_bounds"] = [ | |
| {"numerator": value.numerator, "denominator": value.denominator} | |
| for value in (origin + start * step, origin + end * step) | |
| ] | |
| return result | |
| def windows(rows, root, split, context, text="none", shift=TEXT_SHIFT): | |
| return TokenWindows( | |
| [ | |
| (row, 0, row["tokens"]["valid_frames"]) | |
| for row in rows | |
| if row["split"] == split | |
| ], | |
| root, | |
| context, | |
| text, | |
| shift, | |
| ) | |
| def read_selection(path, manifest, evidence_path): | |
| """Recompute the data owner's policy rather than trusting claimed qualification.""" | |
| from duplex.data.contiguous import build_selection | |
| selection = json.loads(Path(path).read_text()) | |
| evidence = [ | |
| json.loads(line) | |
| for line in Path(evidence_path).read_text().splitlines() | |
| if line.strip() | |
| ] | |
| expected = build_selection( | |
| read_manifest(manifest), sha256(manifest), evidence, sha256(evidence_path) | |
| ) | |
| if selection != expected: | |
| raise ValueError("Selection does not match frozen pool, evidence and policy") | |
| return selection | |
| def selected_windows( | |
| rows, root, split, context, selection, text="none", shift=TEXT_SHIFT | |
| ): | |
| """Final chunking of verified slices, without joining gaps or re-encoding.""" | |
| indexed = { | |
| ( | |
| r["recording_id"], | |
| r["window"]["start_sample"], | |
| r["window"]["end_sample"], | |
| r["tokens"]["sha256"], | |
| ): r | selection["recordings"][r["recording_id"]] | |
| for r in rows | |
| } | |
| result = [] | |
| for item in selection["selections"]: | |
| key = ( | |
| item["recording_id"], | |
| item["window"]["start_sample"], | |
| item["window"]["end_sample"], | |
| item["tokens_sha256"], | |
| ) | |
| row = indexed[key] | |
| if row["split"] != split: | |
| continue | |
| start, end = item["start_frame"], item["end_frame"] | |
| if ( | |
| type(start) is not int | |
| or type(end) is not int | |
| or not 0 <= start < end <= row["tokens"]["valid_frames"] | |
| ): | |
| raise ValueError("Selected frame bounds exceed valid frames") | |
| result.append((row, start, end)) | |
| return TokenWindows(result, root, context, text, shift) | |
| def length_buckets(items, width): | |
| """Window indices keyed by padded length: the window's position bound less its | |
| flush step, rounded up to a multiple of width. Without text that is its frames.""" | |
| buckets = {} | |
| for index in range(len(items)): | |
| frames = min(-(-(items.positions(index) - 1) // width) * width, items.context) | |
| buckets.setdefault(frames, []).append(index) | |
| return buckets | |
| # Bucketed batches pad audio to a multiple of this, keeping few shapes. | |
| AUDIO_PAD = 64 | |
| def batch(items, device, text="none", frames=None): | |
| """Pad windows to the longest, or, when a bucket of `frames` temporal positions | |
| fixes the shape, to frames + 1 positions and audio to the longest window rounded | |
| up to AUDIO_PAD frames (a bucket counts text positions, which hold no audio). | |
| Returns tokens [batch, audio frames, 16], the [batch, audio frames, tracks] mask | |
| of real frames of present tracks, and the text layout (None without text). | |
| Under `one` only track 2's text is kept; under `none` no text.""" | |
| from .network import TEXT_OPTIONS, collate_text, text_layout | |
| lengths = [len(x) for x in items] | |
| width = max(lengths) | |
| if frames: | |
| width = min(frames, -(-width // AUDIO_PAD) * AUDIO_PAD) | |
| tokens = torch.zeros((len(items), width, 16), dtype=torch.long, device=device) | |
| valid = torch.zeros((len(items), width, 2), dtype=torch.bool, device=device) | |
| for i, item in enumerate(items): | |
| tokens[i, : len(item)] = item.audio.to(device) | |
| valid[i, : len(item)] = torch.tensor(item.present, device=device) | |
| if text == "none": | |
| return tokens, valid, None | |
| kept = TEXT_OPTIONS[text] | |
| layouts = [ | |
| text_layout( | |
| len(item), [t if j in kept else None for j, t in enumerate(item.text)] | |
| ) | |
| for item in items | |
| ] | |
| positions = frames + 1 if frames else max(len(x["source"]) for x in layouts) | |
| return tokens, valid, collate_text(layouts, width, positions, device) | |