Download scripts/duplex/model/hf_release.py from Trelis/duplex-0: direct link, hf CLI and curl.
- Browser
- Download file 22.8 kB
-
https://huggingface.co/Trelis/duplex-0/resolve/main/scripts/duplex/model/hf_release.py
- Command line
-
hf download hf://Trelis/duplex-0/scripts/duplex/model/hf_release.py
-
curl -L -o hf_release.py https://huggingface.co/Trelis/duplex-0/resolve/main/scripts/duplex/model/hf_release.py
22.8 kB
| """Verify the published duplex token datasets and write one trainer manifest per stage. | |
| Reads the private Hugging Face layout (`manifest.json`, `<run>/train.parquet`, `<run>/validation.parquet`, `<run>/test.parquet`, `<run>/report.json`, | |
| `<run>/tokens.npz`) downloaded at pinned revisions. Checks every file against `manifest.json`, every | |
| token array against its row (member hash, dtype, `[8, frames]` shape, code range, frame count from the | |
| 24 kHz sample count) and each separated speaker track the same way. Each array is extracted byte for byte | |
| to `<out>/tokens/<key>.npy`, so its hash stays the row's, and the stage manifests use the trainer's | |
| schema-version-2 rows (`tokens.py`): two audio tracks and two text tracks per span, assigned as training | |
| readiness describes. Each text track is the words starting in the span's complete frames, tokenized with | |
| Moshi's SentencePiece model, as int32 `(frame, id)` rows in `<out>/text/`. | |
| A row's published `split` field decides its split. The data owner gives a group one split in every | |
| dataset, so a span with different splits in different datasets fails the build. Revisions without it hold out one in 20 groups by a | |
| salted hash (a group is the podcast feed or the YODAS recording), the same groups in every stage, so the | |
| nested sets (conversational within mid-training within podcast pretraining) cannot leak. Every stage | |
| manifest holds every validation span, tagged in `validation_sets` with the stages whose datasets hold it. | |
| `--stages` builds a subset of the stages and reads only their datasets; a stage left out is absent from | |
| `validation_sets` and from the nesting check. | |
| """ | |
| import argparse | |
| import hashlib | |
| import io | |
| import json | |
| import math | |
| import zipfile | |
| from pathlib import Path | |
| import numpy as np | |
| CODEC = { | |
| "repo": "kyutai/mimi", | |
| "sample_rate": 24000, | |
| "codebooks": 8, | |
| "cardinality": 2048, | |
| "fps": 12.5, | |
| } | |
| HOP = 1920 | |
| TOKENIZER = ( | |
| "kyutai/moshiko-pytorch-bf16", | |
| "tokenizer_spm_32k_3.model", | |
| "2bfc9ae6e89079a5cc7ed2a68436010d91a3d289", | |
| ) | |
| STAGES = { | |
| "pretraining": ("duplex-pre-training-yodas3-mimi", "duplex-pre-training-podcasts-mimi"), | |
| "mid_training": ("duplex-mid-training-podcasts-mimi",), | |
| "conversational": ("duplex-conversational-podcasts-mimi",), | |
| } | |
| def digest(data): | |
| return hashlib.sha256(data).hexdigest() | |
| def group(row): | |
| """Podcast feeds share hosts across episodes, so a feed is the unit held out; YODAS has no uploader.""" | |
| return row.get("split_group") or row["source"].get("rss_url") or row["recording_id"] | |
| def held_out(name, salt, every): | |
| return int(digest(f"{salt}:{name}".encode()), 16) % every == 0 | |
| def check_array(data, record, where, problems): | |
| """Returns the array when its bytes, dtype, shape, range and frame count match the record.""" | |
| if digest(data) != record["sha256"]: | |
| problems.append({**where, "problem": "token hash mismatch"}) | |
| return None | |
| array = np.load(io.BytesIO(data), allow_pickle=False) | |
| # Independent 60 s pieces (rows before 14 November) and pieces with dropped context both | |
| # join to one frame per started 1,920 samples. | |
| expected = -(-record["samples_24k"] // HOP) | |
| if array.dtype != np.uint16 or array.shape != (8, record["frames"]): | |
| problems.append({**where, "problem": f"array {array.dtype} {array.shape}"}) | |
| return None | |
| if record["frames"] != expected: | |
| problems.append( | |
| {**where, "problem": f"frames {record['frames']} != {expected}"} | |
| ) | |
| return None | |
| if array.size and int(array.max()) >= 2048: | |
| problems.append({**where, "problem": "code out of range"}) | |
| return None | |
| return array | |
| def row_files(folder): | |
| """A run folder's row files: Parquet per split, or the JSON Lines layouts of revisions before 17 November.""" | |
| for suffix in ("parquet", "jsonl"): | |
| if found := [p for p in (folder / f"{name}.{suffix}" for name in ("train", "validation", "test")) if p.exists()]: | |
| return found | |
| return [folder / "spans.jsonl"] | |
| def read_rows(file): | |
| """Yields a row file's rows as dicts, a few at a time; a Parquet row holds null where a JSON row had no key.""" | |
| if file.suffix == ".parquet": | |
| import pyarrow.parquet as pq | |
| for batch in pq.ParquetFile(file).iter_batches(batch_size=50): | |
| yield from batch.to_pylist() | |
| else: | |
| with file.open() as lines: | |
| yield from (json.loads(line) for line in lines if line.strip()) | |
| def audit_repo(path): | |
| """Checks a repository and returns (manifest, rows, problems, totals). rows yields | |
| (run, row) pairs one run at a time, so a repository need not fit in memory; | |
| problems and totals are complete once rows is exhausted, and a row's own | |
| problems are recorded before it is yielded.""" | |
| manifest = json.loads((path / "manifest.json").read_text()) | |
| problems, totals = [], {} | |
| def rows(): | |
| for name, info in manifest["files"].items(): | |
| data = (path / name).read_bytes() | |
| if len(data) != info["bytes"] or digest(data) != info["sha256"]: | |
| problems.append({"file": name, "problem": "file hash or size mismatch"}) | |
| for run, expected in manifest["runs"].items(): | |
| # A run with no spans has no Parquet files; a run that expects spans must have its row file. | |
| spans = [ | |
| row | |
| for file in row_files(path / run) | |
| if expected["spans"] or file.exists() | |
| for row in read_rows(file) | |
| ] | |
| archive = zipfile.ZipFile(path / run / "tokens.npz") | |
| members = {n.removesuffix(".npy") for n in archive.namelist()} | |
| used = set() | |
| hours = sum(s["seconds"] for s in spans) / 3600 | |
| stats = totals.setdefault( | |
| run, | |
| { | |
| "spans": len(spans), | |
| "hours": hours, | |
| "manifest_spans": expected["spans"], | |
| "manifest_hours": expected["hours"], | |
| "stored_frames": 0, | |
| "valid_frames": 0, | |
| "track_arrays": 0, | |
| "separation": {}, | |
| }, | |
| ) | |
| if ( | |
| len(spans) != expected["spans"] | |
| or abs(hours - expected["hours"]) > 0.006 | |
| ): | |
| problems.append( | |
| {"run": run, "problem": "span count or hours differ from manifest"} | |
| ) | |
| for row in spans: | |
| where = { | |
| "run": run, | |
| "recording_id": row["recording_id"], | |
| "start_sample": row["start_sample"], | |
| } | |
| tokens = row["tokens"] | |
| if tokens.get("status") != "ok" or tokens["key"] not in members: | |
| problems.append( | |
| { | |
| **where, | |
| "problem": f"tokens {tokens.get('status')}, array missing", | |
| } | |
| ) | |
| continue | |
| data = archive.read(tokens["key"] + ".npy") | |
| used.add(tokens["key"]) | |
| array = check_array(data, tokens, where, problems) | |
| if array is None: | |
| continue | |
| rate = row["audio"]["sample_rate"] | |
| span = row["end_sample"] - row["start_sample"] | |
| if abs(tokens["samples_24k"] - span * 24000 / rate) > 1: | |
| problems.append( | |
| {**where, "problem": "24 kHz length disagrees with span"} | |
| ) | |
| stats["stored_frames"] += tokens["frames"] | |
| stats["valid_frames"] += tokens["samples_24k"] // HOP | |
| separation = row.get("separation") or {"status": "not_run"} | |
| status = separation["status"] | |
| stats["separation"][status] = stats["separation"].get(status, 0) + 1 | |
| # Separated tracks of revisions before `tracks`, then the published tracks; | |
| # a one-speaker row's track 1 is the mixed record, already checked. | |
| published = [ | |
| t["tokens"] | |
| for t in row.get("tracks") or [] | |
| if t["status"] == "ok" and t["tokens"]["key"] != tokens["key"] | |
| ] | |
| listed = separation.get("tracks") or [] | |
| seen_keys = {t.get("key") for t in listed} | |
| for track in listed + [ | |
| t for t in published if t["key"] not in seen_keys | |
| ]: | |
| if track.get("status") != "ok" or track["key"] not in members: | |
| problems.append( | |
| { | |
| **where, | |
| "problem": f"track {track.get('status')}, array missing", | |
| } | |
| ) | |
| continue | |
| used.add(track["key"]) | |
| stats["track_arrays"] += 1 | |
| if ( | |
| check_array( | |
| archive.read(track["key"] + ".npy"), track, where, problems | |
| ) | |
| is None | |
| ): | |
| continue | |
| if track["frames"] != tokens["frames"]: | |
| problems.append( | |
| { | |
| **where, | |
| "problem": "track frames differ from mixed stream", | |
| } | |
| ) | |
| yield run, row | |
| if members - used: | |
| problems.append( | |
| { | |
| "run": run, | |
| "problem": f"{len(members - used)} unreferenced arrays", | |
| } | |
| ) | |
| return manifest, rows(), problems, totals | |
| def text_rows(words, valid, tokenize): | |
| """(frame, id) rows of the words starting in the first `valid` frames, in word order.""" | |
| rows = [] | |
| for word in words: | |
| text = word["word"].strip() | |
| frame = math.floor(word["start_s"] * 12.5) | |
| # The transcription service can time a punctuation mark on its own. | |
| if not 0 <= frame < valid or not any(c.isalnum() for c in text): | |
| continue | |
| rows.extend((frame, i) for i in tokenize(text)) | |
| return np.array(rows, dtype=np.int32).reshape(-1, 2) | |
| def tracks_of(row): | |
| """Audio token records and word lists of the span's two tracks. | |
| A word list None is absent text; [] is text present with no words. From the | |
| published `tracks` field when the revision has it, else from `separation`. | |
| """ | |
| words = row.get("words") if row.get("transcription_status") == "ok" else None | |
| published = row.get("tracks") | |
| if published is not None: | |
| audio, text = [], [] | |
| for track in published: | |
| audio.append(track["tokens"] if track["status"] == "ok" else None) | |
| if track["status"] == "empty": | |
| text.append([]) | |
| elif track["status"] != "ok" or words is None or track["speaker"] is None: | |
| text.append(None) | |
| else: | |
| text.append(track["words"]) | |
| # Spans with no published track, such as more than two diarized speakers, | |
| # train as the mixed audio on track 1 without text. | |
| if not any(audio): | |
| return [row["tokens"], None], [None, None] | |
| return audio, text | |
| separation = row.get("separation") or {} | |
| found = separation.get("tracks") or [] | |
| if ( | |
| separation.get("status") == "ok" | |
| and len(found) == 2 | |
| and all(t.get("status") == "ok" for t in found) | |
| ): | |
| assignment = separation.get("assignment") or {} | |
| speakers = assignment.get("speakers") or [] | |
| if words is None or not assignment.get("resolved") or len(speakers) != 2: | |
| return found, [None, None] | |
| return found, [[w for w in words if w["speaker"] == s] for s in speakers] | |
| speakers = (row.get("stats") or {}).get("speakers") | |
| if words is not None and speakers == 1: | |
| return [row["tokens"], None], [words, []] | |
| return [row["tokens"], None], [None, None] | |
| def valid_frames(row): | |
| """Complete frames inside the source window on the source clock, as the loader requires.""" | |
| rate = row["audio"]["sample_rate"] | |
| valid = row["tokens"]["samples_24k"] // HOP | |
| while valid * rate * 2 > (row["end_sample"] - row["start_sample"]) * 25: | |
| valid -= 1 | |
| return valid | |
| def trainer_row(row, run, repo, split, revision, tokenize, out): | |
| tokens = row["tokens"] | |
| valid = valid_frames(row) | |
| audio, words = tracks_of(row) | |
| text = [] | |
| for index, track in enumerate(words, 1): | |
| if track is None: | |
| text.append(None) | |
| continue | |
| array = text_rows(track, valid, tokenize) | |
| if not len(array): | |
| text.append({"count": 0}) | |
| continue | |
| path = Path("text") / f"{tokens['key']}-text{index}.npy" | |
| buffer = io.BytesIO() | |
| np.save(buffer, array, allow_pickle=False) | |
| (out / path).write_bytes(buffer.getvalue()) | |
| text.append( | |
| { | |
| "path": str(path), | |
| "sha256": digest(buffer.getvalue()), | |
| "count": len(array), | |
| } | |
| ) | |
| return { | |
| "schema_version": 2, | |
| "recording_id": row["recording_id"], | |
| "source_group": group(row), | |
| "split": split, | |
| "split_status": "published_group_split" | |
| if "split" in row | |
| else "provisional_feed_and_recording_holdout", | |
| "source": row["source"], | |
| "audio": row["audio"], | |
| "metadata": { | |
| # Revisions published before 17 November carry these; later ones keep them out of the rows. | |
| "policy": row.get("policy"), | |
| "qualification_status": row.get("qualification_status"), | |
| }, | |
| "window": { | |
| "start_sample": row["start_sample"], | |
| "end_sample": row["end_sample"], | |
| }, | |
| "codec": { | |
| **CODEC, | |
| "revision": revision, | |
| "implementation": tokens["codec"]["codec_implementation"], | |
| }, | |
| "tokens": { | |
| "frames": tokens["frames"], | |
| "valid_frames": valid, | |
| "dtype": "uint16", | |
| "piece_frame_boundaries": tokens.get("piece_frame_boundaries") or [], | |
| "mixed_sha256": tokens["sha256"], | |
| "tracks": [ | |
| None | |
| if t is None | |
| else {"path": f"tokens/{t['key']}.npy", "sha256": t["sha256"]} | |
| for t in audio | |
| ], | |
| }, | |
| "text": text, | |
| "span": { | |
| "repo": repo, | |
| "run": run, | |
| "key": tokens["key"], | |
| "dataset": row["dataset"], | |
| }, | |
| } | |
| def check_splits(staged): | |
| """Fail when one span has different splits in different datasets: a stage would | |
| train on a span another stage holds out.""" | |
| splits = {} | |
| for stage_rows in staged.values(): | |
| for identity, row in stage_rows.items(): | |
| splits.setdefault(identity, set()).add(row["split"]) | |
| mixed = {k: sorted(v) for k, v in splits.items() if len(v) > 1} | |
| if mixed: | |
| examples = list(mixed.items())[:5] | |
| raise ValueError( | |
| f"{len(mixed)} spans have different splits across datasets: {examples}" | |
| ) | |
| def main(): | |
| p = argparse.ArgumentParser(description=__doc__) | |
| p.add_argument( | |
| "--raw", type=Path, required=True, help="Directory holding the downloaded repos" | |
| ) | |
| p.add_argument("--out", type=Path, required=True) | |
| p.add_argument("--salt", default="nano-s2s-sanity-v1") | |
| p.add_argument( | |
| "--every", type=int, default=20, help="Hold out one group in this many" | |
| ) | |
| p.add_argument( | |
| "--tokenizer", type=Path, help="Moshi's SentencePiece model; default: download" | |
| ) | |
| p.add_argument( | |
| "--stages", | |
| nargs="+", | |
| choices=list(STAGES), | |
| default=list(STAGES), | |
| help="Stages to build; each needs its datasets under --raw. Default: all", | |
| ) | |
| args = p.parse_args() | |
| stages = {stage: STAGES[stage] for stage in STAGES if stage in args.stages} | |
| import sentencepiece | |
| tokenizer = args.tokenizer | |
| if tokenizer is None: | |
| from huggingface_hub import hf_hub_download | |
| tokenizer = hf_hub_download(TOKENIZER[0], TOKENIZER[1], revision=TOKENIZER[2]) | |
| model = sentencepiece.SentencePieceProcessor(model_file=str(tokenizer)) | |
| tokenize = model.encode | |
| for folder in ("tokens", "text"): | |
| (args.out / folder).mkdir(parents=True, exist_ok=True) | |
| audit, by_span, revisions, sources = {}, {}, set(), {} | |
| staged = {stage: {} for stage in stages} | |
| for stage, repos in stages.items(): | |
| for repo in repos: | |
| _, rows, problems, totals = audit_repo(args.raw / repo) | |
| audit[repo] = {"problems": problems, "runs": totals} | |
| archives = {} | |
| for run, row in rows: | |
| if any( | |
| p.get("recording_id") == row["recording_id"] | |
| and p.get("start_sample") == row["start_sample"] | |
| for p in problems | |
| ): | |
| continue | |
| revisions.add(row["tokens"]["codec"]["codec_revision"]) | |
| identity = (row["recording_id"], row["start_sample"], row["end_sample"]) | |
| seen = by_span.setdefault(identity, (row["tokens"]["sha256"], set())) | |
| if seen[0] != row["tokens"]["sha256"]: | |
| problems.append( | |
| { | |
| "run": run, | |
| "recording_id": row["recording_id"], | |
| "problem": "same span has different tokens across repos", | |
| } | |
| ) | |
| seen[1].add(repo) | |
| sources.setdefault(identity, set()).add(stage) | |
| archive = archives.get(run) or zipfile.ZipFile( | |
| args.raw / repo / run / "tokens.npz" | |
| ) | |
| archives[run] = archive | |
| audio, _ = tracks_of(row) | |
| for record in [row["tokens"], *audio]: | |
| if record is None: | |
| continue | |
| target = args.out / "tokens" / f"{record['key']}.npy" | |
| if not target.exists(): | |
| target.write_bytes(archive.read(record["key"] + ".npy")) | |
| if "split" in row: | |
| split = row["split"] | |
| else: | |
| split = ( | |
| "validation" | |
| if held_out(group(row), args.salt, args.every) | |
| else "train" | |
| ) | |
| staged[stage][identity] = trainer_row( | |
| row, run, repo, split, None, tokenize, args.out | |
| ) | |
| check_splits(staged) | |
| if len(revisions) != 1: | |
| raise ValueError(f"Expected one codec revision, found {sorted(revisions)}") | |
| (revision,) = revisions | |
| # Nesting: each conversational span is a mid-training span and each mid-training span a podcast pretraining span. | |
| nesting = {} | |
| for inner, outer in ( | |
| ("duplex-conversational-podcasts-mimi", "duplex-mid-training-podcasts-mimi"), | |
| ("duplex-mid-training-podcasts-mimi", "duplex-pre-training-podcasts-mimi"), | |
| ): | |
| if not {inner, outer} <= {repo for repos in stages.values() for repo in repos}: | |
| continue | |
| inside = [repos for _, repos in by_span.values() if inner in repos] | |
| nesting[f"{inner} in {outer}"] = { | |
| "spans": len(inside), | |
| "missing": sum(outer not in repos for repos in inside), | |
| } | |
| # Every stage scores every validation set: its manifest holds every validation span. | |
| held = {} | |
| for stage in stages: | |
| for identity, row in staged[stage].items(): | |
| if row["split"] == "validation": | |
| held.setdefault(identity, row) | |
| for identity, row in held.items(): | |
| row["validation_sets"] = sorted(sources[identity]) | |
| summary = { | |
| "salt": args.salt, | |
| "every": args.every, | |
| "codec_revision": revision, | |
| "tokenizer": { | |
| "repo": TOKENIZER[0], | |
| "file": TOKENIZER[1], | |
| "revision": TOKENIZER[2], | |
| }, | |
| "audit": audit, | |
| "nesting": nesting, | |
| "stages": {}, | |
| } | |
| for stage in stages: | |
| rows = [r for r in staged[stage].values() if r["split"] != "validation"] | |
| rows += list(held.values()) | |
| lines = [] | |
| for row in rows: | |
| row["codec"]["revision"] = revision | |
| lines.append(json.dumps(row, sort_keys=True) + "\n") | |
| path = args.out / f"{stage}.jsonl" | |
| path.write_text("".join(lines)) | |
| split = {} | |
| for row in rows: | |
| name = row["split"] | |
| if name == "validation": | |
| name = "validation:" + ",".join(row["validation_sets"]) | |
| entry = split.setdefault( | |
| name, | |
| { | |
| "spans": 0, | |
| "hours": 0.0, | |
| "groups": set(), | |
| "recordings": set(), | |
| "two_tracks": 0, | |
| "text_tracks": 0, | |
| "text_tokens": 0, | |
| }, | |
| ) | |
| entry["spans"] += 1 | |
| entry["hours"] += row["tokens"]["valid_frames"] / 12.5 / 3600 | |
| entry["groups"].add(row["source_group"]) | |
| entry["recordings"].add(row["recording_id"]) | |
| entry["two_tracks"] += all(row["tokens"]["tracks"]) | |
| entry["text_tracks"] += sum(t is not None for t in row["text"]) | |
| entry["text_tokens"] += sum(t["count"] for t in row["text"] if t) | |
| summary["stages"][stage] = { | |
| "manifest": path.name, | |
| "manifest_sha256": digest(path.read_bytes()), | |
| "splits": { | |
| k: {**v, "groups": len(v["groups"]), "recordings": len(v["recordings"])} | |
| for k, v in sorted(split.items()) | |
| }, | |
| } | |
| (args.out / "summary.json").write_text(json.dumps(summary, indent=1) + "\n") | |
| print( | |
| json.dumps( | |
| { | |
| "stages": summary["stages"], | |
| "nesting": nesting, | |
| "problems": {r: a["problems"][:20] for r, a in audit.items()}, | |
| "problem_counts": {r: len(a["problems"]) for r, a in audit.items()}, | |
| }, | |
| indent=1, | |
| ) | |
| ) | |
| if __name__ == "__main__": | |
| main() | |