Data step builds from the public datasets
Browse files- README.md +1 -1
- scripts/README.md +3 -5
- scripts/duplex/model/hf_release.py +17 -4
README.md
CHANGED
|
@@ -30,7 +30,7 @@ This page is the training report for the runs of 17 November 2026. The training
|
|
| 30 |
### Data
|
| 31 |
|
| 32 |
- **Pre-training:** 749 hours of English speech, 408 hours from YODAS3 and 341 hours from public podcasts.
|
| 33 |
-
- **Conversational:** 156 hours of two-speaker podcast conversation with frequent turn-taking and a separate audio track per speaker. This is a subset of the pre-training data
|
| 34 |
- Whole recordings and podcast feeds are held out for validation and testing, about 4 hours each per source.
|
| 35 |
|
| 36 |
### Schedule
|
|
|
|
| 30 |
### Data
|
| 31 |
|
| 32 |
- **Pre-training:** 749 hours of English speech, 408 hours from YODAS3 and 341 hours from public podcasts.
|
| 33 |
+
- **Conversational:** 156 hours of two-speaker podcast conversation with frequent turn-taking and a separate audio track per speaker. This is a subset of the pre-training data.
|
| 34 |
- Whole recordings and podcast feeds are held out for validation and testing, about 4 hours each per source.
|
| 35 |
|
| 36 |
### Schedule
|
scripts/README.md
CHANGED
|
@@ -23,15 +23,13 @@ Python 3.12 with `torch` (the runs used 2.8.0 with CUDA 12.8), `numpy`, `sentenc
|
|
| 23 |
Download the datasets into one directory, one folder per dataset named as on the Hub, then build the training release:
|
| 24 |
|
| 25 |
```bash
|
| 26 |
-
for name in pre-training-yodas3 pre-training-podcasts
|
| 27 |
hf download --repo-type dataset Trelis/duplex-$name-mimi --local-dir raw/duplex-$name-mimi
|
| 28 |
done
|
| 29 |
-
python -m duplex.model.hf_release --raw raw --out release
|
| 30 |
```
|
| 31 |
|
| 32 |
-
The
|
| 33 |
-
|
| 34 |
-
The release holds `pretraining.jsonl`, `mid_training.jsonl` and `conversational.jsonl`. No stage of this report trains on mid-training; the script still reads it to check that the sets nest.
|
| 35 |
|
| 36 |
## Training
|
| 37 |
|
|
|
|
| 23 |
Download the datasets into one directory, one folder per dataset named as on the Hub, then build the training release:
|
| 24 |
|
| 25 |
```bash
|
| 26 |
+
for name in pre-training-yodas3 pre-training-podcasts conversational-podcasts; do
|
| 27 |
hf download --repo-type dataset Trelis/duplex-$name-mimi --local-dir raw/duplex-$name-mimi
|
| 28 |
done
|
| 29 |
+
python -m duplex.model.hf_release --raw raw --out release --stages pretraining conversational
|
| 30 |
```
|
| 31 |
|
| 32 |
+
The release holds `pretraining.jsonl` and `conversational.jsonl`. The `mid_training` stage of the script reads a dataset that is not public; no run of this report trained on it. The runs were built with all three stages, which gives the same training rows and also tags held-out spans with a mid-training validation set.
|
|
|
|
|
|
|
| 33 |
|
| 34 |
## Training
|
| 35 |
|
scripts/duplex/model/hf_release.py
CHANGED
|
@@ -14,6 +14,9 @@ dataset, so a span with different splits in different datasets fails the build.
|
|
| 14 |
salted hash (a group is the podcast feed or the YODAS recording), the same groups in every stage, so the
|
| 15 |
nested sets (conversational within mid-training within podcast pretraining) cannot leak. Every stage
|
| 16 |
manifest holds every validation span, tagged in `validation_sets` with the stages whose datasets hold it.
|
|
|
|
|
|
|
|
|
|
| 17 |
"""
|
| 18 |
|
| 19 |
import argparse
|
|
@@ -388,7 +391,15 @@ def main():
|
|
| 388 |
p.add_argument(
|
| 389 |
"--tokenizer", type=Path, help="Moshi's SentencePiece model; default: download"
|
| 390 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 391 |
args = p.parse_args()
|
|
|
|
| 392 |
import sentencepiece
|
| 393 |
|
| 394 |
tokenizer = args.tokenizer
|
|
@@ -401,8 +412,8 @@ def main():
|
|
| 401 |
for folder in ("tokens", "text"):
|
| 402 |
(args.out / folder).mkdir(parents=True, exist_ok=True)
|
| 403 |
audit, by_span, revisions, sources = {}, {}, set(), {}
|
| 404 |
-
staged = {stage: {} for stage in
|
| 405 |
-
for stage, repos in
|
| 406 |
for repo in repos:
|
| 407 |
_, rows, problems, totals = audit_repo(args.raw / repo)
|
| 408 |
audit[repo] = {"problems": problems, "runs": totals}
|
|
@@ -459,6 +470,8 @@ def main():
|
|
| 459 |
("duplex-conversational-podcasts-mimi", "duplex-mid-training-podcasts-mimi"),
|
| 460 |
("duplex-mid-training-podcasts-mimi", "duplex-pre-training-podcasts-mimi"),
|
| 461 |
):
|
|
|
|
|
|
|
| 462 |
inside = [repos for _, repos in by_span.values() if inner in repos]
|
| 463 |
nesting[f"{inner} in {outer}"] = {
|
| 464 |
"spans": len(inside),
|
|
@@ -466,7 +479,7 @@ def main():
|
|
| 466 |
}
|
| 467 |
# Every stage scores every validation set: its manifest holds every validation span.
|
| 468 |
held = {}
|
| 469 |
-
for stage in
|
| 470 |
for identity, row in staged[stage].items():
|
| 471 |
if row["split"] == "validation":
|
| 472 |
held.setdefault(identity, row)
|
|
@@ -485,7 +498,7 @@ def main():
|
|
| 485 |
"nesting": nesting,
|
| 486 |
"stages": {},
|
| 487 |
}
|
| 488 |
-
for stage in
|
| 489 |
rows = [r for r in staged[stage].values() if r["split"] != "validation"]
|
| 490 |
rows += list(held.values())
|
| 491 |
lines = []
|
|
|
|
| 14 |
salted hash (a group is the podcast feed or the YODAS recording), the same groups in every stage, so the
|
| 15 |
nested sets (conversational within mid-training within podcast pretraining) cannot leak. Every stage
|
| 16 |
manifest holds every validation span, tagged in `validation_sets` with the stages whose datasets hold it.
|
| 17 |
+
|
| 18 |
+
`--stages` builds a subset of the stages and reads only their datasets; a stage left out is absent from
|
| 19 |
+
`validation_sets` and from the nesting check.
|
| 20 |
"""
|
| 21 |
|
| 22 |
import argparse
|
|
|
|
| 391 |
p.add_argument(
|
| 392 |
"--tokenizer", type=Path, help="Moshi's SentencePiece model; default: download"
|
| 393 |
)
|
| 394 |
+
p.add_argument(
|
| 395 |
+
"--stages",
|
| 396 |
+
nargs="+",
|
| 397 |
+
choices=list(STAGES),
|
| 398 |
+
default=list(STAGES),
|
| 399 |
+
help="Stages to build; each needs its datasets under --raw. Default: all",
|
| 400 |
+
)
|
| 401 |
args = p.parse_args()
|
| 402 |
+
stages = {stage: STAGES[stage] for stage in STAGES if stage in args.stages}
|
| 403 |
import sentencepiece
|
| 404 |
|
| 405 |
tokenizer = args.tokenizer
|
|
|
|
| 412 |
for folder in ("tokens", "text"):
|
| 413 |
(args.out / folder).mkdir(parents=True, exist_ok=True)
|
| 414 |
audit, by_span, revisions, sources = {}, {}, set(), {}
|
| 415 |
+
staged = {stage: {} for stage in stages}
|
| 416 |
+
for stage, repos in stages.items():
|
| 417 |
for repo in repos:
|
| 418 |
_, rows, problems, totals = audit_repo(args.raw / repo)
|
| 419 |
audit[repo] = {"problems": problems, "runs": totals}
|
|
|
|
| 470 |
("duplex-conversational-podcasts-mimi", "duplex-mid-training-podcasts-mimi"),
|
| 471 |
("duplex-mid-training-podcasts-mimi", "duplex-pre-training-podcasts-mimi"),
|
| 472 |
):
|
| 473 |
+
if not {inner, outer} <= {repo for repos in stages.values() for repo in repos}:
|
| 474 |
+
continue
|
| 475 |
inside = [repos for _, repos in by_span.values() if inner in repos]
|
| 476 |
nesting[f"{inner} in {outer}"] = {
|
| 477 |
"spans": len(inside),
|
|
|
|
| 479 |
}
|
| 480 |
# Every stage scores every validation set: its manifest holds every validation span.
|
| 481 |
held = {}
|
| 482 |
+
for stage in stages:
|
| 483 |
for identity, row in staged[stage].items():
|
| 484 |
if row["split"] == "validation":
|
| 485 |
held.setdefault(identity, row)
|
|
|
|
| 498 |
"nesting": nesting,
|
| 499 |
"stages": {},
|
| 500 |
}
|
| 501 |
+
for stage in stages:
|
| 502 |
rows = [r for r in staged[stage].values() if r["split"] != "validation"]
|
| 503 |
rows += list(held.values())
|
| 504 |
lines = []
|