Download scripts/join_annotation_runs.py from 77ethers/alpa: direct link, hf CLI and curl.
- Browser
- Download file 4.47 kB
-
https://huggingface.co/77ethers/alpa/resolve/main/scripts/join_annotation_runs.py
- Command line
-
hf download hf://77ethers/alpa/scripts/join_annotation_runs.py
-
curl -L -o join_annotation_runs.py https://huggingface.co/77ethers/alpa/resolve/main/scripts/join_annotation_runs.py
4.47 kB
| """Join several annotation runs into one training set that alpa.train accepts. | |
| python scripts/join_annotation_runs.py --runs male_a female male_b --output joint | |
| Each run is a folder holding ``annotations/`` (from alpa.annotate) and ``features/`` (from alpa.prepare). The trainer | |
| refuses rows from different annotation runs because their settings could differ, so this script first checks that | |
| the runs were made with the same features, front end, recogniser, alignment settings and limits. It then relabels | |
| the rows with one joint identity and hard-links the feature files into one folder (same disk; no copies). Splits | |
| are kept, and the script stops if any training row shares its text with a held-out row of any run. | |
| """ | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| SAME = ("annotation_version", "acoustic", "frontend", "aligner", "alignment", "dataset_revision") | |
| SPLITS = ("train", "validation", "test") | |
| def join(runs, output): | |
| metas = {run: json.loads((run / "annotations/annotations.json").read_text()) for run in runs} | |
| for key in SAME: | |
| if len({json.dumps(meta["identity"].get(key), sort_keys=True) for meta in metas.values()}) != 1: | |
| raise ValueError(f"the runs differ in {key}") | |
| if len({json.dumps(meta["limits"], sort_keys=True) for meta in metas.values()}) != 1: | |
| raise ValueError("the runs differ in their acceptance limits") | |
| if not all(meta.get("complete") for meta in metas.values()): | |
| raise ValueError("an annotation run is incomplete") | |
| fingerprint = hashlib.sha256(json.dumps({"joined": sorted(meta["fingerprint"] for meta in metas.values())}).encode()).hexdigest() | |
| rows = {split: [] for split in SPLITS} | |
| for run in runs: | |
| for split in SPLITS: | |
| for line in (run / f"annotations/{split}.jsonl").read_text(encoding="utf-8").split("\n"): | |
| if not line.strip(): | |
| continue | |
| row = json.loads(line) | |
| if row["annotation"] != metas[run]["fingerprint"] or row["split"] != split: | |
| raise ValueError(f"unexpected row in {run}: {row['id']}") | |
| source, target = run / "features" / row["path"], output / "features" / row["path"] | |
| target.parent.mkdir(parents=True, exist_ok=True) | |
| if not target.exists(): | |
| os.link(source, target) | |
| row["annotation"] = fingerprint | |
| rows[split].append(row) | |
| ids = [row["id"] for split in SPLITS for row in rows[split]] | |
| if len(ids) != len(set(ids)): | |
| raise ValueError("the runs share clip ids") | |
| held = {row["text_hash"] for split in ("validation", "test") for row in rows[split]} | |
| leaks = [row["id"] for row in rows["train"] if row["text_hash"] in held] | |
| if leaks: | |
| raise ValueError(f"{len(leaks)} training rows share their text with a held-out row, e.g. {leaks[:3]}") | |
| (output / "annotations").mkdir(parents=True, exist_ok=True) | |
| for split, items in rows.items(): | |
| (output / f"annotations/{split}.jsonl").write_text("".join(json.dumps(row, ensure_ascii=False) + "\n" for row in items), encoding="utf-8") | |
| first = next(iter(metas.values())) | |
| meta = {"identity": {"joined_from": [meta["fingerprint"] for meta in metas.values()], **{key: first["identity"][key] for key in SAME}}, | |
| "fingerprint": fingerprint, "complete": True, "acoustic_root": str((output / "features").resolve()), "limits": first["limits"], | |
| "counts": {split: len(items) for split, items in rows.items()}} | |
| (output / "annotations/annotations.json").write_text(json.dumps(meta, ensure_ascii=False, indent=1), encoding="utf-8") | |
| return rows | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| parser.add_argument("--runs", nargs="+", type=Path, required=True, help="folders that each hold annotations/ and features/") | |
| parser.add_argument("--output", type=Path, required=True) | |
| args = parser.parse_args() | |
| rows = join(args.runs, args.output) | |
| for split, items in rows.items(): | |
| by = {} | |
| for row in items: | |
| clips, frames = by.get(row["speaker"], (0, 0)) | |
| by[row["speaker"]] = (clips + 1, frames + row["frames"]) | |
| print(split, {voice: f"{clips} clips, {frames * 256 / 24000 / 3600:.2f} h" for voice, (clips, frames) in by.items()}) | |
| if __name__ == "__main__": | |
| main() | |