"""Join several annotation runs into one training set that alpa.train accepts. python scripts/join_annotation_runs.py --runs male_a female male_b --output joint Each run is a folder holding ``annotations/`` (from alpa.annotate) and ``features/`` (from alpa.prepare). The trainer refuses rows from different annotation runs because their settings could differ, so this script first checks that the runs were made with the same features, front end, recogniser, alignment settings and limits. It then relabels the rows with one joint identity and hard-links the feature files into one folder (same disk; no copies). Splits are kept, and the script stops if any training row shares its text with a held-out row of any run. """ import argparse import hashlib import json import os from pathlib import Path SAME = ("annotation_version", "acoustic", "frontend", "aligner", "alignment", "dataset_revision") SPLITS = ("train", "validation", "test") def join(runs, output): metas = {run: json.loads((run / "annotations/annotations.json").read_text()) for run in runs} for key in SAME: if len({json.dumps(meta["identity"].get(key), sort_keys=True) for meta in metas.values()}) != 1: raise ValueError(f"the runs differ in {key}") if len({json.dumps(meta["limits"], sort_keys=True) for meta in metas.values()}) != 1: raise ValueError("the runs differ in their acceptance limits") if not all(meta.get("complete") for meta in metas.values()): raise ValueError("an annotation run is incomplete") fingerprint = hashlib.sha256(json.dumps({"joined": sorted(meta["fingerprint"] for meta in metas.values())}).encode()).hexdigest() rows = {split: [] for split in SPLITS} for run in runs: for split in SPLITS: for line in (run / f"annotations/{split}.jsonl").read_text(encoding="utf-8").split("\n"): if not line.strip(): continue row = json.loads(line) if row["annotation"] != metas[run]["fingerprint"] or row["split"] != split: raise ValueError(f"unexpected row in {run}: {row['id']}") source, target = run / "features" / row["path"], output / "features" / row["path"] target.parent.mkdir(parents=True, exist_ok=True) if not target.exists(): os.link(source, target) row["annotation"] = fingerprint rows[split].append(row) ids = [row["id"] for split in SPLITS for row in rows[split]] if len(ids) != len(set(ids)): raise ValueError("the runs share clip ids") held = {row["text_hash"] for split in ("validation", "test") for row in rows[split]} leaks = [row["id"] for row in rows["train"] if row["text_hash"] in held] if leaks: raise ValueError(f"{len(leaks)} training rows share their text with a held-out row, e.g. {leaks[:3]}") (output / "annotations").mkdir(parents=True, exist_ok=True) for split, items in rows.items(): (output / f"annotations/{split}.jsonl").write_text("".join(json.dumps(row, ensure_ascii=False) + "\n" for row in items), encoding="utf-8") first = next(iter(metas.values())) meta = {"identity": {"joined_from": [meta["fingerprint"] for meta in metas.values()], **{key: first["identity"][key] for key in SAME}}, "fingerprint": fingerprint, "complete": True, "acoustic_root": str((output / "features").resolve()), "limits": first["limits"], "counts": {split: len(items) for split, items in rows.items()}} (output / "annotations/annotations.json").write_text(json.dumps(meta, ensure_ascii=False, indent=1), encoding="utf-8") return rows def main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--runs", nargs="+", type=Path, required=True, help="folders that each hold annotations/ and features/") parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() rows = join(args.runs, args.output) for split, items in rows.items(): by = {} for row in items: clips, frames = by.get(row["speaker"], (0, 0)) by[row["speaker"]] = (clips + 1, frames + row["frames"]) print(split, {voice: f"{clips} clips, {frames * 256 / 24000 / 3600:.2f} h" for voice, (clips, frames) in by.items()}) if __name__ == "__main__": main()