from __future__ import annotations import argparse import json import os import subprocess from concurrent.futures import ProcessPoolExecutor, as_completed from pathlib import Path from typing import Any import numpy as np from tqdm import tqdm FORMAT_VERSION = "psi0_audio_wav_v1" def ffmpeg_bin() -> str: return os.environ.get("PSI_FFMPEG_BIN", "ffmpeg") def ffprobe_bin() -> str: return os.environ.get("PSI_FFPROBE_BIN", "ffprobe") def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description=( "Materialize deterministic 16 kHz mono float32 WAV inputs and rewrite " "their JSONL manifest." ) ) parser.add_argument("--dataset-root", type=Path, required=True) parser.add_argument("--metadata-jsonl", type=Path, required=True) parser.add_argument("--output-root", type=Path, required=True) parser.add_argument("--output-metadata-jsonl", default="metadata.jsonl") parser.add_argument("--input-kind", choices=("auto", "npy", "media"), required=True) parser.add_argument("--input-path-key", default="audio_path") parser.add_argument("--output-path-key", default="audio_path") parser.add_argument( "--absolute-path-keys", default="converted_path,motion_dir,motion_path", help="Existing relative paths under these keys are made absolute before writing the new manifest.", ) parser.add_argument( "--strip-input-prefix", default="", help="Optional relative prefix removed from the input path before placing it under output/audio/.", ) parser.add_argument("--output-audio-dir", default="audio") parser.add_argument("--sample-rate", type=int, default=16000) parser.add_argument("--num-workers", type=int, default=32) parser.add_argument("--limit", type=int, default=0) parser.add_argument( "--skip-media-without-audio", action=argparse.BooleanOptionalAction, default=False, help="Skip manifest media fallbacks that have no audio stream, matching the historical exporter.", ) parser.add_argument("--overwrite", action="store_true") return parser.parse_args() def resolve_path(path: Path, root: Path) -> Path: return path.expanduser().resolve() if path.is_absolute() else (root / path).expanduser().resolve() def load_rows(path: Path, limit: int) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] with path.open("r", encoding="utf-8") as handle: for line_no, line in enumerate(handle, start=1): line = line.strip() if not line: continue try: rows.append(json.loads(line)) except json.JSONDecodeError as exc: raise ValueError(f"Bad JSONL at {path}:{line_no}") from exc if limit > 0 and len(rows) >= limit: break return rows def load_npy(path: Path) -> np.ndarray: try: audio = np.load(path, allow_pickle=False) except ValueError: audio = np.load(path, allow_pickle=True) if isinstance(audio, np.ndarray) and audio.dtype == object: audio = np.asarray(audio.tolist()) audio = np.asarray(audio, dtype=np.float32).reshape(-1).copy() if audio.size == 0: raise RuntimeError(f"Empty NPY audio: {path}") return audio def decode_media(path: Path, sample_rate: int) -> np.ndarray: command = [ ffmpeg_bin(), "-v", "error", "-threads", "1", "-i", str(path), "-ac", "1", "-ar", str(sample_rate), "-f", "f32le", "-", ] process = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) if process.returncode != 0: message = process.stderr.decode("utf-8", errors="ignore").strip() raise RuntimeError(f"ffmpeg failed to decode {path}: {message}") audio = np.frombuffer(process.stdout, dtype=np.float32).copy() if audio.size == 0: raise RuntimeError(f"Empty media audio: {path}") return audio def has_audio_stream(path: Path) -> bool: process = subprocess.run( [ ffprobe_bin(), "-v", "error", "-select_streams", "a:0", "-show_entries", "stream=index", "-of", "csv=p=0", str(path), ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, ) return process.returncode == 0 and bool(process.stdout.strip()) def load_input(path: Path, input_kind: str, sample_rate: int) -> np.ndarray: if input_kind == "auto": input_kind = "npy" if path.suffix.lower() == ".npy" else "media" return load_npy(path) if input_kind == "npy" else decode_media(path, sample_rate) def write_float_wav(path: Path, audio: np.ndarray, sample_rate: int) -> None: command = [ ffmpeg_bin(), "-v", "error", "-threads", "1", "-f", "f32le", "-ar", str(sample_rate), "-ac", "1", "-i", "-", "-map_metadata", "-1", "-c:a", "pcm_f32le", "-f", "wav", "-y", str(path), ] process = subprocess.run( command, input=np.ascontiguousarray(audio, dtype=np.float32).tobytes(), stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, check=False, ) if process.returncode != 0: message = process.stderr.decode("utf-8", errors="ignore").strip() raise RuntimeError(f"ffmpeg failed to write {path}: {message}") def verify_wav(path: Path, expected: np.ndarray, sample_rate: int) -> None: actual = decode_media(path, sample_rate) if actual.shape != expected.shape or not np.array_equal(actual, expected): max_abs = float("inf") if actual.shape == expected.shape and actual.size: max_abs = float(np.max(np.abs(actual - expected))) raise RuntimeError( f"WAV round-trip mismatch: {path}; expected={expected.shape}, " f"actual={actual.shape}, max_abs={max_abs}" ) def materialize_one( input_path_value: str, output_path_value: str, input_kind: str, sample_rate: int, overwrite: bool, ) -> tuple[str, str]: input_path = Path(input_path_value) output_path = Path(output_path_value) audio = load_input(input_path, input_kind, sample_rate) if output_path.exists() and not overwrite: verify_wav(output_path, audio, sample_rate) return str(output_path), "verified" output_path.parent.mkdir(parents=True, exist_ok=True) temporary = output_path.with_name(f".{output_path.name}.{os.getpid()}.tmp.wav") try: write_float_wav(temporary, audio, sample_rate) verify_wav(temporary, audio, sample_rate) os.replace(temporary, output_path) finally: try: temporary.unlink() except FileNotFoundError: pass return str(output_path), "written" def relative_input_path(path: Path, dataset_root: Path, strip_prefix: Path | None) -> Path: try: relative = path.relative_to(dataset_root) except ValueError as exc: raise ValueError(f"Input path is outside dataset root: {path} vs {dataset_root}") from exc if strip_prefix is not None: try: relative = relative.relative_to(strip_prefix) except ValueError: # Historical manifests may fall back from a missing NPY to the # original media path. Keep that relative path and decode it with # the exact legacy ffmpeg parameters. pass return relative def write_json_atomic(path: Path, payload: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") os.replace(temporary, path) def write_jsonl_atomic(path: Path, rows: list[dict[str, Any]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") with temporary.open("w", encoding="utf-8") as handle: for row in rows: handle.write(json.dumps(row, ensure_ascii=False) + "\n") os.replace(temporary, path) def reusable_complete_output( marker_path: Path, output_manifest: Path, expected_marker: dict[str, Any], ) -> bool: if not marker_path.is_file() or not output_manifest.is_file(): return False try: marker = json.loads(marker_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return False for key, value in expected_marker.items(): if marker.get(key) != value: return False expected_rows = int(marker.get("rows", -1)) found_rows = 0 with output_manifest.open("r", encoding="utf-8") as handle: for line in handle: if not line.strip(): continue row = json.loads(line) audio_value = row.get(expected_marker["output_path_key"]) if not audio_value or not (Path(expected_marker["output_root"]) / str(audio_value)).is_file(): return False found_rows += 1 return found_rows == expected_rows def main() -> None: args = parse_args() if args.sample_rate <= 0 or args.num_workers <= 0 or args.limit < 0: raise ValueError("sample-rate/num-workers must be positive and limit must be non-negative") dataset_root = args.dataset_root.expanduser().resolve() metadata_path = resolve_path(args.metadata_jsonl, dataset_root) output_root = args.output_root.expanduser().resolve() output_manifest = resolve_path(Path(args.output_metadata_jsonl), output_root) marker_path = output_root / ".audio_wav_complete.json" strip_prefix = Path(args.strip_input_prefix) if str(args.strip_input_prefix).strip() else None absolute_path_keys = tuple( value.strip() for value in str(args.absolute_path_keys).split(",") if value.strip() ) source_stat = metadata_path.stat() expected_marker = { "format": FORMAT_VERSION, "dataset_root": str(dataset_root), "metadata_jsonl": str(metadata_path), "metadata_size": int(source_stat.st_size), "metadata_mtime_ns": int(source_stat.st_mtime_ns), "output_root": str(output_root), "input_kind": str(args.input_kind), "input_path_key": str(args.input_path_key), "output_path_key": str(args.output_path_key), "absolute_path_keys": list(absolute_path_keys), "strip_input_prefix": str(strip_prefix or ""), "output_audio_dir": str(args.output_audio_dir), "sample_rate": int(args.sample_rate), "limit": int(args.limit), "skip_media_without_audio": bool(args.skip_media_without_audio), } if not args.overwrite and reusable_complete_output(marker_path, output_manifest, expected_marker): marker = json.loads(marker_path.read_text(encoding="utf-8")) print( f"[audio wav] reuse complete source: rows={marker['rows']} files={marker['files']} " f"root={output_root}" ) return rows = load_rows(metadata_path, args.limit) output_rows: list[dict[str, Any]] = [] jobs_by_output: dict[str, tuple[Path, Path]] = {} skipped_no_audio = 0 for row_no, original_row in enumerate(rows): input_value = original_row.get(args.input_path_key) if input_value in (None, ""): raise ValueError(f"Missing {args.input_path_key!r} in {metadata_path} row {row_no}") input_path = resolve_path(Path(str(input_value)), dataset_root) if not input_path.is_file(): raise FileNotFoundError(f"Audio input does not exist: {input_path}") is_media_input = args.input_kind == "media" or ( args.input_kind == "auto" and input_path.suffix.lower() != ".npy" ) if is_media_input and args.skip_media_without_audio and not has_audio_stream(input_path): skipped_no_audio += 1 continue relative = relative_input_path(input_path, dataset_root, strip_prefix) output_relative = Path(args.output_audio_dir) / relative.with_suffix(".wav") output_path = output_root / output_relative output_key = str(output_path) previous = jobs_by_output.get(output_key) if previous is not None and previous[0] != input_path: raise RuntimeError(f"Two inputs map to the same WAV: {previous[0]} and {input_path}") jobs_by_output[output_key] = (input_path, output_path) row = dict(original_row) for key in absolute_path_keys: value = row.get(key) if value in (None, ""): continue row[key] = str(resolve_path(Path(str(value)), dataset_root)) row[args.output_path_key] = output_relative.as_posix() output_rows.append(row) written = 0 verified = 0 failures: list[str] = [] with ProcessPoolExecutor(max_workers=args.num_workers) as executor: futures = [ executor.submit( materialize_one, str(input_path), str(output_path), str(args.input_kind), int(args.sample_rate), bool(args.overwrite), ) for input_path, output_path in jobs_by_output.values() ] for future in tqdm(as_completed(futures), total=len(futures), desc="Audio WAV", unit="file"): try: _, status = future.result() except Exception as exc: # noqa: BLE001 - aggregate worker failures with context failures.append(str(exc)) continue if status == "written": written += 1 else: verified += 1 if failures: preview = "\n".join(failures[:10]) raise RuntimeError(f"Failed to materialize {len(failures)} WAV files:\n{preview}") write_jsonl_atomic(output_manifest, output_rows) marker = { **expected_marker, "rows": len(output_rows), "files": len(jobs_by_output), "written": written, "verified_existing": verified, "skipped_no_audio": skipped_no_audio, "wav_codec": "pcm_f32le", } write_json_atomic(marker_path, marker) print( f"[audio wav] complete rows={len(output_rows)} files={len(jobs_by_output)} " f"written={written} verified={verified} skipped_no_audio={skipped_no_audio} " f"manifest={output_manifest}" ) if __name__ == "__main__": main()