Download scripts/data/materialize_audio_wav.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 14.9 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/materialize_audio_wav.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/data/materialize_audio_wav.py
-
curl -L -o materialize_audio_wav.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/materialize_audio_wav.py
14.9 kB
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import subprocess | |
| from concurrent.futures import ProcessPoolExecutor, as_completed | |
| from pathlib import Path | |
| from typing import Any | |
| import numpy as np | |
| from tqdm import tqdm | |
| FORMAT_VERSION = "psi0_audio_wav_v1" | |
| def ffmpeg_bin() -> str: | |
| return os.environ.get("PSI_FFMPEG_BIN", "ffmpeg") | |
| def ffprobe_bin() -> str: | |
| return os.environ.get("PSI_FFPROBE_BIN", "ffprobe") | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser( | |
| description=( | |
| "Materialize deterministic 16 kHz mono float32 WAV inputs and rewrite " | |
| "their JSONL manifest." | |
| ) | |
| ) | |
| parser.add_argument("--dataset-root", type=Path, required=True) | |
| parser.add_argument("--metadata-jsonl", type=Path, required=True) | |
| parser.add_argument("--output-root", type=Path, required=True) | |
| parser.add_argument("--output-metadata-jsonl", default="metadata.jsonl") | |
| parser.add_argument("--input-kind", choices=("auto", "npy", "media"), required=True) | |
| parser.add_argument("--input-path-key", default="audio_path") | |
| parser.add_argument("--output-path-key", default="audio_path") | |
| parser.add_argument( | |
| "--absolute-path-keys", | |
| default="converted_path,motion_dir,motion_path", | |
| help="Existing relative paths under these keys are made absolute before writing the new manifest.", | |
| ) | |
| parser.add_argument( | |
| "--strip-input-prefix", | |
| default="", | |
| help="Optional relative prefix removed from the input path before placing it under output/audio/.", | |
| ) | |
| parser.add_argument("--output-audio-dir", default="audio") | |
| parser.add_argument("--sample-rate", type=int, default=16000) | |
| parser.add_argument("--num-workers", type=int, default=32) | |
| parser.add_argument("--limit", type=int, default=0) | |
| parser.add_argument( | |
| "--skip-media-without-audio", | |
| action=argparse.BooleanOptionalAction, | |
| default=False, | |
| help="Skip manifest media fallbacks that have no audio stream, matching the historical exporter.", | |
| ) | |
| parser.add_argument("--overwrite", action="store_true") | |
| return parser.parse_args() | |
| def resolve_path(path: Path, root: Path) -> Path: | |
| return path.expanduser().resolve() if path.is_absolute() else (root / path).expanduser().resolve() | |
| def load_rows(path: Path, limit: int) -> list[dict[str, Any]]: | |
| rows: list[dict[str, Any]] = [] | |
| with path.open("r", encoding="utf-8") as handle: | |
| for line_no, line in enumerate(handle, start=1): | |
| line = line.strip() | |
| if not line: | |
| continue | |
| try: | |
| rows.append(json.loads(line)) | |
| except json.JSONDecodeError as exc: | |
| raise ValueError(f"Bad JSONL at {path}:{line_no}") from exc | |
| if limit > 0 and len(rows) >= limit: | |
| break | |
| return rows | |
| def load_npy(path: Path) -> np.ndarray: | |
| try: | |
| audio = np.load(path, allow_pickle=False) | |
| except ValueError: | |
| audio = np.load(path, allow_pickle=True) | |
| if isinstance(audio, np.ndarray) and audio.dtype == object: | |
| audio = np.asarray(audio.tolist()) | |
| audio = np.asarray(audio, dtype=np.float32).reshape(-1).copy() | |
| if audio.size == 0: | |
| raise RuntimeError(f"Empty NPY audio: {path}") | |
| return audio | |
| def decode_media(path: Path, sample_rate: int) -> np.ndarray: | |
| command = [ | |
| ffmpeg_bin(), | |
| "-v", | |
| "error", | |
| "-threads", | |
| "1", | |
| "-i", | |
| str(path), | |
| "-ac", | |
| "1", | |
| "-ar", | |
| str(sample_rate), | |
| "-f", | |
| "f32le", | |
| "-", | |
| ] | |
| process = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) | |
| if process.returncode != 0: | |
| message = process.stderr.decode("utf-8", errors="ignore").strip() | |
| raise RuntimeError(f"ffmpeg failed to decode {path}: {message}") | |
| audio = np.frombuffer(process.stdout, dtype=np.float32).copy() | |
| if audio.size == 0: | |
| raise RuntimeError(f"Empty media audio: {path}") | |
| return audio | |
| def has_audio_stream(path: Path) -> bool: | |
| process = subprocess.run( | |
| [ | |
| ffprobe_bin(), | |
| "-v", | |
| "error", | |
| "-select_streams", | |
| "a:0", | |
| "-show_entries", | |
| "stream=index", | |
| "-of", | |
| "csv=p=0", | |
| str(path), | |
| ], | |
| stdout=subprocess.PIPE, | |
| stderr=subprocess.PIPE, | |
| check=False, | |
| ) | |
| return process.returncode == 0 and bool(process.stdout.strip()) | |
| def load_input(path: Path, input_kind: str, sample_rate: int) -> np.ndarray: | |
| if input_kind == "auto": | |
| input_kind = "npy" if path.suffix.lower() == ".npy" else "media" | |
| return load_npy(path) if input_kind == "npy" else decode_media(path, sample_rate) | |
| def write_float_wav(path: Path, audio: np.ndarray, sample_rate: int) -> None: | |
| command = [ | |
| ffmpeg_bin(), | |
| "-v", | |
| "error", | |
| "-threads", | |
| "1", | |
| "-f", | |
| "f32le", | |
| "-ar", | |
| str(sample_rate), | |
| "-ac", | |
| "1", | |
| "-i", | |
| "-", | |
| "-map_metadata", | |
| "-1", | |
| "-c:a", | |
| "pcm_f32le", | |
| "-f", | |
| "wav", | |
| "-y", | |
| str(path), | |
| ] | |
| process = subprocess.run( | |
| command, | |
| input=np.ascontiguousarray(audio, dtype=np.float32).tobytes(), | |
| stdout=subprocess.DEVNULL, | |
| stderr=subprocess.PIPE, | |
| check=False, | |
| ) | |
| if process.returncode != 0: | |
| message = process.stderr.decode("utf-8", errors="ignore").strip() | |
| raise RuntimeError(f"ffmpeg failed to write {path}: {message}") | |
| def verify_wav(path: Path, expected: np.ndarray, sample_rate: int) -> None: | |
| actual = decode_media(path, sample_rate) | |
| if actual.shape != expected.shape or not np.array_equal(actual, expected): | |
| max_abs = float("inf") | |
| if actual.shape == expected.shape and actual.size: | |
| max_abs = float(np.max(np.abs(actual - expected))) | |
| raise RuntimeError( | |
| f"WAV round-trip mismatch: {path}; expected={expected.shape}, " | |
| f"actual={actual.shape}, max_abs={max_abs}" | |
| ) | |
| def materialize_one( | |
| input_path_value: str, | |
| output_path_value: str, | |
| input_kind: str, | |
| sample_rate: int, | |
| overwrite: bool, | |
| ) -> tuple[str, str]: | |
| input_path = Path(input_path_value) | |
| output_path = Path(output_path_value) | |
| audio = load_input(input_path, input_kind, sample_rate) | |
| if output_path.exists() and not overwrite: | |
| verify_wav(output_path, audio, sample_rate) | |
| return str(output_path), "verified" | |
| output_path.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = output_path.with_name(f".{output_path.name}.{os.getpid()}.tmp.wav") | |
| try: | |
| write_float_wav(temporary, audio, sample_rate) | |
| verify_wav(temporary, audio, sample_rate) | |
| os.replace(temporary, output_path) | |
| finally: | |
| try: | |
| temporary.unlink() | |
| except FileNotFoundError: | |
| pass | |
| return str(output_path), "written" | |
| def relative_input_path(path: Path, dataset_root: Path, strip_prefix: Path | None) -> Path: | |
| try: | |
| relative = path.relative_to(dataset_root) | |
| except ValueError as exc: | |
| raise ValueError(f"Input path is outside dataset root: {path} vs {dataset_root}") from exc | |
| if strip_prefix is not None: | |
| try: | |
| relative = relative.relative_to(strip_prefix) | |
| except ValueError: | |
| # Historical manifests may fall back from a missing NPY to the | |
| # original media path. Keep that relative path and decode it with | |
| # the exact legacy ffmpeg parameters. | |
| pass | |
| return relative | |
| def write_json_atomic(path: Path, payload: Any) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") | |
| temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") | |
| os.replace(temporary, path) | |
| def write_jsonl_atomic(path: Path, rows: list[dict[str, Any]]) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") | |
| with temporary.open("w", encoding="utf-8") as handle: | |
| for row in rows: | |
| handle.write(json.dumps(row, ensure_ascii=False) + "\n") | |
| os.replace(temporary, path) | |
| def reusable_complete_output( | |
| marker_path: Path, | |
| output_manifest: Path, | |
| expected_marker: dict[str, Any], | |
| ) -> bool: | |
| if not marker_path.is_file() or not output_manifest.is_file(): | |
| return False | |
| try: | |
| marker = json.loads(marker_path.read_text(encoding="utf-8")) | |
| except (OSError, json.JSONDecodeError): | |
| return False | |
| for key, value in expected_marker.items(): | |
| if marker.get(key) != value: | |
| return False | |
| expected_rows = int(marker.get("rows", -1)) | |
| found_rows = 0 | |
| with output_manifest.open("r", encoding="utf-8") as handle: | |
| for line in handle: | |
| if not line.strip(): | |
| continue | |
| row = json.loads(line) | |
| audio_value = row.get(expected_marker["output_path_key"]) | |
| if not audio_value or not (Path(expected_marker["output_root"]) / str(audio_value)).is_file(): | |
| return False | |
| found_rows += 1 | |
| return found_rows == expected_rows | |
| def main() -> None: | |
| args = parse_args() | |
| if args.sample_rate <= 0 or args.num_workers <= 0 or args.limit < 0: | |
| raise ValueError("sample-rate/num-workers must be positive and limit must be non-negative") | |
| dataset_root = args.dataset_root.expanduser().resolve() | |
| metadata_path = resolve_path(args.metadata_jsonl, dataset_root) | |
| output_root = args.output_root.expanduser().resolve() | |
| output_manifest = resolve_path(Path(args.output_metadata_jsonl), output_root) | |
| marker_path = output_root / ".audio_wav_complete.json" | |
| strip_prefix = Path(args.strip_input_prefix) if str(args.strip_input_prefix).strip() else None | |
| absolute_path_keys = tuple( | |
| value.strip() for value in str(args.absolute_path_keys).split(",") if value.strip() | |
| ) | |
| source_stat = metadata_path.stat() | |
| expected_marker = { | |
| "format": FORMAT_VERSION, | |
| "dataset_root": str(dataset_root), | |
| "metadata_jsonl": str(metadata_path), | |
| "metadata_size": int(source_stat.st_size), | |
| "metadata_mtime_ns": int(source_stat.st_mtime_ns), | |
| "output_root": str(output_root), | |
| "input_kind": str(args.input_kind), | |
| "input_path_key": str(args.input_path_key), | |
| "output_path_key": str(args.output_path_key), | |
| "absolute_path_keys": list(absolute_path_keys), | |
| "strip_input_prefix": str(strip_prefix or ""), | |
| "output_audio_dir": str(args.output_audio_dir), | |
| "sample_rate": int(args.sample_rate), | |
| "limit": int(args.limit), | |
| "skip_media_without_audio": bool(args.skip_media_without_audio), | |
| } | |
| if not args.overwrite and reusable_complete_output(marker_path, output_manifest, expected_marker): | |
| marker = json.loads(marker_path.read_text(encoding="utf-8")) | |
| print( | |
| f"[audio wav] reuse complete source: rows={marker['rows']} files={marker['files']} " | |
| f"root={output_root}" | |
| ) | |
| return | |
| rows = load_rows(metadata_path, args.limit) | |
| output_rows: list[dict[str, Any]] = [] | |
| jobs_by_output: dict[str, tuple[Path, Path]] = {} | |
| skipped_no_audio = 0 | |
| for row_no, original_row in enumerate(rows): | |
| input_value = original_row.get(args.input_path_key) | |
| if input_value in (None, ""): | |
| raise ValueError(f"Missing {args.input_path_key!r} in {metadata_path} row {row_no}") | |
| input_path = resolve_path(Path(str(input_value)), dataset_root) | |
| if not input_path.is_file(): | |
| raise FileNotFoundError(f"Audio input does not exist: {input_path}") | |
| is_media_input = args.input_kind == "media" or ( | |
| args.input_kind == "auto" and input_path.suffix.lower() != ".npy" | |
| ) | |
| if is_media_input and args.skip_media_without_audio and not has_audio_stream(input_path): | |
| skipped_no_audio += 1 | |
| continue | |
| relative = relative_input_path(input_path, dataset_root, strip_prefix) | |
| output_relative = Path(args.output_audio_dir) / relative.with_suffix(".wav") | |
| output_path = output_root / output_relative | |
| output_key = str(output_path) | |
| previous = jobs_by_output.get(output_key) | |
| if previous is not None and previous[0] != input_path: | |
| raise RuntimeError(f"Two inputs map to the same WAV: {previous[0]} and {input_path}") | |
| jobs_by_output[output_key] = (input_path, output_path) | |
| row = dict(original_row) | |
| for key in absolute_path_keys: | |
| value = row.get(key) | |
| if value in (None, ""): | |
| continue | |
| row[key] = str(resolve_path(Path(str(value)), dataset_root)) | |
| row[args.output_path_key] = output_relative.as_posix() | |
| output_rows.append(row) | |
| written = 0 | |
| verified = 0 | |
| failures: list[str] = [] | |
| with ProcessPoolExecutor(max_workers=args.num_workers) as executor: | |
| futures = [ | |
| executor.submit( | |
| materialize_one, | |
| str(input_path), | |
| str(output_path), | |
| str(args.input_kind), | |
| int(args.sample_rate), | |
| bool(args.overwrite), | |
| ) | |
| for input_path, output_path in jobs_by_output.values() | |
| ] | |
| for future in tqdm(as_completed(futures), total=len(futures), desc="Audio WAV", unit="file"): | |
| try: | |
| _, status = future.result() | |
| except Exception as exc: # noqa: BLE001 - aggregate worker failures with context | |
| failures.append(str(exc)) | |
| continue | |
| if status == "written": | |
| written += 1 | |
| else: | |
| verified += 1 | |
| if failures: | |
| preview = "\n".join(failures[:10]) | |
| raise RuntimeError(f"Failed to materialize {len(failures)} WAV files:\n{preview}") | |
| write_jsonl_atomic(output_manifest, output_rows) | |
| marker = { | |
| **expected_marker, | |
| "rows": len(output_rows), | |
| "files": len(jobs_by_output), | |
| "written": written, | |
| "verified_existing": verified, | |
| "skipped_no_audio": skipped_no_audio, | |
| "wav_codec": "pcm_f32le", | |
| } | |
| write_json_atomic(marker_path, marker) | |
| print( | |
| f"[audio wav] complete rows={len(output_rows)} files={len(jobs_by_output)} " | |
| f"written={written} verified={verified} skipped_no_audio={skipped_no_audio} " | |
| f"manifest={output_manifest}" | |
| ) | |
| if __name__ == "__main__": | |
| main() | |