code-HFD / scripts /data /materialize_audio_wav.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
14.9 kB
from __future__ import annotations
import argparse
import json
import os
import subprocess
from concurrent.futures import ProcessPoolExecutor, as_completed
from pathlib import Path
from typing import Any
import numpy as np
from tqdm import tqdm
FORMAT_VERSION = "psi0_audio_wav_v1"
def ffmpeg_bin() -> str:
return os.environ.get("PSI_FFMPEG_BIN", "ffmpeg")
def ffprobe_bin() -> str:
return os.environ.get("PSI_FFPROBE_BIN", "ffprobe")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description=(
"Materialize deterministic 16 kHz mono float32 WAV inputs and rewrite "
"their JSONL manifest."
)
)
parser.add_argument("--dataset-root", type=Path, required=True)
parser.add_argument("--metadata-jsonl", type=Path, required=True)
parser.add_argument("--output-root", type=Path, required=True)
parser.add_argument("--output-metadata-jsonl", default="metadata.jsonl")
parser.add_argument("--input-kind", choices=("auto", "npy", "media"), required=True)
parser.add_argument("--input-path-key", default="audio_path")
parser.add_argument("--output-path-key", default="audio_path")
parser.add_argument(
"--absolute-path-keys",
default="converted_path,motion_dir,motion_path",
help="Existing relative paths under these keys are made absolute before writing the new manifest.",
)
parser.add_argument(
"--strip-input-prefix",
default="",
help="Optional relative prefix removed from the input path before placing it under output/audio/.",
)
parser.add_argument("--output-audio-dir", default="audio")
parser.add_argument("--sample-rate", type=int, default=16000)
parser.add_argument("--num-workers", type=int, default=32)
parser.add_argument("--limit", type=int, default=0)
parser.add_argument(
"--skip-media-without-audio",
action=argparse.BooleanOptionalAction,
default=False,
help="Skip manifest media fallbacks that have no audio stream, matching the historical exporter.",
)
parser.add_argument("--overwrite", action="store_true")
return parser.parse_args()
def resolve_path(path: Path, root: Path) -> Path:
return path.expanduser().resolve() if path.is_absolute() else (root / path).expanduser().resolve()
def load_rows(path: Path, limit: int) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
with path.open("r", encoding="utf-8") as handle:
for line_no, line in enumerate(handle, start=1):
line = line.strip()
if not line:
continue
try:
rows.append(json.loads(line))
except json.JSONDecodeError as exc:
raise ValueError(f"Bad JSONL at {path}:{line_no}") from exc
if limit > 0 and len(rows) >= limit:
break
return rows
def load_npy(path: Path) -> np.ndarray:
try:
audio = np.load(path, allow_pickle=False)
except ValueError:
audio = np.load(path, allow_pickle=True)
if isinstance(audio, np.ndarray) and audio.dtype == object:
audio = np.asarray(audio.tolist())
audio = np.asarray(audio, dtype=np.float32).reshape(-1).copy()
if audio.size == 0:
raise RuntimeError(f"Empty NPY audio: {path}")
return audio
def decode_media(path: Path, sample_rate: int) -> np.ndarray:
command = [
ffmpeg_bin(),
"-v",
"error",
"-threads",
"1",
"-i",
str(path),
"-ac",
"1",
"-ar",
str(sample_rate),
"-f",
"f32le",
"-",
]
process = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
if process.returncode != 0:
message = process.stderr.decode("utf-8", errors="ignore").strip()
raise RuntimeError(f"ffmpeg failed to decode {path}: {message}")
audio = np.frombuffer(process.stdout, dtype=np.float32).copy()
if audio.size == 0:
raise RuntimeError(f"Empty media audio: {path}")
return audio
def has_audio_stream(path: Path) -> bool:
process = subprocess.run(
[
ffprobe_bin(),
"-v",
"error",
"-select_streams",
"a:0",
"-show_entries",
"stream=index",
"-of",
"csv=p=0",
str(path),
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=False,
)
return process.returncode == 0 and bool(process.stdout.strip())
def load_input(path: Path, input_kind: str, sample_rate: int) -> np.ndarray:
if input_kind == "auto":
input_kind = "npy" if path.suffix.lower() == ".npy" else "media"
return load_npy(path) if input_kind == "npy" else decode_media(path, sample_rate)
def write_float_wav(path: Path, audio: np.ndarray, sample_rate: int) -> None:
command = [
ffmpeg_bin(),
"-v",
"error",
"-threads",
"1",
"-f",
"f32le",
"-ar",
str(sample_rate),
"-ac",
"1",
"-i",
"-",
"-map_metadata",
"-1",
"-c:a",
"pcm_f32le",
"-f",
"wav",
"-y",
str(path),
]
process = subprocess.run(
command,
input=np.ascontiguousarray(audio, dtype=np.float32).tobytes(),
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
check=False,
)
if process.returncode != 0:
message = process.stderr.decode("utf-8", errors="ignore").strip()
raise RuntimeError(f"ffmpeg failed to write {path}: {message}")
def verify_wav(path: Path, expected: np.ndarray, sample_rate: int) -> None:
actual = decode_media(path, sample_rate)
if actual.shape != expected.shape or not np.array_equal(actual, expected):
max_abs = float("inf")
if actual.shape == expected.shape and actual.size:
max_abs = float(np.max(np.abs(actual - expected)))
raise RuntimeError(
f"WAV round-trip mismatch: {path}; expected={expected.shape}, "
f"actual={actual.shape}, max_abs={max_abs}"
)
def materialize_one(
input_path_value: str,
output_path_value: str,
input_kind: str,
sample_rate: int,
overwrite: bool,
) -> tuple[str, str]:
input_path = Path(input_path_value)
output_path = Path(output_path_value)
audio = load_input(input_path, input_kind, sample_rate)
if output_path.exists() and not overwrite:
verify_wav(output_path, audio, sample_rate)
return str(output_path), "verified"
output_path.parent.mkdir(parents=True, exist_ok=True)
temporary = output_path.with_name(f".{output_path.name}.{os.getpid()}.tmp.wav")
try:
write_float_wav(temporary, audio, sample_rate)
verify_wav(temporary, audio, sample_rate)
os.replace(temporary, output_path)
finally:
try:
temporary.unlink()
except FileNotFoundError:
pass
return str(output_path), "written"
def relative_input_path(path: Path, dataset_root: Path, strip_prefix: Path | None) -> Path:
try:
relative = path.relative_to(dataset_root)
except ValueError as exc:
raise ValueError(f"Input path is outside dataset root: {path} vs {dataset_root}") from exc
if strip_prefix is not None:
try:
relative = relative.relative_to(strip_prefix)
except ValueError:
# Historical manifests may fall back from a missing NPY to the
# original media path. Keep that relative path and decode it with
# the exact legacy ffmpeg parameters.
pass
return relative
def write_json_atomic(path: Path, payload: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
os.replace(temporary, path)
def write_jsonl_atomic(path: Path, rows: list[dict[str, Any]]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
with temporary.open("w", encoding="utf-8") as handle:
for row in rows:
handle.write(json.dumps(row, ensure_ascii=False) + "\n")
os.replace(temporary, path)
def reusable_complete_output(
marker_path: Path,
output_manifest: Path,
expected_marker: dict[str, Any],
) -> bool:
if not marker_path.is_file() or not output_manifest.is_file():
return False
try:
marker = json.loads(marker_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return False
for key, value in expected_marker.items():
if marker.get(key) != value:
return False
expected_rows = int(marker.get("rows", -1))
found_rows = 0
with output_manifest.open("r", encoding="utf-8") as handle:
for line in handle:
if not line.strip():
continue
row = json.loads(line)
audio_value = row.get(expected_marker["output_path_key"])
if not audio_value or not (Path(expected_marker["output_root"]) / str(audio_value)).is_file():
return False
found_rows += 1
return found_rows == expected_rows
def main() -> None:
args = parse_args()
if args.sample_rate <= 0 or args.num_workers <= 0 or args.limit < 0:
raise ValueError("sample-rate/num-workers must be positive and limit must be non-negative")
dataset_root = args.dataset_root.expanduser().resolve()
metadata_path = resolve_path(args.metadata_jsonl, dataset_root)
output_root = args.output_root.expanduser().resolve()
output_manifest = resolve_path(Path(args.output_metadata_jsonl), output_root)
marker_path = output_root / ".audio_wav_complete.json"
strip_prefix = Path(args.strip_input_prefix) if str(args.strip_input_prefix).strip() else None
absolute_path_keys = tuple(
value.strip() for value in str(args.absolute_path_keys).split(",") if value.strip()
)
source_stat = metadata_path.stat()
expected_marker = {
"format": FORMAT_VERSION,
"dataset_root": str(dataset_root),
"metadata_jsonl": str(metadata_path),
"metadata_size": int(source_stat.st_size),
"metadata_mtime_ns": int(source_stat.st_mtime_ns),
"output_root": str(output_root),
"input_kind": str(args.input_kind),
"input_path_key": str(args.input_path_key),
"output_path_key": str(args.output_path_key),
"absolute_path_keys": list(absolute_path_keys),
"strip_input_prefix": str(strip_prefix or ""),
"output_audio_dir": str(args.output_audio_dir),
"sample_rate": int(args.sample_rate),
"limit": int(args.limit),
"skip_media_without_audio": bool(args.skip_media_without_audio),
}
if not args.overwrite and reusable_complete_output(marker_path, output_manifest, expected_marker):
marker = json.loads(marker_path.read_text(encoding="utf-8"))
print(
f"[audio wav] reuse complete source: rows={marker['rows']} files={marker['files']} "
f"root={output_root}"
)
return
rows = load_rows(metadata_path, args.limit)
output_rows: list[dict[str, Any]] = []
jobs_by_output: dict[str, tuple[Path, Path]] = {}
skipped_no_audio = 0
for row_no, original_row in enumerate(rows):
input_value = original_row.get(args.input_path_key)
if input_value in (None, ""):
raise ValueError(f"Missing {args.input_path_key!r} in {metadata_path} row {row_no}")
input_path = resolve_path(Path(str(input_value)), dataset_root)
if not input_path.is_file():
raise FileNotFoundError(f"Audio input does not exist: {input_path}")
is_media_input = args.input_kind == "media" or (
args.input_kind == "auto" and input_path.suffix.lower() != ".npy"
)
if is_media_input and args.skip_media_without_audio and not has_audio_stream(input_path):
skipped_no_audio += 1
continue
relative = relative_input_path(input_path, dataset_root, strip_prefix)
output_relative = Path(args.output_audio_dir) / relative.with_suffix(".wav")
output_path = output_root / output_relative
output_key = str(output_path)
previous = jobs_by_output.get(output_key)
if previous is not None and previous[0] != input_path:
raise RuntimeError(f"Two inputs map to the same WAV: {previous[0]} and {input_path}")
jobs_by_output[output_key] = (input_path, output_path)
row = dict(original_row)
for key in absolute_path_keys:
value = row.get(key)
if value in (None, ""):
continue
row[key] = str(resolve_path(Path(str(value)), dataset_root))
row[args.output_path_key] = output_relative.as_posix()
output_rows.append(row)
written = 0
verified = 0
failures: list[str] = []
with ProcessPoolExecutor(max_workers=args.num_workers) as executor:
futures = [
executor.submit(
materialize_one,
str(input_path),
str(output_path),
str(args.input_kind),
int(args.sample_rate),
bool(args.overwrite),
)
for input_path, output_path in jobs_by_output.values()
]
for future in tqdm(as_completed(futures), total=len(futures), desc="Audio WAV", unit="file"):
try:
_, status = future.result()
except Exception as exc: # noqa: BLE001 - aggregate worker failures with context
failures.append(str(exc))
continue
if status == "written":
written += 1
else:
verified += 1
if failures:
preview = "\n".join(failures[:10])
raise RuntimeError(f"Failed to materialize {len(failures)} WAV files:\n{preview}")
write_jsonl_atomic(output_manifest, output_rows)
marker = {
**expected_marker,
"rows": len(output_rows),
"files": len(jobs_by_output),
"written": written,
"verified_existing": verified,
"skipped_no_audio": skipped_no_audio,
"wav_codec": "pcm_f32le",
}
write_json_atomic(marker_path, marker)
print(
f"[audio wav] complete rows={len(output_rows)} files={len(jobs_by_output)} "
f"written={written} verified={verified} skipped_no_audio={skipped_no_audio} "
f"manifest={output_manifest}"
)
if __name__ == "__main__":
main()