from __future__ import annotations import argparse import json import os import subprocess from concurrent.futures import ProcessPoolExecutor, as_completed from pathlib import Path from typing import Any import numpy as np from tqdm import tqdm INPUT_AUDIO_EXTENSIONS = {".wav", ".flac", ".mp3", ".m4a", ".aac", ".ogg"} def ffmpeg_bin() -> str: return os.environ.get("PSI_FFMPEG_BIN", "ffmpeg") def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Extract audio npy cache for multimodal training.") parser.add_argument( "--mode", choices=("video_manifest", "audio_dir_manifest", "legacy_motion_video"), default="video_manifest", help="Conversion mode.", ) parser.add_argument( "--dataset-root", type=Path, default=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/2264_with_audio"), help="Dataset root that contains metadata and/or audio inputs.", ) parser.add_argument( "--metadata-jsonl", type=str, default="metadata_convert.jsonl", help="Input metadata jsonl, relative to dataset root unless absolute.", ) parser.add_argument( "--output-dir", type=str, default="audio_npy_yth", help="Output audio npy directory, relative to dataset root unless absolute.", ) parser.add_argument( "--output-metadata-jsonl", type=str, default="metadata_audio_npy.jsonl", help="Output metadata jsonl with audio_path filled in.", ) parser.add_argument("--video-path-key", type=str, default="video_path") parser.add_argument("--input-audio-path-key", type=str, default="audio_path") parser.add_argument("--audio-path-key", type=str, default="audio_path") parser.add_argument("--motion-dir-key", type=str, default="converted_path") parser.add_argument("--root-dir", type=Path, default=None, help="Legacy motion root.") parser.add_argument("--video-dir", type=Path, default=None, help="Legacy video root.") parser.add_argument("--sample-rate", type=int, default=16000) parser.add_argument("--num-workers", type=int, default=32) parser.add_argument("--overwrite", action="store_true") return parser.parse_args() def resolve_path(path_value: str | Path, dataset_root: Path) -> Path: path = Path(path_value) if path.is_absolute(): return path return dataset_root / path def relative_to_root(path: Path, dataset_root: Path) -> Path: try: return path.relative_to(dataset_root) except ValueError: return Path(path.name) def output_rel_path_for_file(input_path: Path, dataset_root: Path, output_dir: str | Path) -> Path: return Path(output_dir) / relative_to_root(input_path, dataset_root).with_suffix(".npy") def output_rel_path_for_dir(input_dir: Path, dataset_root: Path, output_dir: str | Path) -> Path: return Path(output_dir) / relative_to_root(input_dir, dataset_root) def extract_one( input_path_str: str, output_path_str: str, sample_rate: int, overwrite: bool, ) -> tuple[str, str, str]: input_path = Path(input_path_str) output_path = Path(output_path_str) if output_path.exists() and not overwrite: return str(input_path), str(output_path), "skip" output_path.parent.mkdir(parents=True, exist_ok=True) cmd = [ ffmpeg_bin(), "-v", "error", "-threads", "1", "-i", str(input_path), "-ac", "1", "-ar", str(sample_rate), "-f", "f32le", "-", ] proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) if proc.returncode != 0: msg = proc.stderr.decode("utf-8", errors="ignore").strip() return str(input_path), msg or "ffmpeg failed", "fail" audio = np.frombuffer(proc.stdout, dtype=np.float32) if audio.size == 0: return str(input_path), "empty decoded audio", "fail" tmp_path = output_path.with_name(f".{output_path.name}.{os.getpid()}.tmp.npy") np.save(tmp_path, audio) os.replace(tmp_path, output_path) return str(input_path), str(output_path), "ok" def load_rows(metadata_path: Path) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] with metadata_path.open("r", encoding="utf-8") as f: for line_no, line in enumerate(f, start=1): line = line.strip() if not line: continue try: rows.append(json.loads(line)) except json.JSONDecodeError as exc: raise ValueError(f"Bad jsonl at {metadata_path}:{line_no}") from exc return rows def count_csv_rows(csv_path: Path) -> int: with csv_path.open("r", encoding="utf-8", errors="ignore") as f: return max(0, sum(1 for _ in f) - 1) def list_legacy_motion_dirs(root_dir: Path) -> list[Path]: return sorted(p.parent for p in root_dir.rglob("joint_pos.csv")) def build_video_manifest_jobs( rows: list[dict[str, Any]], *, dataset_root: Path, output_dir: str, video_path_key: str, ) -> tuple[dict[str, tuple[Path, Path, Path]], list[dict[str, Any]]]: unique_inputs: dict[str, tuple[Path, Path, Path]] = {} row_specs: list[dict[str, Any]] = [] for row in rows: video_value = row.get(video_path_key) if not video_value: row_specs.append({"kind": "missing"}) continue input_path = resolve_path(str(video_value), dataset_root) rel_audio_path = output_rel_path_for_file(input_path, dataset_root, output_dir) output_path = resolve_path(rel_audio_path, dataset_root) unique_inputs[str(input_path)] = (input_path, output_path, rel_audio_path) row_specs.append( { "kind": "file", "input_key": str(input_path), "fallback_value": str(video_value), } ) return unique_inputs, row_specs def build_audio_dir_manifest_jobs( rows: list[dict[str, Any]], *, dataset_root: Path, output_dir: str, input_audio_path_key: str, ) -> tuple[dict[str, tuple[Path, Path]], list[dict[str, Any]]]: unique_inputs: dict[str, tuple[Path, Path]] = {} row_specs: list[dict[str, Any]] = [] for row in rows: audio_value = row.get(input_audio_path_key) if not audio_value: row_specs.append({"kind": "missing"}) continue source_path = resolve_path(str(audio_value), dataset_root) if source_path.is_dir(): input_files = sorted( p for p in source_path.iterdir() if p.is_file() and p.suffix.lower() in INPUT_AUDIO_EXTENSIONS ) rel_output_dir = output_rel_path_for_dir(source_path, dataset_root, output_dir) output_dir_path = resolve_path(rel_output_dir, dataset_root) for input_file in input_files: rel_output_file = output_rel_path_for_file(input_file, dataset_root, output_dir) output_file = resolve_path(rel_output_file, dataset_root) unique_inputs[str(input_file)] = (input_file, output_file) row_specs.append( { "kind": "dir", "output_value": str(rel_output_dir), "output_path": output_dir_path, "fallback_value": str(audio_value), } ) else: rel_output_file = output_rel_path_for_file(source_path, dataset_root, output_dir) output_file = resolve_path(rel_output_file, dataset_root) unique_inputs[str(source_path)] = (source_path, output_file) row_specs.append( { "kind": "file", "output_value": str(rel_output_file), "output_path": output_file, "fallback_value": str(audio_value), } ) return unique_inputs, row_specs def build_legacy_motion_video_jobs( *, motion_root: Path, dataset_root: Path, video_dir: Path, output_dir: str, motion_dir_key: str, audio_path_key: str, video_path_key: str, ) -> tuple[list[dict[str, Any]], dict[str, tuple[Path, Path, Path]]]: rows: list[dict[str, Any]] = [] unique_inputs: dict[str, tuple[Path, Path, Path]] = {} motion_dirs = list_legacy_motion_dirs(motion_root) for row_id, motion_dir in enumerate(tqdm(motion_dirs, desc="Scan legacy dance audio", unit="motion")): video_path = video_dir / f"{motion_dir.name}.mp4" rel_audio_path = output_rel_path_for_file(video_path, dataset_root, output_dir) output_path = resolve_path(rel_audio_path, dataset_root) unique_inputs[str(video_path)] = (video_path, output_path, rel_audio_path) rows.append( { "id": f"{row_id:08d}", "name": motion_dir.name, motion_dir_key: str(motion_dir), video_path_key: str(video_path), audio_path_key: str(rel_audio_path), "n_frames": count_csv_rows(motion_dir / "joint_pos.csv"), } ) return rows, unique_inputs def run_extraction_tasks( unique_inputs: dict[str, tuple[Path, Path] | tuple[Path, Path, Path]], *, sample_rate: int, num_workers: int, overwrite: bool, ) -> tuple[int, int, int, list[tuple[str, str]]]: ok = 0 skipped = 0 failed = 0 failures: list[tuple[str, str]] = [] with ProcessPoolExecutor(max_workers=max(1, int(num_workers))) as executor: futures = [ executor.submit( extract_one, str(spec[0]), str(spec[1]), int(sample_rate), bool(overwrite), ) for spec in unique_inputs.values() ] for future in tqdm(as_completed(futures), total=len(futures), desc="Extract audio npy", unit="file"): input_path, info, status = future.result() if status == "ok": ok += 1 elif status == "skip": skipped += 1 else: failed += 1 failures.append((input_path, info)) return ok, skipped, failed, failures def write_video_manifest_metadata( output_metadata_path: Path, rows: list[dict[str, Any]], row_specs: list[dict[str, Any]], unique_inputs: dict[str, tuple[Path, Path, Path]], *, audio_path_key: str, ) -> None: output_metadata_path.parent.mkdir(parents=True, exist_ok=True) with output_metadata_path.open("w", encoding="utf-8") as f: for row, spec in zip(rows, row_specs): if spec.get("kind") == "file": _, output_path, rel_audio_path = unique_inputs[spec["input_key"]] row[audio_path_key] = str(rel_audio_path) if output_path.exists() else spec["fallback_value"] f.write(json.dumps(row, ensure_ascii=False) + "\n") def write_audio_dir_manifest_metadata( output_metadata_path: Path, rows: list[dict[str, Any]], row_specs: list[dict[str, Any]], *, audio_path_key: str, ) -> None: output_metadata_path.parent.mkdir(parents=True, exist_ok=True) with output_metadata_path.open("w", encoding="utf-8") as f: for row, spec in zip(rows, row_specs): kind = spec.get("kind") if kind in ("file", "dir"): output_path = spec["output_path"] if kind == "dir": has_output = output_path.is_dir() and any( p.is_file() and p.suffix.lower() == ".npy" for p in output_path.iterdir() ) else: has_output = output_path.exists() row[audio_path_key] = spec["output_value"] if has_output else spec["fallback_value"] f.write(json.dumps(row, ensure_ascii=False) + "\n") def write_failure_log(output_metadata_path: Path, failures: list[tuple[str, str]]) -> None: if not failures: return failure_log = output_metadata_path.with_suffix(".failures.log") with failure_log.open("w", encoding="utf-8") as f: for input_path, msg in failures: f.write(f"{input_path}\t{msg}\n") print(f"[audio_npy_yth] failure log: {failure_log}") def main() -> None: args = parse_args() dataset_root = args.dataset_root.resolve() output_root = resolve_path(args.output_dir, dataset_root) output_metadata_path = resolve_path(args.output_metadata_jsonl, dataset_root) print(f"[audio_npy_yth] mode: {args.mode}") print(f"[audio_npy_yth] dataset root: {dataset_root}") print(f"[audio_npy_yth] output root: {output_root}") print(f"[audio_npy_yth] output metadata: {output_metadata_path}") if args.mode == "video_manifest": metadata_path = resolve_path(args.metadata_jsonl, dataset_root) rows = load_rows(metadata_path) unique_inputs, row_specs = build_video_manifest_jobs( rows, dataset_root=dataset_root, output_dir=args.output_dir, video_path_key=args.video_path_key, ) print(f"[audio_npy_yth] metadata: {metadata_path}") print(f"[audio_npy_yth] unique videos: {len(unique_inputs)}") ok, skipped, failed, failures = run_extraction_tasks( unique_inputs, sample_rate=args.sample_rate, num_workers=args.num_workers, overwrite=args.overwrite, ) write_video_manifest_metadata( output_metadata_path, rows, row_specs, unique_inputs, audio_path_key=args.audio_path_key, ) print(f"[audio_npy_yth] ok={ok} skip={skipped} fail={failed}") write_failure_log(output_metadata_path, failures) return if args.mode == "audio_dir_manifest": metadata_path = resolve_path(args.metadata_jsonl, dataset_root) rows = load_rows(metadata_path) unique_inputs, row_specs = build_audio_dir_manifest_jobs( rows, dataset_root=dataset_root, output_dir=args.output_dir, input_audio_path_key=args.input_audio_path_key, ) print(f"[audio_npy_yth] metadata: {metadata_path}") print(f"[audio_npy_yth] unique audio files: {len(unique_inputs)}") ok, skipped, failed, failures = run_extraction_tasks( unique_inputs, sample_rate=args.sample_rate, num_workers=args.num_workers, overwrite=args.overwrite, ) write_audio_dir_manifest_metadata( output_metadata_path, rows, row_specs, audio_path_key=args.audio_path_key, ) print(f"[audio_npy_yth] ok={ok} skip={skipped} fail={failed}") write_failure_log(output_metadata_path, failures) return if args.root_dir is None or args.video_dir is None: raise ValueError("--root-dir and --video-dir are required for legacy_motion_video mode") rows, unique_inputs = build_legacy_motion_video_jobs( motion_root=args.root_dir.resolve(), dataset_root=dataset_root, video_dir=args.video_dir.resolve(), output_dir=args.output_dir, motion_dir_key=args.motion_dir_key, audio_path_key=args.audio_path_key, video_path_key=args.video_path_key, ) print(f"[audio_npy_yth] legacy motions: {len(rows)}") print(f"[audio_npy_yth] unique videos: {len(unique_inputs)}") ok, skipped, failed, failures = run_extraction_tasks( unique_inputs, sample_rate=args.sample_rate, num_workers=args.num_workers, overwrite=args.overwrite, ) output_metadata_path.parent.mkdir(parents=True, exist_ok=True) with output_metadata_path.open("w", encoding="utf-8") as f: for row in rows: video_input = Path(row[args.video_path_key]) _, output_path, rel_audio_path = unique_inputs[str(video_input)] row[args.audio_path_key] = str(rel_audio_path) if output_path.exists() else str(video_input) f.write(json.dumps(row, ensure_ascii=False) + "\n") print(f"[audio_npy_yth] ok={ok} skip={skipped} fail={failed}") write_failure_log(output_metadata_path, failures) if __name__ == "__main__": main()