Download scripts/data/extract_audio_npy_cache.py from Tinghong-Ye22/code-Psi0-yth-v23: direct link, hf CLI and curl.
- Browser
- Download file 16.6 kB
-
https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v23/resolve/main/scripts/data/extract_audio_npy_cache.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Psi0-yth-v23/scripts/data/extract_audio_npy_cache.py
-
curl -L -o extract_audio_npy_cache.py https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v23/resolve/main/scripts/data/extract_audio_npy_cache.py
16.6 kB
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import subprocess | |
| from concurrent.futures import ProcessPoolExecutor, as_completed | |
| from pathlib import Path | |
| from typing import Any | |
| import numpy as np | |
| from tqdm import tqdm | |
| INPUT_AUDIO_EXTENSIONS = {".wav", ".flac", ".mp3", ".m4a", ".aac", ".ogg"} | |
| def ffmpeg_bin() -> str: | |
| return os.environ.get("PSI_FFMPEG_BIN", "ffmpeg") | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser(description="Extract audio npy cache for multimodal training.") | |
| parser.add_argument( | |
| "--mode", | |
| choices=("video_manifest", "audio_dir_manifest", "legacy_motion_video"), | |
| default="video_manifest", | |
| help="Conversion mode.", | |
| ) | |
| parser.add_argument( | |
| "--dataset-root", | |
| type=Path, | |
| default=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/2264_with_audio"), | |
| help="Dataset root that contains metadata and/or audio inputs.", | |
| ) | |
| parser.add_argument( | |
| "--metadata-jsonl", | |
| type=str, | |
| default="metadata_convert.jsonl", | |
| help="Input metadata jsonl, relative to dataset root unless absolute.", | |
| ) | |
| parser.add_argument( | |
| "--output-dir", | |
| type=str, | |
| default="audio_npy_yth", | |
| help="Output audio npy directory, relative to dataset root unless absolute.", | |
| ) | |
| parser.add_argument( | |
| "--output-metadata-jsonl", | |
| type=str, | |
| default="metadata_audio_npy.jsonl", | |
| help="Output metadata jsonl with audio_path filled in.", | |
| ) | |
| parser.add_argument("--video-path-key", type=str, default="video_path") | |
| parser.add_argument("--input-audio-path-key", type=str, default="audio_path") | |
| parser.add_argument("--audio-path-key", type=str, default="audio_path") | |
| parser.add_argument("--motion-dir-key", type=str, default="converted_path") | |
| parser.add_argument("--root-dir", type=Path, default=None, help="Legacy motion root.") | |
| parser.add_argument("--video-dir", type=Path, default=None, help="Legacy video root.") | |
| parser.add_argument("--sample-rate", type=int, default=16000) | |
| parser.add_argument("--num-workers", type=int, default=32) | |
| parser.add_argument("--overwrite", action="store_true") | |
| return parser.parse_args() | |
| def resolve_path(path_value: str | Path, dataset_root: Path) -> Path: | |
| path = Path(path_value) | |
| if path.is_absolute(): | |
| return path | |
| return dataset_root / path | |
| def relative_to_root(path: Path, dataset_root: Path) -> Path: | |
| try: | |
| return path.relative_to(dataset_root) | |
| except ValueError: | |
| return Path(path.name) | |
| def output_rel_path_for_file(input_path: Path, dataset_root: Path, output_dir: str | Path) -> Path: | |
| return Path(output_dir) / relative_to_root(input_path, dataset_root).with_suffix(".npy") | |
| def output_rel_path_for_dir(input_dir: Path, dataset_root: Path, output_dir: str | Path) -> Path: | |
| return Path(output_dir) / relative_to_root(input_dir, dataset_root) | |
| def extract_one( | |
| input_path_str: str, | |
| output_path_str: str, | |
| sample_rate: int, | |
| overwrite: bool, | |
| ) -> tuple[str, str, str]: | |
| input_path = Path(input_path_str) | |
| output_path = Path(output_path_str) | |
| if output_path.exists() and not overwrite: | |
| return str(input_path), str(output_path), "skip" | |
| output_path.parent.mkdir(parents=True, exist_ok=True) | |
| cmd = [ | |
| ffmpeg_bin(), | |
| "-v", | |
| "error", | |
| "-threads", | |
| "1", | |
| "-i", | |
| str(input_path), | |
| "-ac", | |
| "1", | |
| "-ar", | |
| str(sample_rate), | |
| "-f", | |
| "f32le", | |
| "-", | |
| ] | |
| proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) | |
| if proc.returncode != 0: | |
| msg = proc.stderr.decode("utf-8", errors="ignore").strip() | |
| return str(input_path), msg or "ffmpeg failed", "fail" | |
| audio = np.frombuffer(proc.stdout, dtype=np.float32) | |
| if audio.size == 0: | |
| return str(input_path), "empty decoded audio", "fail" | |
| tmp_path = output_path.with_name(f".{output_path.name}.{os.getpid()}.tmp.npy") | |
| np.save(tmp_path, audio) | |
| os.replace(tmp_path, output_path) | |
| return str(input_path), str(output_path), "ok" | |
| def load_rows(metadata_path: Path) -> list[dict[str, Any]]: | |
| rows: list[dict[str, Any]] = [] | |
| with metadata_path.open("r", encoding="utf-8") as f: | |
| for line_no, line in enumerate(f, start=1): | |
| line = line.strip() | |
| if not line: | |
| continue | |
| try: | |
| rows.append(json.loads(line)) | |
| except json.JSONDecodeError as exc: | |
| raise ValueError(f"Bad jsonl at {metadata_path}:{line_no}") from exc | |
| return rows | |
| def count_csv_rows(csv_path: Path) -> int: | |
| with csv_path.open("r", encoding="utf-8", errors="ignore") as f: | |
| return max(0, sum(1 for _ in f) - 1) | |
| def list_legacy_motion_dirs(root_dir: Path) -> list[Path]: | |
| return sorted(p.parent for p in root_dir.rglob("joint_pos.csv")) | |
| def build_video_manifest_jobs( | |
| rows: list[dict[str, Any]], | |
| *, | |
| dataset_root: Path, | |
| output_dir: str, | |
| video_path_key: str, | |
| ) -> tuple[dict[str, tuple[Path, Path, Path]], list[dict[str, Any]]]: | |
| unique_inputs: dict[str, tuple[Path, Path, Path]] = {} | |
| row_specs: list[dict[str, Any]] = [] | |
| for row in rows: | |
| video_value = row.get(video_path_key) | |
| if not video_value: | |
| row_specs.append({"kind": "missing"}) | |
| continue | |
| input_path = resolve_path(str(video_value), dataset_root) | |
| rel_audio_path = output_rel_path_for_file(input_path, dataset_root, output_dir) | |
| output_path = resolve_path(rel_audio_path, dataset_root) | |
| unique_inputs[str(input_path)] = (input_path, output_path, rel_audio_path) | |
| row_specs.append( | |
| { | |
| "kind": "file", | |
| "input_key": str(input_path), | |
| "fallback_value": str(video_value), | |
| } | |
| ) | |
| return unique_inputs, row_specs | |
| def build_audio_dir_manifest_jobs( | |
| rows: list[dict[str, Any]], | |
| *, | |
| dataset_root: Path, | |
| output_dir: str, | |
| input_audio_path_key: str, | |
| ) -> tuple[dict[str, tuple[Path, Path]], list[dict[str, Any]]]: | |
| unique_inputs: dict[str, tuple[Path, Path]] = {} | |
| row_specs: list[dict[str, Any]] = [] | |
| for row in rows: | |
| audio_value = row.get(input_audio_path_key) | |
| if not audio_value: | |
| row_specs.append({"kind": "missing"}) | |
| continue | |
| source_path = resolve_path(str(audio_value), dataset_root) | |
| if source_path.is_dir(): | |
| input_files = sorted( | |
| p for p in source_path.iterdir() | |
| if p.is_file() and p.suffix.lower() in INPUT_AUDIO_EXTENSIONS | |
| ) | |
| rel_output_dir = output_rel_path_for_dir(source_path, dataset_root, output_dir) | |
| output_dir_path = resolve_path(rel_output_dir, dataset_root) | |
| for input_file in input_files: | |
| rel_output_file = output_rel_path_for_file(input_file, dataset_root, output_dir) | |
| output_file = resolve_path(rel_output_file, dataset_root) | |
| unique_inputs[str(input_file)] = (input_file, output_file) | |
| row_specs.append( | |
| { | |
| "kind": "dir", | |
| "output_value": str(rel_output_dir), | |
| "output_path": output_dir_path, | |
| "fallback_value": str(audio_value), | |
| } | |
| ) | |
| else: | |
| rel_output_file = output_rel_path_for_file(source_path, dataset_root, output_dir) | |
| output_file = resolve_path(rel_output_file, dataset_root) | |
| unique_inputs[str(source_path)] = (source_path, output_file) | |
| row_specs.append( | |
| { | |
| "kind": "file", | |
| "output_value": str(rel_output_file), | |
| "output_path": output_file, | |
| "fallback_value": str(audio_value), | |
| } | |
| ) | |
| return unique_inputs, row_specs | |
| def build_legacy_motion_video_jobs( | |
| *, | |
| motion_root: Path, | |
| dataset_root: Path, | |
| video_dir: Path, | |
| output_dir: str, | |
| motion_dir_key: str, | |
| audio_path_key: str, | |
| video_path_key: str, | |
| ) -> tuple[list[dict[str, Any]], dict[str, tuple[Path, Path, Path]]]: | |
| rows: list[dict[str, Any]] = [] | |
| unique_inputs: dict[str, tuple[Path, Path, Path]] = {} | |
| motion_dirs = list_legacy_motion_dirs(motion_root) | |
| for row_id, motion_dir in enumerate(tqdm(motion_dirs, desc="Scan legacy dance audio", unit="motion")): | |
| video_path = video_dir / f"{motion_dir.name}.mp4" | |
| rel_audio_path = output_rel_path_for_file(video_path, dataset_root, output_dir) | |
| output_path = resolve_path(rel_audio_path, dataset_root) | |
| unique_inputs[str(video_path)] = (video_path, output_path, rel_audio_path) | |
| rows.append( | |
| { | |
| "id": f"{row_id:08d}", | |
| "name": motion_dir.name, | |
| motion_dir_key: str(motion_dir), | |
| video_path_key: str(video_path), | |
| audio_path_key: str(rel_audio_path), | |
| "n_frames": count_csv_rows(motion_dir / "joint_pos.csv"), | |
| } | |
| ) | |
| return rows, unique_inputs | |
| def run_extraction_tasks( | |
| unique_inputs: dict[str, tuple[Path, Path] | tuple[Path, Path, Path]], | |
| *, | |
| sample_rate: int, | |
| num_workers: int, | |
| overwrite: bool, | |
| ) -> tuple[int, int, int, list[tuple[str, str]]]: | |
| ok = 0 | |
| skipped = 0 | |
| failed = 0 | |
| failures: list[tuple[str, str]] = [] | |
| with ProcessPoolExecutor(max_workers=max(1, int(num_workers))) as executor: | |
| futures = [ | |
| executor.submit( | |
| extract_one, | |
| str(spec[0]), | |
| str(spec[1]), | |
| int(sample_rate), | |
| bool(overwrite), | |
| ) | |
| for spec in unique_inputs.values() | |
| ] | |
| for future in tqdm(as_completed(futures), total=len(futures), desc="Extract audio npy", unit="file"): | |
| input_path, info, status = future.result() | |
| if status == "ok": | |
| ok += 1 | |
| elif status == "skip": | |
| skipped += 1 | |
| else: | |
| failed += 1 | |
| failures.append((input_path, info)) | |
| return ok, skipped, failed, failures | |
| def write_video_manifest_metadata( | |
| output_metadata_path: Path, | |
| rows: list[dict[str, Any]], | |
| row_specs: list[dict[str, Any]], | |
| unique_inputs: dict[str, tuple[Path, Path, Path]], | |
| *, | |
| audio_path_key: str, | |
| ) -> None: | |
| output_metadata_path.parent.mkdir(parents=True, exist_ok=True) | |
| with output_metadata_path.open("w", encoding="utf-8") as f: | |
| for row, spec in zip(rows, row_specs): | |
| if spec.get("kind") == "file": | |
| _, output_path, rel_audio_path = unique_inputs[spec["input_key"]] | |
| row[audio_path_key] = str(rel_audio_path) if output_path.exists() else spec["fallback_value"] | |
| f.write(json.dumps(row, ensure_ascii=False) + "\n") | |
| def write_audio_dir_manifest_metadata( | |
| output_metadata_path: Path, | |
| rows: list[dict[str, Any]], | |
| row_specs: list[dict[str, Any]], | |
| *, | |
| audio_path_key: str, | |
| ) -> None: | |
| output_metadata_path.parent.mkdir(parents=True, exist_ok=True) | |
| with output_metadata_path.open("w", encoding="utf-8") as f: | |
| for row, spec in zip(rows, row_specs): | |
| kind = spec.get("kind") | |
| if kind in ("file", "dir"): | |
| output_path = spec["output_path"] | |
| if kind == "dir": | |
| has_output = output_path.is_dir() and any( | |
| p.is_file() and p.suffix.lower() == ".npy" | |
| for p in output_path.iterdir() | |
| ) | |
| else: | |
| has_output = output_path.exists() | |
| row[audio_path_key] = spec["output_value"] if has_output else spec["fallback_value"] | |
| f.write(json.dumps(row, ensure_ascii=False) + "\n") | |
| def write_failure_log(output_metadata_path: Path, failures: list[tuple[str, str]]) -> None: | |
| if not failures: | |
| return | |
| failure_log = output_metadata_path.with_suffix(".failures.log") | |
| with failure_log.open("w", encoding="utf-8") as f: | |
| for input_path, msg in failures: | |
| f.write(f"{input_path}\t{msg}\n") | |
| print(f"[audio_npy_yth] failure log: {failure_log}") | |
| def main() -> None: | |
| args = parse_args() | |
| dataset_root = args.dataset_root.resolve() | |
| output_root = resolve_path(args.output_dir, dataset_root) | |
| output_metadata_path = resolve_path(args.output_metadata_jsonl, dataset_root) | |
| print(f"[audio_npy_yth] mode: {args.mode}") | |
| print(f"[audio_npy_yth] dataset root: {dataset_root}") | |
| print(f"[audio_npy_yth] output root: {output_root}") | |
| print(f"[audio_npy_yth] output metadata: {output_metadata_path}") | |
| if args.mode == "video_manifest": | |
| metadata_path = resolve_path(args.metadata_jsonl, dataset_root) | |
| rows = load_rows(metadata_path) | |
| unique_inputs, row_specs = build_video_manifest_jobs( | |
| rows, | |
| dataset_root=dataset_root, | |
| output_dir=args.output_dir, | |
| video_path_key=args.video_path_key, | |
| ) | |
| print(f"[audio_npy_yth] metadata: {metadata_path}") | |
| print(f"[audio_npy_yth] unique videos: {len(unique_inputs)}") | |
| ok, skipped, failed, failures = run_extraction_tasks( | |
| unique_inputs, | |
| sample_rate=args.sample_rate, | |
| num_workers=args.num_workers, | |
| overwrite=args.overwrite, | |
| ) | |
| write_video_manifest_metadata( | |
| output_metadata_path, | |
| rows, | |
| row_specs, | |
| unique_inputs, | |
| audio_path_key=args.audio_path_key, | |
| ) | |
| print(f"[audio_npy_yth] ok={ok} skip={skipped} fail={failed}") | |
| write_failure_log(output_metadata_path, failures) | |
| return | |
| if args.mode == "audio_dir_manifest": | |
| metadata_path = resolve_path(args.metadata_jsonl, dataset_root) | |
| rows = load_rows(metadata_path) | |
| unique_inputs, row_specs = build_audio_dir_manifest_jobs( | |
| rows, | |
| dataset_root=dataset_root, | |
| output_dir=args.output_dir, | |
| input_audio_path_key=args.input_audio_path_key, | |
| ) | |
| print(f"[audio_npy_yth] metadata: {metadata_path}") | |
| print(f"[audio_npy_yth] unique audio files: {len(unique_inputs)}") | |
| ok, skipped, failed, failures = run_extraction_tasks( | |
| unique_inputs, | |
| sample_rate=args.sample_rate, | |
| num_workers=args.num_workers, | |
| overwrite=args.overwrite, | |
| ) | |
| write_audio_dir_manifest_metadata( | |
| output_metadata_path, | |
| rows, | |
| row_specs, | |
| audio_path_key=args.audio_path_key, | |
| ) | |
| print(f"[audio_npy_yth] ok={ok} skip={skipped} fail={failed}") | |
| write_failure_log(output_metadata_path, failures) | |
| return | |
| if args.root_dir is None or args.video_dir is None: | |
| raise ValueError("--root-dir and --video-dir are required for legacy_motion_video mode") | |
| rows, unique_inputs = build_legacy_motion_video_jobs( | |
| motion_root=args.root_dir.resolve(), | |
| dataset_root=dataset_root, | |
| video_dir=args.video_dir.resolve(), | |
| output_dir=args.output_dir, | |
| motion_dir_key=args.motion_dir_key, | |
| audio_path_key=args.audio_path_key, | |
| video_path_key=args.video_path_key, | |
| ) | |
| print(f"[audio_npy_yth] legacy motions: {len(rows)}") | |
| print(f"[audio_npy_yth] unique videos: {len(unique_inputs)}") | |
| ok, skipped, failed, failures = run_extraction_tasks( | |
| unique_inputs, | |
| sample_rate=args.sample_rate, | |
| num_workers=args.num_workers, | |
| overwrite=args.overwrite, | |
| ) | |
| output_metadata_path.parent.mkdir(parents=True, exist_ok=True) | |
| with output_metadata_path.open("w", encoding="utf-8") as f: | |
| for row in rows: | |
| video_input = Path(row[args.video_path_key]) | |
| _, output_path, rel_audio_path = unique_inputs[str(video_input)] | |
| row[args.audio_path_key] = str(rel_audio_path) if output_path.exists() else str(video_input) | |
| f.write(json.dumps(row, ensure_ascii=False) + "\n") | |
| print(f"[audio_npy_yth] ok={ok} skip={skipped} fail={failed}") | |
| write_failure_log(output_metadata_path, failures) | |
| if __name__ == "__main__": | |
| main() | |