code-Psi0-yth-v21 / scripts /data /extract_audio_npy_cache.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
c4f8e4a verified
Raw History Blame Contribute Delete
16.6 kB
from __future__ import annotations
import argparse
import json
import os
import subprocess
from concurrent.futures import ProcessPoolExecutor, as_completed
from pathlib import Path
from typing import Any
import numpy as np
from tqdm import tqdm
INPUT_AUDIO_EXTENSIONS = {".wav", ".flac", ".mp3", ".m4a", ".aac", ".ogg"}
def ffmpeg_bin() -> str:
return os.environ.get("PSI_FFMPEG_BIN", "ffmpeg")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Extract audio npy cache for multimodal training.")
parser.add_argument(
"--mode",
choices=("video_manifest", "audio_dir_manifest", "legacy_motion_video"),
default="video_manifest",
help="Conversion mode.",
)
parser.add_argument(
"--dataset-root",
type=Path,
default=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/2264_with_audio"),
help="Dataset root that contains metadata and/or audio inputs.",
)
parser.add_argument(
"--metadata-jsonl",
type=str,
default="metadata_convert.jsonl",
help="Input metadata jsonl, relative to dataset root unless absolute.",
)
parser.add_argument(
"--output-dir",
type=str,
default="audio_npy_yth",
help="Output audio npy directory, relative to dataset root unless absolute.",
)
parser.add_argument(
"--output-metadata-jsonl",
type=str,
default="metadata_audio_npy.jsonl",
help="Output metadata jsonl with audio_path filled in.",
)
parser.add_argument("--video-path-key", type=str, default="video_path")
parser.add_argument("--input-audio-path-key", type=str, default="audio_path")
parser.add_argument("--audio-path-key", type=str, default="audio_path")
parser.add_argument("--motion-dir-key", type=str, default="converted_path")
parser.add_argument("--root-dir", type=Path, default=None, help="Legacy motion root.")
parser.add_argument("--video-dir", type=Path, default=None, help="Legacy video root.")
parser.add_argument("--sample-rate", type=int, default=16000)
parser.add_argument("--num-workers", type=int, default=32)
parser.add_argument("--overwrite", action="store_true")
return parser.parse_args()
def resolve_path(path_value: str | Path, dataset_root: Path) -> Path:
path = Path(path_value)
if path.is_absolute():
return path
return dataset_root / path
def relative_to_root(path: Path, dataset_root: Path) -> Path:
try:
return path.relative_to(dataset_root)
except ValueError:
return Path(path.name)
def output_rel_path_for_file(input_path: Path, dataset_root: Path, output_dir: str | Path) -> Path:
return Path(output_dir) / relative_to_root(input_path, dataset_root).with_suffix(".npy")
def output_rel_path_for_dir(input_dir: Path, dataset_root: Path, output_dir: str | Path) -> Path:
return Path(output_dir) / relative_to_root(input_dir, dataset_root)
def extract_one(
input_path_str: str,
output_path_str: str,
sample_rate: int,
overwrite: bool,
) -> tuple[str, str, str]:
input_path = Path(input_path_str)
output_path = Path(output_path_str)
if output_path.exists() and not overwrite:
return str(input_path), str(output_path), "skip"
output_path.parent.mkdir(parents=True, exist_ok=True)
cmd = [
ffmpeg_bin(),
"-v",
"error",
"-threads",
"1",
"-i",
str(input_path),
"-ac",
"1",
"-ar",
str(sample_rate),
"-f",
"f32le",
"-",
]
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
if proc.returncode != 0:
msg = proc.stderr.decode("utf-8", errors="ignore").strip()
return str(input_path), msg or "ffmpeg failed", "fail"
audio = np.frombuffer(proc.stdout, dtype=np.float32)
if audio.size == 0:
return str(input_path), "empty decoded audio", "fail"
tmp_path = output_path.with_name(f".{output_path.name}.{os.getpid()}.tmp.npy")
np.save(tmp_path, audio)
os.replace(tmp_path, output_path)
return str(input_path), str(output_path), "ok"
def load_rows(metadata_path: Path) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
with metadata_path.open("r", encoding="utf-8") as f:
for line_no, line in enumerate(f, start=1):
line = line.strip()
if not line:
continue
try:
rows.append(json.loads(line))
except json.JSONDecodeError as exc:
raise ValueError(f"Bad jsonl at {metadata_path}:{line_no}") from exc
return rows
def count_csv_rows(csv_path: Path) -> int:
with csv_path.open("r", encoding="utf-8", errors="ignore") as f:
return max(0, sum(1 for _ in f) - 1)
def list_legacy_motion_dirs(root_dir: Path) -> list[Path]:
return sorted(p.parent for p in root_dir.rglob("joint_pos.csv"))
def build_video_manifest_jobs(
rows: list[dict[str, Any]],
*,
dataset_root: Path,
output_dir: str,
video_path_key: str,
) -> tuple[dict[str, tuple[Path, Path, Path]], list[dict[str, Any]]]:
unique_inputs: dict[str, tuple[Path, Path, Path]] = {}
row_specs: list[dict[str, Any]] = []
for row in rows:
video_value = row.get(video_path_key)
if not video_value:
row_specs.append({"kind": "missing"})
continue
input_path = resolve_path(str(video_value), dataset_root)
rel_audio_path = output_rel_path_for_file(input_path, dataset_root, output_dir)
output_path = resolve_path(rel_audio_path, dataset_root)
unique_inputs[str(input_path)] = (input_path, output_path, rel_audio_path)
row_specs.append(
{
"kind": "file",
"input_key": str(input_path),
"fallback_value": str(video_value),
}
)
return unique_inputs, row_specs
def build_audio_dir_manifest_jobs(
rows: list[dict[str, Any]],
*,
dataset_root: Path,
output_dir: str,
input_audio_path_key: str,
) -> tuple[dict[str, tuple[Path, Path]], list[dict[str, Any]]]:
unique_inputs: dict[str, tuple[Path, Path]] = {}
row_specs: list[dict[str, Any]] = []
for row in rows:
audio_value = row.get(input_audio_path_key)
if not audio_value:
row_specs.append({"kind": "missing"})
continue
source_path = resolve_path(str(audio_value), dataset_root)
if source_path.is_dir():
input_files = sorted(
p for p in source_path.iterdir()
if p.is_file() and p.suffix.lower() in INPUT_AUDIO_EXTENSIONS
)
rel_output_dir = output_rel_path_for_dir(source_path, dataset_root, output_dir)
output_dir_path = resolve_path(rel_output_dir, dataset_root)
for input_file in input_files:
rel_output_file = output_rel_path_for_file(input_file, dataset_root, output_dir)
output_file = resolve_path(rel_output_file, dataset_root)
unique_inputs[str(input_file)] = (input_file, output_file)
row_specs.append(
{
"kind": "dir",
"output_value": str(rel_output_dir),
"output_path": output_dir_path,
"fallback_value": str(audio_value),
}
)
else:
rel_output_file = output_rel_path_for_file(source_path, dataset_root, output_dir)
output_file = resolve_path(rel_output_file, dataset_root)
unique_inputs[str(source_path)] = (source_path, output_file)
row_specs.append(
{
"kind": "file",
"output_value": str(rel_output_file),
"output_path": output_file,
"fallback_value": str(audio_value),
}
)
return unique_inputs, row_specs
def build_legacy_motion_video_jobs(
*,
motion_root: Path,
dataset_root: Path,
video_dir: Path,
output_dir: str,
motion_dir_key: str,
audio_path_key: str,
video_path_key: str,
) -> tuple[list[dict[str, Any]], dict[str, tuple[Path, Path, Path]]]:
rows: list[dict[str, Any]] = []
unique_inputs: dict[str, tuple[Path, Path, Path]] = {}
motion_dirs = list_legacy_motion_dirs(motion_root)
for row_id, motion_dir in enumerate(tqdm(motion_dirs, desc="Scan legacy dance audio", unit="motion")):
video_path = video_dir / f"{motion_dir.name}.mp4"
rel_audio_path = output_rel_path_for_file(video_path, dataset_root, output_dir)
output_path = resolve_path(rel_audio_path, dataset_root)
unique_inputs[str(video_path)] = (video_path, output_path, rel_audio_path)
rows.append(
{
"id": f"{row_id:08d}",
"name": motion_dir.name,
motion_dir_key: str(motion_dir),
video_path_key: str(video_path),
audio_path_key: str(rel_audio_path),
"n_frames": count_csv_rows(motion_dir / "joint_pos.csv"),
}
)
return rows, unique_inputs
def run_extraction_tasks(
unique_inputs: dict[str, tuple[Path, Path] | tuple[Path, Path, Path]],
*,
sample_rate: int,
num_workers: int,
overwrite: bool,
) -> tuple[int, int, int, list[tuple[str, str]]]:
ok = 0
skipped = 0
failed = 0
failures: list[tuple[str, str]] = []
with ProcessPoolExecutor(max_workers=max(1, int(num_workers))) as executor:
futures = [
executor.submit(
extract_one,
str(spec[0]),
str(spec[1]),
int(sample_rate),
bool(overwrite),
)
for spec in unique_inputs.values()
]
for future in tqdm(as_completed(futures), total=len(futures), desc="Extract audio npy", unit="file"):
input_path, info, status = future.result()
if status == "ok":
ok += 1
elif status == "skip":
skipped += 1
else:
failed += 1
failures.append((input_path, info))
return ok, skipped, failed, failures
def write_video_manifest_metadata(
output_metadata_path: Path,
rows: list[dict[str, Any]],
row_specs: list[dict[str, Any]],
unique_inputs: dict[str, tuple[Path, Path, Path]],
*,
audio_path_key: str,
) -> None:
output_metadata_path.parent.mkdir(parents=True, exist_ok=True)
with output_metadata_path.open("w", encoding="utf-8") as f:
for row, spec in zip(rows, row_specs):
if spec.get("kind") == "file":
_, output_path, rel_audio_path = unique_inputs[spec["input_key"]]
row[audio_path_key] = str(rel_audio_path) if output_path.exists() else spec["fallback_value"]
f.write(json.dumps(row, ensure_ascii=False) + "\n")
def write_audio_dir_manifest_metadata(
output_metadata_path: Path,
rows: list[dict[str, Any]],
row_specs: list[dict[str, Any]],
*,
audio_path_key: str,
) -> None:
output_metadata_path.parent.mkdir(parents=True, exist_ok=True)
with output_metadata_path.open("w", encoding="utf-8") as f:
for row, spec in zip(rows, row_specs):
kind = spec.get("kind")
if kind in ("file", "dir"):
output_path = spec["output_path"]
if kind == "dir":
has_output = output_path.is_dir() and any(
p.is_file() and p.suffix.lower() == ".npy"
for p in output_path.iterdir()
)
else:
has_output = output_path.exists()
row[audio_path_key] = spec["output_value"] if has_output else spec["fallback_value"]
f.write(json.dumps(row, ensure_ascii=False) + "\n")
def write_failure_log(output_metadata_path: Path, failures: list[tuple[str, str]]) -> None:
if not failures:
return
failure_log = output_metadata_path.with_suffix(".failures.log")
with failure_log.open("w", encoding="utf-8") as f:
for input_path, msg in failures:
f.write(f"{input_path}\t{msg}\n")
print(f"[audio_npy_yth] failure log: {failure_log}")
def main() -> None:
args = parse_args()
dataset_root = args.dataset_root.resolve()
output_root = resolve_path(args.output_dir, dataset_root)
output_metadata_path = resolve_path(args.output_metadata_jsonl, dataset_root)
print(f"[audio_npy_yth] mode: {args.mode}")
print(f"[audio_npy_yth] dataset root: {dataset_root}")
print(f"[audio_npy_yth] output root: {output_root}")
print(f"[audio_npy_yth] output metadata: {output_metadata_path}")
if args.mode == "video_manifest":
metadata_path = resolve_path(args.metadata_jsonl, dataset_root)
rows = load_rows(metadata_path)
unique_inputs, row_specs = build_video_manifest_jobs(
rows,
dataset_root=dataset_root,
output_dir=args.output_dir,
video_path_key=args.video_path_key,
)
print(f"[audio_npy_yth] metadata: {metadata_path}")
print(f"[audio_npy_yth] unique videos: {len(unique_inputs)}")
ok, skipped, failed, failures = run_extraction_tasks(
unique_inputs,
sample_rate=args.sample_rate,
num_workers=args.num_workers,
overwrite=args.overwrite,
)
write_video_manifest_metadata(
output_metadata_path,
rows,
row_specs,
unique_inputs,
audio_path_key=args.audio_path_key,
)
print(f"[audio_npy_yth] ok={ok} skip={skipped} fail={failed}")
write_failure_log(output_metadata_path, failures)
return
if args.mode == "audio_dir_manifest":
metadata_path = resolve_path(args.metadata_jsonl, dataset_root)
rows = load_rows(metadata_path)
unique_inputs, row_specs = build_audio_dir_manifest_jobs(
rows,
dataset_root=dataset_root,
output_dir=args.output_dir,
input_audio_path_key=args.input_audio_path_key,
)
print(f"[audio_npy_yth] metadata: {metadata_path}")
print(f"[audio_npy_yth] unique audio files: {len(unique_inputs)}")
ok, skipped, failed, failures = run_extraction_tasks(
unique_inputs,
sample_rate=args.sample_rate,
num_workers=args.num_workers,
overwrite=args.overwrite,
)
write_audio_dir_manifest_metadata(
output_metadata_path,
rows,
row_specs,
audio_path_key=args.audio_path_key,
)
print(f"[audio_npy_yth] ok={ok} skip={skipped} fail={failed}")
write_failure_log(output_metadata_path, failures)
return
if args.root_dir is None or args.video_dir is None:
raise ValueError("--root-dir and --video-dir are required for legacy_motion_video mode")
rows, unique_inputs = build_legacy_motion_video_jobs(
motion_root=args.root_dir.resolve(),
dataset_root=dataset_root,
video_dir=args.video_dir.resolve(),
output_dir=args.output_dir,
motion_dir_key=args.motion_dir_key,
audio_path_key=args.audio_path_key,
video_path_key=args.video_path_key,
)
print(f"[audio_npy_yth] legacy motions: {len(rows)}")
print(f"[audio_npy_yth] unique videos: {len(unique_inputs)}")
ok, skipped, failed, failures = run_extraction_tasks(
unique_inputs,
sample_rate=args.sample_rate,
num_workers=args.num_workers,
overwrite=args.overwrite,
)
output_metadata_path.parent.mkdir(parents=True, exist_ok=True)
with output_metadata_path.open("w", encoding="utf-8") as f:
for row in rows:
video_input = Path(row[args.video_path_key])
_, output_path, rel_audio_path = unique_inputs[str(video_input)]
row[args.audio_path_key] = str(rel_audio_path) if output_path.exists() else str(video_input)
f.write(json.dumps(row, ensure_ascii=False) + "\n")
print(f"[audio_npy_yth] ok={ok} skip={skipped} fail={failed}")
write_failure_log(output_metadata_path, failures)
if __name__ == "__main__":
main()