Buckets:
| """Re-encode mp4s in place with a dense GOP (`-g 15`) so random-frame seeks | |
| land on or near an I-frame. Fixes 3-5x dataloader slowdown when the source | |
| videos were encoded with large GOPs (e.g. NVENC `-preset p1` default GOP=300). | |
| Runs many ffmpeg jobs in parallel via ProcessPoolExecutor. Overwrites originals | |
| atomically via a temp file next to each source. | |
| """ | |
| from __future__ import annotations | |
| import argparse, os, shutil, subprocess, sys | |
| from concurrent.futures import ProcessPoolExecutor, as_completed | |
| from pathlib import Path | |
| def _has_nvenc() -> bool: | |
| try: | |
| out = subprocess.run( | |
| ["ffmpeg", "-hide_banner", "-encoders"], | |
| capture_output=True, text=True, check=True, | |
| ).stdout | |
| return "h264_nvenc" in out | |
| except Exception: | |
| return False | |
| def _reencode(job): | |
| src, gop, encoder = job | |
| src = Path(src) | |
| tmp = src.with_suffix(".tmp.mp4") | |
| if encoder == "h264_nvenc": | |
| vcodec = [ | |
| "-c:v", "h264_nvenc", | |
| "-preset", "p4", "-tune", "hq", | |
| "-rc", "constqp", "-qp", "20", | |
| "-g", str(gop), "-keyint_min", str(gop), | |
| "-bf", "0", | |
| ] | |
| else: | |
| vcodec = [ | |
| "-c:v", "libx264", | |
| "-preset", "veryfast", "-crf", "20", | |
| "-g", str(gop), "-keyint_min", str(gop), | |
| "-bf", "0", | |
| ] | |
| cmd = [ | |
| "ffmpeg", "-hide_banner", "-loglevel", "error", "-y", | |
| "-i", str(src), | |
| *vcodec, | |
| "-video_track_timescale", "15000", | |
| "-movflags", "+faststart", | |
| str(tmp), | |
| ] | |
| try: | |
| subprocess.run(cmd, check=True) | |
| os.replace(tmp, src) | |
| return (str(src), True, "") | |
| except subprocess.CalledProcessError as e: | |
| if tmp.exists(): | |
| tmp.unlink() | |
| return (str(src), False, str(e)) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--root", required=True, help="Dir to search recursively for *.mp4") | |
| ap.add_argument("--gop", type=int, default=15) | |
| ap.add_argument("--jobs", type=int, default=8) | |
| ap.add_argument("--encoder", choices=["auto", "h264_nvenc", "libx264"], default="auto") | |
| args = ap.parse_args() | |
| root = Path(args.root).expanduser().resolve() | |
| videos = sorted(root.rglob("*.mp4")) | |
| if not videos: | |
| print(f"[warn] no mp4 under {root}") | |
| return 1 | |
| encoder = args.encoder | |
| if encoder == "auto": | |
| encoder = "h264_nvenc" if _has_nvenc() else "libx264" | |
| print(f"[info] {len(videos)} files, gop={args.gop}, encoder={encoder}, jobs={args.jobs}") | |
| jobs = [(str(v), args.gop, encoder) for v in videos] | |
| ok = fail = 0 | |
| with ProcessPoolExecutor(max_workers=args.jobs) as ex: | |
| futs = [ex.submit(_reencode, j) for j in jobs] | |
| for i, fut in enumerate(as_completed(futs), 1): | |
| src, success, err = fut.result() | |
| if success: | |
| ok += 1 | |
| else: | |
| fail += 1 | |
| print(f"[fail] {src}: {err}", file=sys.stderr) | |
| if i % 20 == 0 or i == len(futs): | |
| print(f"[progress] {i}/{len(futs)} ok={ok} fail={fail}") | |
| print(f"[done] ok={ok} fail={fail}") | |
| return 0 if fail == 0 else 2 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |
Xet Storage Details
- Size:
- 3.27 kB
- Xet hash:
- 7c3724414ff5307ec06e888b675b99ba080376ffe0f5af96e6e2eb62b4e9b0bf
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.