Nadeem-Shoukath115113/opensarm / tools /reencode_dense_gop.py
Nadeem-Shoukath115113's picture
download
raw
3.27 kB
"""Re-encode mp4s in place with a dense GOP (`-g 15`) so random-frame seeks
land on or near an I-frame. Fixes 3-5x dataloader slowdown when the source
videos were encoded with large GOPs (e.g. NVENC `-preset p1` default GOP=300).
Runs many ffmpeg jobs in parallel via ProcessPoolExecutor. Overwrites originals
atomically via a temp file next to each source.
"""
from __future__ import annotations
import argparse, os, shutil, subprocess, sys
from concurrent.futures import ProcessPoolExecutor, as_completed
from pathlib import Path
def _has_nvenc() -> bool:
try:
out = subprocess.run(
["ffmpeg", "-hide_banner", "-encoders"],
capture_output=True, text=True, check=True,
).stdout
return "h264_nvenc" in out
except Exception:
return False
def _reencode(job):
src, gop, encoder = job
src = Path(src)
tmp = src.with_suffix(".tmp.mp4")
if encoder == "h264_nvenc":
vcodec = [
"-c:v", "h264_nvenc",
"-preset", "p4", "-tune", "hq",
"-rc", "constqp", "-qp", "20",
"-g", str(gop), "-keyint_min", str(gop),
"-bf", "0",
]
else:
vcodec = [
"-c:v", "libx264",
"-preset", "veryfast", "-crf", "20",
"-g", str(gop), "-keyint_min", str(gop),
"-bf", "0",
]
cmd = [
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
"-i", str(src),
*vcodec,
"-video_track_timescale", "15000",
"-movflags", "+faststart",
str(tmp),
]
try:
subprocess.run(cmd, check=True)
os.replace(tmp, src)
return (str(src), True, "")
except subprocess.CalledProcessError as e:
if tmp.exists():
tmp.unlink()
return (str(src), False, str(e))
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--root", required=True, help="Dir to search recursively for *.mp4")
ap.add_argument("--gop", type=int, default=15)
ap.add_argument("--jobs", type=int, default=8)
ap.add_argument("--encoder", choices=["auto", "h264_nvenc", "libx264"], default="auto")
args = ap.parse_args()
root = Path(args.root).expanduser().resolve()
videos = sorted(root.rglob("*.mp4"))
if not videos:
print(f"[warn] no mp4 under {root}")
return 1
encoder = args.encoder
if encoder == "auto":
encoder = "h264_nvenc" if _has_nvenc() else "libx264"
print(f"[info] {len(videos)} files, gop={args.gop}, encoder={encoder}, jobs={args.jobs}")
jobs = [(str(v), args.gop, encoder) for v in videos]
ok = fail = 0
with ProcessPoolExecutor(max_workers=args.jobs) as ex:
futs = [ex.submit(_reencode, j) for j in jobs]
for i, fut in enumerate(as_completed(futs), 1):
src, success, err = fut.result()
if success:
ok += 1
else:
fail += 1
print(f"[fail] {src}: {err}", file=sys.stderr)
if i % 20 == 0 or i == len(futs):
print(f"[progress] {i}/{len(futs)} ok={ok} fail={fail}")
print(f"[done] ok={ok} fail={fail}")
return 0 if fail == 0 else 2
if __name__ == "__main__":
sys.exit(main())

Xet Storage Details

Size:
3.27 kB
·
Xet hash:
7c3724414ff5307ec06e888b675b99ba080376ffe0f5af96e6e2eb62b4e9b0bf

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.