Nadeem-Shoukath115113/opensarm / tools /reencode_allintra.py
Nadeem-Shoukath115113's picture
download
raw
3.53 kB
#!/usr/bin/env python
"""Re-encode a dataset's videos all-intra so random frame access is cheap.
Training samples frames scattered across an episode. With a normal GOP the
decoder must walk from the preceding keyframe to every requested frame, so one
random frame costs ~GOP/2 decodes. Encoding with gop_size=1 makes every frame a
keyframe: one decode per requested frame.
Files are replaced atomically via os.replace, so hardlinks to the original
download are broken rather than followed - the source dataset is never modified.
python tools/reencode_allintra.py --dataset-root <dir> [--crf 23] [--jobs 14]
"""
from __future__ import annotations
import argparse
import os
import shutil
import sys
import time
from concurrent.futures import ProcessPoolExecutor, as_completed
from fractions import Fraction
from pathlib import Path
def reencode(path_str: str, crf: int, preset: str) -> tuple[str, int, float]:
import av
path = Path(path_str)
tmp = path.with_suffix(".allintra.tmp.mp4")
t0 = time.time()
src = av.open(str(path))
s_in = src.streams.video[0]
s_in.thread_type = "AUTO"
fps = int(round(float(s_in.average_rate)))
dst = av.open(str(tmp), mode="w")
s_out = dst.add_stream("libx264", rate=fps)
s_out.width, s_out.height = s_in.codec_context.width, s_in.codec_context.height
s_out.pix_fmt = "yuv420p"
s_out.time_base = Fraction(1, fps)
s_out.gop_size = 1
s_out.options = {"crf": str(crf), "preset": preset, "g": "1"}
n = 0
for frame in src.decode(video=0):
out = frame.reformat(format="yuv420p")
out.pts = n
out.time_base = Fraction(1, fps)
for pkt in s_out.encode(out):
dst.mux(pkt)
n += 1
for pkt in s_out.encode():
dst.mux(pkt)
dst.close()
src.close()
os.replace(tmp, path) # breaks the hardlink; the source keeps its own
return (path.name, n, time.time() - t0)
def main() -> None:
ap = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
ap.add_argument("--dataset-root", required=True)
ap.add_argument("--crf", type=int, default=23)
ap.add_argument("--preset", default="veryfast")
ap.add_argument("--jobs", type=int, default=max(1, (os.cpu_count() or 4) - 2))
args = ap.parse_args()
root = Path(args.dataset_root).expanduser().resolve()
videos = sorted((root / "videos").rglob("*.mp4"))
if not videos:
raise SystemExit(f"no videos under {root}/videos")
before = sum(v.stat().st_size for v in videos)
free = shutil.disk_usage(root).free
print(f"{len(videos)} videos, {before/1e9:.1f} GB now, {free/1e9:.0f} GB free")
print(f"re-encoding all-intra (crf={args.crf}, preset={args.preset}) with {args.jobs} jobs", flush=True)
t0 = time.time()
done = frames = 0
with ProcessPoolExecutor(max_workers=args.jobs) as ex:
futs = [ex.submit(reencode, str(v), args.crf, args.preset) for v in videos]
for f in as_completed(futs):
_, n, _ = f.result()
done += 1
frames += n
if done % 25 == 0 or done == len(videos):
el = time.time() - t0
print(f" {done}/{len(videos)} videos, {frames} frames, {el:.0f}s "
f"(eta {el/done*(len(videos)-done):.0f}s)", flush=True)
after = sum(v.stat().st_size for v in videos)
print(f"\ndone in {time.time()-t0:.0f}s | {before/1e9:.1f} GB -> {after/1e9:.1f} GB")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
3.53 kB
·
Xet hash:
ec509bf69aed32402b5fb118becc2b23fdad66efa010235870d33e355f0e71aa

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.