Buckets:
| #!/usr/bin/env python | |
| """Re-encode a dataset's videos all-intra so random frame access is cheap. | |
| Training samples frames scattered across an episode. With a normal GOP the | |
| decoder must walk from the preceding keyframe to every requested frame, so one | |
| random frame costs ~GOP/2 decodes. Encoding with gop_size=1 makes every frame a | |
| keyframe: one decode per requested frame. | |
| Files are replaced atomically via os.replace, so hardlinks to the original | |
| download are broken rather than followed - the source dataset is never modified. | |
| python tools/reencode_allintra.py --dataset-root <dir> [--crf 23] [--jobs 14] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import os | |
| import shutil | |
| import sys | |
| import time | |
| from concurrent.futures import ProcessPoolExecutor, as_completed | |
| from fractions import Fraction | |
| from pathlib import Path | |
| def reencode(path_str: str, crf: int, preset: str) -> tuple[str, int, float]: | |
| import av | |
| path = Path(path_str) | |
| tmp = path.with_suffix(".allintra.tmp.mp4") | |
| t0 = time.time() | |
| src = av.open(str(path)) | |
| s_in = src.streams.video[0] | |
| s_in.thread_type = "AUTO" | |
| fps = int(round(float(s_in.average_rate))) | |
| dst = av.open(str(tmp), mode="w") | |
| s_out = dst.add_stream("libx264", rate=fps) | |
| s_out.width, s_out.height = s_in.codec_context.width, s_in.codec_context.height | |
| s_out.pix_fmt = "yuv420p" | |
| s_out.time_base = Fraction(1, fps) | |
| s_out.gop_size = 1 | |
| s_out.options = {"crf": str(crf), "preset": preset, "g": "1"} | |
| n = 0 | |
| for frame in src.decode(video=0): | |
| out = frame.reformat(format="yuv420p") | |
| out.pts = n | |
| out.time_base = Fraction(1, fps) | |
| for pkt in s_out.encode(out): | |
| dst.mux(pkt) | |
| n += 1 | |
| for pkt in s_out.encode(): | |
| dst.mux(pkt) | |
| dst.close() | |
| src.close() | |
| os.replace(tmp, path) # breaks the hardlink; the source keeps its own | |
| return (path.name, n, time.time() - t0) | |
| def main() -> None: | |
| ap = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) | |
| ap.add_argument("--dataset-root", required=True) | |
| ap.add_argument("--crf", type=int, default=23) | |
| ap.add_argument("--preset", default="veryfast") | |
| ap.add_argument("--jobs", type=int, default=max(1, (os.cpu_count() or 4) - 2)) | |
| args = ap.parse_args() | |
| root = Path(args.dataset_root).expanduser().resolve() | |
| videos = sorted((root / "videos").rglob("*.mp4")) | |
| if not videos: | |
| raise SystemExit(f"no videos under {root}/videos") | |
| before = sum(v.stat().st_size for v in videos) | |
| free = shutil.disk_usage(root).free | |
| print(f"{len(videos)} videos, {before/1e9:.1f} GB now, {free/1e9:.0f} GB free") | |
| print(f"re-encoding all-intra (crf={args.crf}, preset={args.preset}) with {args.jobs} jobs", flush=True) | |
| t0 = time.time() | |
| done = frames = 0 | |
| with ProcessPoolExecutor(max_workers=args.jobs) as ex: | |
| futs = [ex.submit(reencode, str(v), args.crf, args.preset) for v in videos] | |
| for f in as_completed(futs): | |
| _, n, _ = f.result() | |
| done += 1 | |
| frames += n | |
| if done % 25 == 0 or done == len(videos): | |
| el = time.time() - t0 | |
| print(f" {done}/{len(videos)} videos, {frames} frames, {el:.0f}s " | |
| f"(eta {el/done*(len(videos)-done):.0f}s)", flush=True) | |
| after = sum(v.stat().st_size for v in videos) | |
| print(f"\ndone in {time.time()-t0:.0f}s | {before/1e9:.1f} GB -> {after/1e9:.1f} GB") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 3.53 kB
- Xet hash:
- ec509bf69aed32402b5fb118becc2b23fdad66efa010235870d33e355f0e71aa
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.