from __future__ import annotations import argparse import hashlib import json import os import socket import time import uuid from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, ThreadPoolExecutor, as_completed, wait from itertools import islice from pathlib import Path import numpy as np from tqdm import tqdm from transformers import Qwen2_5OmniProcessor from psi.data.seed.seed_video_motion_dataset import ( load_video_array, load_video_clip_array, video_embed_cache_path, ) REQUIRED_CACHE_KEYS = ( "pixel_values_videos", "video_grid_thw", "video_second_per_grid", "video_sample_fps", "video_size_pixels", "source_video_fps", "video_duration_sec", ) _WORKER_PROCESSOR = None _WORKER_MODEL_PATH = "" _SOURCE_PATH_EXISTS: dict[str, bool] = {} def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Cache Qwen2.5-Omni video processor outputs.") parser.add_argument("--video-root", type=str, default="") parser.add_argument("--dataset-root", type=str, default="") parser.add_argument("--metadata-jsonl", type=str, default="") parser.add_argument("--video-path-key", type=str, default="video_path") parser.add_argument("--embed-root", type=str, required=True) parser.add_argument("--model-path", type=str, required=True) parser.add_argument("--video-sample-fps", type=float, default=1.0) parser.add_argument("--video-size-pixels", type=int, default=50176) parser.add_argument("--video-decoder", type=str, default="ffmpeg", choices=["opencv", "ffmpeg"]) parser.add_argument("--dtype", type=str, default="float16", choices=["float16", "float32"]) parser.add_argument("--num-workers", type=int, default=1) parser.add_argument("--save-compressed", action="store_true", default=False) parser.add_argument( "--existing-cache-check", choices=["strict", "size"], default=os.environ.get("PSI_EXISTING_CACHE_CHECK", "strict"), help="How to validate existing npz caches before skipping. strict opens npz; size only checks final file size.", ) parser.add_argument("--limit", type=int, default=0) parser.add_argument("--overwrite", action="store_true", default=False) parser.add_argument("--shard-num", type=int, default=1) parser.add_argument("--shard-id", type=int, default=0) parser.add_argument("--input-already-sharded", action="store_true", default=False) parser.add_argument( "--direct-source-clips", action="store_true", default=False, help=( "Read source_video_path + chunk_start_sec/duration from metadata, but write cache " "to the canonical path derived from video_path." ), ) return parser.parse_args() def _resolve_path(path_value: str, base_dir: Path) -> Path: path = Path(str(path_value)) return path if path.is_absolute() else base_dir / path def resolve_metadata_context(args: argparse.Namespace) -> tuple[Path, Path]: dataset_root = Path(args.dataset_root) if args.dataset_root else Path(args.metadata_jsonl).parent metadata_path = Path(args.metadata_jsonl) if not metadata_path.is_absolute(): metadata_path = dataset_root / metadata_path if not metadata_path.exists(): raise FileNotFoundError(f"metadata_jsonl does not exist: {metadata_path}") return dataset_root, metadata_path def build_video_task_from_row( row: dict, args: argparse.Namespace, dataset_root: Path, metadata_path: Path, line_no: int, ) -> dict: video_value = row.get(args.video_path_key) or row.get("video_path") or row.get("original_video") if not video_value: raise ValueError(f"Missing video path at {metadata_path}:{line_no}") video_path = _resolve_path(str(video_value), dataset_root) task = {"video_path": video_path} if args.direct_source_clips: source_value = row.get("source_video_path") if not source_value: raise ValueError(f"Missing source_video_path at {metadata_path}:{line_no}") source_video_path = _resolve_path(str(source_value), dataset_root) source_key = str(source_video_path) exists = _SOURCE_PATH_EXISTS.get(source_key) if exists is None: exists = source_video_path.exists() _SOURCE_PATH_EXISTS[source_key] = exists if not exists: raise FileNotFoundError(f"Source video file does not exist: {source_video_path}") task.update( { "decode_video_path": source_video_path, "clip_start_sec": float(row.get("chunk_start_sec") or 0.0), "clip_duration_sec": float(row.get("duration") or row.get("video_duration_sec") or 0.0), } ) else: if not video_path.exists(): raise FileNotFoundError(f"Video file does not exist: {video_path}") task["decode_video_path"] = video_path return task def iter_metadata_tasks_stream( args: argparse.Namespace, start_index: int = 0, ): dataset_root, metadata_path = resolve_metadata_context(args) stop_index = int(args.limit) if args.limit > 0 else None with metadata_path.open("r", encoding="utf-8") as f: iterator = islice(f, start_index, None) if start_index > 0 else f for task_index, line in enumerate(iterator, start=start_index): if stop_index is not None and task_index >= stop_index: break line = line.strip() if not line: continue row = json.loads(line) task = build_video_task_from_row( row=row, args=args, dataset_root=dataset_root, metadata_path=metadata_path, line_no=task_index + 1, ) task["task_index"] = task_index yield task def iter_video_tasks(args: argparse.Namespace) -> list[dict]: if args.metadata_jsonl: dataset_root, metadata_path = resolve_metadata_context(args) tasks = [] seen = set() with metadata_path.open("r", encoding="utf-8") as f: for line_no, line in enumerate(f, start=1): line = line.strip() if not line: continue row = json.loads(line) task = build_video_task_from_row( row=row, args=args, dataset_root=dataset_root, metadata_path=metadata_path, line_no=line_no, ) video_path = task["video_path"] key = str(video_embed_cache_path( video_path, args.embed_root, video_sample_fps=args.video_sample_fps, video_size_pixels=args.video_size_pixels, )) if key not in seen: seen.add(key) tasks.append(task) return tasks if not args.video_root: raise ValueError("Either --metadata-jsonl or --video-root is required") video_root = Path(args.video_root) if not video_root.exists(): raise FileNotFoundError(f"video_root does not exist: {video_root}") return [{"video_path": path, "decode_video_path": path} for path in sorted(video_root.glob("*.mp4"))] def is_valid_cache(path: Path) -> bool: if not path.exists() or path.stat().st_size <= 0: return False try: with np.load(path, allow_pickle=False) as cache: for key in REQUIRED_CACHE_KEYS: if key not in cache: return False pixel_values = cache["pixel_values_videos"] grid = cache["video_grid_thw"] if pixel_values.size <= 0 or grid.size <= 0: return False except Exception: return False return True def is_existing_cache_ready(path: Path, check_mode: str) -> bool: if not path.exists() or path.stat().st_size <= 0: return False if check_mode == "size": return True return is_valid_cache(path) def is_existing_cache_ready_fast(path: Path, check_mode: str) -> bool: if check_mode == "size": try: return path.stat().st_size > 0 except FileNotFoundError: return False return is_existing_cache_ready(path, check_mode) def remove_path(path: Path) -> None: try: path.unlink() except FileNotFoundError: pass def stable_hash_mod(value: str, modulo: int) -> int: digest = hashlib.sha1(value.encode("utf-8")).digest() return int.from_bytes(digest[:8], "big") % int(modulo) def shard_manifest_path(embed_root: Path, shard_id: int, shard_num: int) -> Path: return embed_root / f"manifest.shard-{shard_id:05d}-of-{shard_num:05d}.json" def resume_state_path(embed_root: Path, shard_id: int, shard_num: int) -> Path: if shard_num > 1: return embed_root / f"resume.shard-{shard_id:05d}-of-{shard_num:05d}.json" return embed_root / "resume.json" def write_json_atomic(path: Path, payload: dict) -> None: tmp_path = path.with_name(f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp") tmp_path.write_text( json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8", ) tmp_path.replace(path) def merge_shard_manifests(embed_root: Path, shard_num: int) -> bool: shard_paths = [shard_manifest_path(embed_root, shard_id, shard_num) for shard_id in range(shard_num)] if any(not path.exists() for path in shard_paths): return False manifests = [] for path in shard_paths: try: manifests.append(json.loads(path.read_text(encoding="utf-8"))) except Exception: return False if not manifests: return False merged = dict(manifests[0]) merged["count"] = int(sum(int(item.get("count") or 0) for item in manifests)) if all(bool(item.get("input_already_sharded")) for item in manifests): merged["total_count"] = int(sum(int(item.get("count") or 0) for item in manifests)) else: merged["total_count"] = int(max(int(item.get("total_count") or 0) for item in manifests)) merged["shard_id"] = None merged["shard_num"] = int(shard_num) merged["shards"] = [path.name for path in shard_paths] write_json_atomic(embed_root / "manifest.json", merged) return True def maybe_load_json(path: Path) -> dict | None: if not path.exists(): return None try: return json.loads(path.read_text(encoding="utf-8")) except Exception: return None def load_known_total_count(args: argparse.Namespace, embed_root: Path) -> int | None: manifest_path = ( shard_manifest_path(embed_root, args.shard_id, args.shard_num) if args.shard_num > 1 else embed_root / "manifest.json" ) payload = maybe_load_json(manifest_path) if payload and str(payload.get("metadata_jsonl")) == str(args.metadata_jsonl): count = int(payload.get("count") or 0) if count > 0: return min(count, int(args.limit)) if args.limit > 0 else count return None def count_metadata_rows(args: argparse.Namespace) -> int: _, metadata_path = resolve_metadata_context(args) count = 0 with metadata_path.open("rb") as f: for chunk in iter(lambda: f.read(1024 * 1024 * 16), b""): count += chunk.count(b"\n") if args.limit > 0: return min(count, int(args.limit)) return count def dense_source_key(row: dict) -> str: for key in ("name", "id", "video_path"): value = row.get(key) if isinstance(value, str) and value: return value.split(".chunk", 1)[0] return "" def cache_name_from_row(row: dict, args: argparse.Namespace) -> str: video_value = row.get(args.video_path_key) or row.get("video_path") or row.get("original_video") fps_tag = str(float(args.video_sample_fps)).replace(".", "p") return f"{Path(str(video_value)).stem}.fps{fps_tag}.size{int(args.video_size_pixels)}.npz" def compatible_resume_payload( payload: dict | None, args: argparse.Namespace, embed_root: Path, *, shard_id: int, shard_num: int, ) -> bool: if not payload: return False expected = { "dataset_root": str(args.dataset_root), "embed_root": str(embed_root), "video_sample_fps": float(args.video_sample_fps), "video_size_pixels": int(args.video_size_pixels), "shard_id": int(shard_id), "shard_num": int(shard_num), "input_already_sharded": bool(args.input_already_sharded), "direct_source_clips": bool(args.direct_source_clips), } return all(payload.get(key) == value for key, value in expected.items()) def parse_resume_state_name(path: Path) -> tuple[int, int] | None: name = path.name prefix = "resume.shard-" if not name.startswith(prefix) or "-of-" not in name or not name.endswith(".json"): return None try: left, right = name[len(prefix):-len(".json")].split("-of-", 1) return int(left), int(right) except ValueError: return None def infer_frontier_from_parent_resume(args: argparse.Namespace, embed_root: Path) -> int: if args.shard_num <= 1: return 0 dataset_root, _ = resolve_metadata_context(args) candidates = [] for resume_path in sorted(embed_root.glob("resume.shard-*-of-*.json")): parsed = parse_resume_state_name(resume_path) if parsed is None: continue parent_id, parent_num = parsed if parent_num == args.shard_num: continue payload = maybe_load_json(resume_path) if not compatible_resume_payload( payload, args, embed_root, shard_id=parent_id, shard_num=parent_num, ): continue parent_frontier = max(0, int(payload.get("frontier") or 0)) if parent_frontier <= 0: continue parent_metadata = dataset_root / str(payload.get("metadata_jsonl") or "") if not parent_metadata.exists(): parent_metadata = dataset_root / f"metadata.shard-{parent_id:05d}-of-{parent_num:05d}.jsonl" if not parent_metadata.exists(): continue candidates.append((parent_num, parent_id, parent_frontier, parent_metadata)) if not candidates: return 0 completed_cache_names = set() for parent_num, parent_id, parent_frontier, parent_metadata in candidates: with parent_metadata.open("r", encoding="utf-8") as f: iterator = tqdm( islice(f, parent_frontier), total=parent_frontier, desc=f"load resume {parent_id}/{parent_num}", ) for line in iterator: line = line.strip() if not line: continue completed_cache_names.add(cache_name_from_row(json.loads(line), args)) inherited = 0 for video_task in tqdm(iter_metadata_tasks_stream(args), desc="inherit resume"): cache_name = video_embed_cache_path( Path(video_task["video_path"]), embed_root, video_sample_fps=args.video_sample_fps, video_size_pixels=args.video_size_pixels, ).name if cache_name not in completed_cache_names: break inherited = int(video_task["task_index"]) + 1 print( f"[RESUME] inherited frontier={inherited} from " f"{len(candidates)} compatible previous shard resume file(s)" ) return inherited def write_manifest( args: argparse.Namespace, embed_root: Path, total_video_paths: int, ) -> None: manifest = { "video_root": str(args.video_root), "dataset_root": str(args.dataset_root), "metadata_jsonl": str(args.metadata_jsonl), "embed_root": str(embed_root), "model_path": str(args.model_path), "video_sample_fps": float(args.video_sample_fps), "video_size_pixels": int(args.video_size_pixels), "video_decoder": str(args.video_decoder), "dtype": args.dtype, "num_workers": int(args.num_workers), "save_compressed": bool(args.save_compressed), "existing_cache_check": str(args.existing_cache_check), "count": int(total_video_paths), "total_count": int(total_video_paths), "shard_id": int(args.shard_id), "shard_num": int(args.shard_num), "input_already_sharded": bool(args.input_already_sharded), } manifest_path = ( shard_manifest_path(embed_root, args.shard_id, args.shard_num) if args.shard_num > 1 else embed_root / "manifest.json" ) write_json_atomic(manifest_path, manifest) if args.shard_num > 1: merge_shard_manifests(embed_root, args.shard_num) def fast_resume_enabled(args: argparse.Namespace) -> bool: return bool(args.metadata_jsonl and args.input_already_sharded and args.direct_source_clips) def load_resume_frontier(args: argparse.Namespace, embed_root: Path) -> tuple[int, int | None, Path]: state_path = resume_state_path(embed_root, args.shard_id, args.shard_num) payload = maybe_load_json(state_path) if not compatible_resume_payload( payload, args, embed_root, shard_id=args.shard_id, shard_num=args.shard_num, ) or not payload or payload.get("metadata_jsonl") != str(args.metadata_jsonl): return infer_frontier_from_parent_resume(args, embed_root), None, state_path frontier = max(0, int(payload.get("frontier") or 0)) total_count = int(payload.get("total_count") or 0) or None if args.limit > 0: frontier = min(frontier, int(args.limit)) if total_count is not None: total_count = min(total_count, int(args.limit)) return frontier, total_count, state_path class ResumeTracker: def __init__( self, *, path: Path, args: argparse.Namespace, embed_root: Path, frontier: int, total_count: int | None, ) -> None: self.path = path self.args = args self.embed_root = embed_root self.frontier = max(0, int(frontier)) self.total_count = int(total_count) if total_count is not None else None self._completed: set[int] = set() self._dirty = True self._last_flush_time = 0.0 def mark_complete(self, task_index: int | None) -> None: if task_index is None or task_index < 0 or task_index < self.frontier: return self._completed.add(int(task_index)) advanced = False while self.frontier in self._completed: self._completed.remove(self.frontier) self.frontier += 1 advanced = True if advanced: self._dirty = True def maybe_set_total_count(self, total_count: int) -> None: total_count = int(total_count) if self.total_count != total_count: self.total_count = total_count self._dirty = True def flush(self, force: bool = False) -> None: now = time.time() if not force and (not self._dirty or now - self._last_flush_time < 10.0): return payload = { "dataset_root": str(self.args.dataset_root), "metadata_jsonl": str(self.args.metadata_jsonl), "embed_root": str(self.embed_root), "video_sample_fps": float(self.args.video_sample_fps), "video_size_pixels": int(self.args.video_size_pixels), "shard_id": int(self.args.shard_id), "shard_num": int(self.args.shard_num), "input_already_sharded": bool(self.args.input_already_sharded), "direct_source_clips": bool(self.args.direct_source_clips), "frontier": int(self.frontier), "total_count": int(self.total_count) if self.total_count is not None else None, "updated_at": now, } write_json_atomic(self.path, payload) self._dirty = False self._last_flush_time = now def extend_resume_frontier( args: argparse.Namespace, embed_root: Path, frontier: int, ) -> int: if args.overwrite: return 0 current_frontier = max(0, int(frontier)) check_workers = max(1, int(os.environ.get("RESUME_SCAN_NUM_WORKERS", "32"))) batch_size = max(check_workers, int(os.environ.get("RESUME_SCAN_BATCH_SIZE", str(check_workers * 8)))) iterator = iter_metadata_tasks_stream(args, start_index=current_frontier) progress = tqdm(desc=f"resume frontier x{check_workers}") def check_task(video_task: dict) -> tuple[int, bool, Path]: video_path = Path(video_task["video_path"]) cache_path = video_embed_cache_path( video_path, embed_root, video_sample_fps=args.video_sample_fps, video_size_pixels=args.video_size_pixels, ) return ( int(video_task["task_index"]), is_existing_cache_ready_fast(cache_path, args.existing_cache_check), cache_path, ) with ThreadPoolExecutor(max_workers=check_workers) as executor: while True: batch = list(islice(iterator, batch_size)) if not batch: break results = list(executor.map(check_task, batch)) for task_index, ready, cache_path in results: progress.update(1) if ready: current_frontier = task_index + 1 continue if cache_path.exists(): remove_path(cache_path) progress.close() return current_frontier progress.close() return current_frontier def acquire_lock(lock_path: Path) -> int | None: lock_payload = { "pid": os.getpid(), "host": socket.gethostname(), "time": time.time(), } while True: try: fd = os.open(str(lock_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY) os.write(fd, json.dumps(lock_payload).encode("utf-8")) return fd except FileExistsError: try: payload = json.loads(lock_path.read_text(encoding="utf-8")) pid = int(payload.get("pid") or -1) except Exception: pid = -1 if pid <= 0 or not Path(f"/proc/{pid}").exists(): remove_path(lock_path) continue return None def release_lock(fd: int | None, lock_path: Path) -> None: if fd is not None: try: os.close(fd) except OSError: pass remove_path(lock_path) def init_worker(model_path: str) -> None: global _WORKER_PROCESSOR, _WORKER_MODEL_PATH _WORKER_MODEL_PATH = model_path _WORKER_PROCESSOR = Qwen2_5OmniProcessor.from_pretrained(model_path, use_fast=False) def get_worker_processor() -> Qwen2_5OmniProcessor: global _WORKER_PROCESSOR if _WORKER_PROCESSOR is None: if not _WORKER_MODEL_PATH: raise RuntimeError("Worker processor is not initialized") _WORKER_PROCESSOR = Qwen2_5OmniProcessor.from_pretrained(_WORKER_MODEL_PATH, use_fast=False) return _WORKER_PROCESSOR def cache_one_video(task: dict) -> dict: video_path = Path(task["video_path"]) decode_video_path = Path(task.get("decode_video_path") or video_path) embed_root = Path(task["embed_root"]) cache_path = video_embed_cache_path( video_path, embed_root, video_sample_fps=float(task["video_sample_fps"]), video_size_pixels=int(task["video_size_pixels"]), ) overwrite = bool(task["overwrite"]) existing_cache_check = str(task.get("existing_cache_check", "strict")) task_index = int(task.get("task_index", -1)) result_base = {"path": str(cache_path), "task_index": task_index} if cache_path.exists() and not overwrite: if is_existing_cache_ready(cache_path, existing_cache_check): return {"status": "skipped", **result_base} remove_path(cache_path) lock_path = cache_path.with_name(f"{cache_path.name}.lock") lock_fd = acquire_lock(lock_path) if lock_fd is None: if cache_path.exists() and is_existing_cache_ready(cache_path, existing_cache_check): return {"status": "skipped", **result_base} return {"status": "locked", **result_base} tmp_path = cache_path.with_name(f".{cache_path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp") try: if cache_path.exists() and not overwrite: if is_existing_cache_ready(cache_path, existing_cache_check): return {"status": "skipped", **result_base} remove_path(cache_path) if bool(task.get("direct_source_clip", False)): video, source_video_fps, video_duration_sec = load_video_clip_array( decode_video_path, start_sec=float(task["clip_start_sec"]), duration_sec=float(task["clip_duration_sec"]), video_sample_fps=float(task["video_sample_fps"]), video_size_pixels=int(task["video_size_pixels"]), video_decoder=str(task["video_decoder"]), ) else: video, source_video_fps, video_duration_sec = load_video_array( decode_video_path, video_sample_fps=float(task["video_sample_fps"]), video_size_pixels=int(task["video_size_pixels"]), video_decoder=str(task["video_decoder"]), ) processor = get_worker_processor() video_inputs = processor.video_processor( videos=[video], size={ "shortest_edge": int(task["video_size_pixels"]), "longest_edge": int(task["video_size_pixels"]), }, ) video_second_per_grid = np.asarray( [float(processor.video_processor.temporal_patch_size) / float(task["video_sample_fps"])], dtype=np.float32, ) dtype = np.float16 if str(task["dtype"]) == "float16" else np.float32 payload = { "pixel_values_videos": np.asarray(video_inputs["pixel_values_videos"], dtype=dtype), "video_grid_thw": np.asarray(video_inputs["video_grid_thw"], dtype=np.int64), "video_second_per_grid": video_second_per_grid, "video_sample_fps": np.asarray([float(task["video_sample_fps"])], dtype=np.float32), "video_size_pixels": np.asarray([int(task["video_size_pixels"])], dtype=np.int64), "source_video_fps": np.asarray([float(source_video_fps)], dtype=np.float32), "video_duration_sec": np.asarray([float(video_duration_sec)], dtype=np.float32), } with tmp_path.open("wb") as f: if bool(task["save_compressed"]): np.savez_compressed(f, **payload) else: np.savez(f, **payload) if not is_valid_cache(tmp_path): raise RuntimeError(f"Invalid cache was written: {tmp_path}") tmp_path.replace(cache_path) return {"status": "cached", **result_base} except Exception as exc: remove_path(tmp_path) return {"status": "failed", "error": str(exc), **result_base} finally: release_lock(lock_fd, lock_path) def build_cache_task(args: argparse.Namespace, video_task: dict, embed_root: Path) -> dict: task = { "video_path": str(video_task["video_path"]), "decode_video_path": str(video_task.get("decode_video_path") or video_task["video_path"]), "embed_root": str(embed_root), "video_sample_fps": float(args.video_sample_fps), "video_size_pixels": int(args.video_size_pixels), "video_decoder": str(args.video_decoder), "dtype": str(args.dtype), "save_compressed": bool(args.save_compressed), "overwrite": bool(args.overwrite), "existing_cache_check": str(args.existing_cache_check), } if "clip_start_sec" in video_task: task.update( { "direct_source_clip": True, "clip_start_sec": float(video_task["clip_start_sec"]), "clip_duration_sec": float(video_task["clip_duration_sec"]), } ) if "task_index" in video_task: task["task_index"] = int(video_task["task_index"]) return task def update_result_counters( result: dict, tracker: ResumeTracker, counters: dict[str, int], ) -> None: status = result["status"] task_index = result.get("task_index") if status == "cached": counters["cached"] += 1 tracker.mark_complete(task_index) elif status == "skipped": counters["skipped"] += 1 tracker.mark_complete(task_index) elif status == "locked": counters["locked"] += 1 else: counters["failed"] += 1 print(f"[FAILED] {result['path']}: {result.get('error', '')}") tracker.flush() def run_fast_resume_cache(args: argparse.Namespace, embed_root: Path) -> None: frontier, known_total_count, state_path = load_resume_frontier(args, embed_root) if known_total_count is None: known_total_count = load_known_total_count(args, embed_root) if known_total_count is None: known_total_count = count_metadata_rows(args) write_manifest(args, embed_root, known_total_count) extended_frontier = extend_resume_frontier(args, embed_root, frontier) tracker = ResumeTracker( path=state_path, args=args, embed_root=embed_root, frontier=extended_frontier, total_count=known_total_count, ) tracker.flush(force=True) if known_total_count is not None: print( f"[SHARD] cache shard={args.shard_id}/{args.shard_num} mode=pre-sharded " f"resume={extended_frontier}/{known_total_count}" ) write_manifest(args, embed_root, known_total_count) else: print( f"[SHARD] cache shard={args.shard_id}/{args.shard_num} mode=pre-sharded " f"resume={extended_frontier}/?" ) counters = { "cached": 0, "skipped": 0, "locked": 0, "failed": 0, "invalid": 0, } total_hint = None if tracker.total_count is not None: total_hint = max(0, int(tracker.total_count) - int(extended_frontier)) if args.num_workers == 1: progress = tqdm(total=total_hint, desc="cache omni video") total_video_paths = extended_frontier for video_task in iter_metadata_tasks_stream(args, start_index=extended_frontier): total_video_paths = int(video_task["task_index"]) + 1 task = build_cache_task(args, video_task, embed_root) result = cache_one_video(task) update_result_counters(result, tracker, counters) progress.update(1) progress.close() else: max_in_flight = max(int(args.num_workers) * 4, int(args.num_workers)) progress = tqdm(total=total_hint, desc=f"cache omni video x{args.num_workers}") total_video_paths = extended_frontier with ProcessPoolExecutor( max_workers=args.num_workers, initializer=init_worker, initargs=(args.model_path,), ) as executor: pending_futures: dict = {} task_iterator = iter_metadata_tasks_stream(args, start_index=extended_frontier) exhausted = False while True: while not exhausted and len(pending_futures) < max_in_flight: try: video_task = next(task_iterator) except StopIteration: exhausted = True break total_video_paths = int(video_task["task_index"]) + 1 task = build_cache_task(args, video_task, embed_root) future = executor.submit(cache_one_video, task) pending_futures[future] = None if not pending_futures: break done, _ = wait(tuple(pending_futures.keys()), return_when=FIRST_COMPLETED) for future in done: pending_futures.pop(future, None) result = future.result() update_result_counters(result, tracker, counters) progress.update(1) progress.close() if tracker.total_count is None: tracker.maybe_set_total_count(total_video_paths) else: tracker.maybe_set_total_count(max(int(tracker.total_count), int(total_video_paths))) tracker.flush(force=True) if total_video_paths > 0: write_manifest(args, embed_root, total_video_paths if tracker.total_count is None else int(tracker.total_count)) print( f"cached={counters['cached']} skipped={counters['skipped']} invalid_removed={counters['invalid']} " f"locked={counters['locked']} failed={counters['failed']} embed_root={embed_root}" ) if counters["failed"] > 0: raise SystemExit(1) if counters["locked"] > 0: raise SystemExit(2) def main() -> None: args = parse_args() if args.num_workers <= 0: raise ValueError("--num-workers must be positive") if args.shard_num <= 0: raise ValueError("--shard-num must be positive") if args.shard_id < 0 or args.shard_id >= args.shard_num: raise ValueError("--shard-id must satisfy 0 <= shard_id < shard_num") embed_root = Path(args.embed_root) embed_root.mkdir(parents=True, exist_ok=True) if fast_resume_enabled(args): run_fast_resume_cache(args, embed_root) return video_tasks = iter_video_tasks(args) if args.limit > 0: video_tasks = video_tasks[: args.limit] total_video_paths = len(video_tasks) if args.shard_num > 1 and not args.input_already_sharded: video_tasks = [ video_task for video_task in video_tasks if stable_hash_mod(str(video_task["video_path"]), args.shard_num) == args.shard_id ] if args.shard_num > 1: mode = "pre-sharded" if args.input_already_sharded else "hash" print( f"[SHARD] cache shard={args.shard_id}/{args.shard_num} mode={mode} " f"videos={len(video_tasks)}/{total_video_paths}" ) write_manifest(args, embed_root, total_video_paths) pending_tasks = [] cached = 0 skipped = 0 locked = 0 failed = 0 invalid = 0 scan_progress = tqdm(video_tasks, total=len(video_tasks), desc="scan cache state") for video_task in scan_progress: video_path = Path(video_task["video_path"]) cache_path = video_embed_cache_path( video_path, embed_root, video_sample_fps=args.video_sample_fps, video_size_pixels=args.video_size_pixels, ) if cache_path.exists() and not args.overwrite: if is_existing_cache_ready(cache_path, args.existing_cache_check): skipped += 1 continue remove_path(cache_path) invalid += 1 pending_tasks.append(build_cache_task(args, video_task, embed_root)) if pending_tasks: if args.num_workers == 1: init_worker(args.model_path) iterator = (cache_one_video(task) for task in pending_tasks) progress = tqdm(iterator, total=len(pending_tasks), desc="cache omni video") for result in progress: status = result["status"] if status == "cached": cached += 1 elif status == "skipped": skipped += 1 elif status == "locked": locked += 1 else: failed += 1 print(f"[FAILED] {result['path']}: {result.get('error', '')}") else: with ProcessPoolExecutor( max_workers=args.num_workers, initializer=init_worker, initargs=(args.model_path,), ) as executor: futures = [executor.submit(cache_one_video, task) for task in pending_tasks] for future in tqdm( as_completed(futures), total=len(futures), desc=f"cache omni video x{args.num_workers}", ): result = future.result() status = result["status"] if status == "cached": cached += 1 elif status == "skipped": skipped += 1 elif status == "locked": locked += 1 else: failed += 1 print(f"[FAILED] {result['path']}: {result.get('error', '')}") print( f"cached={cached} skipped={skipped} invalid_removed={invalid} " f"locked={locked} failed={failed} embed_root={embed_root}" ) if failed > 0: raise SystemExit(1) if locked > 0: raise SystemExit(2) if __name__ == "__main__": main()