Download cache_video_omni_embeds.py from Tinghong-Ye22/code-Psi0-yth-v21: direct link, hf CLI and curl.
- Browser
- Download file 37.5 kB
-
https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v21/resolve/main/cache_video_omni_embeds.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Psi0-yth-v21/cache_video_omni_embeds.py
-
curl -L -o cache_video_omni_embeds.py https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v21/resolve/main/cache_video_omni_embeds.py
37.5 kB
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| import socket | |
| import time | |
| import uuid | |
| from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, ThreadPoolExecutor, as_completed, wait | |
| from itertools import islice | |
| from pathlib import Path | |
| import numpy as np | |
| from tqdm import tqdm | |
| from transformers import Qwen2_5OmniProcessor | |
| from psi.data.seed.seed_video_motion_dataset import ( | |
| load_video_array, | |
| load_video_clip_array, | |
| video_embed_cache_path, | |
| ) | |
| REQUIRED_CACHE_KEYS = ( | |
| "pixel_values_videos", | |
| "video_grid_thw", | |
| "video_second_per_grid", | |
| "video_sample_fps", | |
| "video_size_pixels", | |
| "source_video_fps", | |
| "video_duration_sec", | |
| ) | |
| _WORKER_PROCESSOR = None | |
| _WORKER_MODEL_PATH = "" | |
| _SOURCE_PATH_EXISTS: dict[str, bool] = {} | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser(description="Cache Qwen2.5-Omni video processor outputs.") | |
| parser.add_argument("--video-root", type=str, default="") | |
| parser.add_argument("--dataset-root", type=str, default="") | |
| parser.add_argument("--metadata-jsonl", type=str, default="") | |
| parser.add_argument("--video-path-key", type=str, default="video_path") | |
| parser.add_argument("--embed-root", type=str, required=True) | |
| parser.add_argument("--model-path", type=str, required=True) | |
| parser.add_argument("--video-sample-fps", type=float, default=1.0) | |
| parser.add_argument("--video-size-pixels", type=int, default=50176) | |
| parser.add_argument("--video-decoder", type=str, default="ffmpeg", choices=["opencv", "ffmpeg"]) | |
| parser.add_argument("--dtype", type=str, default="float16", choices=["float16", "float32"]) | |
| parser.add_argument("--num-workers", type=int, default=1) | |
| parser.add_argument("--save-compressed", action="store_true", default=False) | |
| parser.add_argument( | |
| "--existing-cache-check", | |
| choices=["strict", "size"], | |
| default=os.environ.get("PSI_EXISTING_CACHE_CHECK", "strict"), | |
| help="How to validate existing npz caches before skipping. strict opens npz; size only checks final file size.", | |
| ) | |
| parser.add_argument("--limit", type=int, default=0) | |
| parser.add_argument("--overwrite", action="store_true", default=False) | |
| parser.add_argument("--shard-num", type=int, default=1) | |
| parser.add_argument("--shard-id", type=int, default=0) | |
| parser.add_argument("--input-already-sharded", action="store_true", default=False) | |
| parser.add_argument( | |
| "--direct-source-clips", | |
| action="store_true", | |
| default=False, | |
| help=( | |
| "Read source_video_path + chunk_start_sec/duration from metadata, but write cache " | |
| "to the canonical path derived from video_path." | |
| ), | |
| ) | |
| return parser.parse_args() | |
| def _resolve_path(path_value: str, base_dir: Path) -> Path: | |
| path = Path(str(path_value)) | |
| return path if path.is_absolute() else base_dir / path | |
| def resolve_metadata_context(args: argparse.Namespace) -> tuple[Path, Path]: | |
| dataset_root = Path(args.dataset_root) if args.dataset_root else Path(args.metadata_jsonl).parent | |
| metadata_path = Path(args.metadata_jsonl) | |
| if not metadata_path.is_absolute(): | |
| metadata_path = dataset_root / metadata_path | |
| if not metadata_path.exists(): | |
| raise FileNotFoundError(f"metadata_jsonl does not exist: {metadata_path}") | |
| return dataset_root, metadata_path | |
| def build_video_task_from_row( | |
| row: dict, | |
| args: argparse.Namespace, | |
| dataset_root: Path, | |
| metadata_path: Path, | |
| line_no: int, | |
| ) -> dict: | |
| video_value = row.get(args.video_path_key) or row.get("video_path") or row.get("original_video") | |
| if not video_value: | |
| raise ValueError(f"Missing video path at {metadata_path}:{line_no}") | |
| video_path = _resolve_path(str(video_value), dataset_root) | |
| task = {"video_path": video_path} | |
| if args.direct_source_clips: | |
| source_value = row.get("source_video_path") | |
| if not source_value: | |
| raise ValueError(f"Missing source_video_path at {metadata_path}:{line_no}") | |
| source_video_path = _resolve_path(str(source_value), dataset_root) | |
| source_key = str(source_video_path) | |
| exists = _SOURCE_PATH_EXISTS.get(source_key) | |
| if exists is None: | |
| exists = source_video_path.exists() | |
| _SOURCE_PATH_EXISTS[source_key] = exists | |
| if not exists: | |
| raise FileNotFoundError(f"Source video file does not exist: {source_video_path}") | |
| task.update( | |
| { | |
| "decode_video_path": source_video_path, | |
| "clip_start_sec": float(row.get("chunk_start_sec") or 0.0), | |
| "clip_duration_sec": float(row.get("duration") or row.get("video_duration_sec") or 0.0), | |
| } | |
| ) | |
| else: | |
| if not video_path.exists(): | |
| raise FileNotFoundError(f"Video file does not exist: {video_path}") | |
| task["decode_video_path"] = video_path | |
| return task | |
| def iter_metadata_tasks_stream( | |
| args: argparse.Namespace, | |
| start_index: int = 0, | |
| ): | |
| dataset_root, metadata_path = resolve_metadata_context(args) | |
| stop_index = int(args.limit) if args.limit > 0 else None | |
| with metadata_path.open("r", encoding="utf-8") as f: | |
| iterator = islice(f, start_index, None) if start_index > 0 else f | |
| for task_index, line in enumerate(iterator, start=start_index): | |
| if stop_index is not None and task_index >= stop_index: | |
| break | |
| line = line.strip() | |
| if not line: | |
| continue | |
| row = json.loads(line) | |
| task = build_video_task_from_row( | |
| row=row, | |
| args=args, | |
| dataset_root=dataset_root, | |
| metadata_path=metadata_path, | |
| line_no=task_index + 1, | |
| ) | |
| task["task_index"] = task_index | |
| yield task | |
| def iter_video_tasks(args: argparse.Namespace) -> list[dict]: | |
| if args.metadata_jsonl: | |
| dataset_root, metadata_path = resolve_metadata_context(args) | |
| tasks = [] | |
| seen = set() | |
| with metadata_path.open("r", encoding="utf-8") as f: | |
| for line_no, line in enumerate(f, start=1): | |
| line = line.strip() | |
| if not line: | |
| continue | |
| row = json.loads(line) | |
| task = build_video_task_from_row( | |
| row=row, | |
| args=args, | |
| dataset_root=dataset_root, | |
| metadata_path=metadata_path, | |
| line_no=line_no, | |
| ) | |
| video_path = task["video_path"] | |
| key = str(video_embed_cache_path( | |
| video_path, | |
| args.embed_root, | |
| video_sample_fps=args.video_sample_fps, | |
| video_size_pixels=args.video_size_pixels, | |
| )) | |
| if key not in seen: | |
| seen.add(key) | |
| tasks.append(task) | |
| return tasks | |
| if not args.video_root: | |
| raise ValueError("Either --metadata-jsonl or --video-root is required") | |
| video_root = Path(args.video_root) | |
| if not video_root.exists(): | |
| raise FileNotFoundError(f"video_root does not exist: {video_root}") | |
| return [{"video_path": path, "decode_video_path": path} for path in sorted(video_root.glob("*.mp4"))] | |
| def is_valid_cache(path: Path) -> bool: | |
| if not path.exists() or path.stat().st_size <= 0: | |
| return False | |
| try: | |
| with np.load(path, allow_pickle=False) as cache: | |
| for key in REQUIRED_CACHE_KEYS: | |
| if key not in cache: | |
| return False | |
| pixel_values = cache["pixel_values_videos"] | |
| grid = cache["video_grid_thw"] | |
| if pixel_values.size <= 0 or grid.size <= 0: | |
| return False | |
| except Exception: | |
| return False | |
| return True | |
| def is_existing_cache_ready(path: Path, check_mode: str) -> bool: | |
| if not path.exists() or path.stat().st_size <= 0: | |
| return False | |
| if check_mode == "size": | |
| return True | |
| return is_valid_cache(path) | |
| def is_existing_cache_ready_fast(path: Path, check_mode: str) -> bool: | |
| if check_mode == "size": | |
| try: | |
| return path.stat().st_size > 0 | |
| except FileNotFoundError: | |
| return False | |
| return is_existing_cache_ready(path, check_mode) | |
| def remove_path(path: Path) -> None: | |
| try: | |
| path.unlink() | |
| except FileNotFoundError: | |
| pass | |
| def stable_hash_mod(value: str, modulo: int) -> int: | |
| digest = hashlib.sha1(value.encode("utf-8")).digest() | |
| return int.from_bytes(digest[:8], "big") % int(modulo) | |
| def shard_manifest_path(embed_root: Path, shard_id: int, shard_num: int) -> Path: | |
| return embed_root / f"manifest.shard-{shard_id:05d}-of-{shard_num:05d}.json" | |
| def resume_state_path(embed_root: Path, shard_id: int, shard_num: int) -> Path: | |
| if shard_num > 1: | |
| return embed_root / f"resume.shard-{shard_id:05d}-of-{shard_num:05d}.json" | |
| return embed_root / "resume.json" | |
| def write_json_atomic(path: Path, payload: dict) -> None: | |
| tmp_path = path.with_name(f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp") | |
| tmp_path.write_text( | |
| json.dumps(payload, ensure_ascii=False, indent=2), | |
| encoding="utf-8", | |
| ) | |
| tmp_path.replace(path) | |
| def merge_shard_manifests(embed_root: Path, shard_num: int) -> bool: | |
| shard_paths = [shard_manifest_path(embed_root, shard_id, shard_num) for shard_id in range(shard_num)] | |
| if any(not path.exists() for path in shard_paths): | |
| return False | |
| manifests = [] | |
| for path in shard_paths: | |
| try: | |
| manifests.append(json.loads(path.read_text(encoding="utf-8"))) | |
| except Exception: | |
| return False | |
| if not manifests: | |
| return False | |
| merged = dict(manifests[0]) | |
| merged["count"] = int(sum(int(item.get("count") or 0) for item in manifests)) | |
| if all(bool(item.get("input_already_sharded")) for item in manifests): | |
| merged["total_count"] = int(sum(int(item.get("count") or 0) for item in manifests)) | |
| else: | |
| merged["total_count"] = int(max(int(item.get("total_count") or 0) for item in manifests)) | |
| merged["shard_id"] = None | |
| merged["shard_num"] = int(shard_num) | |
| merged["shards"] = [path.name for path in shard_paths] | |
| write_json_atomic(embed_root / "manifest.json", merged) | |
| return True | |
| def maybe_load_json(path: Path) -> dict | None: | |
| if not path.exists(): | |
| return None | |
| try: | |
| return json.loads(path.read_text(encoding="utf-8")) | |
| except Exception: | |
| return None | |
| def load_known_total_count(args: argparse.Namespace, embed_root: Path) -> int | None: | |
| manifest_path = ( | |
| shard_manifest_path(embed_root, args.shard_id, args.shard_num) | |
| if args.shard_num > 1 | |
| else embed_root / "manifest.json" | |
| ) | |
| payload = maybe_load_json(manifest_path) | |
| if payload and str(payload.get("metadata_jsonl")) == str(args.metadata_jsonl): | |
| count = int(payload.get("count") or 0) | |
| if count > 0: | |
| return min(count, int(args.limit)) if args.limit > 0 else count | |
| return None | |
| def count_metadata_rows(args: argparse.Namespace) -> int: | |
| _, metadata_path = resolve_metadata_context(args) | |
| count = 0 | |
| with metadata_path.open("rb") as f: | |
| for chunk in iter(lambda: f.read(1024 * 1024 * 16), b""): | |
| count += chunk.count(b"\n") | |
| if args.limit > 0: | |
| return min(count, int(args.limit)) | |
| return count | |
| def dense_source_key(row: dict) -> str: | |
| for key in ("name", "id", "video_path"): | |
| value = row.get(key) | |
| if isinstance(value, str) and value: | |
| return value.split(".chunk", 1)[0] | |
| return "" | |
| def cache_name_from_row(row: dict, args: argparse.Namespace) -> str: | |
| video_value = row.get(args.video_path_key) or row.get("video_path") or row.get("original_video") | |
| fps_tag = str(float(args.video_sample_fps)).replace(".", "p") | |
| return f"{Path(str(video_value)).stem}.fps{fps_tag}.size{int(args.video_size_pixels)}.npz" | |
| def compatible_resume_payload( | |
| payload: dict | None, | |
| args: argparse.Namespace, | |
| embed_root: Path, | |
| *, | |
| shard_id: int, | |
| shard_num: int, | |
| ) -> bool: | |
| if not payload: | |
| return False | |
| expected = { | |
| "dataset_root": str(args.dataset_root), | |
| "embed_root": str(embed_root), | |
| "video_sample_fps": float(args.video_sample_fps), | |
| "video_size_pixels": int(args.video_size_pixels), | |
| "shard_id": int(shard_id), | |
| "shard_num": int(shard_num), | |
| "input_already_sharded": bool(args.input_already_sharded), | |
| "direct_source_clips": bool(args.direct_source_clips), | |
| } | |
| return all(payload.get(key) == value for key, value in expected.items()) | |
| def parse_resume_state_name(path: Path) -> tuple[int, int] | None: | |
| name = path.name | |
| prefix = "resume.shard-" | |
| if not name.startswith(prefix) or "-of-" not in name or not name.endswith(".json"): | |
| return None | |
| try: | |
| left, right = name[len(prefix):-len(".json")].split("-of-", 1) | |
| return int(left), int(right) | |
| except ValueError: | |
| return None | |
| def infer_frontier_from_parent_resume(args: argparse.Namespace, embed_root: Path) -> int: | |
| if args.shard_num <= 1: | |
| return 0 | |
| dataset_root, _ = resolve_metadata_context(args) | |
| candidates = [] | |
| for resume_path in sorted(embed_root.glob("resume.shard-*-of-*.json")): | |
| parsed = parse_resume_state_name(resume_path) | |
| if parsed is None: | |
| continue | |
| parent_id, parent_num = parsed | |
| if parent_num == args.shard_num: | |
| continue | |
| payload = maybe_load_json(resume_path) | |
| if not compatible_resume_payload( | |
| payload, | |
| args, | |
| embed_root, | |
| shard_id=parent_id, | |
| shard_num=parent_num, | |
| ): | |
| continue | |
| parent_frontier = max(0, int(payload.get("frontier") or 0)) | |
| if parent_frontier <= 0: | |
| continue | |
| parent_metadata = dataset_root / str(payload.get("metadata_jsonl") or "") | |
| if not parent_metadata.exists(): | |
| parent_metadata = dataset_root / f"metadata.shard-{parent_id:05d}-of-{parent_num:05d}.jsonl" | |
| if not parent_metadata.exists(): | |
| continue | |
| candidates.append((parent_num, parent_id, parent_frontier, parent_metadata)) | |
| if not candidates: | |
| return 0 | |
| completed_cache_names = set() | |
| for parent_num, parent_id, parent_frontier, parent_metadata in candidates: | |
| with parent_metadata.open("r", encoding="utf-8") as f: | |
| iterator = tqdm( | |
| islice(f, parent_frontier), | |
| total=parent_frontier, | |
| desc=f"load resume {parent_id}/{parent_num}", | |
| ) | |
| for line in iterator: | |
| line = line.strip() | |
| if not line: | |
| continue | |
| completed_cache_names.add(cache_name_from_row(json.loads(line), args)) | |
| inherited = 0 | |
| for video_task in tqdm(iter_metadata_tasks_stream(args), desc="inherit resume"): | |
| cache_name = video_embed_cache_path( | |
| Path(video_task["video_path"]), | |
| embed_root, | |
| video_sample_fps=args.video_sample_fps, | |
| video_size_pixels=args.video_size_pixels, | |
| ).name | |
| if cache_name not in completed_cache_names: | |
| break | |
| inherited = int(video_task["task_index"]) + 1 | |
| print( | |
| f"[RESUME] inherited frontier={inherited} from " | |
| f"{len(candidates)} compatible previous shard resume file(s)" | |
| ) | |
| return inherited | |
| def write_manifest( | |
| args: argparse.Namespace, | |
| embed_root: Path, | |
| total_video_paths: int, | |
| ) -> None: | |
| manifest = { | |
| "video_root": str(args.video_root), | |
| "dataset_root": str(args.dataset_root), | |
| "metadata_jsonl": str(args.metadata_jsonl), | |
| "embed_root": str(embed_root), | |
| "model_path": str(args.model_path), | |
| "video_sample_fps": float(args.video_sample_fps), | |
| "video_size_pixels": int(args.video_size_pixels), | |
| "video_decoder": str(args.video_decoder), | |
| "dtype": args.dtype, | |
| "num_workers": int(args.num_workers), | |
| "save_compressed": bool(args.save_compressed), | |
| "existing_cache_check": str(args.existing_cache_check), | |
| "count": int(total_video_paths), | |
| "total_count": int(total_video_paths), | |
| "shard_id": int(args.shard_id), | |
| "shard_num": int(args.shard_num), | |
| "input_already_sharded": bool(args.input_already_sharded), | |
| } | |
| manifest_path = ( | |
| shard_manifest_path(embed_root, args.shard_id, args.shard_num) | |
| if args.shard_num > 1 | |
| else embed_root / "manifest.json" | |
| ) | |
| write_json_atomic(manifest_path, manifest) | |
| if args.shard_num > 1: | |
| merge_shard_manifests(embed_root, args.shard_num) | |
| def fast_resume_enabled(args: argparse.Namespace) -> bool: | |
| return bool(args.metadata_jsonl and args.input_already_sharded and args.direct_source_clips) | |
| def load_resume_frontier(args: argparse.Namespace, embed_root: Path) -> tuple[int, int | None, Path]: | |
| state_path = resume_state_path(embed_root, args.shard_id, args.shard_num) | |
| payload = maybe_load_json(state_path) | |
| if not compatible_resume_payload( | |
| payload, | |
| args, | |
| embed_root, | |
| shard_id=args.shard_id, | |
| shard_num=args.shard_num, | |
| ) or not payload or payload.get("metadata_jsonl") != str(args.metadata_jsonl): | |
| return infer_frontier_from_parent_resume(args, embed_root), None, state_path | |
| frontier = max(0, int(payload.get("frontier") or 0)) | |
| total_count = int(payload.get("total_count") or 0) or None | |
| if args.limit > 0: | |
| frontier = min(frontier, int(args.limit)) | |
| if total_count is not None: | |
| total_count = min(total_count, int(args.limit)) | |
| return frontier, total_count, state_path | |
| class ResumeTracker: | |
| def __init__( | |
| self, | |
| *, | |
| path: Path, | |
| args: argparse.Namespace, | |
| embed_root: Path, | |
| frontier: int, | |
| total_count: int | None, | |
| ) -> None: | |
| self.path = path | |
| self.args = args | |
| self.embed_root = embed_root | |
| self.frontier = max(0, int(frontier)) | |
| self.total_count = int(total_count) if total_count is not None else None | |
| self._completed: set[int] = set() | |
| self._dirty = True | |
| self._last_flush_time = 0.0 | |
| def mark_complete(self, task_index: int | None) -> None: | |
| if task_index is None or task_index < 0 or task_index < self.frontier: | |
| return | |
| self._completed.add(int(task_index)) | |
| advanced = False | |
| while self.frontier in self._completed: | |
| self._completed.remove(self.frontier) | |
| self.frontier += 1 | |
| advanced = True | |
| if advanced: | |
| self._dirty = True | |
| def maybe_set_total_count(self, total_count: int) -> None: | |
| total_count = int(total_count) | |
| if self.total_count != total_count: | |
| self.total_count = total_count | |
| self._dirty = True | |
| def flush(self, force: bool = False) -> None: | |
| now = time.time() | |
| if not force and (not self._dirty or now - self._last_flush_time < 10.0): | |
| return | |
| payload = { | |
| "dataset_root": str(self.args.dataset_root), | |
| "metadata_jsonl": str(self.args.metadata_jsonl), | |
| "embed_root": str(self.embed_root), | |
| "video_sample_fps": float(self.args.video_sample_fps), | |
| "video_size_pixels": int(self.args.video_size_pixels), | |
| "shard_id": int(self.args.shard_id), | |
| "shard_num": int(self.args.shard_num), | |
| "input_already_sharded": bool(self.args.input_already_sharded), | |
| "direct_source_clips": bool(self.args.direct_source_clips), | |
| "frontier": int(self.frontier), | |
| "total_count": int(self.total_count) if self.total_count is not None else None, | |
| "updated_at": now, | |
| } | |
| write_json_atomic(self.path, payload) | |
| self._dirty = False | |
| self._last_flush_time = now | |
| def extend_resume_frontier( | |
| args: argparse.Namespace, | |
| embed_root: Path, | |
| frontier: int, | |
| ) -> int: | |
| if args.overwrite: | |
| return 0 | |
| current_frontier = max(0, int(frontier)) | |
| check_workers = max(1, int(os.environ.get("RESUME_SCAN_NUM_WORKERS", "32"))) | |
| batch_size = max(check_workers, int(os.environ.get("RESUME_SCAN_BATCH_SIZE", str(check_workers * 8)))) | |
| iterator = iter_metadata_tasks_stream(args, start_index=current_frontier) | |
| progress = tqdm(desc=f"resume frontier x{check_workers}") | |
| def check_task(video_task: dict) -> tuple[int, bool, Path]: | |
| video_path = Path(video_task["video_path"]) | |
| cache_path = video_embed_cache_path( | |
| video_path, | |
| embed_root, | |
| video_sample_fps=args.video_sample_fps, | |
| video_size_pixels=args.video_size_pixels, | |
| ) | |
| return ( | |
| int(video_task["task_index"]), | |
| is_existing_cache_ready_fast(cache_path, args.existing_cache_check), | |
| cache_path, | |
| ) | |
| with ThreadPoolExecutor(max_workers=check_workers) as executor: | |
| while True: | |
| batch = list(islice(iterator, batch_size)) | |
| if not batch: | |
| break | |
| results = list(executor.map(check_task, batch)) | |
| for task_index, ready, cache_path in results: | |
| progress.update(1) | |
| if ready: | |
| current_frontier = task_index + 1 | |
| continue | |
| if cache_path.exists(): | |
| remove_path(cache_path) | |
| progress.close() | |
| return current_frontier | |
| progress.close() | |
| return current_frontier | |
| def acquire_lock(lock_path: Path) -> int | None: | |
| lock_payload = { | |
| "pid": os.getpid(), | |
| "host": socket.gethostname(), | |
| "time": time.time(), | |
| } | |
| while True: | |
| try: | |
| fd = os.open(str(lock_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY) | |
| os.write(fd, json.dumps(lock_payload).encode("utf-8")) | |
| return fd | |
| except FileExistsError: | |
| try: | |
| payload = json.loads(lock_path.read_text(encoding="utf-8")) | |
| pid = int(payload.get("pid") or -1) | |
| except Exception: | |
| pid = -1 | |
| if pid <= 0 or not Path(f"/proc/{pid}").exists(): | |
| remove_path(lock_path) | |
| continue | |
| return None | |
| def release_lock(fd: int | None, lock_path: Path) -> None: | |
| if fd is not None: | |
| try: | |
| os.close(fd) | |
| except OSError: | |
| pass | |
| remove_path(lock_path) | |
| def init_worker(model_path: str) -> None: | |
| global _WORKER_PROCESSOR, _WORKER_MODEL_PATH | |
| _WORKER_MODEL_PATH = model_path | |
| _WORKER_PROCESSOR = Qwen2_5OmniProcessor.from_pretrained(model_path, use_fast=False) | |
| def get_worker_processor() -> Qwen2_5OmniProcessor: | |
| global _WORKER_PROCESSOR | |
| if _WORKER_PROCESSOR is None: | |
| if not _WORKER_MODEL_PATH: | |
| raise RuntimeError("Worker processor is not initialized") | |
| _WORKER_PROCESSOR = Qwen2_5OmniProcessor.from_pretrained(_WORKER_MODEL_PATH, use_fast=False) | |
| return _WORKER_PROCESSOR | |
| def cache_one_video(task: dict) -> dict: | |
| video_path = Path(task["video_path"]) | |
| decode_video_path = Path(task.get("decode_video_path") or video_path) | |
| embed_root = Path(task["embed_root"]) | |
| cache_path = video_embed_cache_path( | |
| video_path, | |
| embed_root, | |
| video_sample_fps=float(task["video_sample_fps"]), | |
| video_size_pixels=int(task["video_size_pixels"]), | |
| ) | |
| overwrite = bool(task["overwrite"]) | |
| existing_cache_check = str(task.get("existing_cache_check", "strict")) | |
| task_index = int(task.get("task_index", -1)) | |
| result_base = {"path": str(cache_path), "task_index": task_index} | |
| if cache_path.exists() and not overwrite: | |
| if is_existing_cache_ready(cache_path, existing_cache_check): | |
| return {"status": "skipped", **result_base} | |
| remove_path(cache_path) | |
| lock_path = cache_path.with_name(f"{cache_path.name}.lock") | |
| lock_fd = acquire_lock(lock_path) | |
| if lock_fd is None: | |
| if cache_path.exists() and is_existing_cache_ready(cache_path, existing_cache_check): | |
| return {"status": "skipped", **result_base} | |
| return {"status": "locked", **result_base} | |
| tmp_path = cache_path.with_name(f".{cache_path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp") | |
| try: | |
| if cache_path.exists() and not overwrite: | |
| if is_existing_cache_ready(cache_path, existing_cache_check): | |
| return {"status": "skipped", **result_base} | |
| remove_path(cache_path) | |
| if bool(task.get("direct_source_clip", False)): | |
| video, source_video_fps, video_duration_sec = load_video_clip_array( | |
| decode_video_path, | |
| start_sec=float(task["clip_start_sec"]), | |
| duration_sec=float(task["clip_duration_sec"]), | |
| video_sample_fps=float(task["video_sample_fps"]), | |
| video_size_pixels=int(task["video_size_pixels"]), | |
| video_decoder=str(task["video_decoder"]), | |
| ) | |
| else: | |
| video, source_video_fps, video_duration_sec = load_video_array( | |
| decode_video_path, | |
| video_sample_fps=float(task["video_sample_fps"]), | |
| video_size_pixels=int(task["video_size_pixels"]), | |
| video_decoder=str(task["video_decoder"]), | |
| ) | |
| processor = get_worker_processor() | |
| video_inputs = processor.video_processor( | |
| videos=[video], | |
| size={ | |
| "shortest_edge": int(task["video_size_pixels"]), | |
| "longest_edge": int(task["video_size_pixels"]), | |
| }, | |
| ) | |
| video_second_per_grid = np.asarray( | |
| [float(processor.video_processor.temporal_patch_size) / float(task["video_sample_fps"])], | |
| dtype=np.float32, | |
| ) | |
| dtype = np.float16 if str(task["dtype"]) == "float16" else np.float32 | |
| payload = { | |
| "pixel_values_videos": np.asarray(video_inputs["pixel_values_videos"], dtype=dtype), | |
| "video_grid_thw": np.asarray(video_inputs["video_grid_thw"], dtype=np.int64), | |
| "video_second_per_grid": video_second_per_grid, | |
| "video_sample_fps": np.asarray([float(task["video_sample_fps"])], dtype=np.float32), | |
| "video_size_pixels": np.asarray([int(task["video_size_pixels"])], dtype=np.int64), | |
| "source_video_fps": np.asarray([float(source_video_fps)], dtype=np.float32), | |
| "video_duration_sec": np.asarray([float(video_duration_sec)], dtype=np.float32), | |
| } | |
| with tmp_path.open("wb") as f: | |
| if bool(task["save_compressed"]): | |
| np.savez_compressed(f, **payload) | |
| else: | |
| np.savez(f, **payload) | |
| if not is_valid_cache(tmp_path): | |
| raise RuntimeError(f"Invalid cache was written: {tmp_path}") | |
| tmp_path.replace(cache_path) | |
| return {"status": "cached", **result_base} | |
| except Exception as exc: | |
| remove_path(tmp_path) | |
| return {"status": "failed", "error": str(exc), **result_base} | |
| finally: | |
| release_lock(lock_fd, lock_path) | |
| def build_cache_task(args: argparse.Namespace, video_task: dict, embed_root: Path) -> dict: | |
| task = { | |
| "video_path": str(video_task["video_path"]), | |
| "decode_video_path": str(video_task.get("decode_video_path") or video_task["video_path"]), | |
| "embed_root": str(embed_root), | |
| "video_sample_fps": float(args.video_sample_fps), | |
| "video_size_pixels": int(args.video_size_pixels), | |
| "video_decoder": str(args.video_decoder), | |
| "dtype": str(args.dtype), | |
| "save_compressed": bool(args.save_compressed), | |
| "overwrite": bool(args.overwrite), | |
| "existing_cache_check": str(args.existing_cache_check), | |
| } | |
| if "clip_start_sec" in video_task: | |
| task.update( | |
| { | |
| "direct_source_clip": True, | |
| "clip_start_sec": float(video_task["clip_start_sec"]), | |
| "clip_duration_sec": float(video_task["clip_duration_sec"]), | |
| } | |
| ) | |
| if "task_index" in video_task: | |
| task["task_index"] = int(video_task["task_index"]) | |
| return task | |
| def update_result_counters( | |
| result: dict, | |
| tracker: ResumeTracker, | |
| counters: dict[str, int], | |
| ) -> None: | |
| status = result["status"] | |
| task_index = result.get("task_index") | |
| if status == "cached": | |
| counters["cached"] += 1 | |
| tracker.mark_complete(task_index) | |
| elif status == "skipped": | |
| counters["skipped"] += 1 | |
| tracker.mark_complete(task_index) | |
| elif status == "locked": | |
| counters["locked"] += 1 | |
| else: | |
| counters["failed"] += 1 | |
| print(f"[FAILED] {result['path']}: {result.get('error', '')}") | |
| tracker.flush() | |
| def run_fast_resume_cache(args: argparse.Namespace, embed_root: Path) -> None: | |
| frontier, known_total_count, state_path = load_resume_frontier(args, embed_root) | |
| if known_total_count is None: | |
| known_total_count = load_known_total_count(args, embed_root) | |
| if known_total_count is None: | |
| known_total_count = count_metadata_rows(args) | |
| write_manifest(args, embed_root, known_total_count) | |
| extended_frontier = extend_resume_frontier(args, embed_root, frontier) | |
| tracker = ResumeTracker( | |
| path=state_path, | |
| args=args, | |
| embed_root=embed_root, | |
| frontier=extended_frontier, | |
| total_count=known_total_count, | |
| ) | |
| tracker.flush(force=True) | |
| if known_total_count is not None: | |
| print( | |
| f"[SHARD] cache shard={args.shard_id}/{args.shard_num} mode=pre-sharded " | |
| f"resume={extended_frontier}/{known_total_count}" | |
| ) | |
| write_manifest(args, embed_root, known_total_count) | |
| else: | |
| print( | |
| f"[SHARD] cache shard={args.shard_id}/{args.shard_num} mode=pre-sharded " | |
| f"resume={extended_frontier}/?" | |
| ) | |
| counters = { | |
| "cached": 0, | |
| "skipped": 0, | |
| "locked": 0, | |
| "failed": 0, | |
| "invalid": 0, | |
| } | |
| total_hint = None | |
| if tracker.total_count is not None: | |
| total_hint = max(0, int(tracker.total_count) - int(extended_frontier)) | |
| if args.num_workers == 1: | |
| progress = tqdm(total=total_hint, desc="cache omni video") | |
| total_video_paths = extended_frontier | |
| for video_task in iter_metadata_tasks_stream(args, start_index=extended_frontier): | |
| total_video_paths = int(video_task["task_index"]) + 1 | |
| task = build_cache_task(args, video_task, embed_root) | |
| result = cache_one_video(task) | |
| update_result_counters(result, tracker, counters) | |
| progress.update(1) | |
| progress.close() | |
| else: | |
| max_in_flight = max(int(args.num_workers) * 4, int(args.num_workers)) | |
| progress = tqdm(total=total_hint, desc=f"cache omni video x{args.num_workers}") | |
| total_video_paths = extended_frontier | |
| with ProcessPoolExecutor( | |
| max_workers=args.num_workers, | |
| initializer=init_worker, | |
| initargs=(args.model_path,), | |
| ) as executor: | |
| pending_futures: dict = {} | |
| task_iterator = iter_metadata_tasks_stream(args, start_index=extended_frontier) | |
| exhausted = False | |
| while True: | |
| while not exhausted and len(pending_futures) < max_in_flight: | |
| try: | |
| video_task = next(task_iterator) | |
| except StopIteration: | |
| exhausted = True | |
| break | |
| total_video_paths = int(video_task["task_index"]) + 1 | |
| task = build_cache_task(args, video_task, embed_root) | |
| future = executor.submit(cache_one_video, task) | |
| pending_futures[future] = None | |
| if not pending_futures: | |
| break | |
| done, _ = wait(tuple(pending_futures.keys()), return_when=FIRST_COMPLETED) | |
| for future in done: | |
| pending_futures.pop(future, None) | |
| result = future.result() | |
| update_result_counters(result, tracker, counters) | |
| progress.update(1) | |
| progress.close() | |
| if tracker.total_count is None: | |
| tracker.maybe_set_total_count(total_video_paths) | |
| else: | |
| tracker.maybe_set_total_count(max(int(tracker.total_count), int(total_video_paths))) | |
| tracker.flush(force=True) | |
| if total_video_paths > 0: | |
| write_manifest(args, embed_root, total_video_paths if tracker.total_count is None else int(tracker.total_count)) | |
| print( | |
| f"cached={counters['cached']} skipped={counters['skipped']} invalid_removed={counters['invalid']} " | |
| f"locked={counters['locked']} failed={counters['failed']} embed_root={embed_root}" | |
| ) | |
| if counters["failed"] > 0: | |
| raise SystemExit(1) | |
| if counters["locked"] > 0: | |
| raise SystemExit(2) | |
| def main() -> None: | |
| args = parse_args() | |
| if args.num_workers <= 0: | |
| raise ValueError("--num-workers must be positive") | |
| if args.shard_num <= 0: | |
| raise ValueError("--shard-num must be positive") | |
| if args.shard_id < 0 or args.shard_id >= args.shard_num: | |
| raise ValueError("--shard-id must satisfy 0 <= shard_id < shard_num") | |
| embed_root = Path(args.embed_root) | |
| embed_root.mkdir(parents=True, exist_ok=True) | |
| if fast_resume_enabled(args): | |
| run_fast_resume_cache(args, embed_root) | |
| return | |
| video_tasks = iter_video_tasks(args) | |
| if args.limit > 0: | |
| video_tasks = video_tasks[: args.limit] | |
| total_video_paths = len(video_tasks) | |
| if args.shard_num > 1 and not args.input_already_sharded: | |
| video_tasks = [ | |
| video_task for video_task in video_tasks | |
| if stable_hash_mod(str(video_task["video_path"]), args.shard_num) == args.shard_id | |
| ] | |
| if args.shard_num > 1: | |
| mode = "pre-sharded" if args.input_already_sharded else "hash" | |
| print( | |
| f"[SHARD] cache shard={args.shard_id}/{args.shard_num} mode={mode} " | |
| f"videos={len(video_tasks)}/{total_video_paths}" | |
| ) | |
| write_manifest(args, embed_root, total_video_paths) | |
| pending_tasks = [] | |
| cached = 0 | |
| skipped = 0 | |
| locked = 0 | |
| failed = 0 | |
| invalid = 0 | |
| scan_progress = tqdm(video_tasks, total=len(video_tasks), desc="scan cache state") | |
| for video_task in scan_progress: | |
| video_path = Path(video_task["video_path"]) | |
| cache_path = video_embed_cache_path( | |
| video_path, | |
| embed_root, | |
| video_sample_fps=args.video_sample_fps, | |
| video_size_pixels=args.video_size_pixels, | |
| ) | |
| if cache_path.exists() and not args.overwrite: | |
| if is_existing_cache_ready(cache_path, args.existing_cache_check): | |
| skipped += 1 | |
| continue | |
| remove_path(cache_path) | |
| invalid += 1 | |
| pending_tasks.append(build_cache_task(args, video_task, embed_root)) | |
| if pending_tasks: | |
| if args.num_workers == 1: | |
| init_worker(args.model_path) | |
| iterator = (cache_one_video(task) for task in pending_tasks) | |
| progress = tqdm(iterator, total=len(pending_tasks), desc="cache omni video") | |
| for result in progress: | |
| status = result["status"] | |
| if status == "cached": | |
| cached += 1 | |
| elif status == "skipped": | |
| skipped += 1 | |
| elif status == "locked": | |
| locked += 1 | |
| else: | |
| failed += 1 | |
| print(f"[FAILED] {result['path']}: {result.get('error', '')}") | |
| else: | |
| with ProcessPoolExecutor( | |
| max_workers=args.num_workers, | |
| initializer=init_worker, | |
| initargs=(args.model_path,), | |
| ) as executor: | |
| futures = [executor.submit(cache_one_video, task) for task in pending_tasks] | |
| for future in tqdm( | |
| as_completed(futures), | |
| total=len(futures), | |
| desc=f"cache omni video x{args.num_workers}", | |
| ): | |
| result = future.result() | |
| status = result["status"] | |
| if status == "cached": | |
| cached += 1 | |
| elif status == "skipped": | |
| skipped += 1 | |
| elif status == "locked": | |
| locked += 1 | |
| else: | |
| failed += 1 | |
| print(f"[FAILED] {result['path']}: {result.get('error', '')}") | |
| print( | |
| f"cached={cached} skipped={skipped} invalid_removed={invalid} " | |
| f"locked={locked} failed={failed} embed_root={embed_root}" | |
| ) | |
| if failed > 0: | |
| raise SystemExit(1) | |
| if locked > 0: | |
| raise SystemExit(2) | |
| if __name__ == "__main__": | |
| main() | |