code-Psi0-yth-v21 / cache_video_omni_embeds.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
d8aa1ac verified
Raw History Blame Contribute Delete
37.5 kB
from __future__ import annotations
import argparse
import hashlib
import json
import os
import socket
import time
import uuid
from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, ThreadPoolExecutor, as_completed, wait
from itertools import islice
from pathlib import Path
import numpy as np
from tqdm import tqdm
from transformers import Qwen2_5OmniProcessor
from psi.data.seed.seed_video_motion_dataset import (
load_video_array,
load_video_clip_array,
video_embed_cache_path,
)
REQUIRED_CACHE_KEYS = (
"pixel_values_videos",
"video_grid_thw",
"video_second_per_grid",
"video_sample_fps",
"video_size_pixels",
"source_video_fps",
"video_duration_sec",
)
_WORKER_PROCESSOR = None
_WORKER_MODEL_PATH = ""
_SOURCE_PATH_EXISTS: dict[str, bool] = {}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Cache Qwen2.5-Omni video processor outputs.")
parser.add_argument("--video-root", type=str, default="")
parser.add_argument("--dataset-root", type=str, default="")
parser.add_argument("--metadata-jsonl", type=str, default="")
parser.add_argument("--video-path-key", type=str, default="video_path")
parser.add_argument("--embed-root", type=str, required=True)
parser.add_argument("--model-path", type=str, required=True)
parser.add_argument("--video-sample-fps", type=float, default=1.0)
parser.add_argument("--video-size-pixels", type=int, default=50176)
parser.add_argument("--video-decoder", type=str, default="ffmpeg", choices=["opencv", "ffmpeg"])
parser.add_argument("--dtype", type=str, default="float16", choices=["float16", "float32"])
parser.add_argument("--num-workers", type=int, default=1)
parser.add_argument("--save-compressed", action="store_true", default=False)
parser.add_argument(
"--existing-cache-check",
choices=["strict", "size"],
default=os.environ.get("PSI_EXISTING_CACHE_CHECK", "strict"),
help="How to validate existing npz caches before skipping. strict opens npz; size only checks final file size.",
)
parser.add_argument("--limit", type=int, default=0)
parser.add_argument("--overwrite", action="store_true", default=False)
parser.add_argument("--shard-num", type=int, default=1)
parser.add_argument("--shard-id", type=int, default=0)
parser.add_argument("--input-already-sharded", action="store_true", default=False)
parser.add_argument(
"--direct-source-clips",
action="store_true",
default=False,
help=(
"Read source_video_path + chunk_start_sec/duration from metadata, but write cache "
"to the canonical path derived from video_path."
),
)
return parser.parse_args()
def _resolve_path(path_value: str, base_dir: Path) -> Path:
path = Path(str(path_value))
return path if path.is_absolute() else base_dir / path
def resolve_metadata_context(args: argparse.Namespace) -> tuple[Path, Path]:
dataset_root = Path(args.dataset_root) if args.dataset_root else Path(args.metadata_jsonl).parent
metadata_path = Path(args.metadata_jsonl)
if not metadata_path.is_absolute():
metadata_path = dataset_root / metadata_path
if not metadata_path.exists():
raise FileNotFoundError(f"metadata_jsonl does not exist: {metadata_path}")
return dataset_root, metadata_path
def build_video_task_from_row(
row: dict,
args: argparse.Namespace,
dataset_root: Path,
metadata_path: Path,
line_no: int,
) -> dict:
video_value = row.get(args.video_path_key) or row.get("video_path") or row.get("original_video")
if not video_value:
raise ValueError(f"Missing video path at {metadata_path}:{line_no}")
video_path = _resolve_path(str(video_value), dataset_root)
task = {"video_path": video_path}
if args.direct_source_clips:
source_value = row.get("source_video_path")
if not source_value:
raise ValueError(f"Missing source_video_path at {metadata_path}:{line_no}")
source_video_path = _resolve_path(str(source_value), dataset_root)
source_key = str(source_video_path)
exists = _SOURCE_PATH_EXISTS.get(source_key)
if exists is None:
exists = source_video_path.exists()
_SOURCE_PATH_EXISTS[source_key] = exists
if not exists:
raise FileNotFoundError(f"Source video file does not exist: {source_video_path}")
task.update(
{
"decode_video_path": source_video_path,
"clip_start_sec": float(row.get("chunk_start_sec") or 0.0),
"clip_duration_sec": float(row.get("duration") or row.get("video_duration_sec") or 0.0),
}
)
else:
if not video_path.exists():
raise FileNotFoundError(f"Video file does not exist: {video_path}")
task["decode_video_path"] = video_path
return task
def iter_metadata_tasks_stream(
args: argparse.Namespace,
start_index: int = 0,
):
dataset_root, metadata_path = resolve_metadata_context(args)
stop_index = int(args.limit) if args.limit > 0 else None
with metadata_path.open("r", encoding="utf-8") as f:
iterator = islice(f, start_index, None) if start_index > 0 else f
for task_index, line in enumerate(iterator, start=start_index):
if stop_index is not None and task_index >= stop_index:
break
line = line.strip()
if not line:
continue
row = json.loads(line)
task = build_video_task_from_row(
row=row,
args=args,
dataset_root=dataset_root,
metadata_path=metadata_path,
line_no=task_index + 1,
)
task["task_index"] = task_index
yield task
def iter_video_tasks(args: argparse.Namespace) -> list[dict]:
if args.metadata_jsonl:
dataset_root, metadata_path = resolve_metadata_context(args)
tasks = []
seen = set()
with metadata_path.open("r", encoding="utf-8") as f:
for line_no, line in enumerate(f, start=1):
line = line.strip()
if not line:
continue
row = json.loads(line)
task = build_video_task_from_row(
row=row,
args=args,
dataset_root=dataset_root,
metadata_path=metadata_path,
line_no=line_no,
)
video_path = task["video_path"]
key = str(video_embed_cache_path(
video_path,
args.embed_root,
video_sample_fps=args.video_sample_fps,
video_size_pixels=args.video_size_pixels,
))
if key not in seen:
seen.add(key)
tasks.append(task)
return tasks
if not args.video_root:
raise ValueError("Either --metadata-jsonl or --video-root is required")
video_root = Path(args.video_root)
if not video_root.exists():
raise FileNotFoundError(f"video_root does not exist: {video_root}")
return [{"video_path": path, "decode_video_path": path} for path in sorted(video_root.glob("*.mp4"))]
def is_valid_cache(path: Path) -> bool:
if not path.exists() or path.stat().st_size <= 0:
return False
try:
with np.load(path, allow_pickle=False) as cache:
for key in REQUIRED_CACHE_KEYS:
if key not in cache:
return False
pixel_values = cache["pixel_values_videos"]
grid = cache["video_grid_thw"]
if pixel_values.size <= 0 or grid.size <= 0:
return False
except Exception:
return False
return True
def is_existing_cache_ready(path: Path, check_mode: str) -> bool:
if not path.exists() or path.stat().st_size <= 0:
return False
if check_mode == "size":
return True
return is_valid_cache(path)
def is_existing_cache_ready_fast(path: Path, check_mode: str) -> bool:
if check_mode == "size":
try:
return path.stat().st_size > 0
except FileNotFoundError:
return False
return is_existing_cache_ready(path, check_mode)
def remove_path(path: Path) -> None:
try:
path.unlink()
except FileNotFoundError:
pass
def stable_hash_mod(value: str, modulo: int) -> int:
digest = hashlib.sha1(value.encode("utf-8")).digest()
return int.from_bytes(digest[:8], "big") % int(modulo)
def shard_manifest_path(embed_root: Path, shard_id: int, shard_num: int) -> Path:
return embed_root / f"manifest.shard-{shard_id:05d}-of-{shard_num:05d}.json"
def resume_state_path(embed_root: Path, shard_id: int, shard_num: int) -> Path:
if shard_num > 1:
return embed_root / f"resume.shard-{shard_id:05d}-of-{shard_num:05d}.json"
return embed_root / "resume.json"
def write_json_atomic(path: Path, payload: dict) -> None:
tmp_path = path.with_name(f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp")
tmp_path.write_text(
json.dumps(payload, ensure_ascii=False, indent=2),
encoding="utf-8",
)
tmp_path.replace(path)
def merge_shard_manifests(embed_root: Path, shard_num: int) -> bool:
shard_paths = [shard_manifest_path(embed_root, shard_id, shard_num) for shard_id in range(shard_num)]
if any(not path.exists() for path in shard_paths):
return False
manifests = []
for path in shard_paths:
try:
manifests.append(json.loads(path.read_text(encoding="utf-8")))
except Exception:
return False
if not manifests:
return False
merged = dict(manifests[0])
merged["count"] = int(sum(int(item.get("count") or 0) for item in manifests))
if all(bool(item.get("input_already_sharded")) for item in manifests):
merged["total_count"] = int(sum(int(item.get("count") or 0) for item in manifests))
else:
merged["total_count"] = int(max(int(item.get("total_count") or 0) for item in manifests))
merged["shard_id"] = None
merged["shard_num"] = int(shard_num)
merged["shards"] = [path.name for path in shard_paths]
write_json_atomic(embed_root / "manifest.json", merged)
return True
def maybe_load_json(path: Path) -> dict | None:
if not path.exists():
return None
try:
return json.loads(path.read_text(encoding="utf-8"))
except Exception:
return None
def load_known_total_count(args: argparse.Namespace, embed_root: Path) -> int | None:
manifest_path = (
shard_manifest_path(embed_root, args.shard_id, args.shard_num)
if args.shard_num > 1
else embed_root / "manifest.json"
)
payload = maybe_load_json(manifest_path)
if payload and str(payload.get("metadata_jsonl")) == str(args.metadata_jsonl):
count = int(payload.get("count") or 0)
if count > 0:
return min(count, int(args.limit)) if args.limit > 0 else count
return None
def count_metadata_rows(args: argparse.Namespace) -> int:
_, metadata_path = resolve_metadata_context(args)
count = 0
with metadata_path.open("rb") as f:
for chunk in iter(lambda: f.read(1024 * 1024 * 16), b""):
count += chunk.count(b"\n")
if args.limit > 0:
return min(count, int(args.limit))
return count
def dense_source_key(row: dict) -> str:
for key in ("name", "id", "video_path"):
value = row.get(key)
if isinstance(value, str) and value:
return value.split(".chunk", 1)[0]
return ""
def cache_name_from_row(row: dict, args: argparse.Namespace) -> str:
video_value = row.get(args.video_path_key) or row.get("video_path") or row.get("original_video")
fps_tag = str(float(args.video_sample_fps)).replace(".", "p")
return f"{Path(str(video_value)).stem}.fps{fps_tag}.size{int(args.video_size_pixels)}.npz"
def compatible_resume_payload(
payload: dict | None,
args: argparse.Namespace,
embed_root: Path,
*,
shard_id: int,
shard_num: int,
) -> bool:
if not payload:
return False
expected = {
"dataset_root": str(args.dataset_root),
"embed_root": str(embed_root),
"video_sample_fps": float(args.video_sample_fps),
"video_size_pixels": int(args.video_size_pixels),
"shard_id": int(shard_id),
"shard_num": int(shard_num),
"input_already_sharded": bool(args.input_already_sharded),
"direct_source_clips": bool(args.direct_source_clips),
}
return all(payload.get(key) == value for key, value in expected.items())
def parse_resume_state_name(path: Path) -> tuple[int, int] | None:
name = path.name
prefix = "resume.shard-"
if not name.startswith(prefix) or "-of-" not in name or not name.endswith(".json"):
return None
try:
left, right = name[len(prefix):-len(".json")].split("-of-", 1)
return int(left), int(right)
except ValueError:
return None
def infer_frontier_from_parent_resume(args: argparse.Namespace, embed_root: Path) -> int:
if args.shard_num <= 1:
return 0
dataset_root, _ = resolve_metadata_context(args)
candidates = []
for resume_path in sorted(embed_root.glob("resume.shard-*-of-*.json")):
parsed = parse_resume_state_name(resume_path)
if parsed is None:
continue
parent_id, parent_num = parsed
if parent_num == args.shard_num:
continue
payload = maybe_load_json(resume_path)
if not compatible_resume_payload(
payload,
args,
embed_root,
shard_id=parent_id,
shard_num=parent_num,
):
continue
parent_frontier = max(0, int(payload.get("frontier") or 0))
if parent_frontier <= 0:
continue
parent_metadata = dataset_root / str(payload.get("metadata_jsonl") or "")
if not parent_metadata.exists():
parent_metadata = dataset_root / f"metadata.shard-{parent_id:05d}-of-{parent_num:05d}.jsonl"
if not parent_metadata.exists():
continue
candidates.append((parent_num, parent_id, parent_frontier, parent_metadata))
if not candidates:
return 0
completed_cache_names = set()
for parent_num, parent_id, parent_frontier, parent_metadata in candidates:
with parent_metadata.open("r", encoding="utf-8") as f:
iterator = tqdm(
islice(f, parent_frontier),
total=parent_frontier,
desc=f"load resume {parent_id}/{parent_num}",
)
for line in iterator:
line = line.strip()
if not line:
continue
completed_cache_names.add(cache_name_from_row(json.loads(line), args))
inherited = 0
for video_task in tqdm(iter_metadata_tasks_stream(args), desc="inherit resume"):
cache_name = video_embed_cache_path(
Path(video_task["video_path"]),
embed_root,
video_sample_fps=args.video_sample_fps,
video_size_pixels=args.video_size_pixels,
).name
if cache_name not in completed_cache_names:
break
inherited = int(video_task["task_index"]) + 1
print(
f"[RESUME] inherited frontier={inherited} from "
f"{len(candidates)} compatible previous shard resume file(s)"
)
return inherited
def write_manifest(
args: argparse.Namespace,
embed_root: Path,
total_video_paths: int,
) -> None:
manifest = {
"video_root": str(args.video_root),
"dataset_root": str(args.dataset_root),
"metadata_jsonl": str(args.metadata_jsonl),
"embed_root": str(embed_root),
"model_path": str(args.model_path),
"video_sample_fps": float(args.video_sample_fps),
"video_size_pixels": int(args.video_size_pixels),
"video_decoder": str(args.video_decoder),
"dtype": args.dtype,
"num_workers": int(args.num_workers),
"save_compressed": bool(args.save_compressed),
"existing_cache_check": str(args.existing_cache_check),
"count": int(total_video_paths),
"total_count": int(total_video_paths),
"shard_id": int(args.shard_id),
"shard_num": int(args.shard_num),
"input_already_sharded": bool(args.input_already_sharded),
}
manifest_path = (
shard_manifest_path(embed_root, args.shard_id, args.shard_num)
if args.shard_num > 1
else embed_root / "manifest.json"
)
write_json_atomic(manifest_path, manifest)
if args.shard_num > 1:
merge_shard_manifests(embed_root, args.shard_num)
def fast_resume_enabled(args: argparse.Namespace) -> bool:
return bool(args.metadata_jsonl and args.input_already_sharded and args.direct_source_clips)
def load_resume_frontier(args: argparse.Namespace, embed_root: Path) -> tuple[int, int | None, Path]:
state_path = resume_state_path(embed_root, args.shard_id, args.shard_num)
payload = maybe_load_json(state_path)
if not compatible_resume_payload(
payload,
args,
embed_root,
shard_id=args.shard_id,
shard_num=args.shard_num,
) or not payload or payload.get("metadata_jsonl") != str(args.metadata_jsonl):
return infer_frontier_from_parent_resume(args, embed_root), None, state_path
frontier = max(0, int(payload.get("frontier") or 0))
total_count = int(payload.get("total_count") or 0) or None
if args.limit > 0:
frontier = min(frontier, int(args.limit))
if total_count is not None:
total_count = min(total_count, int(args.limit))
return frontier, total_count, state_path
class ResumeTracker:
def __init__(
self,
*,
path: Path,
args: argparse.Namespace,
embed_root: Path,
frontier: int,
total_count: int | None,
) -> None:
self.path = path
self.args = args
self.embed_root = embed_root
self.frontier = max(0, int(frontier))
self.total_count = int(total_count) if total_count is not None else None
self._completed: set[int] = set()
self._dirty = True
self._last_flush_time = 0.0
def mark_complete(self, task_index: int | None) -> None:
if task_index is None or task_index < 0 or task_index < self.frontier:
return
self._completed.add(int(task_index))
advanced = False
while self.frontier in self._completed:
self._completed.remove(self.frontier)
self.frontier += 1
advanced = True
if advanced:
self._dirty = True
def maybe_set_total_count(self, total_count: int) -> None:
total_count = int(total_count)
if self.total_count != total_count:
self.total_count = total_count
self._dirty = True
def flush(self, force: bool = False) -> None:
now = time.time()
if not force and (not self._dirty or now - self._last_flush_time < 10.0):
return
payload = {
"dataset_root": str(self.args.dataset_root),
"metadata_jsonl": str(self.args.metadata_jsonl),
"embed_root": str(self.embed_root),
"video_sample_fps": float(self.args.video_sample_fps),
"video_size_pixels": int(self.args.video_size_pixels),
"shard_id": int(self.args.shard_id),
"shard_num": int(self.args.shard_num),
"input_already_sharded": bool(self.args.input_already_sharded),
"direct_source_clips": bool(self.args.direct_source_clips),
"frontier": int(self.frontier),
"total_count": int(self.total_count) if self.total_count is not None else None,
"updated_at": now,
}
write_json_atomic(self.path, payload)
self._dirty = False
self._last_flush_time = now
def extend_resume_frontier(
args: argparse.Namespace,
embed_root: Path,
frontier: int,
) -> int:
if args.overwrite:
return 0
current_frontier = max(0, int(frontier))
check_workers = max(1, int(os.environ.get("RESUME_SCAN_NUM_WORKERS", "32")))
batch_size = max(check_workers, int(os.environ.get("RESUME_SCAN_BATCH_SIZE", str(check_workers * 8))))
iterator = iter_metadata_tasks_stream(args, start_index=current_frontier)
progress = tqdm(desc=f"resume frontier x{check_workers}")
def check_task(video_task: dict) -> tuple[int, bool, Path]:
video_path = Path(video_task["video_path"])
cache_path = video_embed_cache_path(
video_path,
embed_root,
video_sample_fps=args.video_sample_fps,
video_size_pixels=args.video_size_pixels,
)
return (
int(video_task["task_index"]),
is_existing_cache_ready_fast(cache_path, args.existing_cache_check),
cache_path,
)
with ThreadPoolExecutor(max_workers=check_workers) as executor:
while True:
batch = list(islice(iterator, batch_size))
if not batch:
break
results = list(executor.map(check_task, batch))
for task_index, ready, cache_path in results:
progress.update(1)
if ready:
current_frontier = task_index + 1
continue
if cache_path.exists():
remove_path(cache_path)
progress.close()
return current_frontier
progress.close()
return current_frontier
def acquire_lock(lock_path: Path) -> int | None:
lock_payload = {
"pid": os.getpid(),
"host": socket.gethostname(),
"time": time.time(),
}
while True:
try:
fd = os.open(str(lock_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
os.write(fd, json.dumps(lock_payload).encode("utf-8"))
return fd
except FileExistsError:
try:
payload = json.loads(lock_path.read_text(encoding="utf-8"))
pid = int(payload.get("pid") or -1)
except Exception:
pid = -1
if pid <= 0 or not Path(f"/proc/{pid}").exists():
remove_path(lock_path)
continue
return None
def release_lock(fd: int | None, lock_path: Path) -> None:
if fd is not None:
try:
os.close(fd)
except OSError:
pass
remove_path(lock_path)
def init_worker(model_path: str) -> None:
global _WORKER_PROCESSOR, _WORKER_MODEL_PATH
_WORKER_MODEL_PATH = model_path
_WORKER_PROCESSOR = Qwen2_5OmniProcessor.from_pretrained(model_path, use_fast=False)
def get_worker_processor() -> Qwen2_5OmniProcessor:
global _WORKER_PROCESSOR
if _WORKER_PROCESSOR is None:
if not _WORKER_MODEL_PATH:
raise RuntimeError("Worker processor is not initialized")
_WORKER_PROCESSOR = Qwen2_5OmniProcessor.from_pretrained(_WORKER_MODEL_PATH, use_fast=False)
return _WORKER_PROCESSOR
def cache_one_video(task: dict) -> dict:
video_path = Path(task["video_path"])
decode_video_path = Path(task.get("decode_video_path") or video_path)
embed_root = Path(task["embed_root"])
cache_path = video_embed_cache_path(
video_path,
embed_root,
video_sample_fps=float(task["video_sample_fps"]),
video_size_pixels=int(task["video_size_pixels"]),
)
overwrite = bool(task["overwrite"])
existing_cache_check = str(task.get("existing_cache_check", "strict"))
task_index = int(task.get("task_index", -1))
result_base = {"path": str(cache_path), "task_index": task_index}
if cache_path.exists() and not overwrite:
if is_existing_cache_ready(cache_path, existing_cache_check):
return {"status": "skipped", **result_base}
remove_path(cache_path)
lock_path = cache_path.with_name(f"{cache_path.name}.lock")
lock_fd = acquire_lock(lock_path)
if lock_fd is None:
if cache_path.exists() and is_existing_cache_ready(cache_path, existing_cache_check):
return {"status": "skipped", **result_base}
return {"status": "locked", **result_base}
tmp_path = cache_path.with_name(f".{cache_path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp")
try:
if cache_path.exists() and not overwrite:
if is_existing_cache_ready(cache_path, existing_cache_check):
return {"status": "skipped", **result_base}
remove_path(cache_path)
if bool(task.get("direct_source_clip", False)):
video, source_video_fps, video_duration_sec = load_video_clip_array(
decode_video_path,
start_sec=float(task["clip_start_sec"]),
duration_sec=float(task["clip_duration_sec"]),
video_sample_fps=float(task["video_sample_fps"]),
video_size_pixels=int(task["video_size_pixels"]),
video_decoder=str(task["video_decoder"]),
)
else:
video, source_video_fps, video_duration_sec = load_video_array(
decode_video_path,
video_sample_fps=float(task["video_sample_fps"]),
video_size_pixels=int(task["video_size_pixels"]),
video_decoder=str(task["video_decoder"]),
)
processor = get_worker_processor()
video_inputs = processor.video_processor(
videos=[video],
size={
"shortest_edge": int(task["video_size_pixels"]),
"longest_edge": int(task["video_size_pixels"]),
},
)
video_second_per_grid = np.asarray(
[float(processor.video_processor.temporal_patch_size) / float(task["video_sample_fps"])],
dtype=np.float32,
)
dtype = np.float16 if str(task["dtype"]) == "float16" else np.float32
payload = {
"pixel_values_videos": np.asarray(video_inputs["pixel_values_videos"], dtype=dtype),
"video_grid_thw": np.asarray(video_inputs["video_grid_thw"], dtype=np.int64),
"video_second_per_grid": video_second_per_grid,
"video_sample_fps": np.asarray([float(task["video_sample_fps"])], dtype=np.float32),
"video_size_pixels": np.asarray([int(task["video_size_pixels"])], dtype=np.int64),
"source_video_fps": np.asarray([float(source_video_fps)], dtype=np.float32),
"video_duration_sec": np.asarray([float(video_duration_sec)], dtype=np.float32),
}
with tmp_path.open("wb") as f:
if bool(task["save_compressed"]):
np.savez_compressed(f, **payload)
else:
np.savez(f, **payload)
if not is_valid_cache(tmp_path):
raise RuntimeError(f"Invalid cache was written: {tmp_path}")
tmp_path.replace(cache_path)
return {"status": "cached", **result_base}
except Exception as exc:
remove_path(tmp_path)
return {"status": "failed", "error": str(exc), **result_base}
finally:
release_lock(lock_fd, lock_path)
def build_cache_task(args: argparse.Namespace, video_task: dict, embed_root: Path) -> dict:
task = {
"video_path": str(video_task["video_path"]),
"decode_video_path": str(video_task.get("decode_video_path") or video_task["video_path"]),
"embed_root": str(embed_root),
"video_sample_fps": float(args.video_sample_fps),
"video_size_pixels": int(args.video_size_pixels),
"video_decoder": str(args.video_decoder),
"dtype": str(args.dtype),
"save_compressed": bool(args.save_compressed),
"overwrite": bool(args.overwrite),
"existing_cache_check": str(args.existing_cache_check),
}
if "clip_start_sec" in video_task:
task.update(
{
"direct_source_clip": True,
"clip_start_sec": float(video_task["clip_start_sec"]),
"clip_duration_sec": float(video_task["clip_duration_sec"]),
}
)
if "task_index" in video_task:
task["task_index"] = int(video_task["task_index"])
return task
def update_result_counters(
result: dict,
tracker: ResumeTracker,
counters: dict[str, int],
) -> None:
status = result["status"]
task_index = result.get("task_index")
if status == "cached":
counters["cached"] += 1
tracker.mark_complete(task_index)
elif status == "skipped":
counters["skipped"] += 1
tracker.mark_complete(task_index)
elif status == "locked":
counters["locked"] += 1
else:
counters["failed"] += 1
print(f"[FAILED] {result['path']}: {result.get('error', '')}")
tracker.flush()
def run_fast_resume_cache(args: argparse.Namespace, embed_root: Path) -> None:
frontier, known_total_count, state_path = load_resume_frontier(args, embed_root)
if known_total_count is None:
known_total_count = load_known_total_count(args, embed_root)
if known_total_count is None:
known_total_count = count_metadata_rows(args)
write_manifest(args, embed_root, known_total_count)
extended_frontier = extend_resume_frontier(args, embed_root, frontier)
tracker = ResumeTracker(
path=state_path,
args=args,
embed_root=embed_root,
frontier=extended_frontier,
total_count=known_total_count,
)
tracker.flush(force=True)
if known_total_count is not None:
print(
f"[SHARD] cache shard={args.shard_id}/{args.shard_num} mode=pre-sharded "
f"resume={extended_frontier}/{known_total_count}"
)
write_manifest(args, embed_root, known_total_count)
else:
print(
f"[SHARD] cache shard={args.shard_id}/{args.shard_num} mode=pre-sharded "
f"resume={extended_frontier}/?"
)
counters = {
"cached": 0,
"skipped": 0,
"locked": 0,
"failed": 0,
"invalid": 0,
}
total_hint = None
if tracker.total_count is not None:
total_hint = max(0, int(tracker.total_count) - int(extended_frontier))
if args.num_workers == 1:
progress = tqdm(total=total_hint, desc="cache omni video")
total_video_paths = extended_frontier
for video_task in iter_metadata_tasks_stream(args, start_index=extended_frontier):
total_video_paths = int(video_task["task_index"]) + 1
task = build_cache_task(args, video_task, embed_root)
result = cache_one_video(task)
update_result_counters(result, tracker, counters)
progress.update(1)
progress.close()
else:
max_in_flight = max(int(args.num_workers) * 4, int(args.num_workers))
progress = tqdm(total=total_hint, desc=f"cache omni video x{args.num_workers}")
total_video_paths = extended_frontier
with ProcessPoolExecutor(
max_workers=args.num_workers,
initializer=init_worker,
initargs=(args.model_path,),
) as executor:
pending_futures: dict = {}
task_iterator = iter_metadata_tasks_stream(args, start_index=extended_frontier)
exhausted = False
while True:
while not exhausted and len(pending_futures) < max_in_flight:
try:
video_task = next(task_iterator)
except StopIteration:
exhausted = True
break
total_video_paths = int(video_task["task_index"]) + 1
task = build_cache_task(args, video_task, embed_root)
future = executor.submit(cache_one_video, task)
pending_futures[future] = None
if not pending_futures:
break
done, _ = wait(tuple(pending_futures.keys()), return_when=FIRST_COMPLETED)
for future in done:
pending_futures.pop(future, None)
result = future.result()
update_result_counters(result, tracker, counters)
progress.update(1)
progress.close()
if tracker.total_count is None:
tracker.maybe_set_total_count(total_video_paths)
else:
tracker.maybe_set_total_count(max(int(tracker.total_count), int(total_video_paths)))
tracker.flush(force=True)
if total_video_paths > 0:
write_manifest(args, embed_root, total_video_paths if tracker.total_count is None else int(tracker.total_count))
print(
f"cached={counters['cached']} skipped={counters['skipped']} invalid_removed={counters['invalid']} "
f"locked={counters['locked']} failed={counters['failed']} embed_root={embed_root}"
)
if counters["failed"] > 0:
raise SystemExit(1)
if counters["locked"] > 0:
raise SystemExit(2)
def main() -> None:
args = parse_args()
if args.num_workers <= 0:
raise ValueError("--num-workers must be positive")
if args.shard_num <= 0:
raise ValueError("--shard-num must be positive")
if args.shard_id < 0 or args.shard_id >= args.shard_num:
raise ValueError("--shard-id must satisfy 0 <= shard_id < shard_num")
embed_root = Path(args.embed_root)
embed_root.mkdir(parents=True, exist_ok=True)
if fast_resume_enabled(args):
run_fast_resume_cache(args, embed_root)
return
video_tasks = iter_video_tasks(args)
if args.limit > 0:
video_tasks = video_tasks[: args.limit]
total_video_paths = len(video_tasks)
if args.shard_num > 1 and not args.input_already_sharded:
video_tasks = [
video_task for video_task in video_tasks
if stable_hash_mod(str(video_task["video_path"]), args.shard_num) == args.shard_id
]
if args.shard_num > 1:
mode = "pre-sharded" if args.input_already_sharded else "hash"
print(
f"[SHARD] cache shard={args.shard_id}/{args.shard_num} mode={mode} "
f"videos={len(video_tasks)}/{total_video_paths}"
)
write_manifest(args, embed_root, total_video_paths)
pending_tasks = []
cached = 0
skipped = 0
locked = 0
failed = 0
invalid = 0
scan_progress = tqdm(video_tasks, total=len(video_tasks), desc="scan cache state")
for video_task in scan_progress:
video_path = Path(video_task["video_path"])
cache_path = video_embed_cache_path(
video_path,
embed_root,
video_sample_fps=args.video_sample_fps,
video_size_pixels=args.video_size_pixels,
)
if cache_path.exists() and not args.overwrite:
if is_existing_cache_ready(cache_path, args.existing_cache_check):
skipped += 1
continue
remove_path(cache_path)
invalid += 1
pending_tasks.append(build_cache_task(args, video_task, embed_root))
if pending_tasks:
if args.num_workers == 1:
init_worker(args.model_path)
iterator = (cache_one_video(task) for task in pending_tasks)
progress = tqdm(iterator, total=len(pending_tasks), desc="cache omni video")
for result in progress:
status = result["status"]
if status == "cached":
cached += 1
elif status == "skipped":
skipped += 1
elif status == "locked":
locked += 1
else:
failed += 1
print(f"[FAILED] {result['path']}: {result.get('error', '')}")
else:
with ProcessPoolExecutor(
max_workers=args.num_workers,
initializer=init_worker,
initargs=(args.model_path,),
) as executor:
futures = [executor.submit(cache_one_video, task) for task in pending_tasks]
for future in tqdm(
as_completed(futures),
total=len(futures),
desc=f"cache omni video x{args.num_workers}",
):
result = future.result()
status = result["status"]
if status == "cached":
cached += 1
elif status == "skipped":
skipped += 1
elif status == "locked":
locked += 1
else:
failed += 1
print(f"[FAILED] {result['path']}: {result.get('error', '')}")
print(
f"cached={cached} skipped={skipped} invalid_removed={invalid} "
f"locked={locked} failed={failed} embed_root={embed_root}"
)
if failed > 0:
raise SystemExit(1)
if locked > 0:
raise SystemExit(2)
if __name__ == "__main__":
main()