code-HFD / scripts /data /cache_omni_hidden_states.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
136 kB
from __future__ import annotations import argparse import gc import hashlib import json import math import os import random import sqlite3 import subprocess import time import traceback import warnings import wave from collections import deque from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait from dataclasses import dataclass from pathlib import Path from typing import Any, Iterable, Iterator import numpy as np import pandas as pd import torch import torch.distributed as dist from safetensors import safe_open from safetensors.torch import save_file from tqdm import tqdm from transformers import AutoConfig, Qwen2_5OmniProcessor, Qwen2_5OmniThinkerForConditionalGeneration from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import Qwen2_5OmniProcessorKwargs from psi.data.seed.full_duplex_audio_contract import ( AUDIO_COMMAND_STREAMING_PROMPT, HFD_AUDIO_SOURCE_FORMAT, HFD_ONLINE_ENCODER_ATTENTION, HFD_ONLINE_ENCODER_BATCH_SIZE, HFD_ONLINE_ENCODER_CONTRACT, HFD_ONLINE_ENCODER_DTYPE, encoder_contract_payload, qwen_model_fingerprint, write_json_atomic, ) from psi.data.seed.seed_audio_motion_dataset import ( _AUDIO_DANCE_PROMPT_POOLS, ffmpeg_bin, ffprobe_bin, ) from psi.data.seed.motion_frame_index import read_motion_frame_count from psi.data.seed.seed_video_motion_dataset import _VIDEO_MOTION_PROMPT_POOLS from psi.data.seed.text_prompt import ( DEFAULT_SEED_TEXT_FIELDS, collect_seed_text_candidates, compose_seed_motion_prompt_from_candidates, ) OMNI_DEFAULT_SYSTEM_PROMPT = ( "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, " "capable of perceiving auditory and visual inputs, as well as generating text and speech." ) ACTIVE_MODEL_FINGERPRINT = "" SUPPORTED_TASKS = ( "text", "audio_text", "audio_only", "audio_command_text", "audio_command_only", "video_clip_text", "video_clip_only", ) DEFAULT_AUDIO_FILE_EXTENSIONS = (".wav", ".flac", ".mp3", ".m4a", ".aac", ".ogg", ".npy") @dataclass(frozen=True) class SourcePreset: name: str modality: str dataset_root: Path manifest_path: Path allowed_tasks: tuple[str, ...] instruction_key: str = "" audio_path_key: str = "audio_path" video_path_key: str = "video_path" video_embed_dir: Path | None = None is_audio_dir_manifest: bool = False audio_file_extensions: tuple[str, ...] = DEFAULT_AUDIO_FILE_EXTENSIONS text_fields: tuple[str, ...] = tuple(DEFAULT_SEED_TEXT_FIELDS) SOURCE_PRESETS: dict[str, SourcePreset] = { "seed_text": SourcePreset( name="seed_text", modality="text", dataset_root=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all"), manifest_path=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/metadata/seed_metadata_v004.csv"), allowed_tasks=("text",), ), "2264_with_audio": SourcePreset( name="2264_with_audio", modality="audio", dataset_root=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_audio_wav_v26/2264_with_audio"), manifest_path=Path( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_audio_wav_v26/2264_with_audio/metadata.jsonl" ), allowed_tasks=("audio_text", "audio_only"), audio_file_extensions=(".wav",), ), "psi0_dance_audio": SourcePreset( name="psi0_dance_audio", modality="audio", dataset_root=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_audio_wav_v26/psi0_dance_audio"), manifest_path=Path( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_audio_wav_v26/psi0_dance_audio/metadata.jsonl" ), allowed_tasks=("audio_text", "audio_only"), audio_file_extensions=(".wav",), ), "qwen3_tts_seed_audio_command": SourcePreset( name="qwen3_tts_seed_audio_command", modality="audio", dataset_root=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts"), manifest_path=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/metadata.jsonl"), allowed_tasks=("audio_command_text", "audio_command_only"), instruction_key="content_short_description", is_audio_dir_manifest=True, audio_file_extensions=(".wav",), ), "dance_1723_video_10s_clip50": SourcePreset( name="dance_1723_video_10s_clip50", modality="video_clip", dataset_root=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/dance_1723_video_10s"), manifest_path=Path( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/dance_1723_video_10s/metadata.jsonl" ), allowed_tasks=("video_clip_text", "video_clip_only"), instruction_key="video_caption", video_embed_dir=Path( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/dance_1723_video_10s/embeds" ), ), "motion_example_0506_demo_clip50": SourcePreset( name="motion_example_0506_demo_clip50", modality="video_clip", dataset_root=Path( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/motion_example_0506_demo" ), manifest_path=Path( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/motion_example_0506_demo/metadata.jsonl" ), allowed_tasks=("video_clip_text", "video_clip_only"), instruction_key="video_caption", video_embed_dir=Path( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/motion_example_0506_demo/embeds" ), ), "2264_with_audio_clip50": SourcePreset( name="2264_with_audio_clip50", modality="video_clip", dataset_root=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/2264_with_audio"), manifest_path=Path( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/2264_with_audio/metadata.jsonl" ), allowed_tasks=("video_clip_text", "video_clip_only"), instruction_key="video_caption", video_embed_dir=Path( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/2264_with_audio/embeds" ), ), } _VIDEO_EMBED_INFO_CACHE: dict[str, dict[str, Any]] = {} _AUDIO_RESAMPLED_SAMPLE_COUNT_CACHE: dict[tuple[str, int, int, int], int] = {} @dataclass class ExportItem: source_name: str task_name: str modality: str input_path: Path output_path: Path instruction: str manifest_id: str rel_source_path: str audio_start_sec: float = 0.0 audio_duration_sec: float = 0.0 full_audio: bool = True audio_sampling_rate: int = 16000 shard_id: int = -1 source_key: str = "" source_output_path: Path | None = None source_ordinal: int = -1 source_item_index: int = -1 source_item_count: int = 0 frame_id: int = -1 source_step_frames: int = 0 motion_fps: int = 50 def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Cache Qwen2.5-Omni thinker hidden states for stable multimodal tasks.") parser.add_argument( "--mode", choices=("estimate", "benchmark", "export", "status", "verify"), default="estimate", ) parser.add_argument("--model-path", default="/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B") parser.add_argument( "--output-root", default="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_omni_hidden_states_v23", ) parser.add_argument("--tasks", default="text,audio_text,audio_only,audio_command_text,audio_command_only,video_clip_text,video_clip_only") parser.add_argument( "--sources", default=( "seed_text,2264_with_audio,psi0_dance_audio,qwen3_tts_seed_audio_command," "dance_1723_video_10s_clip50,motion_example_0506_demo_clip50,2264_with_audio_clip50" ), ) parser.add_argument( "--source-config-file", type=Path, default=None, help="Optional JSON source preset list. Paths may be relative to the JSON file.", ) parser.add_argument( "--source-config-json", default="", help="Inline equivalent of --source-config-file; intended for small automated checks.", ) parser.add_argument("--motion-fps", type=int, default=50) parser.add_argument("--audio-sampling-rate", type=int, default=16000) parser.add_argument("--audio-window-seconds", type=float, default=0.6) parser.add_argument("--audio-left-context-seconds", type=float, default=0.0) parser.add_argument("--audio-sample-step-frames", type=int, default=1) parser.add_argument("--audio-command-window-seconds", type=float, default=0.6) parser.add_argument("--audio-command-sample-step-frames", type=int, default=20) parser.add_argument( "--audio-command-full-audio", action=argparse.BooleanOptionalAction, default=True, help=( "Keep each audio-command WAV whole (legacy behavior). Pass " "--no-audio-command-full-audio to emit complete fixed-length windows " "on the WAV timeline." ), ) parser.add_argument( "--audio-frame-count-source", choices=("motion_csv", "metadata"), default="motion_csv", help=( "How windowed-audio export decides its frame range. motion_csv follows the " "current training index; metadata exactly reproduces the historical v23 cache." ), ) parser.add_argument("--estimate-samples-per-group", type=int, default=32) parser.add_argument("--benchmark-samples", type=int, default=256) parser.add_argument("--limit", type=int, default=0) parser.add_argument( "--limit-sources", type=int, default=0, help=( "Limit complete WAV sources in --storage-format source mode. " "Unlike --limit, this never leaves a partially written source." ), ) parser.add_argument( "--verify-sources", type=int, default=64, help=( "Number of complete WAV sources to recompute with the real " "batch-one online encoder in verify mode; 0 verifies all." ), ) parser.add_argument( "--verify-windows-per-source", type=int, default=0, help=( "Optional cap on independently recomputed windows from each source. " "Zero verifies every window. This keeps exact B=1 verification " "practical for long continuous background sessions." ), ) parser.add_argument("--audio-batch-size", type=int, default=16) parser.add_argument("--text-batch-size", type=int, default=16) parser.add_argument("--video-batch-size", type=int, default=8) parser.add_argument("--num-io-workers", type=int, default=8) parser.add_argument("--num-save-workers", type=int, default=2) parser.add_argument("--max-pending-save-batches", type=int, default=2) parser.add_argument("--save-dtype", choices=("bf16", "fp16", "fp32"), default="bf16") parser.add_argument( "--storage-format", choices=("file", "shard", "source"), default="shard", help=( "source stores every window from one audio-command WAV in one " "safetensors file and writes only a compact source-level SQLite index." ), ) parser.add_argument("--shard-size", type=int, default=4096) parser.add_argument("--overwrite", action="store_true") parser.add_argument("--confirm-full-video", action="store_true") return parser.parse_args() def parse_csv_list(raw: str) -> list[str]: return [part.strip() for part in str(raw).split(",") if part.strip()] def _source_config_path(value: Any, base_dir: Path) -> Path: path = Path(str(value)).expanduser() return path if path.is_absolute() else (base_dir / path).resolve() def _source_preset_from_config(raw: dict[str, Any], base_dir: Path) -> SourcePreset: known = { "name", "modality", "dataset_root", "manifest_path", "allowed_tasks", "instruction_key", "audio_path_key", "video_path_key", "video_embed_dir", "is_audio_dir_manifest", "audio_file_extensions", "text_fields", } unknown = sorted(set(raw) - known) if unknown: raise ValueError(f"Unknown source preset fields for {raw.get('name', '<unnamed>')}: {unknown}") required = ("name", "modality", "dataset_root", "manifest_path", "allowed_tasks") missing = [key for key in required if raw.get(key) in (None, "", [])] if missing: raise ValueError(f"Source preset is missing required fields {missing}: {raw}") modality = str(raw["modality"]) if modality not in {"text", "audio", "video_clip"}: raise ValueError(f"Unsupported source modality {modality!r} for {raw['name']!r}") dataset_root = _source_config_path(raw["dataset_root"], base_dir) manifest_path = _source_config_path(raw["manifest_path"], base_dir) video_embed_raw = raw.get("video_embed_dir") video_embed_dir = ( _source_config_path(video_embed_raw, base_dir) if video_embed_raw not in (None, "") else None ) raw_audio_extensions = raw.get("audio_file_extensions", DEFAULT_AUDIO_FILE_EXTENSIONS) if isinstance(raw_audio_extensions, str): raw_audio_extensions = raw_audio_extensions.split(",") audio_file_extensions = tuple( value if value.startswith(".") else f".{value}" for value in (str(item).strip().lower() for item in raw_audio_extensions) if value ) if not audio_file_extensions: raise ValueError(f"audio_file_extensions cannot be empty for {raw['name']!r}") return SourcePreset( name=str(raw["name"]), modality=modality, dataset_root=dataset_root, manifest_path=manifest_path, allowed_tasks=tuple(str(value) for value in raw["allowed_tasks"]), instruction_key=str(raw.get("instruction_key") or ""), audio_path_key=str(raw.get("audio_path_key") or "audio_path"), video_path_key=str(raw.get("video_path_key") or "video_path"), video_embed_dir=video_embed_dir, is_audio_dir_manifest=bool(raw.get("is_audio_dir_manifest", False)), audio_file_extensions=audio_file_extensions, text_fields=tuple(str(value) for value in raw.get("text_fields", DEFAULT_SEED_TEXT_FIELDS)), ) def load_source_presets(args: argparse.Namespace) -> dict[str, SourcePreset]: if args.source_config_file is not None and str(args.source_config_json).strip(): raise ValueError("Use only one of --source-config-file and --source-config-json") if args.source_config_file is None and not str(args.source_config_json).strip(): return dict(SOURCE_PRESETS) if args.source_config_file is not None: config_path = args.source_config_file.expanduser().resolve() payload = json.loads(config_path.read_text(encoding="utf-8")) base_dir = config_path.parent else: payload = json.loads(str(args.source_config_json)) base_dir = Path.cwd() if isinstance(payload, dict): payload = payload.get("sources") if not isinstance(payload, list) or not payload: raise ValueError("Source config must be a non-empty list or {'sources': [...]} object") presets: dict[str, SourcePreset] = {} for raw in payload: if not isinstance(raw, dict): raise ValueError(f"Source config entries must be objects, got {type(raw).__name__}") preset = _source_preset_from_config(raw, base_dir) if preset.name in presets: raise ValueError(f"Duplicate source preset name: {preset.name}") presets[preset.name] = preset return presets def rank() -> int: return int(os.environ.get("RANK", "0")) def world_size() -> int: return int(os.environ.get("WORLD_SIZE", "1")) def is_rank0() -> bool: return rank() == 0 def assign_source_to_least_loaded_rank( rank_window_loads: list[int], source_window_count: int, ) -> int: """Greedily balance complete sources without splitting their time axis.""" if not rank_window_loads: raise ValueError("rank_window_loads must not be empty") count = int(source_window_count) if count <= 0: raise ValueError(f"source_window_count must be positive, got {count}") owner = min( range(len(rank_window_loads)), key=lambda rank_index: ( int(rank_window_loads[rank_index]), rank_index, ), ) rank_window_loads[owner] += count return owner def log(msg: str) -> None: if is_rank0(): print(msg, flush=True) def sanitize_rel_path(path: Path) -> str: return str(path).replace("\\", "/") def count_jsonl_rows(path: Path) -> int: count = 0 with path.open("rb") as f: for chunk in iter(lambda: f.read(1024 * 1024 * 16), b""): count += chunk.count(b"\n") return count def count_csv_data_rows(path: Path) -> int: count = 0 with path.open("rb") as f: for chunk in iter(lambda: f.read(1024 * 1024 * 16), b""): count += chunk.count(b"\n") return max(0, count - 1) def maybe_init_distributed() -> None: if world_size() <= 1 or dist.is_initialized(): return # Cache export workers are independent model replicas; distributed is only # used for tiny progress/stat reductions. Gloo allows multiple workers to # share one GPU without NCCL duplicate-device issues. dist.init_process_group(backend="gloo") def cleanup_distributed() -> None: if dist.is_initialized(): dist.destroy_process_group() def current_device() -> torch.device: if torch.cuda.is_available(): local_rank = int(os.environ.get("LOCAL_RANK", "0")) device_index = local_rank % torch.cuda.device_count() torch.cuda.set_device(device_index) return torch.device(f"cuda:{device_index}") return torch.device("cpu") def save_dtype_from_arg(arg: str) -> torch.dtype: if arg == "bf16": return torch.bfloat16 if arg == "fp16": return torch.float16 return torch.float32 def model_dtype_for_device(device: torch.device, arg: str) -> torch.dtype: if device.type == "cuda": return save_dtype_from_arg(arg) return torch.float32 def load_hidden_size(model_path: str) -> int: cfg = AutoConfig.from_pretrained(model_path, trust_remote_code=True) thinker_cfg = getattr(cfg, "thinker_config", None) if thinker_cfg is None: raise ValueError(f"Model config at {model_path} has no thinker_config") text_cfg = thinker_cfg.get("text_config", thinker_cfg) if isinstance(thinker_cfg, dict) else thinker_cfg.text_config hidden_size = text_cfg.get("hidden_size") if isinstance(text_cfg, dict) else getattr(text_cfg, "hidden_size", None) if hidden_size is None: raise ValueError(f"Cannot find hidden_size in thinker_config.text_config for {model_path}") return int(hidden_size) def load_processor(model_path: str) -> Qwen2_5OmniProcessor: return Qwen2_5OmniProcessor.from_pretrained(model_path) def load_model(model_path: str, device: torch.device, save_dtype: torch.dtype) -> Qwen2_5OmniThinkerForConditionalGeneration: load_kwargs: dict[str, Any] = { "dtype": model_dtype_for_device( device, "bf16" if save_dtype == torch.bfloat16 else "fp16" if save_dtype == torch.float16 else "fp32", ), "device_map": None, } if device.type == "cuda": load_kwargs["attn_implementation"] = "flash_attention_2" model = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained(model_path, **load_kwargs) model.eval().to(device) return model def stable_pool_prompt(prompt_pools: dict[str, list[str]], key: str) -> str: layouts = list(prompt_pools.keys()) layout_digest = hashlib.sha1(f"{key}::layout".encode("utf-8")).digest() layout = layouts[int.from_bytes(layout_digest[:8], "big") % len(layouts)] prompts = prompt_pools[layout] prompt_digest = hashlib.sha1(f"{key}::prompt".encode("utf-8")).digest() return str(prompts[int.from_bytes(prompt_digest[:8], "big") % len(prompts)]).strip() def instruction_identity(row: dict[str, Any]) -> str: for key in ("id", "filename", "name", "audio_path", "video_path"): value = row.get(key) if value not in (None, ""): return str(value) return json.dumps(row, sort_keys=True, ensure_ascii=False) def instruction_for_item(source_name: str, task_name: str, row: dict[str, Any]) -> str: if task_name in ("audio_only", "audio_command_only", "video_clip_only"): return "" key = f"{source_name}::{task_name}::{instruction_identity(row)}" if task_name == "audio_text": return stable_pool_prompt(_AUDIO_DANCE_PROMPT_POOLS, key) if task_name == "audio_command_text": return AUDIO_COMMAND_STREAMING_PROMPT if task_name == "video_clip_text": return stable_pool_prompt(_VIDEO_MOTION_PROMPT_POOLS, key) return "" def stable_text_instruction(row: dict[str, Any], preset: SourcePreset) -> str: key = f"{preset.name}::{row.get('_row_idx', instruction_identity(row))}" seed = int.from_bytes(hashlib.sha1(key.encode("utf-8")).digest()[:8], "big") rng = random.Random(seed) candidates = collect_seed_text_candidates(row, preset.text_fields) fallback = str(row.get("move_name", "seed motion")).strip() or "seed motion" return compose_seed_motion_prompt_from_candidates(candidates, fallback=fallback, rng=rng) def rel_to_root(path: Path, root: Path) -> Path: try: return path.relative_to(root) except ValueError: return Path(path.name) def resolve_audio_output_path(output_root: Path, source_name: str, task_name: str, input_path: Path, dataset_root: Path) -> Path: rel = rel_to_root(input_path, dataset_root).with_suffix(".safetensors") return output_root / "audio" / source_name / task_name / rel def resolve_audio_window_output_path( output_root: Path, source_name: str, task_name: str, input_path: Path, dataset_root: Path, frame_id: int, ) -> Path: rel = rel_to_root(input_path, dataset_root).with_suffix("") rel = rel.parent / f"{rel.name}.f{int(frame_id):08d}.safetensors" return output_root / "audio" / source_name / task_name / rel def resolve_audio_source_output_path( output_root: Path, source_name: str, task_name: str, input_path: Path, dataset_root: Path, ) -> Path: rel = rel_to_root(input_path, dataset_root).with_suffix(".safetensors") return output_root / "audio" / source_name / task_name / "sources" / rel def resolve_video_output_path(output_root: Path, source_name: str, task_name: str, embed_path: Path) -> Path: return output_root / "video_clip" / source_name / task_name / f"{embed_path.stem}.safetensors" def resolve_text_output_path(output_root: Path, source_name: str, task_name: str, motion_dir: Path, dataset_root: Path) -> Path: rel = rel_to_root(motion_dir, dataset_root) return output_root / "text" / source_name / task_name / rel / "hidden.safetensors" def cache_group_dir(output_root: Path, source_name: str, task_name: str, modality: str) -> Path: if modality == "audio": modality_dir = "audio" elif modality == "text": modality_dir = "text" else: modality_dir = "video_clip" return output_root / modality_dir / source_name / task_name def export_item_cache_key(item: ExportItem, output_root: Path) -> str: group_dir = cache_group_dir(output_root, item.source_name, item.task_name, item.modality) try: return sanitize_rel_path(item.output_path.relative_to(group_dir)) except ValueError: return sanitize_rel_path(Path(item.output_path.name)) def shard_paths(output_root: Path, source_name: str, task_name: str, modality: str, shard_rank: int, shard_id: int) -> tuple[Path, Path]: group_dir = cache_group_dir(output_root, source_name, task_name, modality) stem = f"ws{world_size()}.rank{int(shard_rank):02d}.shard{int(shard_id):06d}" shard_dir = group_dir / "shards" return shard_dir / f"{stem}.safetensors", shard_dir / f"{stem}.index.jsonl" def shard_is_complete( output_root: Path, source_name: str, task_name: str, modality: str, shard_rank: int, shard_id: int, expected_count: int, ) -> bool: shard_path, index_path = shard_paths(output_root, source_name, task_name, modality, shard_rank, shard_id) if expected_count <= 0: return False if not shard_path.exists() or not index_path.exists(): return False try: return count_jsonl_rows(index_path) == int(expected_count) except OSError: return False def iter_jsonl_rows(path: Path) -> Iterator[tuple[int, dict[str, Any]]]: with path.open("r", encoding="utf-8") as f: for idx, line in enumerate(f): line = line.strip() if not line: continue yield idx, json.loads(line) def manifest_motion_num_frames(row: dict[str, Any], preset: SourcePreset) -> int: motion_value = row.get("converted_path") or row.get("motion_dir") or row.get("motion_path") if not motion_value: raise ValueError( f"Audio manifest row has no motion path for source={preset.name}: " f"id={row.get('id', row.get('name', ''))}" ) motion_dir = Path(str(motion_value)) if not motion_dir.is_absolute(): motion_dir = preset.dataset_root / motion_dir return read_motion_frame_count(motion_dir) def metadata_num_frames(row: dict[str, Any], motion_fps: int) -> int: """Historical v23 frame-count rule, retained for bitwise cache reproduction.""" for key in ("clip_frames", "motion_chunk_frames", "n_frames", "move_duration_frames"): value = row.get(key) if value not in (None, ""): try: n_frames = int(value) except (TypeError, ValueError): n_frames = 0 if n_frames > 0: return n_frames duration = row.get("duration") if duration not in (None, ""): try: duration_sec = float(duration) except (TypeError, ValueError): duration_sec = 0.0 if duration_sec > 0: return max(1, int(round(duration_sec * float(motion_fps))) - 3) return 0 def audio_num_frames( row: dict[str, Any], preset: SourcePreset, args: argparse.Namespace, ) -> int: if str(args.audio_frame_count_source) == "metadata": return metadata_num_frames(row, int(args.motion_fps)) return manifest_motion_num_frames(row, preset) def read_start_hold_seconds(row: dict[str, Any], preset: SourcePreset) -> float: motion_value = row.get("converted_path") or row.get("motion_dir") or row.get("motion_path") if not motion_value: return 0.0 motion_dir = Path(str(motion_value)) if not motion_dir.is_absolute(): motion_dir = preset.dataset_root / motion_dir info_path = motion_dir / "info.txt" if not info_path.exists(): return 0.0 text = info_path.read_text(encoding="utf-8", errors="ignore") import re match = re.search(r"start_hold_seconds:\s*([0-9.]+)", text) return float(match.group(1)) if match else 0.0 def map_seed_motion_dir(move_g1_path: str, root_dir: Path) -> Path: rel = str(move_g1_path).strip().replace("\\", "/") rel = rel.replace("g1/csv/", "g1/csv_sonic/") rel = rel.replace("/csv/", "/csv_sonic/") if rel.endswith(".csv"): rel = rel[:-4] if rel.startswith("dataset/seed/"): rel = rel[len("dataset/seed/") :] if rel.startswith("seed/"): rel = rel[len("seed/") :] path = Path(rel) return path if path.is_absolute() else root_dir / path def iter_text_items(preset: SourcePreset, task_name: str, output_root: Path) -> Iterator[ExportItem]: usecols = ["move_g1_path", "move_name", "move_duration_frames", *preset.text_fields] usecols_set = set(usecols) metadata = pd.read_csv(preset.manifest_path, usecols=lambda col: col in usecols_set) if "move_g1_path" not in metadata.columns: raise ValueError(f"Text metadata must contain move_g1_path: {preset.manifest_path}") for row_idx, row in metadata.iterrows(): row_dict = row.to_dict() row_dict["_row_idx"] = int(row_idx) motion_dir = map_seed_motion_dir(str(row_dict["move_g1_path"]), preset.dataset_root) yield ExportItem( source_name=preset.name, task_name=task_name, modality="text", input_path=motion_dir, output_path=resolve_text_output_path(output_root, preset.name, task_name, motion_dir, preset.dataset_root), instruction=stable_text_instruction(row_dict, preset), manifest_id=str(row_idx), rel_source_path=sanitize_rel_path(rel_to_root(motion_dir, preset.dataset_root)), full_audio=False, ) def iter_audio_items( preset: SourcePreset, task_name: str, output_root: Path, args: argparse.Namespace | None = None, ) -> Iterator[ExportItem]: if task_name in ("audio_text", "audio_only"): if args is None: raise ValueError("args is required for windowed audio export") step_frames = max(1, int(args.audio_sample_step_frames)) motion_fps = int(args.motion_fps) for row_idx, row in iter_jsonl_rows(preset.manifest_path): raw_path = row.get(preset.audio_path_key) if not raw_path: continue audio_path = preset.dataset_root / str(raw_path) if not audio_path.exists(): continue if audio_path.suffix.lower() != ".npy" and not has_audio_stream(audio_path): continue n_frames = audio_num_frames(row, preset, args) if n_frames <= 0: continue instruction = instruction_for_item(preset.name, task_name, row) start_hold_seconds = read_start_hold_seconds(row, preset) for local_index, frame_id in enumerate(range(0, n_frames, step_frames)): audio_start_sec = ( frame_id / float(motion_fps) - start_hold_seconds - float(args.audio_left_context_seconds) ) yield ExportItem( source_name=preset.name, task_name=task_name, modality="audio", input_path=audio_path, output_path=resolve_audio_window_output_path( output_root, preset.name, task_name, audio_path, preset.dataset_root, frame_id, ), instruction=instruction, manifest_id=str(row.get("id", row.get("name", row_idx))), rel_source_path=f"{sanitize_rel_path(rel_to_root(audio_path, preset.dataset_root))}#f{frame_id}", audio_start_sec=float(audio_start_sec), audio_duration_sec=float(args.audio_window_seconds), full_audio=False, audio_sampling_rate=int(args.audio_sampling_rate), ) return if args is None: raise ValueError("args is required for audio-command export") sampling_rate = int(args.audio_sampling_rate) command_full_audio = bool(args.audio_command_full_audio) source_layout = str(getattr(args, "storage_format", "file")) == "source" source_limit = max(int(getattr(args, "limit_sources", 0)), 0) eligible_source_ordinal = 0 for row_idx, row, instruction, audio_file in iter_unique_manifest_audio_files(preset, task_name): manifest_id = str(row.get("filename", row.get("id", row.get("name", row_idx)))) rel_audio_path = sanitize_rel_path(rel_to_root(audio_file, preset.dataset_root)) if command_full_audio: yield ExportItem( source_name=preset.name, task_name=task_name, modality="audio", input_path=audio_file, output_path=resolve_audio_output_path( output_root, preset.name, task_name, audio_file, preset.dataset_root, ), instruction=instruction, manifest_id=manifest_id, rel_source_path=rel_audio_path, full_audio=True, audio_sampling_rate=sampling_rate, ) continue frame_ids = list( iter_complete_audio_window_frame_ids( audio_file, sampling_rate=sampling_rate, window_seconds=float(args.audio_command_window_seconds), step_frames=int(args.audio_command_sample_step_frames), motion_fps=int(args.motion_fps), ) ) if not frame_ids: continue if source_layout and source_limit > 0 and eligible_source_ordinal >= source_limit: break source_ordinal = eligible_source_ordinal eligible_source_ordinal += 1 source_key = Path(rel_audio_path).with_suffix("").as_posix() source_output_path = resolve_audio_source_output_path( output_root, preset.name, task_name, audio_file, preset.dataset_root, ) for source_item_index, frame_id in enumerate(frame_ids): yield ExportItem( source_name=preset.name, task_name=task_name, modality="audio", input_path=audio_file, output_path=resolve_audio_window_output_path( output_root, preset.name, task_name, audio_file, preset.dataset_root, frame_id, ), instruction=instruction, manifest_id=manifest_id, rel_source_path=f"{rel_audio_path}#f{frame_id}", audio_start_sec=float(frame_id) / float(args.motion_fps), audio_duration_sec=float(args.audio_command_window_seconds), full_audio=False, audio_sampling_rate=sampling_rate, source_key=source_key, source_output_path=source_output_path, source_ordinal=source_ordinal, source_item_index=source_item_index, source_item_count=len(frame_ids), frame_id=int(frame_id), source_step_frames=int(args.audio_command_sample_step_frames), motion_fps=int(args.motion_fps), ) def iter_unique_manifest_audio_files( preset: SourcePreset, task_name: str, ) -> Iterator[tuple[int, dict[str, Any], str, Path]]: """Expand an audio manifest once while preserving its stable first-row prompt.""" seen_files: set[str] = set() seen_audio_dirs: set[str] = set() for row_idx, row in iter_jsonl_rows(preset.manifest_path): raw_path = row.get(preset.audio_path_key) if not raw_path: continue audio_path = preset.dataset_root / str(raw_path) instruction = instruction_for_item(preset.name, task_name, row) if preset.is_audio_dir_manifest: dir_key = str(audio_path) if dir_key in seen_audio_dirs: continue seen_audio_dirs.add(dir_key) if not audio_path.is_dir(): continue audio_files = sorted( path for path in audio_path.iterdir() if path.is_file() and path.suffix.lower() in preset.audio_file_extensions ) else: if not audio_path.exists(): continue if audio_path.suffix.lower() != ".npy" and not has_audio_stream(audio_path): continue audio_files = [audio_path] for audio_file in audio_files: file_key = str(audio_file) if file_key in seen_files: continue seen_files.add(file_key) yield row_idx, row, instruction, audio_file def find_video_embed_path(preset: SourcePreset, row: dict[str, Any]) -> Path | None: assert preset.video_embed_dir is not None video_value = row.get(preset.video_path_key) or row.get("video_path") if not video_value: return None stem = Path(str(video_value)).stem manifest_info = _VIDEO_EMBED_INFO_CACHE.get(preset.name) if manifest_info is None: manifest_path = preset.video_embed_dir / "manifest.json" if manifest_path.exists(): manifest_info = json.loads(manifest_path.read_text(encoding="utf-8")) else: manifest_info = {} _VIDEO_EMBED_INFO_CACHE[preset.name] = manifest_info fps_value = manifest_info.get("video_sample_fps") size_value = manifest_info.get("video_size_pixels") if fps_value is not None and size_value is not None: fps_token = str(fps_value).replace(".", "p") direct_path = preset.video_embed_dir / f"{stem}.fps{fps_token}.size{size_value}.npz" if direct_path.exists(): return direct_path matches = sorted(preset.video_embed_dir.glob(f"{stem}.fps*.size*.npz")) if not matches: return None return matches[0] def iter_video_items(preset: SourcePreset, task_name: str, output_root: Path) -> Iterator[ExportItem]: for row_idx, row in iter_jsonl_rows(preset.manifest_path): instruction = instruction_for_item(preset.name, task_name, row) embed_path = find_video_embed_path(preset, row) if embed_path is None or not embed_path.exists(): continue yield ExportItem( source_name=preset.name, task_name=task_name, modality="video_clip", input_path=embed_path, output_path=resolve_video_output_path(output_root, preset.name, task_name, embed_path), instruction=instruction, manifest_id=str(row.get("id", row.get("name", row_idx))), rel_source_path=sanitize_rel_path(embed_path.name), ) def iter_group_items( preset: SourcePreset, task_name: str, output_root: Path, args: argparse.Namespace | None = None, ) -> Iterator[ExportItem]: if preset.modality == "text": yield from iter_text_items(preset, task_name, output_root) return if preset.modality == "audio": yield from iter_audio_items(preset, task_name, output_root, args) return yield from iter_video_items(preset, task_name, output_root) def count_group_items( preset: SourcePreset, task_name: str, output_root: Path, args: argparse.Namespace | None = None, ) -> int: if str(getattr(args, "storage_format", "")) == "source": # Source mode may apply --limit-sources and must count only complete # WAVs. Reuse the exact iterator so the progress total cannot include # a partial or otherwise ineligible source. return sum(1 for _ in iter_group_items(preset, task_name, output_root, args)) if preset.modality == "text": return count_csv_data_rows(preset.manifest_path) if preset.modality == "video_clip": # Training samples are driven by the video_clip metadata jsonl, not by # the embed manifest. Some embed manifests contain cached entries that # are not present in the current training metadata. return count_jsonl_rows(preset.manifest_path) if preset.modality == "audio" and task_name in ("audio_text", "audio_only"): if args is None: raise ValueError("args is required for windowed audio count") step_frames = max(1, int(args.audio_sample_step_frames)) motion_fps = int(args.motion_fps) total = 0 for _, row in iter_jsonl_rows(preset.manifest_path): raw_path = row.get(preset.audio_path_key) if not raw_path: continue audio_path = preset.dataset_root / str(raw_path) if not audio_path.exists(): continue if audio_path.suffix.lower() != ".npy" and not has_audio_stream(audio_path): continue n_frames = audio_num_frames(row, preset, args) if n_frames <= 0: continue total += max(1, (n_frames + step_frames - 1) // step_frames) return total if preset.modality == "audio" and task_name in ("audio_command_text", "audio_command_only"): if args is None: raise ValueError("args is required for audio-command count") audio_files = ( audio_file for _, _, _, audio_file in iter_unique_manifest_audio_files(preset, task_name) ) if bool(args.audio_command_full_audio): return sum(1 for _ in audio_files) return sum( complete_audio_window_count( audio_file, sampling_rate=int(args.audio_sampling_rate), window_seconds=float(args.audio_command_window_seconds), step_frames=int(args.audio_command_sample_step_frames), motion_fps=int(args.motion_fps), ) for audio_file in audio_files ) return sum(1 for _ in iter_group_items(preset, task_name, output_root, args)) def should_keep_item(global_index: int, total: int, sample_count: int) -> bool: if sample_count <= 0: return False if total <= sample_count: return True step = max(1, total // sample_count) return global_index % step == 0 def sampled_group_items( preset: SourcePreset, task_name: str, output_root: Path, sample_count: int, args: argparse.Namespace | None = None, ) -> list[ExportItem]: sampled: list[ExportItem] = [] for idx, item in enumerate(iter_group_items(preset, task_name, output_root, args)): sampled.append(item) if len(sampled) >= sample_count: break return sampled def source_task_pairs(tasks: list[str], sources: list[str]) -> list[tuple[SourcePreset, str]]: pairs: list[tuple[SourcePreset, str]] = [] for source_name in sources: preset = SOURCE_PRESETS[source_name] for task_name in tasks: if task_name in preset.allowed_tasks: pairs.append((preset, task_name)) return pairs def has_audio_stream(path: Path) -> bool: cmd = [ ffprobe_bin(), "-v", "error", "-select_streams", "a:0", "-show_entries", "stream=index", "-of", "csv=p=0", str(path), ] proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) return proc.returncode == 0 and bool(proc.stdout.strip()) def resampled_audio_num_samples(path: Path, sampling_rate: int) -> int: """Read audio duration cheaply and express it in target-rate samples.""" path = Path(path) sampling_rate = int(sampling_rate) if sampling_rate <= 0: raise ValueError(f"sampling_rate must be positive, got {sampling_rate}") stat = path.stat() cache_key = (str(path.resolve()), sampling_rate, int(stat.st_size), int(stat.st_mtime_ns)) cached = _AUDIO_RESAMPLED_SAMPLE_COUNT_CACHE.get(cache_key) if cached is not None: return cached sample_count = 0 if path.suffix.lower() == ".wav": try: with wave.open(str(path), "rb") as wav_file: source_frames = int(wav_file.getnframes()) source_rate = int(wav_file.getframerate()) if source_frames > 0 and source_rate > 0: sample_count = int(round(source_frames * sampling_rate / float(source_rate))) except (EOFError, OSError, wave.Error): sample_count = 0 if sample_count <= 0: sample_count = int(load_audio_array(path, sampling_rate).shape[0]) if sample_count <= 0: raise RuntimeError(f"Audio contains no samples: {path}") _AUDIO_RESAMPLED_SAMPLE_COUNT_CACHE[cache_key] = sample_count return sample_count def complete_audio_window_count( path: Path, *, sampling_rate: int, window_seconds: float, step_frames: int, motion_fps: int, ) -> int: """Count complete windows only; an incomplete final tail is not padded.""" sampling_rate = int(sampling_rate) step_frames = int(step_frames) motion_fps = int(motion_fps) window_seconds = float(window_seconds) if window_seconds <= 0: raise ValueError(f"window_seconds must be positive, got {window_seconds}") if step_frames <= 0 or motion_fps <= 0: raise ValueError( f"step_frames and motion_fps must be positive, got {step_frames} and {motion_fps}" ) window_samples = max(1, int(round(window_seconds * sampling_rate))) step_samples = max(1, int(round(step_frames * sampling_rate / float(motion_fps)))) total_samples = resampled_audio_num_samples(Path(path), sampling_rate) if total_samples < window_samples: return 0 return int((total_samples - window_samples) // step_samples + 1) def iter_complete_audio_window_frame_ids( path: Path, *, sampling_rate: int, window_seconds: float, step_frames: int, motion_fps: int, ) -> range: count = complete_audio_window_count( path, sampling_rate=sampling_rate, window_seconds=window_seconds, step_frames=step_frames, motion_fps=motion_fps, ) return range(0, count * int(step_frames), int(step_frames)) def load_audio_array(path: Path, sampling_rate: int = 16000) -> np.ndarray: if path.suffix.lower() == ".npy": try: audio = np.load(path, allow_pickle=False) except ValueError: audio = np.load(path, allow_pickle=True) if isinstance(audio, np.ndarray) and audio.dtype == object: audio = np.asarray(audio.tolist()) audio = np.asarray(audio, dtype=np.float32).reshape(-1).copy() else: cmd = [ ffmpeg_bin(), "-v", "error", "-threads", "1", "-i", str(path), "-ac", "1", "-ar", str(int(sampling_rate)), "-f", "f32le", "-", ] proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) if proc.returncode != 0: msg = proc.stderr.decode("utf-8", errors="ignore") raise RuntimeError(f"ffmpeg failed to decode audio from {path}: {msg}") audio = np.frombuffer(proc.stdout, dtype=np.float32).copy() if audio.size == 0: raise RuntimeError(f"Empty audio decoded from {path}") return audio def audio_segment(audio: np.ndarray, start_sec: float, duration_sec: float, sampling_rate: int) -> np.ndarray: num_samples = max(1, int(round(float(duration_sec) * int(sampling_rate)))) start_sample = int(round(float(start_sec) * int(sampling_rate))) end_sample = start_sample + num_samples segment = np.zeros((num_samples,), dtype=np.float32) src_start = max(0, start_sample) src_end = min(audio.shape[0], end_sample) if src_end > src_start: dst_start = src_start - start_sample segment[dst_start : dst_start + (src_end - src_start)] = audio[src_start:src_end] return segment def load_audio_payload(item: ExportItem) -> dict[str, Any]: audio = load_audio_array(item.input_path, item.audio_sampling_rate) if not item.full_audio: audio = audio_segment( audio, item.audio_start_sec, item.audio_duration_sec, item.audio_sampling_rate, ) return { "audio": audio, "instruction": item.instruction, "full_audio": bool(item.full_audio), } def load_audio_payloads( items: list[ExportItem], num_io_workers: int, io_executor: ThreadPoolExecutor | None = None, ) -> list[dict[str, Any]]: unique_paths = list(dict.fromkeys(item.input_path for item in items)) sampling_rate_by_path = {item.input_path: int(item.audio_sampling_rate) for item in items} def load_pair(path: Path) -> tuple[Path, np.ndarray]: return path, load_audio_array(path, sampling_rate_by_path.get(path, 16000)) if num_io_workers <= 1 or len(unique_paths) == 1: audio_by_path = dict(load_pair(path) for path in unique_paths) elif io_executor is not None: audio_by_path = dict(io_executor.map(load_pair, unique_paths)) else: with ThreadPoolExecutor(max_workers=num_io_workers) as executor: audio_by_path = dict(executor.map(load_pair, unique_paths)) payloads: list[dict[str, Any]] = [] for item in items: audio = audio_by_path[item.input_path] if not item.full_audio: audio = audio_segment( audio, item.audio_start_sec, item.audio_duration_sec, item.audio_sampling_rate, ) payloads.append( { "audio": audio, "instruction": item.instruction, "full_audio": bool(item.full_audio), } ) return payloads def load_video_payload(item: ExportItem) -> dict[str, Any]: with np.load(item.input_path, allow_pickle=False) as cache: return { "pixel_values_videos": cache["pixel_values_videos"], "video_grid_thw": cache["video_grid_thw"], "video_second_per_grid": cache["video_second_per_grid"], "instruction": item.instruction, } def load_text_payload(item: ExportItem) -> dict[str, Any]: return {"instruction": item.instruction} def build_audio_text(processor: Qwen2_5OmniProcessor, instructions: list[str]) -> list[str]: texts: list[str] = [] for instruction in instructions: content = [{"type": "audio", "audio": "audio.wav"}] if instruction: content.append({"type": "text", "text": instruction}) messages = [[ {"role": "system", "content": [{"type": "text", "text": OMNI_DEFAULT_SYSTEM_PROMPT}]}, {"role": "user", "content": content}, ]] texts.append(processor.apply_chat_template(messages[0], tokenize=False, add_generation_prompt=True)) return texts def build_video_text(processor: Qwen2_5OmniProcessor, instructions: list[str]) -> list[str]: texts: list[str] = [] for instruction in instructions: content = [{"type": "video", "video": "video.mp4"}] if instruction: content.append({"type": "text", "text": instruction}) messages = [[ {"role": "system", "content": [{"type": "text", "text": OMNI_DEFAULT_SYSTEM_PROMPT}]}, {"role": "user", "content": content}, ]] texts.append(processor.apply_chat_template(messages[0], tokenize=False, add_generation_prompt=True)) return texts def build_text_text(processor: Qwen2_5OmniProcessor, instructions: list[str]) -> list[str]: texts: list[str] = [] for instruction in instructions: messages = [[ {"role": "system", "content": [{"type": "text", "text": OMNI_DEFAULT_SYSTEM_PROMPT}]}, {"role": "user", "content": [{"type": "text", "text": str(instruction).strip()}]}, ]] texts.append(processor.apply_chat_template(messages[0], tokenize=False, add_generation_prompt=True)) return texts def collate_audio_batch( processor: Qwen2_5OmniProcessor, payloads: list[dict[str, Any]], ) -> dict[str, torch.Tensor]: texts = build_audio_text(processor, [payload["instruction"] for payload in payloads]) audios = [np.asarray(payload["audio"], dtype=np.float32) for payload in payloads] if any(bool(payload.get("full_audio", False)) for payload in payloads): single_inputs = [ processor( text=[text], audio=[audio], padding=True, return_tensors="pt", ) for text, audio in zip(texts, audios) ] pad_token_id = int(getattr(processor.tokenizer, "pad_token_id", 0) or 0) input_ids = torch.nn.utils.rnn.pad_sequence( [ item["input_ids"].squeeze(0) if item["input_ids"].dim() == 2 else item["input_ids"] for item in single_inputs ], batch_first=True, padding_value=pad_token_id, ) output: dict[str, torch.Tensor] = { "input_ids": input_ids, "attention_mask": input_ids.ne(pad_token_id).long(), } input_features = [ item["input_features"].squeeze(0) if item["input_features"].dim() == 3 else item["input_features"] for item in single_inputs ] feature_attention_mask = [ item["feature_attention_mask"].squeeze(0) if item["feature_attention_mask"].dim() == 2 else item["feature_attention_mask"] for item in single_inputs ] max_audio_len = max(int(feature.shape[-1]) for feature in input_features) padded_input_features = [] padded_feature_attention_mask = [] for feature, feature_mask in zip(input_features, feature_attention_mask): pad_len = max_audio_len - int(feature.shape[-1]) if pad_len > 0: feature_pad_shape = list(feature.shape) feature_pad_shape[-1] = pad_len feature = torch.cat( [feature, torch.zeros(feature_pad_shape, dtype=feature.dtype, device=feature.device)], dim=-1, ) mask_pad_shape = list(feature_mask.shape) mask_pad_shape[-1] = pad_len feature_mask = torch.cat( [feature_mask, torch.zeros(mask_pad_shape, dtype=feature_mask.dtype, device=feature_mask.device)], dim=-1, ) padded_input_features.append(feature) padded_feature_attention_mask.append(feature_mask) output["input_features"] = torch.stack(padded_input_features).to(dtype=torch.float32) output["feature_attention_mask"] = torch.stack(padded_feature_attention_mask) if "audio_feature_lengths" in single_inputs[0]: output["audio_feature_lengths"] = torch.stack( [ item["audio_feature_lengths"].squeeze(0) if item["audio_feature_lengths"].dim() > 0 else item["audio_feature_lengths"] for item in single_inputs ] ) return output output_kwargs = processor._merge_kwargs( Qwen2_5OmniProcessorKwargs, tokenizer_init_kwargs=processor.tokenizer.init_kwargs, padding=True, return_tensors="pt", ) audio_kwargs = dict(output_kwargs["audio_kwargs"]) feature_extractor = processor.feature_extractor max_audio_samples = max(int(audio.shape[0]) for audio in audios) hop_length = int(feature_extractor.hop_length) right_context = int(feature_extractor.n_fft) max_length = int(math.ceil((max_audio_samples + right_context) / hop_length) * hop_length) max_length = min(max_length, int(feature_extractor.n_samples)) audio_kwargs["padding"] = "max_length" audio_kwargs["max_length"] = max_length audio_kwargs["return_attention_mask"] = True audio_inputs = feature_extractor( audios, **audio_kwargs, ) feature_mask = audio_inputs.pop("attention_mask") input_features = audio_inputs.pop("input_features") max_valid = int(feature_mask.sum(dim=1).max().item()) if 0 < max_valid < int(feature_mask.shape[-1]): feature_mask = feature_mask[:, :max_valid].contiguous() input_features = input_features[..., :max_valid].contiguous() input_lengths = (feature_mask.sum(-1) - 1) // 2 + 1 audio_lengths = iter((input_lengths - 2) // 2 + 1) texts = processor.replace_multimodal_special_tokens( texts, audio_lengths, iter([]), iter([]), video_second_per_grid=iter([]), use_audio_in_video=output_kwargs["videos_kwargs"]["use_audio_in_video"], position_id_per_seconds=output_kwargs["videos_kwargs"]["position_id_per_seconds"], seconds_per_chunk=output_kwargs["videos_kwargs"]["seconds_per_chunk"], ) inputs = processor.tokenizer(texts, **output_kwargs["text_kwargs"]) inputs["feature_attention_mask"] = feature_mask inputs["input_features"] = input_features return inputs def collate_video_batch( processor: Qwen2_5OmniProcessor, payloads: list[dict[str, Any]], ) -> dict[str, torch.Tensor]: texts = build_video_text(processor, [payload["instruction"] for payload in payloads]) grids_for_replace = [payload["video_grid_thw"][0] for payload in payloads] seconds_for_replace = [float(np.asarray(payload["video_second_per_grid"]).reshape(-1)[0]) for payload in payloads] texts = processor.replace_multimodal_special_tokens( texts, iter([]), iter([]), iter(grids_for_replace), iter(seconds_for_replace), False, 25, 2.0, ) text_inputs = processor.tokenizer(texts, padding=True, return_tensors="pt") inputs = {key: value for key, value in text_inputs.items()} inputs["pixel_values_videos"] = torch.cat( [torch.as_tensor(payload["pixel_values_videos"], dtype=torch.float32) for payload in payloads], dim=0, ) inputs["video_grid_thw"] = torch.cat( [torch.as_tensor(payload["video_grid_thw"], dtype=torch.long) for payload in payloads], dim=0, ) inputs["video_second_per_grid"] = torch.cat( [ torch.as_tensor(payload["video_second_per_grid"], dtype=torch.float32).reshape(-1) for payload in payloads ], dim=0, ) return inputs def collate_text_batch( processor: Qwen2_5OmniProcessor, payloads: list[dict[str, Any]], ) -> dict[str, torch.Tensor]: texts = build_text_text(processor, [payload["instruction"] for payload in payloads]) return processor.tokenizer(texts, padding=True, return_tensors="pt") def move_inputs_to_device(inputs: dict[str, Any], device: torch.device) -> dict[str, Any]: return { key: value.to(device, non_blocking=True) if hasattr(value, "to") else value for key, value in inputs.items() } def prepare_inputs_cpu( preset: SourcePreset, items: list[ExportItem], args: argparse.Namespace, processor: Qwen2_5OmniProcessor, io_executor: ThreadPoolExecutor | None = None, ) -> dict[str, torch.Tensor]: payloads = load_payloads( items, int(args.num_io_workers), io_executor=io_executor, ) if preset.modality == "audio": return collate_audio_batch(processor, payloads) if preset.modality == "text": return collate_text_batch(processor, payloads) return collate_video_batch(processor, payloads) def omni_hidden_states_without_lm_head( model: Qwen2_5OmniThinkerForConditionalGeneration, vlm_kwargs: dict[str, torch.Tensor], ) -> torch.Tensor: input_ids = vlm_kwargs.get("input_ids") attention_mask = vlm_kwargs.get("attention_mask") input_features = vlm_kwargs.get("input_features") pixel_values = vlm_kwargs.get("pixel_values") pixel_values_videos = vlm_kwargs.get("pixel_values_videos") image_grid_thw = vlm_kwargs.get("image_grid_thw") video_grid_thw = vlm_kwargs.get("video_grid_thw") feature_attention_mask = vlm_kwargs.get("feature_attention_mask") audio_feature_lengths = vlm_kwargs.get("audio_feature_lengths") video_second_per_grid = vlm_kwargs.get("video_second_per_grid") inputs_embeds = vlm_kwargs.get("inputs_embeds") position_ids = vlm_kwargs.get("position_ids") cache_position = vlm_kwargs.get("cache_position") use_audio_in_video = vlm_kwargs.get("use_audio_in_video") if inputs_embeds is None: inputs_embeds = model.get_input_embeddings()(input_ids) if input_features is not None: audio_features = model.get_audio_features( input_features, feature_attention_mask=feature_attention_mask, audio_feature_lengths=audio_feature_lengths, ) audio_features = audio_features.to(inputs_embeds.device, inputs_embeds.dtype) _, _, audio_mask = model.get_placeholder_mask(input_ids, inputs_embeds=inputs_embeds) inputs_embeds = inputs_embeds.masked_scatter(audio_mask, audio_features) if pixel_values is not None: image_embeds = model.get_image_features(pixel_values, image_grid_thw) image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _, _ = model.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds, ) inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) if pixel_values_videos is not None: video_embeds = model.get_video_features(pixel_values_videos, video_grid_thw) video_embeds = video_embeds.to(inputs_embeds.device, inputs_embeds.dtype) _, video_mask, _ = model.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds, ) inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds) if feature_attention_mask is not None: audio_feature_lengths = torch.sum(feature_attention_mask, dim=1) else: audio_feature_lengths = None if attention_mask is not None and position_ids is None: if cache_position is None or (cache_position is not None and cache_position[0] == 0) or model.rope_deltas is None: delta0 = (1 - attention_mask).sum(dim=-1).unsqueeze(1) position_ids, rope_deltas = model.get_rope_index( input_ids, image_grid_thw, video_grid_thw, attention_mask, use_audio_in_video, audio_feature_lengths, video_second_per_grid, ) model.rope_deltas = rope_deltas - delta0 else: batch_size, seq_length = input_ids.shape delta = cache_position[0] + model.rope_deltas if cache_position is not None else 0 position_ids = torch.arange(seq_length, device=input_ids.device) position_ids = position_ids.view(1, -1).expand(batch_size, -1) position_ids = position_ids.add(delta) position_ids = position_ids.unsqueeze(0).expand(3, -1, -1) outputs = model.model( attention_mask=attention_mask, position_ids=position_ids, past_key_values=vlm_kwargs.get("past_key_values"), inputs_embeds=inputs_embeds, use_cache=False, output_attentions=vlm_kwargs.get("output_attentions"), output_hidden_states=True, return_dict=True, cache_position=cache_position, ) return outputs.hidden_states[-1] def online_encoder_metadata(task_name: str) -> dict[str, str]: if str(task_name) != "audio_command_text": return {} if not ACTIVE_MODEL_FINGERPRINT: raise RuntimeError("Qwen model fingerprint was not initialized") return { "encoder_contract": HFD_ONLINE_ENCODER_CONTRACT, "encoder_batch_size": str(HFD_ONLINE_ENCODER_BATCH_SIZE), "encoder_dtype": HFD_ONLINE_ENCODER_DTYPE, "attention_implementation": HFD_ONLINE_ENCODER_ATTENTION, "model_fingerprint": ACTIVE_MODEL_FINGERPRINT, } def save_hidden_state( item: ExportItem, hidden_states: torch.Tensor, save_dtype: torch.dtype, overwrite: bool, ) -> tuple[bool, int]: if item.output_path.exists() and not overwrite: return False, 0 item.output_path.parent.mkdir(parents=True, exist_ok=True) payload = {"hidden_states": hidden_states.to(dtype=save_dtype).cpu().contiguous()} metadata = { "source_name": item.source_name, "task_name": item.task_name, "modality": item.modality, "manifest_id": item.manifest_id, "source_path": item.rel_source_path, "instruction_sha1": hashlib.sha1(item.instruction.encode("utf-8")).hexdigest(), "full_audio": str(bool(item.full_audio)), "audio_start_sec": f"{float(item.audio_start_sec):.6f}", "audio_duration_sec": f"{float(item.audio_duration_sec):.6f}", **online_encoder_metadata(item.task_name), } tmp_path = item.output_path.with_name(f".{item.output_path.name}.{os.getpid()}.tmp") save_file(payload, str(tmp_path), metadata=metadata) os.replace(tmp_path, item.output_path) return True, item.output_path.stat().st_size def save_hidden_state_cpu( item: ExportItem, hidden_states: torch.Tensor, overwrite: bool, ) -> tuple[bool, int]: if item.output_path.exists() and not overwrite: return False, 0 item.output_path.parent.mkdir(parents=True, exist_ok=True) payload = {"hidden_states": hidden_states.contiguous()} metadata = { "source_name": item.source_name, "task_name": item.task_name, "modality": item.modality, "manifest_id": item.manifest_id, "source_path": item.rel_source_path, "instruction_sha1": hashlib.sha1(item.instruction.encode("utf-8")).hexdigest(), "full_audio": str(bool(item.full_audio)), "audio_start_sec": f"{float(item.audio_start_sec):.6f}", "audio_duration_sec": f"{float(item.audio_duration_sec):.6f}", **online_encoder_metadata(item.task_name), } tmp_path = item.output_path.with_name(f".{item.output_path.name}.{os.getpid()}.tmp") save_file(payload, str(tmp_path), metadata=metadata) os.replace(tmp_path, item.output_path) return True, item.output_path.stat().st_size def load_payloads( items: list[ExportItem], num_io_workers: int, io_executor: ThreadPoolExecutor | None = None, ) -> list[dict[str, Any]]: if items[0].modality == "audio": return load_audio_payloads(items, num_io_workers, io_executor) if items[0].modality == "text": return [load_text_payload(item) for item in items] loader = load_video_payload if num_io_workers <= 1 or len(items) == 1: return [loader(item) for item in items] if io_executor is not None: return list(io_executor.map(loader, items)) with ThreadPoolExecutor(max_workers=num_io_workers) as executor: return list(executor.map(loader, items)) def is_cuda_oom(exc: Exception) -> bool: if isinstance(exc, torch.OutOfMemoryError): return True message = str(exc).lower() return "out of memory" in message and "cuda" in message def save_hidden_state_batch_cpu( items: list[ExportItem], hidden_cpu: torch.Tensor, seq_lens: list[int], overwrite: bool, ) -> dict[str, Any]: stats = { "written": 0, "skipped": 0, "failed": 0, "hidden_bytes": 0, "error": None, } for batch_idx, batch_item in enumerate(items): seq_len = int(seq_lens[batch_idx]) sample_hidden = hidden_cpu[batch_idx, :seq_len, :] try: did_write, file_bytes = save_hidden_state_cpu(batch_item, sample_hidden, overwrite) if did_write: stats["written"] += 1 stats["hidden_bytes"] += int(file_bytes) else: stats["skipped"] += 1 except Exception as exc: # noqa: BLE001 error = f"{batch_item.source_name}/{batch_item.task_name}: {type(exc).__name__}: {exc}" stats["failed"] += 1 if stats["error"] is None: stats["error"] = error if is_rank0(): print(f"[ERROR] {error}", flush=True) traceback.print_exc() return stats def source_hidden_file_is_complete(item: ExportItem) -> bool: path = item.source_output_path if path is None or not path.is_file() or item.source_item_count <= 0: return False try: with safe_open(str(path), framework="pt", device="cpu") as reader: metadata = reader.metadata() or {} required_tensors = {"hidden_states", "offsets", "frame_ids"} if not required_tensors.issubset(set(reader.keys())): return False if metadata.get("format") != HFD_AUDIO_SOURCE_FORMAT: return False if metadata.get("source_key") != item.source_key: return False if metadata.get("source_name") != item.source_name: return False if metadata.get("task_name") != item.task_name: return False for key, value in online_encoder_metadata(item.task_name).items(): if metadata.get(key) != value: return False if metadata.get("instruction_sha1") != hashlib.sha1( item.instruction.encode("utf-8") ).hexdigest(): return False if int(metadata.get("count", "-1")) != int(item.source_item_count): return False if int(metadata.get("step_frames", "-1")) != int(item.source_step_frames): return False if int(metadata.get("motion_fps", "-1")) != int(item.motion_fps): return False if int(metadata.get("sampling_rate", "-1")) != int(item.audio_sampling_rate): return False if not math.isclose( float(metadata.get("window_seconds", "nan")), float(item.audio_duration_sec), rel_tol=0.0, abs_tol=1e-9, ): return False hidden_shape = tuple(reader.get_slice("hidden_states").get_shape()) offsets_shape = tuple(reader.get_slice("offsets").get_shape()) frame_ids_shape = tuple(reader.get_slice("frame_ids").get_shape()) if len(hidden_shape) != 2: return False if offsets_shape != (int(item.source_item_count) + 1,): return False if frame_ids_shape != (int(item.source_item_count),): return False offsets = reader.get_tensor("offsets") frame_ids = reader.get_tensor("frame_ids") if int(offsets[0]) != 0 or int(offsets[-1]) != int(hidden_shape[0]): return False if bool(torch.any(offsets[1:] <= offsets[:-1])): return False expected_frame_ids = torch.arange( int(item.source_item_count), dtype=torch.int64, ) * int(item.source_step_frames) return bool(torch.equal(frame_ids, expected_frame_ids)) except (OSError, RuntimeError, TypeError, ValueError): return False def save_hidden_state_source_cpu( items: list[ExportItem], hiddens: list[torch.Tensor], overwrite: bool, ) -> dict[str, Any]: stats = { "written": 0, "skipped": 0, "failed": 0, "hidden_bytes": 0, "error": None, } if not items: return stats first = items[0] output_path = first.source_output_path expected_count = int(first.source_item_count) if output_path is None: raise ValueError(f"Missing source_output_path for {first.input_path}") if len(items) != expected_count or len(hiddens) != expected_count: raise ValueError( f"Incomplete source {first.source_key}: " f"collected={len(items)} expected={expected_count}" ) if any(item.source_key != first.source_key for item in items): raise ValueError(f"Mixed WAV sources in one source file: {first.source_key}") if [int(item.source_item_index) for item in items] != list(range(expected_count)): raise ValueError(f"Out-of-order windows for source {first.source_key}") if output_path.exists() and not overwrite and source_hidden_file_is_complete(first): stats["skipped"] = expected_count return stats tmp_path = output_path.with_name(f".{output_path.name}.{os.getpid()}.tmp") try: output_path.parent.mkdir(parents=True, exist_ok=True) lengths = [int(hidden.shape[0]) for hidden in hiddens] offsets = [0] for length in lengths: if length <= 0: raise ValueError( f"Empty hidden window in source {first.source_key}" ) offsets.append(offsets[-1] + length) flat_hidden = torch.cat( [hidden.contiguous() for hidden in hiddens], dim=0, ).contiguous() frame_ids = torch.tensor( [int(item.frame_id) for item in items], dtype=torch.int64, ) expected_frame_ids = torch.arange( expected_count, dtype=torch.int64, ) * int(first.source_step_frames) if not torch.equal(frame_ids, expected_frame_ids): raise ValueError( f"Non-contiguous source frames for {first.source_key}: " f"{frame_ids.tolist()}" ) metadata = { "format": HFD_AUDIO_SOURCE_FORMAT, "source_name": first.source_name, "task_name": first.task_name, "source_key": first.source_key, "source_path": first.rel_source_path.split("#", 1)[0], "manifest_id": first.manifest_id, "instruction_sha1": hashlib.sha1( first.instruction.encode("utf-8") ).hexdigest(), "count": str(expected_count), "hidden_dim": str(int(flat_hidden.shape[-1])), "window_seconds": f"{float(first.audio_duration_sec):.9f}", "step_frames": str(int(first.source_step_frames)), "motion_fps": str(int(first.motion_fps)), "sampling_rate": str(int(first.audio_sampling_rate)), **online_encoder_metadata(first.task_name), } save_file( { "hidden_states": flat_hidden, "offsets": torch.tensor(offsets, dtype=torch.int64), "frame_ids": frame_ids, }, str(tmp_path), metadata=metadata, ) os.replace(tmp_path, output_path) stats["written"] = expected_count stats["hidden_bytes"] = int(output_path.stat().st_size) return stats except Exception as exc: # noqa: BLE001 tmp_path.unlink(missing_ok=True) error = ( f"{first.source_name}/{first.task_name}/{first.source_key}: " f"{type(exc).__name__}: {exc}" ) stats["failed"] = expected_count stats["error"] = error if is_rank0(): print(f"[ERROR] {error}", flush=True) traceback.print_exc() return stats def source_index_row( item: ExportItem, output_root: Path, ) -> tuple[str, str, int, float, int, int, int]: source_path = item.source_output_path if source_path is None: raise ValueError(f"Missing source output path for {item.input_path}") source_dir = ( cache_group_dir( output_root, item.source_name, item.task_name, item.modality, ) / "sources" ) relative_path = sanitize_rel_path(source_path.relative_to(source_dir)) return ( item.source_key, relative_path, int(item.source_item_count), float(item.audio_duration_sec), int(item.source_step_frames), int(item.motion_fps), int(item.audio_sampling_rate), ) def build_rank_source_sqlite_index( output_root: Path, source_name: str, task_name: str, modality: str, source_rows: Iterable[tuple[str, str, int, float, int, int, int]], shard_rank: int, ) -> Path: source_dir = ( cache_group_dir(output_root, source_name, task_name, modality) / "sources" ) source_dir.mkdir(parents=True, exist_ok=True) sqlite_path = ( source_dir / f"index.ws{world_size()}.rank{int(shard_rank):02d}.sqlite" ) tmp_path = source_dir / f".{sqlite_path.name}.{os.getpid()}.tmp" tmp_path.unlink(missing_ok=True) rows = sorted(source_rows, key=lambda row: row[0]) conn = sqlite3.connect(str(tmp_path)) try: conn.execute("PRAGMA journal_mode=OFF") conn.execute("PRAGMA synchronous=OFF") conn.execute("PRAGMA temp_store=MEMORY") conn.execute( "CREATE TABLE sources (" "key TEXT PRIMARY KEY, " "path TEXT NOT NULL, " "count INTEGER NOT NULL, " "window_seconds REAL NOT NULL, " "step_frames INTEGER NOT NULL, " "motion_fps INTEGER NOT NULL, " "sampling_rate INTEGER NOT NULL)" ) conn.executemany( "INSERT INTO sources VALUES (?, ?, ?, ?, ?, ?, ?)", rows, ) conn.execute("PRAGMA user_version=1") conn.commit() finally: conn.close() os.replace(tmp_path, sqlite_path) return sqlite_path def save_hidden_state_shard_cpu( output_root: Path, shard_rank: int, shard_id: int, items: list[ExportItem], hiddens: list[torch.Tensor], overwrite: bool, ) -> dict[str, Any]: stats = { "written": 0, "skipped": 0, "failed": 0, "hidden_bytes": 0, "error": None, } if not items: return stats first = items[0] shard_path, index_path = shard_paths( output_root, first.source_name, first.task_name, first.modality, shard_rank, shard_id, ) if shard_path.exists() and index_path.exists() and not overwrite: try: if count_jsonl_rows(index_path) == len(items): stats["skipped"] = len(items) return stats except OSError: pass try: shard_path.parent.mkdir(parents=True, exist_ok=True) lengths = [int(hidden.shape[0]) for hidden in hiddens] offsets = [0] for length in lengths: offsets.append(offsets[-1] + length) flat_hidden = torch.cat([hidden.contiguous() for hidden in hiddens], dim=0).contiguous() offsets_tensor = torch.tensor(offsets, dtype=torch.int64) metadata = { "source_name": first.source_name, "task_name": first.task_name, "modality": first.modality, "rank": str(int(shard_rank)), "shard_id": str(int(shard_id)), "count": str(len(items)), "hidden_dim": str(int(flat_hidden.shape[-1])), } tmp_shard = shard_path.with_name(f".{shard_path.name}.{os.getpid()}.tmp") save_file( { "hidden_states": flat_hidden, "offsets": offsets_tensor, }, str(tmp_shard), metadata=metadata, ) tmp_index = index_path.with_name(f".{index_path.name}.{os.getpid()}.tmp") with tmp_index.open("w", encoding="utf-8") as f: for sample_idx, item in enumerate(items): line = { "key": export_item_cache_key(item, output_root), "shard": shard_path.name, "idx": sample_idx, "offset": offsets[sample_idx], "length": lengths[sample_idx], "source_path": item.rel_source_path, "input_path": str(item.input_path), "manifest_id": item.manifest_id, "instruction_sha1": hashlib.sha1(item.instruction.encode("utf-8")).hexdigest(), "full_audio": bool(item.full_audio), "audio_start_sec": float(item.audio_start_sec), "audio_duration_sec": float(item.audio_duration_sec), } f.write(json.dumps(line, ensure_ascii=False, separators=(",", ":")) + "\n") os.replace(tmp_shard, shard_path) os.replace(tmp_index, index_path) stats["written"] = len(items) stats["hidden_bytes"] = int(shard_path.stat().st_size + index_path.stat().st_size) return stats except Exception as exc: # noqa: BLE001 error = f"{first.source_name}/{first.task_name}/rank{shard_rank}/shard{shard_id}: {type(exc).__name__}: {exc}" stats["failed"] = len(items) stats["error"] = error if is_rank0(): print(f"[ERROR] {error}", flush=True) traceback.print_exc() return stats def build_rank_sqlite_index( output_root: Path, source_name: str, task_name: str, modality: str, shard_rank: int, expected_rows: int | None = None, ) -> Path | None: group_dir = cache_group_dir(output_root, source_name, task_name, modality) shard_dir = group_dir / "shards" if not shard_dir.exists(): return None index_files = sorted(shard_dir.glob(f"*rank{int(shard_rank):02d}.shard*.index.jsonl")) if not index_files: return None sqlite_path = shard_dir / f"index.rank{int(shard_rank):02d}.sqlite" # A no-op resume must not rewrite an otherwise identical SQLite file. # merge_omni_hidden_for_oss fingerprints source indexes, so changing only # the SQLite mtime would invalidate already-finished destination shards. if sqlite_path.exists() and sqlite_path.stat().st_mtime_ns >= max( path.stat().st_mtime_ns for path in index_files ): try: uri = f"file:{sqlite_path}?mode=ro" with sqlite3.connect(uri, uri=True, timeout=60.0) as existing: existing.execute("PRAGMA query_only=ON") row_count = int(existing.execute("SELECT COUNT(*) FROM samples").fetchone()[0]) if expected_rows is None or row_count == int(expected_rows): return sqlite_path except (OSError, sqlite3.Error, TypeError, ValueError): pass tmp_path = shard_dir / f".{sqlite_path.name}.{os.getpid()}.tmp" if tmp_path.exists(): tmp_path.unlink() conn = sqlite3.connect(str(tmp_path)) try: conn.execute("PRAGMA journal_mode=OFF") conn.execute("PRAGMA synchronous=OFF") conn.execute("PRAGMA temp_store=MEMORY") conn.execute( "CREATE TABLE samples (" "key TEXT PRIMARY KEY, " "shard TEXT NOT NULL, " "offset INTEGER NOT NULL, " "length INTEGER NOT NULL)" ) rows: list[tuple[str, str, int, int]] = [] for index_file in index_files: with index_file.open("r", encoding="utf-8") as f: for line in f: line = line.strip() if not line: continue item = json.loads(line) rows.append((str(item["key"]), str(item["shard"]), int(item["offset"]), int(item["length"]))) if len(rows) >= 10000: conn.executemany("INSERT OR REPLACE INTO samples VALUES (?, ?, ?, ?)", rows) rows.clear() if rows: conn.executemany("INSERT OR REPLACE INTO samples VALUES (?, ?, ?, ?)", rows) conn.execute("CREATE INDEX IF NOT EXISTS idx_samples_key ON samples(key)") conn.commit() finally: conn.close() os.replace(tmp_path, sqlite_path) return sqlite_path class ShardWriter: def __init__( self, output_root: Path, shard_rank: int, shard_size: int, overwrite: bool, save_executor: ThreadPoolExecutor, ) -> None: self.output_root = output_root self.shard_rank = int(shard_rank) self.shard_size = max(1, int(shard_size)) self.overwrite = bool(overwrite) self.save_executor = save_executor self.current_shard_id: int | None = None self.items: list[ExportItem] = [] self.hiddens: list[torch.Tensor] = [] def add_batch(self, items: list[ExportItem], hidden_cpu: torch.Tensor, seq_lens: list[int]) -> list[Future[dict[str, Any]]]: futures: list[Future[dict[str, Any]]] = [] for batch_idx, item in enumerate(items): shard_id = int(item.shard_id) if shard_id < 0: raise ValueError(f"Missing shard_id for {item.input_path}") if self.current_shard_id is None: self.current_shard_id = shard_id if shard_id != self.current_shard_id: futures.extend(self.flush()) self.current_shard_id = shard_id seq_len = int(seq_lens[batch_idx]) self.items.append(item) self.hiddens.append(hidden_cpu[batch_idx, :seq_len, :].contiguous()) if len(self.items) >= self.shard_size: futures.extend(self.flush()) return futures def flush(self) -> list[Future[dict[str, Any]]]: if not self.items or self.current_shard_id is None: return [] items = self.items hiddens = self.hiddens shard_id = int(self.current_shard_id) self.items = [] self.hiddens = [] self.current_shard_id = None return [ self.save_executor.submit( save_hidden_state_shard_cpu, self.output_root, self.shard_rank, shard_id, items, hiddens, self.overwrite, ) ] class SourceWriter: """Collect consecutive windows and atomically save one complete WAV.""" def __init__( self, overwrite: bool, save_executor: ThreadPoolExecutor, ) -> None: self.overwrite = bool(overwrite) self.save_executor = save_executor self.current_source_key: str | None = None self.expected_count = 0 self.items: list[ExportItem] = [] self.hiddens: list[torch.Tensor] = [] def add_batch( self, items: list[ExportItem], hidden_cpu: torch.Tensor, seq_lens: list[int], ) -> list[Future[dict[str, Any]]]: futures: list[Future[dict[str, Any]]] = [] for batch_idx, item in enumerate(items): if not item.source_key or item.source_item_count <= 0: raise ValueError( f"Missing source metadata for {item.input_path}" ) if self.current_source_key is None: self.current_source_key = item.source_key self.expected_count = int(item.source_item_count) elif item.source_key != self.current_source_key: futures.extend(self.flush()) self.current_source_key = item.source_key self.expected_count = int(item.source_item_count) if int(item.source_item_count) != self.expected_count: raise ValueError( f"Inconsistent source count for {item.source_key}" ) seq_len = int(seq_lens[batch_idx]) self.items.append(item) self.hiddens.append( hidden_cpu[batch_idx, :seq_len, :].clone( memory_format=torch.contiguous_format ) ) if len(self.items) == self.expected_count: futures.extend(self.flush()) elif len(self.items) > self.expected_count: raise ValueError( f"Too many windows for source {item.source_key}" ) return futures def flush(self) -> list[Future[dict[str, Any]]]: if not self.items: self.current_source_key = None self.expected_count = 0 return [] if len(self.items) != self.expected_count: raise RuntimeError( f"Incomplete source {self.current_source_key}: " f"collected={len(self.items)} expected={self.expected_count}" ) items = self.items hiddens = self.hiddens self.items = [] self.hiddens = [] self.current_source_key = None self.expected_count = 0 return [ self.save_executor.submit( save_hidden_state_source_cpu, items, hiddens, self.overwrite, ) ] def process_export_batch( preset: SourcePreset, task_name: str, items: list[ExportItem], args: argparse.Namespace, processor: Qwen2_5OmniProcessor, model: Qwen2_5OmniThinkerForConditionalGeneration, device: torch.device, save_dtype: torch.dtype, io_executor: ThreadPoolExecutor | None = None, save_executor: ThreadPoolExecutor | None = None, preloaded_payloads: list[dict[str, Any]] | None = None, preloaded_inputs: dict[str, torch.Tensor] | None = None, ) -> dict[str, Any]: payloads: list[dict[str, Any]] | None = None inputs_cpu: dict[str, torch.Tensor] | None = None inputs: dict[str, torch.Tensor] | None = None hidden: torch.Tensor | None = None hidden_cpu: torch.Tensor | None = None try: if preloaded_inputs is not None: inputs_cpu = preloaded_inputs else: payloads = preloaded_payloads if preloaded_payloads is not None else load_payloads( items, int(args.num_io_workers), io_executor=io_executor, ) if preset.modality == "audio": inputs_cpu = collate_audio_batch(processor, payloads) elif preset.modality == "text": inputs_cpu = collate_text_batch(processor, payloads) else: inputs_cpu = collate_video_batch(processor, payloads) inputs = move_inputs_to_device(inputs_cpu, device) with torch.inference_mode(): hidden = omni_hidden_states_without_lm_head(model, inputs) attention_mask = inputs["attention_mask"] seq_lens = [int(v) for v in attention_mask.sum(dim=1).tolist()] hidden_cpu = hidden.to(dtype=save_dtype).cpu().contiguous() stats = { "processed": len(items), "written": 0, "skipped": 0, "failed": 0, "hidden_tokens": int(sum(seq_lens)), "hidden_bytes": 0, "seq_sum": int(sum(seq_lens)), "error": None, "pending_saves": [], "shard_batches": [], "source_batches": [], } overwrite = args.overwrite if args.mode == "export" else True if args.storage_format == "shard": stats["shard_batches"].append((items, hidden_cpu, seq_lens)) hidden_cpu = None elif args.storage_format == "source": stats["source_batches"].append((items, hidden_cpu, seq_lens)) hidden_cpu = None elif save_executor is None: save_stats = save_hidden_state_batch_cpu(items, hidden_cpu, seq_lens, overwrite) stats["written"] += int(save_stats["written"]) stats["skipped"] += int(save_stats["skipped"]) stats["failed"] += int(save_stats["failed"]) stats["hidden_bytes"] += int(save_stats["hidden_bytes"]) stats["error"] = save_stats["error"] else: stats["pending_saves"].append( save_executor.submit(save_hidden_state_batch_cpu, items, hidden_cpu, seq_lens, overwrite) ) del hidden del hidden_cpu del inputs del inputs_cpu del payloads return stats except Exception as exc: # noqa: BLE001 error = f"{preset.name}/{task_name}: {type(exc).__name__}: {exc}" if hidden is not None: del hidden hidden = None if hidden_cpu is not None: del hidden_cpu hidden_cpu = None if inputs is not None: del inputs inputs = None if inputs_cpu is not None: del inputs_cpu inputs_cpu = None if payloads is not None: del payloads payloads = None gc.collect() if device.type == "cuda": torch.cuda.empty_cache() if is_cuda_oom(exc) and len(items) > 1: left_size = len(items) // 2 right_size = len(items) - left_size if is_rank0(): print( f"[WARN] {error} | retry split batch {len(items)} -> {left_size} + {right_size}", flush=True, ) left_stats = process_export_batch( preset, task_name, items[:left_size], args, processor, model, device, save_dtype, io_executor=io_executor, save_executor=save_executor, ) right_stats = process_export_batch( preset, task_name, items[left_size:], args, processor, model, device, save_dtype, io_executor=io_executor, save_executor=save_executor, ) merged = { "processed": left_stats["processed"] + right_stats["processed"], "written": left_stats["written"] + right_stats["written"], "skipped": left_stats["skipped"] + right_stats["skipped"], "failed": left_stats["failed"] + right_stats["failed"], "hidden_tokens": left_stats["hidden_tokens"] + right_stats["hidden_tokens"], "hidden_bytes": left_stats["hidden_bytes"] + right_stats["hidden_bytes"], "seq_sum": left_stats["seq_sum"] + right_stats["seq_sum"], "error": left_stats["error"] or right_stats["error"] or error, "pending_saves": [*left_stats.get("pending_saves", []), *right_stats.get("pending_saves", [])], "shard_batches": [*left_stats.get("shard_batches", []), *right_stats.get("shard_batches", [])], "source_batches": [*left_stats.get("source_batches", []), *right_stats.get("source_batches", [])], } return merged if is_rank0(): print(f"[ERROR] {error}", flush=True) traceback.print_exc() return { "processed": 0, "written": 0, "skipped": 0, "failed": len(items), "hidden_tokens": 0, "hidden_bytes": 0, "seq_sum": 0, "error": error, "pending_saves": [], "shard_batches": [], "source_batches": [], } def collect_completed_save_futures( pending_saves: deque[Future[dict[str, Any]]], *, wait_for_one: bool, ) -> list[dict[str, Any]]: if not pending_saves: return [] future_list = list(pending_saves) if wait_for_one: done, _ = wait(future_list, return_when=FIRST_COMPLETED) else: done, _ = wait(future_list, timeout=0, return_when=FIRST_COMPLETED) if not done: return [] done_ids = {id(fut) for fut in done} results: list[dict[str, Any]] = [] remaining: deque[Future[dict[str, Any]]] = deque() for fut in pending_saves: if id(fut) in done_ids: results.append(fut.result()) else: remaining.append(fut) pending_saves.clear() pending_saves.extend(remaining) return results def bench_or_export_group( preset: SourcePreset, task_name: str, args: argparse.Namespace, processor: Qwen2_5OmniProcessor, model: Qwen2_5OmniThinkerForConditionalGeneration, device: torch.device, save_dtype: torch.dtype, group_output_root: Path, ) -> dict[str, Any]: storage_format = str(args.storage_format) if storage_format == "source": log( f"[{args.mode.upper()}] streaming complete WAV sources without " "a duplicate window-count pass" ) global_total = 0 elif args.mode == "benchmark": global_total = int(args.benchmark_samples) elif args.limit > 0: global_total = int(args.limit) else: log(f"[{args.mode.upper()}] counting {preset.name}/{task_name} ...") global_total = count_group_items(preset, task_name, group_output_root, args) log(f"[{args.mode.upper()}] {preset.name}/{task_name}: total={global_total:,}") local_total = ( None if storage_format == "source" else global_total // world_size() + (1 if rank() < (global_total % world_size()) else 0) ) progress = tqdm( total=local_total, desc=f"{preset.name}/{task_name} rank{rank()}", disable=not is_rank0(), unit="item", ) if preset.modality == "audio": batch_size = int(args.audio_batch_size) elif preset.modality == "text": batch_size = int(args.text_batch_size) else: batch_size = int(args.video_batch_size) group_items: list[ExportItem] = [] processed = 0 written = 0 skipped = 0 failed = 0 hidden_tokens = 0 hidden_bytes = 0 seq_sum = 0 first_error: str | None = None started = time.time() max_pending_save_batches = max(1, int(args.max_pending_save_batches)) pending_saves: deque[Future[dict[str, Any]]] = deque() shard_size = max(1, int(args.shard_size)) complete_shard_cache: dict[int, bool] = {} complete_source_cache: dict[str, bool] = {} source_index_rows: dict[ str, tuple[str, str, int, float, int, int, int], ] = {} def merge_save_stats(save_stats: dict[str, Any]) -> None: nonlocal written, skipped, failed, hidden_bytes, first_error written += int(save_stats["written"]) skipped += int(save_stats["skipped"]) failed += int(save_stats["failed"]) hidden_bytes += int(save_stats["hidden_bytes"]) if first_error is None and save_stats.get("error"): first_error = str(save_stats["error"]) def iter_local_batches() -> Iterator[list[ExportItem]]: nonlocal skipped batch: list[ExportItem] = [] local_index = 0 source_rank_loads = [0 for _ in range(world_size())] active_source_ordinal = -1 active_source_rank = -1 for global_index, item in enumerate(iter_group_items(preset, task_name, group_output_root, args)): if storage_format != "source" and global_index >= global_total: break if storage_format == "source": if item.source_ordinal < 0: raise ValueError( f"Missing source ordinal for {item.input_path}" ) if int(item.source_ordinal) != active_source_ordinal: active_source_ordinal = int(item.source_ordinal) active_source_rank = assign_source_to_least_loaded_rank( source_rank_loads, int(item.source_item_count), ) if active_source_rank != rank(): continue source_index_rows.setdefault( item.source_key, source_index_row(item, group_output_root), ) if args.mode == "export" and not args.overwrite: complete = complete_source_cache.get(item.source_key) if complete is None: complete = source_hidden_file_is_complete(item) complete_source_cache[item.source_key] = complete if complete: skipped += 1 progress.update(1) progress.set_postfix( processed=processed, written=written, skipped=skipped, failed=failed, ) continue else: if global_index % world_size() != rank(): continue if storage_format == "shard": shard_id = local_index // shard_size item.shard_id = int(shard_id) local_index += 1 if args.mode == "export" and not args.overwrite: complete = complete_shard_cache.get(shard_id) if complete is None: expected_count = min(shard_size, max(0, local_total - shard_id * shard_size)) complete = shard_is_complete( group_output_root, preset.name, task_name, preset.modality, rank(), shard_id, expected_count, ) complete_shard_cache[shard_id] = complete if complete: skipped += 1 progress.update(1) progress.set_postfix(processed=processed, written=written, skipped=skipped, failed=failed) continue elif storage_format == "file": if args.mode == "export" and item.output_path.exists() and not args.overwrite: skipped += 1 progress.update(1) progress.set_postfix(processed=processed, written=written, skipped=skipped, failed=failed) continue batch.append(item) if len(batch) >= batch_size: yield batch batch = [] if batch: yield batch with ThreadPoolExecutor(max_workers=max(1, int(args.num_io_workers))) as io_executor, \ ThreadPoolExecutor(max_workers=max(1, int(args.num_save_workers))) as save_executor, \ ThreadPoolExecutor(max_workers=2) as prefetch_executor: shard_writer = ( ShardWriter( group_output_root, rank(), shard_size, args.overwrite if args.mode == "export" else True, save_executor, ) if storage_format == "shard" else None ) source_writer = ( SourceWriter( args.overwrite if args.mode == "export" else True, save_executor, ) if storage_format == "source" else None ) batch_iter = iter_local_batches() current_items = next(batch_iter, None) current_input_future: Future[dict[str, torch.Tensor]] | None = None next_items: list[ExportItem] | None = None next_input_future: Future[dict[str, torch.Tensor]] | None = None if current_items is not None: current_input_future = prefetch_executor.submit( prepare_inputs_cpu, preset, current_items, args, processor, io_executor, ) while current_items is not None and current_input_future is not None: next_items = next(batch_iter, None) if next_items is not None: next_input_future = prefetch_executor.submit( prepare_inputs_cpu, preset, next_items, args, processor, io_executor, ) else: next_input_future = None batch_stats = process_export_batch( preset, task_name, current_items, args, processor, model, device, save_dtype, io_executor=io_executor, save_executor=save_executor, preloaded_inputs=current_input_future.result(), ) processed += int(batch_stats["processed"]) failed += int(batch_stats["failed"]) hidden_tokens += int(batch_stats["hidden_tokens"]) seq_sum += int(batch_stats["seq_sum"]) if first_error is None and batch_stats["error"]: first_error = str(batch_stats["error"]) for fut in batch_stats.get("pending_saves", []): pending_saves.append(fut) for shard_items, shard_hidden_cpu, shard_seq_lens in batch_stats.get("shard_batches", []): if shard_writer is None: raise RuntimeError("shard batch returned while storage_format != shard") for fut in shard_writer.add_batch(shard_items, shard_hidden_cpu, shard_seq_lens): pending_saves.append(fut) del shard_hidden_cpu for source_items, source_hidden_cpu, source_seq_lens in batch_stats.get("source_batches", []): if source_writer is None: raise RuntimeError( "source batch returned while storage_format != source" ) for fut in source_writer.add_batch( source_items, source_hidden_cpu, source_seq_lens, ): pending_saves.append(fut) del source_hidden_cpu while len(pending_saves) >= max_pending_save_batches: completed = collect_completed_save_futures(pending_saves, wait_for_one=True) for save_stats in completed: merge_save_stats(save_stats) for save_stats in collect_completed_save_futures(pending_saves, wait_for_one=False): merge_save_stats(save_stats) progress.update(len(current_items)) progress.set_postfix(processed=processed, written=written, skipped=skipped, failed=failed) current_items = next_items current_input_future = next_input_future if shard_writer is not None: for fut in shard_writer.flush(): pending_saves.append(fut) if source_writer is not None: for fut in source_writer.flush(): pending_saves.append(fut) while pending_saves: for save_stats in collect_completed_save_futures(pending_saves, wait_for_one=True): merge_save_stats(save_stats) progress.close() if storage_format == "shard" and args.mode == "export": sqlite_path = build_rank_sqlite_index( group_output_root, preset.name, task_name, preset.modality, rank(), expected_rows=local_total, ) if sqlite_path is not None and is_rank0(): print(f"[SHARD] rebuilt sqlite index for rank0: {sqlite_path}", flush=True) if storage_format == "source" and args.mode == "export": if failed: raise RuntimeError( f"Refusing to publish source index after {failed} failed windows" ) sqlite_path = build_rank_source_sqlite_index( group_output_root, preset.name, task_name, preset.modality, source_index_rows.values(), rank(), ) if is_rank0(): print( f"[SOURCE] wrote compact source index for rank0: {sqlite_path}", flush=True, ) elapsed = time.time() - started metrics = torch.tensor( [processed, written, skipped, failed, hidden_tokens, hidden_bytes, seq_sum], dtype=torch.float64, ) wall = torch.tensor([elapsed], dtype=torch.float64) # Export is embarrassingly parallel by rank. During resume, ranks can have # very different amounts of work left because complete shards are skipped # per rank. A group-end all_reduce can therefore time out while a slower # rank is still legitimately filling missing shards. Keep export progress # rank-local; completeness is determined by shard/index files, not by this # summary metric. if dist.is_initialized() and args.mode != "export": dist.all_reduce(metrics, op=dist.ReduceOp.SUM) dist.all_reduce(wall, op=dist.ReduceOp.MAX) total_processed, total_written, total_skipped, total_failed, total_tokens, total_bytes, total_seq_sum = metrics.tolist() wall_sec = float(wall.item()) avg_seq_len = (total_seq_sum / total_processed) if total_processed > 0 else 0.0 samples_per_sec = (total_processed / wall_sec) if wall_sec > 0 else 0.0 tokens_per_sec = (total_tokens / wall_sec) if wall_sec > 0 else 0.0 avg_file_mb = (total_bytes / max(total_written, 1)) / (1024 ** 2) group_stats = { "source_name": preset.name, "task_name": task_name, "global_total": ( None if storage_format == "source" else int(global_total) ), "processed": int(total_processed), "written": int(total_written), "skipped": int(total_skipped), "failed": int(total_failed), "elapsed_sec": wall_sec, "samples_per_sec": samples_per_sec, "tokens_per_sec": tokens_per_sec, "avg_seq_len": avg_seq_len, "avg_output_mb": avg_file_mb, "first_error": first_error or "", } return group_stats def estimate_group( preset: SourcePreset, task_name: str, args: argparse.Namespace, processor: Qwen2_5OmniProcessor, hidden_size: int, ) -> dict[str, Any]: sample_items = sampled_group_items( preset, task_name, Path(args.output_root), int(args.estimate_samples_per_group), args, ) if not sample_items: return { "source_name": preset.name, "task_name": task_name, "total_items": 0, "sample_count": 0, } seq_lens: list[int] = [] output_bytes: list[int] = [] payloads = load_payloads(sample_items, int(args.num_io_workers)) sample_progress = tqdm( payloads, total=len(payloads), desc=f"Estimate {preset.name}/{task_name}", disable=not is_rank0(), leave=False, unit="sample", ) if preset.modality == "audio": for payload in sample_progress: texts = build_audio_text(processor, [payload["instruction"]]) inputs = processor( text=texts, audio=[np.asarray(payload["audio"], dtype=np.float32)], padding=True, return_tensors="pt", ) seq_len = int(inputs["input_ids"].shape[1]) seq_lens.append(seq_len) output_bytes.append(seq_len * hidden_size * torch.tensor([], dtype=save_dtype_from_arg(args.save_dtype)).element_size()) elif preset.modality == "text": for payload in sample_progress: texts = build_text_text(processor, [payload["instruction"]]) tokenized = processor.tokenizer(texts, padding=True, return_tensors="pt") seq_len = int(tokenized["input_ids"].shape[1]) seq_lens.append(seq_len) output_bytes.append(seq_len * hidden_size * torch.tensor([], dtype=save_dtype_from_arg(args.save_dtype)).element_size()) else: for payload in sample_progress: texts = build_video_text(processor, [payload["instruction"]]) texts = processor.replace_multimodal_special_tokens( texts, iter([]), iter([]), iter([payload["video_grid_thw"][0]]), iter([float(np.asarray(payload["video_second_per_grid"]).reshape(-1)[0])]), False, 25, 2.0, ) tokenized = processor.tokenizer(texts, padding=True, return_tensors="pt") seq_len = int(tokenized["input_ids"].shape[1]) seq_lens.append(seq_len) output_bytes.append(seq_len * hidden_size * torch.tensor([], dtype=save_dtype_from_arg(args.save_dtype)).element_size()) sample_progress.close() total_items = count_group_items(preset, task_name, Path(args.output_root), args) avg_seq_len = float(sum(seq_lens) / len(seq_lens)) avg_output_bytes = float(sum(output_bytes) / len(output_bytes)) return { "source_name": preset.name, "task_name": task_name, "total_items": int(total_items), "sample_count": len(seq_lens), "avg_seq_len": avg_seq_len, "min_seq_len": int(min(seq_lens)), "max_seq_len": int(max(seq_lens)), "avg_output_mb": avg_output_bytes / (1024 ** 2), "estimated_total_gib": total_items * avg_output_bytes / (1024 ** 3), "estimated_total_tib": total_items * avg_output_bytes / (1024 ** 4), } def ensure_video_export_is_confirmed(args: argparse.Namespace, pairs: list[tuple[SourcePreset, str]]) -> None: if args.mode != "export": return has_video = any(preset.modality == "video_clip" for preset, _ in pairs) if has_video and not args.confirm_full_video: raise ValueError("video_clip export is disabled by default; re-run with --confirm-full-video") def write_report(output_root: Path, mode: str, report: dict[str, Any]) -> Path: report_dir = output_root / "_reports" report_dir.mkdir(parents=True, exist_ok=True) timestamp = time.strftime("%Y%m%d_%H%M%S") path = report_dir / f"{mode}.{timestamp}.json" path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") return path def run_estimate(args: argparse.Namespace, pairs: list[tuple[SourcePreset, str]]) -> None: hidden_size = load_hidden_size(args.model_path) processor = load_processor(args.model_path) report = { "mode": args.mode, "model_path": args.model_path, "hidden_size": hidden_size, "save_dtype": args.save_dtype, "groups": [], } group_progress = tqdm( pairs, total=len(pairs), desc="Estimate groups", disable=not is_rank0(), leave=True, unit="group", ) for preset, task_name in group_progress: stats = estimate_group(preset, task_name, args, processor, hidden_size) report["groups"].append(stats) group_progress.set_postfix( source=preset.name, task=task_name, tib=f"{stats.get('estimated_total_tib', 0):.3f}", ) log( f"[ESTIMATE] {preset.name}/{task_name}: total={stats['total_items']:,} " f"avg_seq={stats.get('avg_seq_len', 0):.2f} avg_mb={stats.get('avg_output_mb', 0):.3f} " f"total_tib={stats.get('estimated_total_tib', 0):.3f}" ) group_progress.close() report_path = write_report(Path(args.output_root), args.mode, report) log(f"[REPORT] wrote {report_path}") def run_status(args: argparse.Namespace, pairs: list[tuple[SourcePreset, str]]) -> None: output_root = Path(args.output_root) report = { "mode": args.mode, "output_root": str(output_root), "storage_format": args.storage_format, "groups": [], } for preset, task_name in tqdm(pairs, desc="Status groups", disable=not is_rank0(), unit="group"): group_dir = cache_group_dir(output_root, preset.name, task_name, preset.modality) total = count_group_items(preset, task_name, output_root, args) if args.storage_format == "shard": shard_dir = group_dir / "shards" existing = 0 shard_count = 0 sqlite_count = 0 if shard_dir.exists(): for index_path in shard_dir.glob("*shard*.index.jsonl"): existing += count_jsonl_rows(index_path) shard_count = sum(1 for _ in shard_dir.glob("*shard*.safetensors")) sqlite_count = sum(1 for _ in shard_dir.glob("index.rank*.sqlite")) tmp_count = sum(1 for _ in group_dir.rglob("*.tmp")) if group_dir.exists() else 0 elif args.storage_format == "source": source_dir = group_dir / "sources" existing = 0 source_count = 0 sqlite_paths = ( sorted(source_dir.glob("index.ws*.rank*.sqlite")) if source_dir.exists() else [] ) seen_keys: set[str] = set() for sqlite_path in sqlite_paths: conn = sqlite3.connect( f"file:{sqlite_path}?mode=ro", uri=True, ) try: for key, count in conn.execute( "SELECT key, count FROM sources" ): if str(key) in seen_keys: continue seen_keys.add(str(key)) source_count += 1 existing += int(count) finally: conn.close() shard_count = source_count sqlite_count = len(sqlite_paths) tmp_count = ( sum(1 for _ in source_dir.rglob("*.tmp")) if source_dir.exists() else 0 ) else: existing = 0 for item in iter_group_items(preset, task_name, output_root, args): if item.output_path.exists(): existing += 1 shard_count = 0 sqlite_count = 0 tmp_count = sum(1 for _ in group_dir.rglob("*.tmp")) if group_dir.exists() else 0 stats = { "source_name": preset.name, "task_name": task_name, "expected": total, "existing": existing, "missing": max(0, total - existing), "tmp": tmp_count, "shards": shard_count, "sqlite_indexes": sqlite_count, } report["groups"].append(stats) log( f"[STATUS] {preset.name}/{task_name}: " f"existing={existing:,} missing={max(0, total - existing):,} expected={total:,} " f"shards={shard_count:,} sqlite={sqlite_count:,} tmp={tmp_count:,}" ) report_path = write_report(output_root, args.mode, report) log(f"[REPORT] wrote {report_path}") def run_benchmark_or_export(args: argparse.Namespace, pairs: list[tuple[SourcePreset, str]]) -> None: maybe_init_distributed() device = current_device() save_dtype = save_dtype_from_arg(args.save_dtype) processor = load_processor(args.model_path) model = load_model(args.model_path, device, save_dtype) group_output_root = ( Path(args.output_root) / "_benchmark_outputs" if args.mode == "benchmark" else Path(args.output_root) ) report = { "mode": args.mode, "model_path": args.model_path, "world_size": world_size(), "rank": rank(), "save_dtype": args.save_dtype, "storage_format": args.storage_format, "shard_size": int(args.shard_size), "audio_command_full_audio": bool(args.audio_command_full_audio), "audio_command_window_seconds": float(args.audio_command_window_seconds), "audio_command_sample_step_frames": int(args.audio_command_sample_step_frames), "limit_sources": int(args.limit_sources), "groups": [], } group_progress = tqdm( pairs, total=len(pairs), desc=f"{args.mode.title()} groups", disable=not is_rank0(), leave=True, unit="group", ) for preset, task_name in group_progress: stats = bench_or_export_group( preset, task_name, args, processor, model, device, save_dtype, group_output_root, ) if is_rank0(): report["groups"].append(stats) group_progress.set_postfix( source=preset.name, task=task_name, samples_per_sec=f"{stats['samples_per_sec']:.2f}", avg_mb=f"{stats['avg_output_mb']:.3f}", ) log( f"[{args.mode.upper()}] {preset.name}/{task_name}: processed={stats['processed']:,} " f"written={stats['written']:,} avg_seq={stats['avg_seq_len']:.2f} " f"samples/s={stats['samples_per_sec']:.2f} tokens/s={stats['tokens_per_sec']:.2f} " f"avg_mb={stats['avg_output_mb']:.3f}" ) group_progress.close() if is_rank0(): report_path = write_report(Path(args.output_root), args.mode, report) log(f"[REPORT] wrote {report_path}") cleanup_distributed() def run_verify_online_hidden( args: argparse.Namespace, pair: tuple[SourcePreset, str], ) -> None: """Recompute saved audio hidden states through the real B=1 online path.""" preset, task_name = pair output_root = Path(args.output_root) contract_path = output_root / "_reports" / "encoder_contract.json" if not contract_path.is_file(): raise FileNotFoundError( f"Missing encoder contract before online verification: {contract_path}" ) recorded_contract = json.loads(contract_path.read_text(encoding="utf-8")) expected_contract = encoder_contract_payload(args.model_path) for key, expected in expected_contract.items(): if recorded_contract.get(key) != expected: raise RuntimeError( f"Encoder contract changed before verification: {key}=" f"{recorded_contract.get(key)!r}, expected={expected!r}" ) device = current_device() processor = load_processor(args.model_path) model = load_model(args.model_path, device, torch.bfloat16) verify_limit = max(int(args.verify_sources), 0) verify_windows_per_source = max( int(args.verify_windows_per_source), 0, ) current_source_key: str | None = None source_audio: np.ndarray | None = None source_flat: torch.Tensor | None = None source_offsets: torch.Tensor | None = None verified_sources = 0 verified_windows = 0 mismatched_windows = 0 max_abs_error = 0.0 first_mismatch = "" items = iter_group_items(preset, task_name, output_root, args) for item in items: source_key = item.source_key or str(item.input_path) if source_key != current_source_key: if verify_limit > 0 and verified_sources >= verify_limit: break current_source_key = source_key verified_sources += 1 source_audio = load_audio_array( item.input_path, item.audio_sampling_rate, ) source_flat = None source_offsets = None if args.storage_format == "source": source_path = item.source_output_path if source_path is None or not source_path.is_file(): raise FileNotFoundError( f"Missing source hidden file: {source_path}" ) with safe_open( str(source_path), framework="pt", device="cpu", ) as reader: metadata = reader.metadata() or {} for key, value in online_encoder_metadata(task_name).items(): if metadata.get(key) != value: raise RuntimeError( f"Source encoder metadata mismatch at " f"{source_path}: {key}={metadata.get(key)!r}, " f"expected={value!r}" ) source_flat = reader.get_tensor("hidden_states") source_offsets = reader.get_tensor("offsets") if source_audio is None: raise RuntimeError(f"Audio was not loaded for {source_key}") if ( verify_windows_per_source > 0 and item.source_item_index is not None and int(item.source_item_index) >= verify_windows_per_source ): continue audio = source_audio if not item.full_audio: audio = audio_segment( audio, item.audio_start_sec, item.audio_duration_sec, item.audio_sampling_rate, ) inputs_cpu = collate_audio_batch( processor, [ { "audio": audio, "instruction": item.instruction, "full_audio": bool(item.full_audio), } ], ) inputs = move_inputs_to_device(inputs_cpu, device) with torch.inference_mode(): hidden = omni_hidden_states_without_lm_head(model, inputs) seq_len = int(inputs["attention_mask"].sum().item()) online = ( hidden[0, :seq_len] .to(dtype=torch.bfloat16) .cpu() .contiguous() ) if args.storage_format == "source": if source_flat is None or source_offsets is None: raise RuntimeError(f"Source hidden was not loaded for {source_key}") source_index = int(item.source_item_index) cached = source_flat[ int(source_offsets[source_index]) : int(source_offsets[source_index + 1]) ] cache_path = item.source_output_path else: cache_path = item.output_path if not cache_path.is_file(): raise FileNotFoundError(cache_path) with safe_open( str(cache_path), framework="pt", device="cpu", ) as reader: metadata = reader.metadata() or {} for key, value in online_encoder_metadata(task_name).items(): if metadata.get(key) != value: raise RuntimeError( f"Encoder metadata mismatch at {cache_path}: " f"{key}={metadata.get(key)!r}, expected={value!r}" ) cached = reader.get_tensor("hidden_states") verified_windows += 1 if tuple(online.shape) != tuple(cached.shape): mismatched_windows += 1 if not first_mismatch: first_mismatch = ( f"{cache_path}: shape {tuple(cached.shape)} != " f"{tuple(online.shape)}" ) continue difference = (online.float() - cached.float()).abs() window_max = float(difference.max().item()) max_abs_error = max(max_abs_error, window_max) if not torch.equal(online, cached): mismatched_windows += 1 if not first_mismatch: first_mismatch = ( f"{cache_path}: source_index={item.source_item_index}, " f"max_abs_error={window_max}" ) report = { **expected_contract, "storage_format": args.storage_format, "verified_sources": verified_sources, "verified_windows": verified_windows, "verified_windows_per_source_limit": verify_windows_per_source, "mismatched_windows": mismatched_windows, "max_abs_error": max_abs_error, "first_mismatch": first_mismatch, "verified_at": time.time(), } report_path = output_root / "_reports" / "online_hidden_verify.json" write_json_atomic(report_path, report) log( "[VERIFY] exact B=1 online equivalence: " f"sources={verified_sources:,} windows={verified_windows:,} " f"mismatched={mismatched_windows:,} max_abs={max_abs_error:g}" ) log(f"[REPORT] wrote {report_path}") if verified_windows <= 0: raise RuntimeError("Online hidden verification selected no windows") if mismatched_windows != 0 or max_abs_error != 0.0: raise RuntimeError( "Saved hidden states are not bitwise equal to online Qwen output: " f"{first_mismatch}" ) def main() -> None: global ACTIVE_MODEL_FINGERPRINT, SOURCE_PRESETS warnings.filterwarnings( "ignore", message="The image processor of type `Qwen2VLImageProcessor` is now loaded as a fast processor by default.*", ) warnings.filterwarnings( "ignore", message="Unrecognized keys in `rope_scaling` for 'rope_type'='default': .*", ) args = parse_args() ACTIVE_MODEL_FINGERPRINT = qwen_model_fingerprint(args.model_path) if int(args.motion_fps) <= 0 or int(args.audio_sampling_rate) <= 0: raise ValueError("motion_fps and audio_sampling_rate must be positive") if not bool(args.audio_command_full_audio): if float(args.audio_command_window_seconds) <= 0: raise ValueError("audio_command_window_seconds must be positive in windowed mode") if int(args.audio_command_sample_step_frames) <= 0: raise ValueError("audio_command_sample_step_frames must be positive in windowed mode") SOURCE_PRESETS = load_source_presets(args) tasks = parse_csv_list(args.tasks) sources = parse_csv_list(args.sources) unknown_tasks = [task for task in tasks if task not in SUPPORTED_TASKS] if unknown_tasks: raise ValueError(f"Unsupported tasks: {unknown_tasks}") unknown_sources = [source for source in sources if source not in SOURCE_PRESETS] if unknown_sources: raise ValueError(f"Unsupported sources: {unknown_sources}") pairs = source_task_pairs(tasks, sources) if not pairs: raise ValueError("No compatible source/task pairs selected") ensure_video_export_is_confirmed(args, pairs) has_audio_command = any(task == "audio_command_text" for _, task in pairs) if has_audio_command and args.mode in ("export", "verify"): if int(args.audio_batch_size) != HFD_ONLINE_ENCODER_BATCH_SIZE: raise ValueError( "Streaming audio-command hidden states must use " f"--audio-batch-size {HFD_ONLINE_ENCODER_BATCH_SIZE}; larger " "batches are not bitwise equal to real one-window inference." ) if str(args.save_dtype) != "bf16": raise ValueError( "Streaming audio-command hidden states must be saved as bf16" ) if args.storage_format == "source": if args.mode == "benchmark": raise ValueError( "--storage-format source uses complete WAVs and does not support " "the window-count based benchmark mode; use export with " "--limit-sources for a representative timed subset." ) if len(pairs) != 1 or pairs[0][0].modality != "audio" or pairs[0][1] != "audio_command_text": raise ValueError( "--storage-format source is intentionally restricted to the " "HFD audio_command_text source" ) if bool(args.audio_command_full_audio): raise ValueError( "--storage-format source requires windowed audio command export" ) if int(args.limit) != 0: raise ValueError( "Use --limit-sources instead of --limit with source storage" ) if ( int(args.motion_fps) != 50 or int(args.audio_sampling_rate) != 16000 or not math.isclose( float(args.audio_command_window_seconds), 0.6, ) or int(args.audio_command_sample_step_frames) != 20 ): raise ValueError( "HFD source storage is fixed to 50 fps, 16 kHz, a 0.6 s " "window and a 20-frame/0.4 s stride" ) output_root = Path(args.output_root) output_root.mkdir(parents=True, exist_ok=True) if args.mode == "export" and is_rank0() and has_audio_command: write_json_atomic( output_root / "_reports" / "encoder_contract.json", encoder_contract_payload(args.model_path), ) log(f"Mode: {args.mode}") log(f"Tasks: {','.join(tasks)}") log(f"Sources: {','.join(sources)}") log(f"Output root: {output_root}") if args.storage_format == "source": log( "Storage: source (one complete WAV per safetensors, " f"limit_sources={int(args.limit_sources)})" ) else: log(f"Storage: {args.storage_format} shard_size={int(args.shard_size)}") log( "Audio command: " f"full_audio={bool(args.audio_command_full_audio)} " f"window={float(args.audio_command_window_seconds):g}s " f"step={int(args.audio_command_sample_step_frames)} motion frames" ) if args.mode == "estimate": run_estimate(args, pairs) return if args.mode == "status": run_status(args, pairs) return if args.mode == "verify": if len(pairs) != 1 or pairs[0][0].modality != "audio": raise ValueError("Online hidden verification requires one audio pair") run_verify_online_hidden(args, pairs[0]) return run_benchmark_or_export(args, pairs) if __name__ == "__main__": main()
from __future__ import annotations
import argparse
import gc
import hashlib
import json
import math
import os
import random
import sqlite3
import subprocess
import time
import traceback
import warnings
import wave
from collections import deque
from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Iterable, Iterator
import numpy as np
import pandas as pd
import torch
import torch.distributed as dist
from safetensors import safe_open
from safetensors.torch import save_file
from tqdm import tqdm
from transformers import AutoConfig, Qwen2_5OmniProcessor, Qwen2_5OmniThinkerForConditionalGeneration
from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import Qwen2_5OmniProcessorKwargs
from psi.data.seed.full_duplex_audio_contract import (
AUDIO_COMMAND_STREAMING_PROMPT,
HFD_AUDIO_SOURCE_FORMAT,
HFD_ONLINE_ENCODER_ATTENTION,
HFD_ONLINE_ENCODER_BATCH_SIZE,
HFD_ONLINE_ENCODER_CONTRACT,
HFD_ONLINE_ENCODER_DTYPE,
encoder_contract_payload,
qwen_model_fingerprint,
write_json_atomic,
)
from psi.data.seed.seed_audio_motion_dataset import (
_AUDIO_DANCE_PROMPT_POOLS,
ffmpeg_bin,
ffprobe_bin,
)
from psi.data.seed.motion_frame_index import read_motion_frame_count
from psi.data.seed.seed_video_motion_dataset import _VIDEO_MOTION_PROMPT_POOLS
from psi.data.seed.text_prompt import (
DEFAULT_SEED_TEXT_FIELDS,
collect_seed_text_candidates,
compose_seed_motion_prompt_from_candidates,
)
OMNI_DEFAULT_SYSTEM_PROMPT = (
"You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, "
"capable of perceiving auditory and visual inputs, as well as generating text and speech."
)
ACTIVE_MODEL_FINGERPRINT = ""
SUPPORTED_TASKS = (
"text",
"audio_text",
"audio_only",
"audio_command_text",
"audio_command_only",
"video_clip_text",
"video_clip_only",
)
DEFAULT_AUDIO_FILE_EXTENSIONS = (".wav", ".flac", ".mp3", ".m4a", ".aac", ".ogg", ".npy")
@dataclass(frozen=True)
class SourcePreset:
name: str
modality: str
dataset_root: Path
manifest_path: Path
allowed_tasks: tuple[str, ...]
instruction_key: str = ""
audio_path_key: str = "audio_path"
video_path_key: str = "video_path"
video_embed_dir: Path | None = None
is_audio_dir_manifest: bool = False
audio_file_extensions: tuple[str, ...] = DEFAULT_AUDIO_FILE_EXTENSIONS
text_fields: tuple[str, ...] = tuple(DEFAULT_SEED_TEXT_FIELDS)
SOURCE_PRESETS: dict[str, SourcePreset] = {
"seed_text": SourcePreset(
name="seed_text",
modality="text",
dataset_root=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all"),
manifest_path=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/metadata/seed_metadata_v004.csv"),
allowed_tasks=("text",),
),
"2264_with_audio": SourcePreset(