Download scripts/data/cache_omni_hidden_states.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 136 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/cache_omni_hidden_states.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/data/cache_omni_hidden_states.py
-
curl -L -o cache_omni_hidden_states.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/cache_omni_hidden_states.py
136 kB
from __future__ import annotations
import argparse
import gc
import hashlib
import json
import math
import os
import random
import sqlite3
import subprocess
import time
import traceback
import warnings
import wave
from collections import deque
from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Iterable, Iterator
import numpy as np
import pandas as pd
import torch
import torch.distributed as dist
from safetensors import safe_open
from safetensors.torch import save_file
from tqdm import tqdm
from transformers import AutoConfig, Qwen2_5OmniProcessor, Qwen2_5OmniThinkerForConditionalGeneration
from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import Qwen2_5OmniProcessorKwargs
from psi.data.seed.full_duplex_audio_contract import (
AUDIO_COMMAND_STREAMING_PROMPT,
HFD_AUDIO_SOURCE_FORMAT,
HFD_ONLINE_ENCODER_ATTENTION,
HFD_ONLINE_ENCODER_BATCH_SIZE,
HFD_ONLINE_ENCODER_CONTRACT,
HFD_ONLINE_ENCODER_DTYPE,
encoder_contract_payload,
qwen_model_fingerprint,
write_json_atomic,
)
from psi.data.seed.seed_audio_motion_dataset import (
_AUDIO_DANCE_PROMPT_POOLS,
ffmpeg_bin,
ffprobe_bin,
)
from psi.data.seed.motion_frame_index import read_motion_frame_count
from psi.data.seed.seed_video_motion_dataset import _VIDEO_MOTION_PROMPT_POOLS
from psi.data.seed.text_prompt import (
DEFAULT_SEED_TEXT_FIELDS,
collect_seed_text_candidates,
compose_seed_motion_prompt_from_candidates,
)
OMNI_DEFAULT_SYSTEM_PROMPT = (
"You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, "
"capable of perceiving auditory and visual inputs, as well as generating text and speech."
)
ACTIVE_MODEL_FINGERPRINT = ""
SUPPORTED_TASKS = (
"text",
"audio_text",
"audio_only",
"audio_command_text",
"audio_command_only",
"video_clip_text",
"video_clip_only",
)
DEFAULT_AUDIO_FILE_EXTENSIONS = (".wav", ".flac", ".mp3", ".m4a", ".aac", ".ogg", ".npy")
@dataclass(frozen=True)
class SourcePreset:
name: str
modality: str
dataset_root: Path
manifest_path: Path
allowed_tasks: tuple[str, ...]
instruction_key: str = ""
audio_path_key: str = "audio_path"
video_path_key: str = "video_path"
video_embed_dir: Path | None = None
is_audio_dir_manifest: bool = False
audio_file_extensions: tuple[str, ...] = DEFAULT_AUDIO_FILE_EXTENSIONS
text_fields: tuple[str, ...] = tuple(DEFAULT_SEED_TEXT_FIELDS)
SOURCE_PRESETS: dict[str, SourcePreset] = {
"seed_text": SourcePreset(
name="seed_text",
modality="text",
dataset_root=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all"),
manifest_path=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/metadata/seed_metadata_v004.csv"),
allowed_tasks=("text",),
),
"2264_with_audio": SourcePreset(
name="2264_with_audio",
modality="audio",
dataset_root=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_audio_wav_v26/2264_with_audio"),
manifest_path=Path(
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_audio_wav_v26/2264_with_audio/metadata.jsonl"
),
allowed_tasks=("audio_text", "audio_only"),
audio_file_extensions=(".wav",),
),
"psi0_dance_audio": SourcePreset(
name="psi0_dance_audio",
modality="audio",
dataset_root=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_audio_wav_v26/psi0_dance_audio"),
manifest_path=Path(
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_audio_wav_v26/psi0_dance_audio/metadata.jsonl"
),
allowed_tasks=("audio_text", "audio_only"),
audio_file_extensions=(".wav",),
),
"qwen3_tts_seed_audio_command": SourcePreset(
name="qwen3_tts_seed_audio_command",
modality="audio",
dataset_root=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts"),
manifest_path=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/metadata.jsonl"),
allowed_tasks=("audio_command_text", "audio_command_only"),
instruction_key="content_short_description",
is_audio_dir_manifest=True,
audio_file_extensions=(".wav",),
),
"dance_1723_video_10s_clip50": SourcePreset(
name="dance_1723_video_10s_clip50",
modality="video_clip",
dataset_root=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/dance_1723_video_10s"),
manifest_path=Path(
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/dance_1723_video_10s/metadata.jsonl"
),
allowed_tasks=("video_clip_text", "video_clip_only"),
instruction_key="video_caption",
video_embed_dir=Path(
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/dance_1723_video_10s/embeds"
),
),
"motion_example_0506_demo_clip50": SourcePreset(
name="motion_example_0506_demo_clip50",
modality="video_clip",
dataset_root=Path(
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/motion_example_0506_demo"
),
manifest_path=Path(
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/motion_example_0506_demo/metadata.jsonl"
),
allowed_tasks=("video_clip_text", "video_clip_only"),
instruction_key="video_caption",
video_embed_dir=Path(
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/motion_example_0506_demo/embeds"
),
),
"2264_with_audio_clip50": SourcePreset(
name="2264_with_audio_clip50",
modality="video_clip",
dataset_root=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/2264_with_audio"),
manifest_path=Path(
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/2264_with_audio/metadata.jsonl"
),
allowed_tasks=("video_clip_text", "video_clip_only"),
instruction_key="video_caption",
video_embed_dir=Path(
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/2264_with_audio/embeds"
),
),
}
_VIDEO_EMBED_INFO_CACHE: dict[str, dict[str, Any]] = {}
_AUDIO_RESAMPLED_SAMPLE_COUNT_CACHE: dict[tuple[str, int, int, int], int] = {}
@dataclass
class ExportItem:
source_name: str
task_name: str
modality: str
input_path: Path
output_path: Path
instruction: str
manifest_id: str
rel_source_path: str
audio_start_sec: float = 0.0
audio_duration_sec: float = 0.0
full_audio: bool = True
audio_sampling_rate: int = 16000
shard_id: int = -1
source_key: str = ""
source_output_path: Path | None = None
source_ordinal: int = -1
source_item_index: int = -1
source_item_count: int = 0
frame_id: int = -1
source_step_frames: int = 0
motion_fps: int = 50
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Cache Qwen2.5-Omni thinker hidden states for stable multimodal tasks.")
parser.add_argument(
"--mode",
choices=("estimate", "benchmark", "export", "status", "verify"),
default="estimate",
)
parser.add_argument("--model-path", default="/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B")
parser.add_argument(
"--output-root",
default="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_omni_hidden_states_v23",
)
parser.add_argument("--tasks", default="text,audio_text,audio_only,audio_command_text,audio_command_only,video_clip_text,video_clip_only")
parser.add_argument(
"--sources",
default=(
"seed_text,2264_with_audio,psi0_dance_audio,qwen3_tts_seed_audio_command,"
"dance_1723_video_10s_clip50,motion_example_0506_demo_clip50,2264_with_audio_clip50"
),
)
parser.add_argument(
"--source-config-file",
type=Path,
default=None,
help="Optional JSON source preset list. Paths may be relative to the JSON file.",
)
parser.add_argument(
"--source-config-json",
default="",
help="Inline equivalent of --source-config-file; intended for small automated checks.",
)
parser.add_argument("--motion-fps", type=int, default=50)
parser.add_argument("--audio-sampling-rate", type=int, default=16000)
parser.add_argument("--audio-window-seconds", type=float, default=0.6)
parser.add_argument("--audio-left-context-seconds", type=float, default=0.0)
parser.add_argument("--audio-sample-step-frames", type=int, default=1)
parser.add_argument("--audio-command-window-seconds", type=float, default=0.6)
parser.add_argument("--audio-command-sample-step-frames", type=int, default=20)
parser.add_argument(
"--audio-command-full-audio",
action=argparse.BooleanOptionalAction,
default=True,
help=(
"Keep each audio-command WAV whole (legacy behavior). Pass "
"--no-audio-command-full-audio to emit complete fixed-length windows "
"on the WAV timeline."
),
)
parser.add_argument(
"--audio-frame-count-source",
choices=("motion_csv", "metadata"),
default="motion_csv",
help=(
"How windowed-audio export decides its frame range. motion_csv follows the "
"current training index; metadata exactly reproduces the historical v23 cache."
),
)
parser.add_argument("--estimate-samples-per-group", type=int, default=32)
parser.add_argument("--benchmark-samples", type=int, default=256)
parser.add_argument("--limit", type=int, default=0)
parser.add_argument(
"--limit-sources",
type=int,
default=0,
help=(
"Limit complete WAV sources in --storage-format source mode. "
"Unlike --limit, this never leaves a partially written source."
),
)
parser.add_argument(
"--verify-sources",
type=int,
default=64,
help=(
"Number of complete WAV sources to recompute with the real "
"batch-one online encoder in verify mode; 0 verifies all."
),
)
parser.add_argument(
"--verify-windows-per-source",
type=int,
default=0,
help=(
"Optional cap on independently recomputed windows from each source. "
"Zero verifies every window. This keeps exact B=1 verification "
"practical for long continuous background sessions."
),
)
parser.add_argument("--audio-batch-size", type=int, default=16)
parser.add_argument("--text-batch-size", type=int, default=16)
parser.add_argument("--video-batch-size", type=int, default=8)
parser.add_argument("--num-io-workers", type=int, default=8)
parser.add_argument("--num-save-workers", type=int, default=2)
parser.add_argument("--max-pending-save-batches", type=int, default=2)
parser.add_argument("--save-dtype", choices=("bf16", "fp16", "fp32"), default="bf16")
parser.add_argument(
"--storage-format",
choices=("file", "shard", "source"),
default="shard",
help=(
"source stores every window from one audio-command WAV in one "
"safetensors file and writes only a compact source-level SQLite index."
),
)
parser.add_argument("--shard-size", type=int, default=4096)
parser.add_argument("--overwrite", action="store_true")
parser.add_argument("--confirm-full-video", action="store_true")
return parser.parse_args()
def parse_csv_list(raw: str) -> list[str]:
return [part.strip() for part in str(raw).split(",") if part.strip()]
def _source_config_path(value: Any, base_dir: Path) -> Path:
path = Path(str(value)).expanduser()
return path if path.is_absolute() else (base_dir / path).resolve()
def _source_preset_from_config(raw: dict[str, Any], base_dir: Path) -> SourcePreset:
known = {
"name",
"modality",
"dataset_root",
"manifest_path",
"allowed_tasks",
"instruction_key",
"audio_path_key",
"video_path_key",
"video_embed_dir",
"is_audio_dir_manifest",
"audio_file_extensions",
"text_fields",
}
unknown = sorted(set(raw) - known)
if unknown:
raise ValueError(f"Unknown source preset fields for {raw.get('name', '<unnamed>')}: {unknown}")
required = ("name", "modality", "dataset_root", "manifest_path", "allowed_tasks")
missing = [key for key in required if raw.get(key) in (None, "", [])]
if missing:
raise ValueError(f"Source preset is missing required fields {missing}: {raw}")
modality = str(raw["modality"])
if modality not in {"text", "audio", "video_clip"}:
raise ValueError(f"Unsupported source modality {modality!r} for {raw['name']!r}")
dataset_root = _source_config_path(raw["dataset_root"], base_dir)
manifest_path = _source_config_path(raw["manifest_path"], base_dir)
video_embed_raw = raw.get("video_embed_dir")
video_embed_dir = (
_source_config_path(video_embed_raw, base_dir)
if video_embed_raw not in (None, "")
else None
)
raw_audio_extensions = raw.get("audio_file_extensions", DEFAULT_AUDIO_FILE_EXTENSIONS)
if isinstance(raw_audio_extensions, str):
raw_audio_extensions = raw_audio_extensions.split(",")
audio_file_extensions = tuple(
value if value.startswith(".") else f".{value}"
for value in (str(item).strip().lower() for item in raw_audio_extensions)
if value
)
if not audio_file_extensions:
raise ValueError(f"audio_file_extensions cannot be empty for {raw['name']!r}")
return SourcePreset(
name=str(raw["name"]),
modality=modality,
dataset_root=dataset_root,
manifest_path=manifest_path,
allowed_tasks=tuple(str(value) for value in raw["allowed_tasks"]),
instruction_key=str(raw.get("instruction_key") or ""),
audio_path_key=str(raw.get("audio_path_key") or "audio_path"),
video_path_key=str(raw.get("video_path_key") or "video_path"),
video_embed_dir=video_embed_dir,
is_audio_dir_manifest=bool(raw.get("is_audio_dir_manifest", False)),
audio_file_extensions=audio_file_extensions,
text_fields=tuple(str(value) for value in raw.get("text_fields", DEFAULT_SEED_TEXT_FIELDS)),
)
def load_source_presets(args: argparse.Namespace) -> dict[str, SourcePreset]:
if args.source_config_file is not None and str(args.source_config_json).strip():
raise ValueError("Use only one of --source-config-file and --source-config-json")
if args.source_config_file is None and not str(args.source_config_json).strip():
return dict(SOURCE_PRESETS)
if args.source_config_file is not None:
config_path = args.source_config_file.expanduser().resolve()
payload = json.loads(config_path.read_text(encoding="utf-8"))
base_dir = config_path.parent
else:
payload = json.loads(str(args.source_config_json))
base_dir = Path.cwd()
if isinstance(payload, dict):
payload = payload.get("sources")
if not isinstance(payload, list) or not payload:
raise ValueError("Source config must be a non-empty list or {'sources': [...]} object")
presets: dict[str, SourcePreset] = {}
for raw in payload:
if not isinstance(raw, dict):
raise ValueError(f"Source config entries must be objects, got {type(raw).__name__}")
preset = _source_preset_from_config(raw, base_dir)
if preset.name in presets:
raise ValueError(f"Duplicate source preset name: {preset.name}")
presets[preset.name] = preset
return presets
def rank() -> int:
return int(os.environ.get("RANK", "0"))
def world_size() -> int:
return int(os.environ.get("WORLD_SIZE", "1"))
def is_rank0() -> bool:
return rank() == 0
def assign_source_to_least_loaded_rank(
rank_window_loads: list[int],
source_window_count: int,
) -> int:
"""Greedily balance complete sources without splitting their time axis."""
if not rank_window_loads:
raise ValueError("rank_window_loads must not be empty")
count = int(source_window_count)
if count <= 0:
raise ValueError(f"source_window_count must be positive, got {count}")
owner = min(
range(len(rank_window_loads)),
key=lambda rank_index: (
int(rank_window_loads[rank_index]),
rank_index,
),
)
rank_window_loads[owner] += count
return owner
def log(msg: str) -> None:
if is_rank0():
print(msg, flush=True)
def sanitize_rel_path(path: Path) -> str:
return str(path).replace("\\", "/")
def count_jsonl_rows(path: Path) -> int:
count = 0
with path.open("rb") as f:
for chunk in iter(lambda: f.read(1024 * 1024 * 16), b""):
count += chunk.count(b"\n")
return count
def count_csv_data_rows(path: Path) -> int:
count = 0
with path.open("rb") as f:
for chunk in iter(lambda: f.read(1024 * 1024 * 16), b""):
count += chunk.count(b"\n")
return max(0, count - 1)
def maybe_init_distributed() -> None:
if world_size() <= 1 or dist.is_initialized():
return
# Cache export workers are independent model replicas; distributed is only
# used for tiny progress/stat reductions. Gloo allows multiple workers to
# share one GPU without NCCL duplicate-device issues.
dist.init_process_group(backend="gloo")
def cleanup_distributed() -> None:
if dist.is_initialized():
dist.destroy_process_group()
def current_device() -> torch.device:
if torch.cuda.is_available():
local_rank = int(os.environ.get("LOCAL_RANK", "0"))
device_index = local_rank % torch.cuda.device_count()
torch.cuda.set_device(device_index)
return torch.device(f"cuda:{device_index}")
return torch.device("cpu")
def save_dtype_from_arg(arg: str) -> torch.dtype:
if arg == "bf16":
return torch.bfloat16
if arg == "fp16":
return torch.float16
return torch.float32
def model_dtype_for_device(device: torch.device, arg: str) -> torch.dtype:
if device.type == "cuda":
return save_dtype_from_arg(arg)
return torch.float32
def load_hidden_size(model_path: str) -> int:
cfg = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
thinker_cfg = getattr(cfg, "thinker_config", None)
if thinker_cfg is None:
raise ValueError(f"Model config at {model_path} has no thinker_config")
text_cfg = thinker_cfg.get("text_config", thinker_cfg) if isinstance(thinker_cfg, dict) else thinker_cfg.text_config
hidden_size = text_cfg.get("hidden_size") if isinstance(text_cfg, dict) else getattr(text_cfg, "hidden_size", None)
if hidden_size is None:
raise ValueError(f"Cannot find hidden_size in thinker_config.text_config for {model_path}")
return int(hidden_size)
def load_processor(model_path: str) -> Qwen2_5OmniProcessor:
return Qwen2_5OmniProcessor.from_pretrained(model_path)
def load_model(model_path: str, device: torch.device, save_dtype: torch.dtype) -> Qwen2_5OmniThinkerForConditionalGeneration:
load_kwargs: dict[str, Any] = {
"dtype": model_dtype_for_device(
device,
"bf16" if save_dtype == torch.bfloat16 else "fp16" if save_dtype == torch.float16 else "fp32",
),
"device_map": None,
}
if device.type == "cuda":
load_kwargs["attn_implementation"] = "flash_attention_2"
model = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained(model_path, **load_kwargs)
model.eval().to(device)
return model
def stable_pool_prompt(prompt_pools: dict[str, list[str]], key: str) -> str:
layouts = list(prompt_pools.keys())
layout_digest = hashlib.sha1(f"{key}::layout".encode("utf-8")).digest()
layout = layouts[int.from_bytes(layout_digest[:8], "big") % len(layouts)]
prompts = prompt_pools[layout]
prompt_digest = hashlib.sha1(f"{key}::prompt".encode("utf-8")).digest()
return str(prompts[int.from_bytes(prompt_digest[:8], "big") % len(prompts)]).strip()
def instruction_identity(row: dict[str, Any]) -> str:
for key in ("id", "filename", "name", "audio_path", "video_path"):
value = row.get(key)
if value not in (None, ""):
return str(value)
return json.dumps(row, sort_keys=True, ensure_ascii=False)
def instruction_for_item(source_name: str, task_name: str, row: dict[str, Any]) -> str:
if task_name in ("audio_only", "audio_command_only", "video_clip_only"):
return ""
key = f"{source_name}::{task_name}::{instruction_identity(row)}"
if task_name == "audio_text":
return stable_pool_prompt(_AUDIO_DANCE_PROMPT_POOLS, key)
if task_name == "audio_command_text":
return AUDIO_COMMAND_STREAMING_PROMPT
if task_name == "video_clip_text":
return stable_pool_prompt(_VIDEO_MOTION_PROMPT_POOLS, key)
return ""
def stable_text_instruction(row: dict[str, Any], preset: SourcePreset) -> str:
key = f"{preset.name}::{row.get('_row_idx', instruction_identity(row))}"
seed = int.from_bytes(hashlib.sha1(key.encode("utf-8")).digest()[:8], "big")
rng = random.Random(seed)
candidates = collect_seed_text_candidates(row, preset.text_fields)
fallback = str(row.get("move_name", "seed motion")).strip() or "seed motion"
return compose_seed_motion_prompt_from_candidates(candidates, fallback=fallback, rng=rng)
def rel_to_root(path: Path, root: Path) -> Path:
try:
return path.relative_to(root)
except ValueError:
return Path(path.name)
def resolve_audio_output_path(output_root: Path, source_name: str, task_name: str, input_path: Path, dataset_root: Path) -> Path:
rel = rel_to_root(input_path, dataset_root).with_suffix(".safetensors")
return output_root / "audio" / source_name / task_name / rel
def resolve_audio_window_output_path(
output_root: Path,
source_name: str,
task_name: str,
input_path: Path,
dataset_root: Path,
frame_id: int,
) -> Path:
rel = rel_to_root(input_path, dataset_root).with_suffix("")
rel = rel.parent / f"{rel.name}.f{int(frame_id):08d}.safetensors"
return output_root / "audio" / source_name / task_name / rel
def resolve_audio_source_output_path(
output_root: Path,
source_name: str,
task_name: str,
input_path: Path,
dataset_root: Path,
) -> Path:
rel = rel_to_root(input_path, dataset_root).with_suffix(".safetensors")
return output_root / "audio" / source_name / task_name / "sources" / rel
def resolve_video_output_path(output_root: Path, source_name: str, task_name: str, embed_path: Path) -> Path:
return output_root / "video_clip" / source_name / task_name / f"{embed_path.stem}.safetensors"
def resolve_text_output_path(output_root: Path, source_name: str, task_name: str, motion_dir: Path, dataset_root: Path) -> Path:
rel = rel_to_root(motion_dir, dataset_root)
return output_root / "text" / source_name / task_name / rel / "hidden.safetensors"
def cache_group_dir(output_root: Path, source_name: str, task_name: str, modality: str) -> Path:
if modality == "audio":
modality_dir = "audio"
elif modality == "text":
modality_dir = "text"
else:
modality_dir = "video_clip"
return output_root / modality_dir / source_name / task_name
def export_item_cache_key(item: ExportItem, output_root: Path) -> str:
group_dir = cache_group_dir(output_root, item.source_name, item.task_name, item.modality)
try:
return sanitize_rel_path(item.output_path.relative_to(group_dir))
except ValueError:
return sanitize_rel_path(Path(item.output_path.name))
def shard_paths(output_root: Path, source_name: str, task_name: str, modality: str, shard_rank: int, shard_id: int) -> tuple[Path, Path]:
group_dir = cache_group_dir(output_root, source_name, task_name, modality)
stem = f"ws{world_size()}.rank{int(shard_rank):02d}.shard{int(shard_id):06d}"
shard_dir = group_dir / "shards"
return shard_dir / f"{stem}.safetensors", shard_dir / f"{stem}.index.jsonl"
def shard_is_complete(
output_root: Path,
source_name: str,
task_name: str,
modality: str,
shard_rank: int,
shard_id: int,
expected_count: int,
) -> bool:
shard_path, index_path = shard_paths(output_root, source_name, task_name, modality, shard_rank, shard_id)
if expected_count <= 0:
return False
if not shard_path.exists() or not index_path.exists():
return False
try:
return count_jsonl_rows(index_path) == int(expected_count)
except OSError:
return False
def iter_jsonl_rows(path: Path) -> Iterator[tuple[int, dict[str, Any]]]:
with path.open("r", encoding="utf-8") as f:
for idx, line in enumerate(f):
line = line.strip()
if not line:
continue
yield idx, json.loads(line)
def manifest_motion_num_frames(row: dict[str, Any], preset: SourcePreset) -> int:
motion_value = row.get("converted_path") or row.get("motion_dir") or row.get("motion_path")
if not motion_value:
raise ValueError(
f"Audio manifest row has no motion path for source={preset.name}: "
f"id={row.get('id', row.get('name', ''))}"
)
motion_dir = Path(str(motion_value))
if not motion_dir.is_absolute():
motion_dir = preset.dataset_root / motion_dir
return read_motion_frame_count(motion_dir)
def metadata_num_frames(row: dict[str, Any], motion_fps: int) -> int:
"""Historical v23 frame-count rule, retained for bitwise cache reproduction."""
for key in ("clip_frames", "motion_chunk_frames", "n_frames", "move_duration_frames"):
value = row.get(key)
if value not in (None, ""):
try:
n_frames = int(value)
except (TypeError, ValueError):
n_frames = 0
if n_frames > 0:
return n_frames
duration = row.get("duration")
if duration not in (None, ""):
try:
duration_sec = float(duration)
except (TypeError, ValueError):
duration_sec = 0.0
if duration_sec > 0:
return max(1, int(round(duration_sec * float(motion_fps))) - 3)
return 0
def audio_num_frames(
row: dict[str, Any],
preset: SourcePreset,
args: argparse.Namespace,
) -> int:
if str(args.audio_frame_count_source) == "metadata":
return metadata_num_frames(row, int(args.motion_fps))
return manifest_motion_num_frames(row, preset)
def read_start_hold_seconds(row: dict[str, Any], preset: SourcePreset) -> float:
motion_value = row.get("converted_path") or row.get("motion_dir") or row.get("motion_path")
if not motion_value:
return 0.0
motion_dir = Path(str(motion_value))
if not motion_dir.is_absolute():
motion_dir = preset.dataset_root / motion_dir
info_path = motion_dir / "info.txt"
if not info_path.exists():
return 0.0
text = info_path.read_text(encoding="utf-8", errors="ignore")
import re
match = re.search(r"start_hold_seconds:\s*([0-9.]+)", text)
return float(match.group(1)) if match else 0.0
def map_seed_motion_dir(move_g1_path: str, root_dir: Path) -> Path:
rel = str(move_g1_path).strip().replace("\\", "/")
rel = rel.replace("g1/csv/", "g1/csv_sonic/")
rel = rel.replace("/csv/", "/csv_sonic/")
if rel.endswith(".csv"):
rel = rel[:-4]
if rel.startswith("dataset/seed/"):
rel = rel[len("dataset/seed/") :]
if rel.startswith("seed/"):
rel = rel[len("seed/") :]
path = Path(rel)
return path if path.is_absolute() else root_dir / path
def iter_text_items(preset: SourcePreset, task_name: str, output_root: Path) -> Iterator[ExportItem]:
usecols = ["move_g1_path", "move_name", "move_duration_frames", *preset.text_fields]
usecols_set = set(usecols)
metadata = pd.read_csv(preset.manifest_path, usecols=lambda col: col in usecols_set)
if "move_g1_path" not in metadata.columns:
raise ValueError(f"Text metadata must contain move_g1_path: {preset.manifest_path}")
for row_idx, row in metadata.iterrows():
row_dict = row.to_dict()
row_dict["_row_idx"] = int(row_idx)
motion_dir = map_seed_motion_dir(str(row_dict["move_g1_path"]), preset.dataset_root)
yield ExportItem(
source_name=preset.name,
task_name=task_name,
modality="text",
input_path=motion_dir,
output_path=resolve_text_output_path(output_root, preset.name, task_name, motion_dir, preset.dataset_root),
instruction=stable_text_instruction(row_dict, preset),
manifest_id=str(row_idx),
rel_source_path=sanitize_rel_path(rel_to_root(motion_dir, preset.dataset_root)),
full_audio=False,
)
def iter_audio_items(
preset: SourcePreset,
task_name: str,
output_root: Path,
args: argparse.Namespace | None = None,
) -> Iterator[ExportItem]:
if task_name in ("audio_text", "audio_only"):
if args is None:
raise ValueError("args is required for windowed audio export")
step_frames = max(1, int(args.audio_sample_step_frames))
motion_fps = int(args.motion_fps)
for row_idx, row in iter_jsonl_rows(preset.manifest_path):
raw_path = row.get(preset.audio_path_key)
if not raw_path:
continue
audio_path = preset.dataset_root / str(raw_path)
if not audio_path.exists():
continue
if audio_path.suffix.lower() != ".npy" and not has_audio_stream(audio_path):
continue
n_frames = audio_num_frames(row, preset, args)
if n_frames <= 0:
continue
instruction = instruction_for_item(preset.name, task_name, row)
start_hold_seconds = read_start_hold_seconds(row, preset)
for local_index, frame_id in enumerate(range(0, n_frames, step_frames)):
audio_start_sec = (
frame_id / float(motion_fps)
- start_hold_seconds
- float(args.audio_left_context_seconds)
)
yield ExportItem(
source_name=preset.name,
task_name=task_name,
modality="audio",
input_path=audio_path,
output_path=resolve_audio_window_output_path(
output_root,
preset.name,
task_name,
audio_path,
preset.dataset_root,
frame_id,
),
instruction=instruction,
manifest_id=str(row.get("id", row.get("name", row_idx))),
rel_source_path=f"{sanitize_rel_path(rel_to_root(audio_path, preset.dataset_root))}#f{frame_id}",
audio_start_sec=float(audio_start_sec),
audio_duration_sec=float(args.audio_window_seconds),
full_audio=False,
audio_sampling_rate=int(args.audio_sampling_rate),
)
return
if args is None:
raise ValueError("args is required for audio-command export")
sampling_rate = int(args.audio_sampling_rate)
command_full_audio = bool(args.audio_command_full_audio)
source_layout = str(getattr(args, "storage_format", "file")) == "source"
source_limit = max(int(getattr(args, "limit_sources", 0)), 0)
eligible_source_ordinal = 0
for row_idx, row, instruction, audio_file in iter_unique_manifest_audio_files(preset, task_name):
manifest_id = str(row.get("filename", row.get("id", row.get("name", row_idx))))
rel_audio_path = sanitize_rel_path(rel_to_root(audio_file, preset.dataset_root))
if command_full_audio:
yield ExportItem(
source_name=preset.name,
task_name=task_name,
modality="audio",
input_path=audio_file,
output_path=resolve_audio_output_path(
output_root,
preset.name,
task_name,
audio_file,
preset.dataset_root,
),
instruction=instruction,
manifest_id=manifest_id,
rel_source_path=rel_audio_path,
full_audio=True,
audio_sampling_rate=sampling_rate,
)
continue
frame_ids = list(
iter_complete_audio_window_frame_ids(
audio_file,
sampling_rate=sampling_rate,
window_seconds=float(args.audio_command_window_seconds),
step_frames=int(args.audio_command_sample_step_frames),
motion_fps=int(args.motion_fps),
)
)
if not frame_ids:
continue
if source_layout and source_limit > 0 and eligible_source_ordinal >= source_limit:
break
source_ordinal = eligible_source_ordinal
eligible_source_ordinal += 1
source_key = Path(rel_audio_path).with_suffix("").as_posix()
source_output_path = resolve_audio_source_output_path(
output_root,
preset.name,
task_name,
audio_file,
preset.dataset_root,
)
for source_item_index, frame_id in enumerate(frame_ids):
yield ExportItem(
source_name=preset.name,
task_name=task_name,
modality="audio",
input_path=audio_file,
output_path=resolve_audio_window_output_path(
output_root,
preset.name,
task_name,
audio_file,
preset.dataset_root,
frame_id,
),
instruction=instruction,
manifest_id=manifest_id,
rel_source_path=f"{rel_audio_path}#f{frame_id}",
audio_start_sec=float(frame_id) / float(args.motion_fps),
audio_duration_sec=float(args.audio_command_window_seconds),
full_audio=False,
audio_sampling_rate=sampling_rate,
source_key=source_key,
source_output_path=source_output_path,
source_ordinal=source_ordinal,
source_item_index=source_item_index,
source_item_count=len(frame_ids),
frame_id=int(frame_id),
source_step_frames=int(args.audio_command_sample_step_frames),
motion_fps=int(args.motion_fps),
)
def iter_unique_manifest_audio_files(
preset: SourcePreset,
task_name: str,
) -> Iterator[tuple[int, dict[str, Any], str, Path]]:
"""Expand an audio manifest once while preserving its stable first-row prompt."""
seen_files: set[str] = set()
seen_audio_dirs: set[str] = set()
for row_idx, row in iter_jsonl_rows(preset.manifest_path):
raw_path = row.get(preset.audio_path_key)
if not raw_path:
continue
audio_path = preset.dataset_root / str(raw_path)
instruction = instruction_for_item(preset.name, task_name, row)
if preset.is_audio_dir_manifest:
dir_key = str(audio_path)
if dir_key in seen_audio_dirs:
continue
seen_audio_dirs.add(dir_key)
if not audio_path.is_dir():
continue
audio_files = sorted(
path
for path in audio_path.iterdir()
if path.is_file() and path.suffix.lower() in preset.audio_file_extensions
)
else:
if not audio_path.exists():
continue
if audio_path.suffix.lower() != ".npy" and not has_audio_stream(audio_path):
continue
audio_files = [audio_path]
for audio_file in audio_files:
file_key = str(audio_file)
if file_key in seen_files:
continue
seen_files.add(file_key)
yield row_idx, row, instruction, audio_file
def find_video_embed_path(preset: SourcePreset, row: dict[str, Any]) -> Path | None:
assert preset.video_embed_dir is not None
video_value = row.get(preset.video_path_key) or row.get("video_path")
if not video_value:
return None
stem = Path(str(video_value)).stem
manifest_info = _VIDEO_EMBED_INFO_CACHE.get(preset.name)
if manifest_info is None:
manifest_path = preset.video_embed_dir / "manifest.json"
if manifest_path.exists():
manifest_info = json.loads(manifest_path.read_text(encoding="utf-8"))
else:
manifest_info = {}
_VIDEO_EMBED_INFO_CACHE[preset.name] = manifest_info
fps_value = manifest_info.get("video_sample_fps")
size_value = manifest_info.get("video_size_pixels")
if fps_value is not None and size_value is not None:
fps_token = str(fps_value).replace(".", "p")
direct_path = preset.video_embed_dir / f"{stem}.fps{fps_token}.size{size_value}.npz"
if direct_path.exists():
return direct_path
matches = sorted(preset.video_embed_dir.glob(f"{stem}.fps*.size*.npz"))
if not matches:
return None
return matches[0]
def iter_video_items(preset: SourcePreset, task_name: str, output_root: Path) -> Iterator[ExportItem]:
for row_idx, row in iter_jsonl_rows(preset.manifest_path):
instruction = instruction_for_item(preset.name, task_name, row)
embed_path = find_video_embed_path(preset, row)
if embed_path is None or not embed_path.exists():
continue
yield ExportItem(
source_name=preset.name,
task_name=task_name,
modality="video_clip",
input_path=embed_path,
output_path=resolve_video_output_path(output_root, preset.name, task_name, embed_path),
instruction=instruction,
manifest_id=str(row.get("id", row.get("name", row_idx))),
rel_source_path=sanitize_rel_path(embed_path.name),
)
def iter_group_items(
preset: SourcePreset,
task_name: str,
output_root: Path,
args: argparse.Namespace | None = None,
) -> Iterator[ExportItem]:
if preset.modality == "text":
yield from iter_text_items(preset, task_name, output_root)
return
if preset.modality == "audio":
yield from iter_audio_items(preset, task_name, output_root, args)
return
yield from iter_video_items(preset, task_name, output_root)
def count_group_items(
preset: SourcePreset,
task_name: str,
output_root: Path,
args: argparse.Namespace | None = None,
) -> int:
if str(getattr(args, "storage_format", "")) == "source":
# Source mode may apply --limit-sources and must count only complete
# WAVs. Reuse the exact iterator so the progress total cannot include
# a partial or otherwise ineligible source.
return sum(1 for _ in iter_group_items(preset, task_name, output_root, args))
if preset.modality == "text":
return count_csv_data_rows(preset.manifest_path)
if preset.modality == "video_clip":
# Training samples are driven by the video_clip metadata jsonl, not by
# the embed manifest. Some embed manifests contain cached entries that
# are not present in the current training metadata.
return count_jsonl_rows(preset.manifest_path)
if preset.modality == "audio" and task_name in ("audio_text", "audio_only"):
if args is None:
raise ValueError("args is required for windowed audio count")
step_frames = max(1, int(args.audio_sample_step_frames))
motion_fps = int(args.motion_fps)
total = 0
for _, row in iter_jsonl_rows(preset.manifest_path):
raw_path = row.get(preset.audio_path_key)
if not raw_path:
continue
audio_path = preset.dataset_root / str(raw_path)
if not audio_path.exists():
continue
if audio_path.suffix.lower() != ".npy" and not has_audio_stream(audio_path):
continue
n_frames = audio_num_frames(row, preset, args)
if n_frames <= 0:
continue
total += max(1, (n_frames + step_frames - 1) // step_frames)
return total
if preset.modality == "audio" and task_name in ("audio_command_text", "audio_command_only"):
if args is None:
raise ValueError("args is required for audio-command count")
audio_files = (
audio_file
for _, _, _, audio_file in iter_unique_manifest_audio_files(preset, task_name)
)
if bool(args.audio_command_full_audio):
return sum(1 for _ in audio_files)
return sum(
complete_audio_window_count(
audio_file,
sampling_rate=int(args.audio_sampling_rate),
window_seconds=float(args.audio_command_window_seconds),
step_frames=int(args.audio_command_sample_step_frames),
motion_fps=int(args.motion_fps),
)
for audio_file in audio_files
)
return sum(1 for _ in iter_group_items(preset, task_name, output_root, args))
def should_keep_item(global_index: int, total: int, sample_count: int) -> bool:
if sample_count <= 0:
return False
if total <= sample_count:
return True
step = max(1, total // sample_count)
return global_index % step == 0
def sampled_group_items(
preset: SourcePreset,
task_name: str,
output_root: Path,
sample_count: int,
args: argparse.Namespace | None = None,
) -> list[ExportItem]:
sampled: list[ExportItem] = []
for idx, item in enumerate(iter_group_items(preset, task_name, output_root, args)):
sampled.append(item)
if len(sampled) >= sample_count:
break
return sampled
def source_task_pairs(tasks: list[str], sources: list[str]) -> list[tuple[SourcePreset, str]]:
pairs: list[tuple[SourcePreset, str]] = []
for source_name in sources:
preset = SOURCE_PRESETS[source_name]
for task_name in tasks:
if task_name in preset.allowed_tasks:
pairs.append((preset, task_name))
return pairs
def has_audio_stream(path: Path) -> bool:
cmd = [
ffprobe_bin(),
"-v",
"error",
"-select_streams",
"a:0",
"-show_entries",
"stream=index",
"-of",
"csv=p=0",
str(path),
]
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
return proc.returncode == 0 and bool(proc.stdout.strip())
def resampled_audio_num_samples(path: Path, sampling_rate: int) -> int:
"""Read audio duration cheaply and express it in target-rate samples."""
path = Path(path)
sampling_rate = int(sampling_rate)
if sampling_rate <= 0:
raise ValueError(f"sampling_rate must be positive, got {sampling_rate}")
stat = path.stat()
cache_key = (str(path.resolve()), sampling_rate, int(stat.st_size), int(stat.st_mtime_ns))
cached = _AUDIO_RESAMPLED_SAMPLE_COUNT_CACHE.get(cache_key)
if cached is not None:
return cached
sample_count = 0
if path.suffix.lower() == ".wav":
try:
with wave.open(str(path), "rb") as wav_file:
source_frames = int(wav_file.getnframes())
source_rate = int(wav_file.getframerate())
if source_frames > 0 and source_rate > 0:
sample_count = int(round(source_frames * sampling_rate / float(source_rate)))
except (EOFError, OSError, wave.Error):
sample_count = 0
if sample_count <= 0:
sample_count = int(load_audio_array(path, sampling_rate).shape[0])
if sample_count <= 0:
raise RuntimeError(f"Audio contains no samples: {path}")
_AUDIO_RESAMPLED_SAMPLE_COUNT_CACHE[cache_key] = sample_count
return sample_count
def complete_audio_window_count(
path: Path,
*,
sampling_rate: int,
window_seconds: float,
step_frames: int,
motion_fps: int,
) -> int:
"""Count complete windows only; an incomplete final tail is not padded."""
sampling_rate = int(sampling_rate)
step_frames = int(step_frames)
motion_fps = int(motion_fps)
window_seconds = float(window_seconds)
if window_seconds <= 0:
raise ValueError(f"window_seconds must be positive, got {window_seconds}")
if step_frames <= 0 or motion_fps <= 0:
raise ValueError(
f"step_frames and motion_fps must be positive, got {step_frames} and {motion_fps}"
)
window_samples = max(1, int(round(window_seconds * sampling_rate)))
step_samples = max(1, int(round(step_frames * sampling_rate / float(motion_fps))))
total_samples = resampled_audio_num_samples(Path(path), sampling_rate)
if total_samples < window_samples:
return 0
return int((total_samples - window_samples) // step_samples + 1)
def iter_complete_audio_window_frame_ids(
path: Path,
*,
sampling_rate: int,
window_seconds: float,
step_frames: int,
motion_fps: int,
) -> range:
count = complete_audio_window_count(
path,
sampling_rate=sampling_rate,
window_seconds=window_seconds,
step_frames=step_frames,
motion_fps=motion_fps,
)
return range(0, count * int(step_frames), int(step_frames))
def load_audio_array(path: Path, sampling_rate: int = 16000) -> np.ndarray:
if path.suffix.lower() == ".npy":
try:
audio = np.load(path, allow_pickle=False)
except ValueError:
audio = np.load(path, allow_pickle=True)
if isinstance(audio, np.ndarray) and audio.dtype == object:
audio = np.asarray(audio.tolist())
audio = np.asarray(audio, dtype=np.float32).reshape(-1).copy()
else:
cmd = [
ffmpeg_bin(),
"-v",
"error",
"-threads",
"1",
"-i",
str(path),
"-ac",
"1",
"-ar",
str(int(sampling_rate)),
"-f",
"f32le",
"-",
]
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
if proc.returncode != 0:
msg = proc.stderr.decode("utf-8", errors="ignore")
raise RuntimeError(f"ffmpeg failed to decode audio from {path}: {msg}")
audio = np.frombuffer(proc.stdout, dtype=np.float32).copy()
if audio.size == 0:
raise RuntimeError(f"Empty audio decoded from {path}")
return audio
def audio_segment(audio: np.ndarray, start_sec: float, duration_sec: float, sampling_rate: int) -> np.ndarray:
num_samples = max(1, int(round(float(duration_sec) * int(sampling_rate))))
start_sample = int(round(float(start_sec) * int(sampling_rate)))
end_sample = start_sample + num_samples
segment = np.zeros((num_samples,), dtype=np.float32)
src_start = max(0, start_sample)
src_end = min(audio.shape[0], end_sample)
if src_end > src_start:
dst_start = src_start - start_sample
segment[dst_start : dst_start + (src_end - src_start)] = audio[src_start:src_end]
return segment
def load_audio_payload(item: ExportItem) -> dict[str, Any]:
audio = load_audio_array(item.input_path, item.audio_sampling_rate)
if not item.full_audio:
audio = audio_segment(
audio,
item.audio_start_sec,
item.audio_duration_sec,
item.audio_sampling_rate,
)
return {
"audio": audio,
"instruction": item.instruction,
"full_audio": bool(item.full_audio),
}
def load_audio_payloads(
items: list[ExportItem],
num_io_workers: int,
io_executor: ThreadPoolExecutor | None = None,
) -> list[dict[str, Any]]:
unique_paths = list(dict.fromkeys(item.input_path for item in items))
sampling_rate_by_path = {item.input_path: int(item.audio_sampling_rate) for item in items}
def load_pair(path: Path) -> tuple[Path, np.ndarray]:
return path, load_audio_array(path, sampling_rate_by_path.get(path, 16000))
if num_io_workers <= 1 or len(unique_paths) == 1:
audio_by_path = dict(load_pair(path) for path in unique_paths)
elif io_executor is not None:
audio_by_path = dict(io_executor.map(load_pair, unique_paths))
else:
with ThreadPoolExecutor(max_workers=num_io_workers) as executor:
audio_by_path = dict(executor.map(load_pair, unique_paths))
payloads: list[dict[str, Any]] = []
for item in items:
audio = audio_by_path[item.input_path]
if not item.full_audio:
audio = audio_segment(
audio,
item.audio_start_sec,
item.audio_duration_sec,
item.audio_sampling_rate,
)
payloads.append(
{
"audio": audio,
"instruction": item.instruction,
"full_audio": bool(item.full_audio),
}
)
return payloads
def load_video_payload(item: ExportItem) -> dict[str, Any]:
with np.load(item.input_path, allow_pickle=False) as cache:
return {
"pixel_values_videos": cache["pixel_values_videos"],
"video_grid_thw": cache["video_grid_thw"],
"video_second_per_grid": cache["video_second_per_grid"],
"instruction": item.instruction,
}
def load_text_payload(item: ExportItem) -> dict[str, Any]:
return {"instruction": item.instruction}
def build_audio_text(processor: Qwen2_5OmniProcessor, instructions: list[str]) -> list[str]:
texts: list[str] = []
for instruction in instructions:
content = [{"type": "audio", "audio": "audio.wav"}]
if instruction:
content.append({"type": "text", "text": instruction})
messages = [[
{"role": "system", "content": [{"type": "text", "text": OMNI_DEFAULT_SYSTEM_PROMPT}]},
{"role": "user", "content": content},
]]
texts.append(processor.apply_chat_template(messages[0], tokenize=False, add_generation_prompt=True))
return texts
def build_video_text(processor: Qwen2_5OmniProcessor, instructions: list[str]) -> list[str]:
texts: list[str] = []
for instruction in instructions:
content = [{"type": "video", "video": "video.mp4"}]
if instruction:
content.append({"type": "text", "text": instruction})
messages = [[
{"role": "system", "content": [{"type": "text", "text": OMNI_DEFAULT_SYSTEM_PROMPT}]},
{"role": "user", "content": content},
]]
texts.append(processor.apply_chat_template(messages[0], tokenize=False, add_generation_prompt=True))
return texts
def build_text_text(processor: Qwen2_5OmniProcessor, instructions: list[str]) -> list[str]:
texts: list[str] = []
for instruction in instructions:
messages = [[
{"role": "system", "content": [{"type": "text", "text": OMNI_DEFAULT_SYSTEM_PROMPT}]},
{"role": "user", "content": [{"type": "text", "text": str(instruction).strip()}]},
]]
texts.append(processor.apply_chat_template(messages[0], tokenize=False, add_generation_prompt=True))
return texts
def collate_audio_batch(
processor: Qwen2_5OmniProcessor,
payloads: list[dict[str, Any]],
) -> dict[str, torch.Tensor]:
texts = build_audio_text(processor, [payload["instruction"] for payload in payloads])
audios = [np.asarray(payload["audio"], dtype=np.float32) for payload in payloads]
if any(bool(payload.get("full_audio", False)) for payload in payloads):
single_inputs = [
processor(
text=[text],
audio=[audio],
padding=True,
return_tensors="pt",
)
for text, audio in zip(texts, audios)
]
pad_token_id = int(getattr(processor.tokenizer, "pad_token_id", 0) or 0)
input_ids = torch.nn.utils.rnn.pad_sequence(
[
item["input_ids"].squeeze(0)
if item["input_ids"].dim() == 2
else item["input_ids"]
for item in single_inputs
],
batch_first=True,
padding_value=pad_token_id,
)
output: dict[str, torch.Tensor] = {
"input_ids": input_ids,
"attention_mask": input_ids.ne(pad_token_id).long(),
}
input_features = [
item["input_features"].squeeze(0)
if item["input_features"].dim() == 3
else item["input_features"]
for item in single_inputs
]
feature_attention_mask = [
item["feature_attention_mask"].squeeze(0)
if item["feature_attention_mask"].dim() == 2
else item["feature_attention_mask"]
for item in single_inputs
]
max_audio_len = max(int(feature.shape[-1]) for feature in input_features)
padded_input_features = []
padded_feature_attention_mask = []
for feature, feature_mask in zip(input_features, feature_attention_mask):
pad_len = max_audio_len - int(feature.shape[-1])
if pad_len > 0:
feature_pad_shape = list(feature.shape)
feature_pad_shape[-1] = pad_len
feature = torch.cat(
[feature, torch.zeros(feature_pad_shape, dtype=feature.dtype, device=feature.device)],
dim=-1,
)
mask_pad_shape = list(feature_mask.shape)
mask_pad_shape[-1] = pad_len
feature_mask = torch.cat(
[feature_mask, torch.zeros(mask_pad_shape, dtype=feature_mask.dtype, device=feature_mask.device)],
dim=-1,
)
padded_input_features.append(feature)
padded_feature_attention_mask.append(feature_mask)
output["input_features"] = torch.stack(padded_input_features).to(dtype=torch.float32)
output["feature_attention_mask"] = torch.stack(padded_feature_attention_mask)
if "audio_feature_lengths" in single_inputs[0]:
output["audio_feature_lengths"] = torch.stack(
[
item["audio_feature_lengths"].squeeze(0)
if item["audio_feature_lengths"].dim() > 0
else item["audio_feature_lengths"]
for item in single_inputs
]
)
return output
output_kwargs = processor._merge_kwargs(
Qwen2_5OmniProcessorKwargs,
tokenizer_init_kwargs=processor.tokenizer.init_kwargs,
padding=True,
return_tensors="pt",
)
audio_kwargs = dict(output_kwargs["audio_kwargs"])
feature_extractor = processor.feature_extractor
max_audio_samples = max(int(audio.shape[0]) for audio in audios)
hop_length = int(feature_extractor.hop_length)
right_context = int(feature_extractor.n_fft)
max_length = int(math.ceil((max_audio_samples + right_context) / hop_length) * hop_length)
max_length = min(max_length, int(feature_extractor.n_samples))
audio_kwargs["padding"] = "max_length"
audio_kwargs["max_length"] = max_length
audio_kwargs["return_attention_mask"] = True
audio_inputs = feature_extractor(
audios,
**audio_kwargs,
)
feature_mask = audio_inputs.pop("attention_mask")
input_features = audio_inputs.pop("input_features")
max_valid = int(feature_mask.sum(dim=1).max().item())
if 0 < max_valid < int(feature_mask.shape[-1]):
feature_mask = feature_mask[:, :max_valid].contiguous()
input_features = input_features[..., :max_valid].contiguous()
input_lengths = (feature_mask.sum(-1) - 1) // 2 + 1
audio_lengths = iter((input_lengths - 2) // 2 + 1)
texts = processor.replace_multimodal_special_tokens(
texts,
audio_lengths,
iter([]),
iter([]),
video_second_per_grid=iter([]),
use_audio_in_video=output_kwargs["videos_kwargs"]["use_audio_in_video"],
position_id_per_seconds=output_kwargs["videos_kwargs"]["position_id_per_seconds"],
seconds_per_chunk=output_kwargs["videos_kwargs"]["seconds_per_chunk"],
)
inputs = processor.tokenizer(texts, **output_kwargs["text_kwargs"])
inputs["feature_attention_mask"] = feature_mask
inputs["input_features"] = input_features
return inputs
def collate_video_batch(
processor: Qwen2_5OmniProcessor,
payloads: list[dict[str, Any]],
) -> dict[str, torch.Tensor]:
texts = build_video_text(processor, [payload["instruction"] for payload in payloads])
grids_for_replace = [payload["video_grid_thw"][0] for payload in payloads]
seconds_for_replace = [float(np.asarray(payload["video_second_per_grid"]).reshape(-1)[0]) for payload in payloads]
texts = processor.replace_multimodal_special_tokens(
texts,
iter([]),
iter([]),
iter(grids_for_replace),
iter(seconds_for_replace),
False,
25,
2.0,
)
text_inputs = processor.tokenizer(texts, padding=True, return_tensors="pt")
inputs = {key: value for key, value in text_inputs.items()}
inputs["pixel_values_videos"] = torch.cat(
[torch.as_tensor(payload["pixel_values_videos"], dtype=torch.float32) for payload in payloads],
dim=0,
)
inputs["video_grid_thw"] = torch.cat(
[torch.as_tensor(payload["video_grid_thw"], dtype=torch.long) for payload in payloads],
dim=0,
)
inputs["video_second_per_grid"] = torch.cat(
[
torch.as_tensor(payload["video_second_per_grid"], dtype=torch.float32).reshape(-1)
for payload in payloads
],
dim=0,
)
return inputs
def collate_text_batch(
processor: Qwen2_5OmniProcessor,
payloads: list[dict[str, Any]],
) -> dict[str, torch.Tensor]:
texts = build_text_text(processor, [payload["instruction"] for payload in payloads])
return processor.tokenizer(texts, padding=True, return_tensors="pt")
def move_inputs_to_device(inputs: dict[str, Any], device: torch.device) -> dict[str, Any]:
return {
key: value.to(device, non_blocking=True) if hasattr(value, "to") else value
for key, value in inputs.items()
}
def prepare_inputs_cpu(
preset: SourcePreset,
items: list[ExportItem],
args: argparse.Namespace,
processor: Qwen2_5OmniProcessor,
io_executor: ThreadPoolExecutor | None = None,
) -> dict[str, torch.Tensor]:
payloads = load_payloads(
items,
int(args.num_io_workers),
io_executor=io_executor,
)
if preset.modality == "audio":
return collate_audio_batch(processor, payloads)
if preset.modality == "text":
return collate_text_batch(processor, payloads)
return collate_video_batch(processor, payloads)
def omni_hidden_states_without_lm_head(
model: Qwen2_5OmniThinkerForConditionalGeneration,
vlm_kwargs: dict[str, torch.Tensor],
) -> torch.Tensor:
input_ids = vlm_kwargs.get("input_ids")
attention_mask = vlm_kwargs.get("attention_mask")
input_features = vlm_kwargs.get("input_features")
pixel_values = vlm_kwargs.get("pixel_values")
pixel_values_videos = vlm_kwargs.get("pixel_values_videos")
image_grid_thw = vlm_kwargs.get("image_grid_thw")
video_grid_thw = vlm_kwargs.get("video_grid_thw")
feature_attention_mask = vlm_kwargs.get("feature_attention_mask")
audio_feature_lengths = vlm_kwargs.get("audio_feature_lengths")
video_second_per_grid = vlm_kwargs.get("video_second_per_grid")
inputs_embeds = vlm_kwargs.get("inputs_embeds")
position_ids = vlm_kwargs.get("position_ids")
cache_position = vlm_kwargs.get("cache_position")
use_audio_in_video = vlm_kwargs.get("use_audio_in_video")
if inputs_embeds is None:
inputs_embeds = model.get_input_embeddings()(input_ids)
if input_features is not None:
audio_features = model.get_audio_features(
input_features,
feature_attention_mask=feature_attention_mask,
audio_feature_lengths=audio_feature_lengths,
)
audio_features = audio_features.to(inputs_embeds.device, inputs_embeds.dtype)
_, _, audio_mask = model.get_placeholder_mask(input_ids, inputs_embeds=inputs_embeds)
inputs_embeds = inputs_embeds.masked_scatter(audio_mask, audio_features)
if pixel_values is not None:
image_embeds = model.get_image_features(pixel_values, image_grid_thw)
image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
image_mask, _, _ = model.get_placeholder_mask(
input_ids,
inputs_embeds=inputs_embeds,
image_features=image_embeds,
)
inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds)
if pixel_values_videos is not None:
video_embeds = model.get_video_features(pixel_values_videos, video_grid_thw)
video_embeds = video_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
_, video_mask, _ = model.get_placeholder_mask(
input_ids,
inputs_embeds=inputs_embeds,
video_features=video_embeds,
)
inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds)
if feature_attention_mask is not None:
audio_feature_lengths = torch.sum(feature_attention_mask, dim=1)
else:
audio_feature_lengths = None
if attention_mask is not None and position_ids is None:
if cache_position is None or (cache_position is not None and cache_position[0] == 0) or model.rope_deltas is None:
delta0 = (1 - attention_mask).sum(dim=-1).unsqueeze(1)
position_ids, rope_deltas = model.get_rope_index(
input_ids,
image_grid_thw,
video_grid_thw,
attention_mask,
use_audio_in_video,
audio_feature_lengths,
video_second_per_grid,
)
model.rope_deltas = rope_deltas - delta0
else:
batch_size, seq_length = input_ids.shape
delta = cache_position[0] + model.rope_deltas if cache_position is not None else 0
position_ids = torch.arange(seq_length, device=input_ids.device)
position_ids = position_ids.view(1, -1).expand(batch_size, -1)
position_ids = position_ids.add(delta)
position_ids = position_ids.unsqueeze(0).expand(3, -1, -1)
outputs = model.model(
attention_mask=attention_mask,
position_ids=position_ids,
past_key_values=vlm_kwargs.get("past_key_values"),
inputs_embeds=inputs_embeds,
use_cache=False,
output_attentions=vlm_kwargs.get("output_attentions"),
output_hidden_states=True,
return_dict=True,
cache_position=cache_position,
)
return outputs.hidden_states[-1]
def online_encoder_metadata(task_name: str) -> dict[str, str]:
if str(task_name) != "audio_command_text":
return {}
if not ACTIVE_MODEL_FINGERPRINT:
raise RuntimeError("Qwen model fingerprint was not initialized")
return {
"encoder_contract": HFD_ONLINE_ENCODER_CONTRACT,
"encoder_batch_size": str(HFD_ONLINE_ENCODER_BATCH_SIZE),
"encoder_dtype": HFD_ONLINE_ENCODER_DTYPE,
"attention_implementation": HFD_ONLINE_ENCODER_ATTENTION,
"model_fingerprint": ACTIVE_MODEL_FINGERPRINT,
}
def save_hidden_state(
item: ExportItem,
hidden_states: torch.Tensor,
save_dtype: torch.dtype,
overwrite: bool,
) -> tuple[bool, int]:
if item.output_path.exists() and not overwrite:
return False, 0
item.output_path.parent.mkdir(parents=True, exist_ok=True)
payload = {"hidden_states": hidden_states.to(dtype=save_dtype).cpu().contiguous()}
metadata = {
"source_name": item.source_name,
"task_name": item.task_name,
"modality": item.modality,
"manifest_id": item.manifest_id,
"source_path": item.rel_source_path,
"instruction_sha1": hashlib.sha1(item.instruction.encode("utf-8")).hexdigest(),
"full_audio": str(bool(item.full_audio)),
"audio_start_sec": f"{float(item.audio_start_sec):.6f}",
"audio_duration_sec": f"{float(item.audio_duration_sec):.6f}",
**online_encoder_metadata(item.task_name),
}
tmp_path = item.output_path.with_name(f".{item.output_path.name}.{os.getpid()}.tmp")
save_file(payload, str(tmp_path), metadata=metadata)
os.replace(tmp_path, item.output_path)
return True, item.output_path.stat().st_size
def save_hidden_state_cpu(
item: ExportItem,
hidden_states: torch.Tensor,
overwrite: bool,
) -> tuple[bool, int]:
if item.output_path.exists() and not overwrite:
return False, 0
item.output_path.parent.mkdir(parents=True, exist_ok=True)
payload = {"hidden_states": hidden_states.contiguous()}
metadata = {
"source_name": item.source_name,
"task_name": item.task_name,
"modality": item.modality,
"manifest_id": item.manifest_id,
"source_path": item.rel_source_path,
"instruction_sha1": hashlib.sha1(item.instruction.encode("utf-8")).hexdigest(),
"full_audio": str(bool(item.full_audio)),
"audio_start_sec": f"{float(item.audio_start_sec):.6f}",
"audio_duration_sec": f"{float(item.audio_duration_sec):.6f}",
**online_encoder_metadata(item.task_name),
}
tmp_path = item.output_path.with_name(f".{item.output_path.name}.{os.getpid()}.tmp")
save_file(payload, str(tmp_path), metadata=metadata)
os.replace(tmp_path, item.output_path)
return True, item.output_path.stat().st_size
def load_payloads(
items: list[ExportItem],
num_io_workers: int,
io_executor: ThreadPoolExecutor | None = None,
) -> list[dict[str, Any]]:
if items[0].modality == "audio":
return load_audio_payloads(items, num_io_workers, io_executor)
if items[0].modality == "text":
return [load_text_payload(item) for item in items]
loader = load_video_payload
if num_io_workers <= 1 or len(items) == 1:
return [loader(item) for item in items]
if io_executor is not None:
return list(io_executor.map(loader, items))
with ThreadPoolExecutor(max_workers=num_io_workers) as executor:
return list(executor.map(loader, items))
def is_cuda_oom(exc: Exception) -> bool:
if isinstance(exc, torch.OutOfMemoryError):
return True
message = str(exc).lower()
return "out of memory" in message and "cuda" in message
def save_hidden_state_batch_cpu(
items: list[ExportItem],
hidden_cpu: torch.Tensor,
seq_lens: list[int],
overwrite: bool,
) -> dict[str, Any]:
stats = {
"written": 0,
"skipped": 0,
"failed": 0,
"hidden_bytes": 0,
"error": None,
}
for batch_idx, batch_item in enumerate(items):
seq_len = int(seq_lens[batch_idx])
sample_hidden = hidden_cpu[batch_idx, :seq_len, :]
try:
did_write, file_bytes = save_hidden_state_cpu(batch_item, sample_hidden, overwrite)
if did_write:
stats["written"] += 1
stats["hidden_bytes"] += int(file_bytes)
else:
stats["skipped"] += 1
except Exception as exc: # noqa: BLE001
error = f"{batch_item.source_name}/{batch_item.task_name}: {type(exc).__name__}: {exc}"
stats["failed"] += 1
if stats["error"] is None:
stats["error"] = error
if is_rank0():
print(f"[ERROR] {error}", flush=True)
traceback.print_exc()
return stats
def source_hidden_file_is_complete(item: ExportItem) -> bool:
path = item.source_output_path
if path is None or not path.is_file() or item.source_item_count <= 0:
return False
try:
with safe_open(str(path), framework="pt", device="cpu") as reader:
metadata = reader.metadata() or {}
required_tensors = {"hidden_states", "offsets", "frame_ids"}
if not required_tensors.issubset(set(reader.keys())):
return False
if metadata.get("format") != HFD_AUDIO_SOURCE_FORMAT:
return False
if metadata.get("source_key") != item.source_key:
return False
if metadata.get("source_name") != item.source_name:
return False
if metadata.get("task_name") != item.task_name:
return False
for key, value in online_encoder_metadata(item.task_name).items():
if metadata.get(key) != value:
return False
if metadata.get("instruction_sha1") != hashlib.sha1(
item.instruction.encode("utf-8")
).hexdigest():
return False
if int(metadata.get("count", "-1")) != int(item.source_item_count):
return False
if int(metadata.get("step_frames", "-1")) != int(item.source_step_frames):
return False
if int(metadata.get("motion_fps", "-1")) != int(item.motion_fps):
return False
if int(metadata.get("sampling_rate", "-1")) != int(item.audio_sampling_rate):
return False
if not math.isclose(
float(metadata.get("window_seconds", "nan")),
float(item.audio_duration_sec),
rel_tol=0.0,
abs_tol=1e-9,
):
return False
hidden_shape = tuple(reader.get_slice("hidden_states").get_shape())
offsets_shape = tuple(reader.get_slice("offsets").get_shape())
frame_ids_shape = tuple(reader.get_slice("frame_ids").get_shape())
if len(hidden_shape) != 2:
return False
if offsets_shape != (int(item.source_item_count) + 1,):
return False
if frame_ids_shape != (int(item.source_item_count),):
return False
offsets = reader.get_tensor("offsets")
frame_ids = reader.get_tensor("frame_ids")
if int(offsets[0]) != 0 or int(offsets[-1]) != int(hidden_shape[0]):
return False
if bool(torch.any(offsets[1:] <= offsets[:-1])):
return False
expected_frame_ids = torch.arange(
int(item.source_item_count),
dtype=torch.int64,
) * int(item.source_step_frames)
return bool(torch.equal(frame_ids, expected_frame_ids))
except (OSError, RuntimeError, TypeError, ValueError):
return False
def save_hidden_state_source_cpu(
items: list[ExportItem],
hiddens: list[torch.Tensor],
overwrite: bool,
) -> dict[str, Any]:
stats = {
"written": 0,
"skipped": 0,
"failed": 0,
"hidden_bytes": 0,
"error": None,
}
if not items:
return stats
first = items[0]
output_path = first.source_output_path
expected_count = int(first.source_item_count)
if output_path is None:
raise ValueError(f"Missing source_output_path for {first.input_path}")
if len(items) != expected_count or len(hiddens) != expected_count:
raise ValueError(
f"Incomplete source {first.source_key}: "
f"collected={len(items)} expected={expected_count}"
)
if any(item.source_key != first.source_key for item in items):
raise ValueError(f"Mixed WAV sources in one source file: {first.source_key}")
if [int(item.source_item_index) for item in items] != list(range(expected_count)):
raise ValueError(f"Out-of-order windows for source {first.source_key}")
if output_path.exists() and not overwrite and source_hidden_file_is_complete(first):
stats["skipped"] = expected_count
return stats
tmp_path = output_path.with_name(f".{output_path.name}.{os.getpid()}.tmp")
try:
output_path.parent.mkdir(parents=True, exist_ok=True)
lengths = [int(hidden.shape[0]) for hidden in hiddens]
offsets = [0]
for length in lengths:
if length <= 0:
raise ValueError(
f"Empty hidden window in source {first.source_key}"
)
offsets.append(offsets[-1] + length)
flat_hidden = torch.cat(
[hidden.contiguous() for hidden in hiddens],
dim=0,
).contiguous()
frame_ids = torch.tensor(
[int(item.frame_id) for item in items],
dtype=torch.int64,
)
expected_frame_ids = torch.arange(
expected_count,
dtype=torch.int64,
) * int(first.source_step_frames)
if not torch.equal(frame_ids, expected_frame_ids):
raise ValueError(
f"Non-contiguous source frames for {first.source_key}: "
f"{frame_ids.tolist()}"
)
metadata = {
"format": HFD_AUDIO_SOURCE_FORMAT,
"source_name": first.source_name,
"task_name": first.task_name,
"source_key": first.source_key,
"source_path": first.rel_source_path.split("#", 1)[0],
"manifest_id": first.manifest_id,
"instruction_sha1": hashlib.sha1(
first.instruction.encode("utf-8")
).hexdigest(),
"count": str(expected_count),
"hidden_dim": str(int(flat_hidden.shape[-1])),
"window_seconds": f"{float(first.audio_duration_sec):.9f}",
"step_frames": str(int(first.source_step_frames)),
"motion_fps": str(int(first.motion_fps)),
"sampling_rate": str(int(first.audio_sampling_rate)),
**online_encoder_metadata(first.task_name),
}
save_file(
{
"hidden_states": flat_hidden,
"offsets": torch.tensor(offsets, dtype=torch.int64),
"frame_ids": frame_ids,
},
str(tmp_path),
metadata=metadata,
)
os.replace(tmp_path, output_path)
stats["written"] = expected_count
stats["hidden_bytes"] = int(output_path.stat().st_size)
return stats
except Exception as exc: # noqa: BLE001
tmp_path.unlink(missing_ok=True)
error = (
f"{first.source_name}/{first.task_name}/{first.source_key}: "
f"{type(exc).__name__}: {exc}"
)
stats["failed"] = expected_count
stats["error"] = error
if is_rank0():
print(f"[ERROR] {error}", flush=True)
traceback.print_exc()
return stats
def source_index_row(
item: ExportItem,
output_root: Path,
) -> tuple[str, str, int, float, int, int, int]:
source_path = item.source_output_path
if source_path is None:
raise ValueError(f"Missing source output path for {item.input_path}")
source_dir = (
cache_group_dir(
output_root,
item.source_name,
item.task_name,
item.modality,
)
/ "sources"
)
relative_path = sanitize_rel_path(source_path.relative_to(source_dir))
return (
item.source_key,
relative_path,
int(item.source_item_count),
float(item.audio_duration_sec),
int(item.source_step_frames),
int(item.motion_fps),
int(item.audio_sampling_rate),
)
def build_rank_source_sqlite_index(
output_root: Path,
source_name: str,
task_name: str,
modality: str,
source_rows: Iterable[tuple[str, str, int, float, int, int, int]],
shard_rank: int,
) -> Path:
source_dir = (
cache_group_dir(output_root, source_name, task_name, modality)
/ "sources"
)
source_dir.mkdir(parents=True, exist_ok=True)
sqlite_path = (
source_dir
/ f"index.ws{world_size()}.rank{int(shard_rank):02d}.sqlite"
)
tmp_path = source_dir / f".{sqlite_path.name}.{os.getpid()}.tmp"
tmp_path.unlink(missing_ok=True)
rows = sorted(source_rows, key=lambda row: row[0])
conn = sqlite3.connect(str(tmp_path))
try:
conn.execute("PRAGMA journal_mode=OFF")
conn.execute("PRAGMA synchronous=OFF")
conn.execute("PRAGMA temp_store=MEMORY")
conn.execute(
"CREATE TABLE sources ("
"key TEXT PRIMARY KEY, "
"path TEXT NOT NULL, "
"count INTEGER NOT NULL, "
"window_seconds REAL NOT NULL, "
"step_frames INTEGER NOT NULL, "
"motion_fps INTEGER NOT NULL, "
"sampling_rate INTEGER NOT NULL)"
)
conn.executemany(
"INSERT INTO sources VALUES (?, ?, ?, ?, ?, ?, ?)",
rows,
)
conn.execute("PRAGMA user_version=1")
conn.commit()
finally:
conn.close()
os.replace(tmp_path, sqlite_path)
return sqlite_path
def save_hidden_state_shard_cpu(
output_root: Path,
shard_rank: int,
shard_id: int,
items: list[ExportItem],
hiddens: list[torch.Tensor],
overwrite: bool,
) -> dict[str, Any]:
stats = {
"written": 0,
"skipped": 0,
"failed": 0,
"hidden_bytes": 0,
"error": None,
}
if not items:
return stats
first = items[0]
shard_path, index_path = shard_paths(
output_root,
first.source_name,
first.task_name,
first.modality,
shard_rank,
shard_id,
)
if shard_path.exists() and index_path.exists() and not overwrite:
try:
if count_jsonl_rows(index_path) == len(items):
stats["skipped"] = len(items)
return stats
except OSError:
pass
try:
shard_path.parent.mkdir(parents=True, exist_ok=True)
lengths = [int(hidden.shape[0]) for hidden in hiddens]
offsets = [0]
for length in lengths:
offsets.append(offsets[-1] + length)
flat_hidden = torch.cat([hidden.contiguous() for hidden in hiddens], dim=0).contiguous()
offsets_tensor = torch.tensor(offsets, dtype=torch.int64)
metadata = {
"source_name": first.source_name,
"task_name": first.task_name,
"modality": first.modality,
"rank": str(int(shard_rank)),
"shard_id": str(int(shard_id)),
"count": str(len(items)),
"hidden_dim": str(int(flat_hidden.shape[-1])),
}
tmp_shard = shard_path.with_name(f".{shard_path.name}.{os.getpid()}.tmp")
save_file(
{
"hidden_states": flat_hidden,
"offsets": offsets_tensor,
},
str(tmp_shard),
metadata=metadata,
)
tmp_index = index_path.with_name(f".{index_path.name}.{os.getpid()}.tmp")
with tmp_index.open("w", encoding="utf-8") as f:
for sample_idx, item in enumerate(items):
line = {
"key": export_item_cache_key(item, output_root),
"shard": shard_path.name,
"idx": sample_idx,
"offset": offsets[sample_idx],
"length": lengths[sample_idx],
"source_path": item.rel_source_path,
"input_path": str(item.input_path),
"manifest_id": item.manifest_id,
"instruction_sha1": hashlib.sha1(item.instruction.encode("utf-8")).hexdigest(),
"full_audio": bool(item.full_audio),
"audio_start_sec": float(item.audio_start_sec),
"audio_duration_sec": float(item.audio_duration_sec),
}
f.write(json.dumps(line, ensure_ascii=False, separators=(",", ":")) + "\n")
os.replace(tmp_shard, shard_path)
os.replace(tmp_index, index_path)
stats["written"] = len(items)
stats["hidden_bytes"] = int(shard_path.stat().st_size + index_path.stat().st_size)
return stats
except Exception as exc: # noqa: BLE001
error = f"{first.source_name}/{first.task_name}/rank{shard_rank}/shard{shard_id}: {type(exc).__name__}: {exc}"
stats["failed"] = len(items)
stats["error"] = error
if is_rank0():
print(f"[ERROR] {error}", flush=True)
traceback.print_exc()
return stats
def build_rank_sqlite_index(
output_root: Path,
source_name: str,
task_name: str,
modality: str,
shard_rank: int,
expected_rows: int | None = None,
) -> Path | None:
group_dir = cache_group_dir(output_root, source_name, task_name, modality)
shard_dir = group_dir / "shards"
if not shard_dir.exists():
return None
index_files = sorted(shard_dir.glob(f"*rank{int(shard_rank):02d}.shard*.index.jsonl"))
if not index_files:
return None
sqlite_path = shard_dir / f"index.rank{int(shard_rank):02d}.sqlite"
# A no-op resume must not rewrite an otherwise identical SQLite file.
# merge_omni_hidden_for_oss fingerprints source indexes, so changing only
# the SQLite mtime would invalidate already-finished destination shards.
if sqlite_path.exists() and sqlite_path.stat().st_mtime_ns >= max(
path.stat().st_mtime_ns for path in index_files
):
try:
uri = f"file:{sqlite_path}?mode=ro"
with sqlite3.connect(uri, uri=True, timeout=60.0) as existing:
existing.execute("PRAGMA query_only=ON")
row_count = int(existing.execute("SELECT COUNT(*) FROM samples").fetchone()[0])
if expected_rows is None or row_count == int(expected_rows):
return sqlite_path
except (OSError, sqlite3.Error, TypeError, ValueError):
pass
tmp_path = shard_dir / f".{sqlite_path.name}.{os.getpid()}.tmp"
if tmp_path.exists():
tmp_path.unlink()
conn = sqlite3.connect(str(tmp_path))
try:
conn.execute("PRAGMA journal_mode=OFF")
conn.execute("PRAGMA synchronous=OFF")
conn.execute("PRAGMA temp_store=MEMORY")
conn.execute(
"CREATE TABLE samples ("
"key TEXT PRIMARY KEY, "
"shard TEXT NOT NULL, "
"offset INTEGER NOT NULL, "
"length INTEGER NOT NULL)"
)
rows: list[tuple[str, str, int, int]] = []
for index_file in index_files:
with index_file.open("r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
item = json.loads(line)
rows.append((str(item["key"]), str(item["shard"]), int(item["offset"]), int(item["length"])))
if len(rows) >= 10000:
conn.executemany("INSERT OR REPLACE INTO samples VALUES (?, ?, ?, ?)", rows)
rows.clear()
if rows:
conn.executemany("INSERT OR REPLACE INTO samples VALUES (?, ?, ?, ?)", rows)
conn.execute("CREATE INDEX IF NOT EXISTS idx_samples_key ON samples(key)")
conn.commit()
finally:
conn.close()
os.replace(tmp_path, sqlite_path)
return sqlite_path
class ShardWriter:
def __init__(
self,
output_root: Path,
shard_rank: int,
shard_size: int,
overwrite: bool,
save_executor: ThreadPoolExecutor,
) -> None:
self.output_root = output_root
self.shard_rank = int(shard_rank)
self.shard_size = max(1, int(shard_size))
self.overwrite = bool(overwrite)
self.save_executor = save_executor
self.current_shard_id: int | None = None
self.items: list[ExportItem] = []
self.hiddens: list[torch.Tensor] = []
def add_batch(self, items: list[ExportItem], hidden_cpu: torch.Tensor, seq_lens: list[int]) -> list[Future[dict[str, Any]]]:
futures: list[Future[dict[str, Any]]] = []
for batch_idx, item in enumerate(items):
shard_id = int(item.shard_id)
if shard_id < 0:
raise ValueError(f"Missing shard_id for {item.input_path}")
if self.current_shard_id is None:
self.current_shard_id = shard_id
if shard_id != self.current_shard_id:
futures.extend(self.flush())
self.current_shard_id = shard_id
seq_len = int(seq_lens[batch_idx])
self.items.append(item)
self.hiddens.append(hidden_cpu[batch_idx, :seq_len, :].contiguous())
if len(self.items) >= self.shard_size:
futures.extend(self.flush())
return futures
def flush(self) -> list[Future[dict[str, Any]]]:
if not self.items or self.current_shard_id is None:
return []
items = self.items
hiddens = self.hiddens
shard_id = int(self.current_shard_id)
self.items = []
self.hiddens = []
self.current_shard_id = None
return [
self.save_executor.submit(
save_hidden_state_shard_cpu,
self.output_root,
self.shard_rank,
shard_id,
items,
hiddens,
self.overwrite,
)
]
class SourceWriter:
"""Collect consecutive windows and atomically save one complete WAV."""
def __init__(
self,
overwrite: bool,
save_executor: ThreadPoolExecutor,
) -> None:
self.overwrite = bool(overwrite)
self.save_executor = save_executor
self.current_source_key: str | None = None
self.expected_count = 0
self.items: list[ExportItem] = []
self.hiddens: list[torch.Tensor] = []
def add_batch(
self,
items: list[ExportItem],
hidden_cpu: torch.Tensor,
seq_lens: list[int],
) -> list[Future[dict[str, Any]]]:
futures: list[Future[dict[str, Any]]] = []
for batch_idx, item in enumerate(items):
if not item.source_key or item.source_item_count <= 0:
raise ValueError(
f"Missing source metadata for {item.input_path}"
)
if self.current_source_key is None:
self.current_source_key = item.source_key
self.expected_count = int(item.source_item_count)
elif item.source_key != self.current_source_key:
futures.extend(self.flush())
self.current_source_key = item.source_key
self.expected_count = int(item.source_item_count)
if int(item.source_item_count) != self.expected_count:
raise ValueError(
f"Inconsistent source count for {item.source_key}"
)
seq_len = int(seq_lens[batch_idx])
self.items.append(item)
self.hiddens.append(
hidden_cpu[batch_idx, :seq_len, :].clone(
memory_format=torch.contiguous_format
)
)
if len(self.items) == self.expected_count:
futures.extend(self.flush())
elif len(self.items) > self.expected_count:
raise ValueError(
f"Too many windows for source {item.source_key}"
)
return futures
def flush(self) -> list[Future[dict[str, Any]]]:
if not self.items:
self.current_source_key = None
self.expected_count = 0
return []
if len(self.items) != self.expected_count:
raise RuntimeError(
f"Incomplete source {self.current_source_key}: "
f"collected={len(self.items)} expected={self.expected_count}"
)
items = self.items
hiddens = self.hiddens
self.items = []
self.hiddens = []
self.current_source_key = None
self.expected_count = 0
return [
self.save_executor.submit(
save_hidden_state_source_cpu,
items,
hiddens,
self.overwrite,
)
]
def process_export_batch(
preset: SourcePreset,
task_name: str,
items: list[ExportItem],
args: argparse.Namespace,
processor: Qwen2_5OmniProcessor,
model: Qwen2_5OmniThinkerForConditionalGeneration,
device: torch.device,
save_dtype: torch.dtype,
io_executor: ThreadPoolExecutor | None = None,
save_executor: ThreadPoolExecutor | None = None,
preloaded_payloads: list[dict[str, Any]] | None = None,
preloaded_inputs: dict[str, torch.Tensor] | None = None,
) -> dict[str, Any]:
payloads: list[dict[str, Any]] | None = None
inputs_cpu: dict[str, torch.Tensor] | None = None
inputs: dict[str, torch.Tensor] | None = None
hidden: torch.Tensor | None = None
hidden_cpu: torch.Tensor | None = None
try:
if preloaded_inputs is not None:
inputs_cpu = preloaded_inputs
else:
payloads = preloaded_payloads if preloaded_payloads is not None else load_payloads(
items,
int(args.num_io_workers),
io_executor=io_executor,
)
if preset.modality == "audio":
inputs_cpu = collate_audio_batch(processor, payloads)
elif preset.modality == "text":
inputs_cpu = collate_text_batch(processor, payloads)
else:
inputs_cpu = collate_video_batch(processor, payloads)
inputs = move_inputs_to_device(inputs_cpu, device)
with torch.inference_mode():
hidden = omni_hidden_states_without_lm_head(model, inputs)
attention_mask = inputs["attention_mask"]
seq_lens = [int(v) for v in attention_mask.sum(dim=1).tolist()]
hidden_cpu = hidden.to(dtype=save_dtype).cpu().contiguous()
stats = {
"processed": len(items),
"written": 0,
"skipped": 0,
"failed": 0,
"hidden_tokens": int(sum(seq_lens)),
"hidden_bytes": 0,
"seq_sum": int(sum(seq_lens)),
"error": None,
"pending_saves": [],
"shard_batches": [],
"source_batches": [],
}
overwrite = args.overwrite if args.mode == "export" else True
if args.storage_format == "shard":
stats["shard_batches"].append((items, hidden_cpu, seq_lens))
hidden_cpu = None
elif args.storage_format == "source":
stats["source_batches"].append((items, hidden_cpu, seq_lens))
hidden_cpu = None
elif save_executor is None:
save_stats = save_hidden_state_batch_cpu(items, hidden_cpu, seq_lens, overwrite)
stats["written"] += int(save_stats["written"])
stats["skipped"] += int(save_stats["skipped"])
stats["failed"] += int(save_stats["failed"])
stats["hidden_bytes"] += int(save_stats["hidden_bytes"])
stats["error"] = save_stats["error"]
else:
stats["pending_saves"].append(
save_executor.submit(save_hidden_state_batch_cpu, items, hidden_cpu, seq_lens, overwrite)
)
del hidden
del hidden_cpu
del inputs
del inputs_cpu
del payloads
return stats
except Exception as exc: # noqa: BLE001
error = f"{preset.name}/{task_name}: {type(exc).__name__}: {exc}"
if hidden is not None:
del hidden
hidden = None
if hidden_cpu is not None:
del hidden_cpu
hidden_cpu = None
if inputs is not None:
del inputs
inputs = None
if inputs_cpu is not None:
del inputs_cpu
inputs_cpu = None
if payloads is not None:
del payloads
payloads = None
gc.collect()
if device.type == "cuda":
torch.cuda.empty_cache()
if is_cuda_oom(exc) and len(items) > 1:
left_size = len(items) // 2
right_size = len(items) - left_size
if is_rank0():
print(
f"[WARN] {error} | retry split batch {len(items)} -> {left_size} + {right_size}",
flush=True,
)
left_stats = process_export_batch(
preset,
task_name,
items[:left_size],
args,
processor,
model,
device,
save_dtype,
io_executor=io_executor,
save_executor=save_executor,
)
right_stats = process_export_batch(
preset,
task_name,
items[left_size:],
args,
processor,
model,
device,
save_dtype,
io_executor=io_executor,
save_executor=save_executor,
)
merged = {
"processed": left_stats["processed"] + right_stats["processed"],
"written": left_stats["written"] + right_stats["written"],
"skipped": left_stats["skipped"] + right_stats["skipped"],
"failed": left_stats["failed"] + right_stats["failed"],
"hidden_tokens": left_stats["hidden_tokens"] + right_stats["hidden_tokens"],
"hidden_bytes": left_stats["hidden_bytes"] + right_stats["hidden_bytes"],
"seq_sum": left_stats["seq_sum"] + right_stats["seq_sum"],
"error": left_stats["error"] or right_stats["error"] or error,
"pending_saves": [*left_stats.get("pending_saves", []), *right_stats.get("pending_saves", [])],
"shard_batches": [*left_stats.get("shard_batches", []), *right_stats.get("shard_batches", [])],
"source_batches": [*left_stats.get("source_batches", []), *right_stats.get("source_batches", [])],
}
return merged
if is_rank0():
print(f"[ERROR] {error}", flush=True)
traceback.print_exc()
return {
"processed": 0,
"written": 0,
"skipped": 0,
"failed": len(items),
"hidden_tokens": 0,
"hidden_bytes": 0,
"seq_sum": 0,
"error": error,
"pending_saves": [],
"shard_batches": [],
"source_batches": [],
}
def collect_completed_save_futures(
pending_saves: deque[Future[dict[str, Any]]],
*,
wait_for_one: bool,
) -> list[dict[str, Any]]:
if not pending_saves:
return []
future_list = list(pending_saves)
if wait_for_one:
done, _ = wait(future_list, return_when=FIRST_COMPLETED)
else:
done, _ = wait(future_list, timeout=0, return_when=FIRST_COMPLETED)
if not done:
return []
done_ids = {id(fut) for fut in done}
results: list[dict[str, Any]] = []
remaining: deque[Future[dict[str, Any]]] = deque()
for fut in pending_saves:
if id(fut) in done_ids:
results.append(fut.result())
else:
remaining.append(fut)
pending_saves.clear()
pending_saves.extend(remaining)
return results
def bench_or_export_group(
preset: SourcePreset,
task_name: str,
args: argparse.Namespace,
processor: Qwen2_5OmniProcessor,
model: Qwen2_5OmniThinkerForConditionalGeneration,
device: torch.device,
save_dtype: torch.dtype,
group_output_root: Path,
) -> dict[str, Any]:
storage_format = str(args.storage_format)
if storage_format == "source":
log(
f"[{args.mode.upper()}] streaming complete WAV sources without "
"a duplicate window-count pass"
)
global_total = 0
elif args.mode == "benchmark":
global_total = int(args.benchmark_samples)
elif args.limit > 0:
global_total = int(args.limit)
else:
log(f"[{args.mode.upper()}] counting {preset.name}/{task_name} ...")
global_total = count_group_items(preset, task_name, group_output_root, args)
log(f"[{args.mode.upper()}] {preset.name}/{task_name}: total={global_total:,}")
local_total = (
None
if storage_format == "source"
else global_total // world_size()
+ (1 if rank() < (global_total % world_size()) else 0)
)
progress = tqdm(
total=local_total,
desc=f"{preset.name}/{task_name} rank{rank()}",
disable=not is_rank0(),
unit="item",
)
if preset.modality == "audio":
batch_size = int(args.audio_batch_size)
elif preset.modality == "text":
batch_size = int(args.text_batch_size)
else:
batch_size = int(args.video_batch_size)
group_items: list[ExportItem] = []
processed = 0
written = 0
skipped = 0
failed = 0
hidden_tokens = 0
hidden_bytes = 0
seq_sum = 0
first_error: str | None = None
started = time.time()
max_pending_save_batches = max(1, int(args.max_pending_save_batches))
pending_saves: deque[Future[dict[str, Any]]] = deque()
shard_size = max(1, int(args.shard_size))
complete_shard_cache: dict[int, bool] = {}
complete_source_cache: dict[str, bool] = {}
source_index_rows: dict[
str,
tuple[str, str, int, float, int, int, int],
] = {}
def merge_save_stats(save_stats: dict[str, Any]) -> None:
nonlocal written, skipped, failed, hidden_bytes, first_error
written += int(save_stats["written"])
skipped += int(save_stats["skipped"])
failed += int(save_stats["failed"])
hidden_bytes += int(save_stats["hidden_bytes"])
if first_error is None and save_stats.get("error"):
first_error = str(save_stats["error"])
def iter_local_batches() -> Iterator[list[ExportItem]]:
nonlocal skipped
batch: list[ExportItem] = []
local_index = 0
source_rank_loads = [0 for _ in range(world_size())]
active_source_ordinal = -1
active_source_rank = -1
for global_index, item in enumerate(iter_group_items(preset, task_name, group_output_root, args)):
if storage_format != "source" and global_index >= global_total:
break
if storage_format == "source":
if item.source_ordinal < 0:
raise ValueError(
f"Missing source ordinal for {item.input_path}"
)
if int(item.source_ordinal) != active_source_ordinal:
active_source_ordinal = int(item.source_ordinal)
active_source_rank = assign_source_to_least_loaded_rank(
source_rank_loads,
int(item.source_item_count),
)
if active_source_rank != rank():
continue
source_index_rows.setdefault(
item.source_key,
source_index_row(item, group_output_root),
)
if args.mode == "export" and not args.overwrite:
complete = complete_source_cache.get(item.source_key)
if complete is None:
complete = source_hidden_file_is_complete(item)
complete_source_cache[item.source_key] = complete
if complete:
skipped += 1
progress.update(1)
progress.set_postfix(
processed=processed,
written=written,
skipped=skipped,
failed=failed,
)
continue
else:
if global_index % world_size() != rank():
continue
if storage_format == "shard":
shard_id = local_index // shard_size
item.shard_id = int(shard_id)
local_index += 1
if args.mode == "export" and not args.overwrite:
complete = complete_shard_cache.get(shard_id)
if complete is None:
expected_count = min(shard_size, max(0, local_total - shard_id * shard_size))
complete = shard_is_complete(
group_output_root,
preset.name,
task_name,
preset.modality,
rank(),
shard_id,
expected_count,
)
complete_shard_cache[shard_id] = complete
if complete:
skipped += 1
progress.update(1)
progress.set_postfix(processed=processed, written=written, skipped=skipped, failed=failed)
continue
elif storage_format == "file":
if args.mode == "export" and item.output_path.exists() and not args.overwrite:
skipped += 1
progress.update(1)
progress.set_postfix(processed=processed, written=written, skipped=skipped, failed=failed)
continue
batch.append(item)
if len(batch) >= batch_size:
yield batch
batch = []
if batch:
yield batch
with ThreadPoolExecutor(max_workers=max(1, int(args.num_io_workers))) as io_executor, \
ThreadPoolExecutor(max_workers=max(1, int(args.num_save_workers))) as save_executor, \
ThreadPoolExecutor(max_workers=2) as prefetch_executor:
shard_writer = (
ShardWriter(
group_output_root,
rank(),
shard_size,
args.overwrite if args.mode == "export" else True,
save_executor,
)
if storage_format == "shard"
else None
)
source_writer = (
SourceWriter(
args.overwrite if args.mode == "export" else True,
save_executor,
)
if storage_format == "source"
else None
)
batch_iter = iter_local_batches()
current_items = next(batch_iter, None)
current_input_future: Future[dict[str, torch.Tensor]] | None = None
next_items: list[ExportItem] | None = None
next_input_future: Future[dict[str, torch.Tensor]] | None = None
if current_items is not None:
current_input_future = prefetch_executor.submit(
prepare_inputs_cpu,
preset,
current_items,
args,
processor,
io_executor,
)
while current_items is not None and current_input_future is not None:
next_items = next(batch_iter, None)
if next_items is not None:
next_input_future = prefetch_executor.submit(
prepare_inputs_cpu,
preset,
next_items,
args,
processor,
io_executor,
)
else:
next_input_future = None
batch_stats = process_export_batch(
preset,
task_name,
current_items,
args,
processor,
model,
device,
save_dtype,
io_executor=io_executor,
save_executor=save_executor,
preloaded_inputs=current_input_future.result(),
)
processed += int(batch_stats["processed"])
failed += int(batch_stats["failed"])
hidden_tokens += int(batch_stats["hidden_tokens"])
seq_sum += int(batch_stats["seq_sum"])
if first_error is None and batch_stats["error"]:
first_error = str(batch_stats["error"])
for fut in batch_stats.get("pending_saves", []):
pending_saves.append(fut)
for shard_items, shard_hidden_cpu, shard_seq_lens in batch_stats.get("shard_batches", []):
if shard_writer is None:
raise RuntimeError("shard batch returned while storage_format != shard")
for fut in shard_writer.add_batch(shard_items, shard_hidden_cpu, shard_seq_lens):
pending_saves.append(fut)
del shard_hidden_cpu
for source_items, source_hidden_cpu, source_seq_lens in batch_stats.get("source_batches", []):
if source_writer is None:
raise RuntimeError(
"source batch returned while storage_format != source"
)
for fut in source_writer.add_batch(
source_items,
source_hidden_cpu,
source_seq_lens,
):
pending_saves.append(fut)
del source_hidden_cpu
while len(pending_saves) >= max_pending_save_batches:
completed = collect_completed_save_futures(pending_saves, wait_for_one=True)
for save_stats in completed:
merge_save_stats(save_stats)
for save_stats in collect_completed_save_futures(pending_saves, wait_for_one=False):
merge_save_stats(save_stats)
progress.update(len(current_items))
progress.set_postfix(processed=processed, written=written, skipped=skipped, failed=failed)
current_items = next_items
current_input_future = next_input_future
if shard_writer is not None:
for fut in shard_writer.flush():
pending_saves.append(fut)
if source_writer is not None:
for fut in source_writer.flush():
pending_saves.append(fut)
while pending_saves:
for save_stats in collect_completed_save_futures(pending_saves, wait_for_one=True):
merge_save_stats(save_stats)
progress.close()
if storage_format == "shard" and args.mode == "export":
sqlite_path = build_rank_sqlite_index(
group_output_root,
preset.name,
task_name,
preset.modality,
rank(),
expected_rows=local_total,
)
if sqlite_path is not None and is_rank0():
print(f"[SHARD] rebuilt sqlite index for rank0: {sqlite_path}", flush=True)
if storage_format == "source" and args.mode == "export":
if failed:
raise RuntimeError(
f"Refusing to publish source index after {failed} failed windows"
)
sqlite_path = build_rank_source_sqlite_index(
group_output_root,
preset.name,
task_name,
preset.modality,
source_index_rows.values(),
rank(),
)
if is_rank0():
print(
f"[SOURCE] wrote compact source index for rank0: {sqlite_path}",
flush=True,
)
elapsed = time.time() - started
metrics = torch.tensor(
[processed, written, skipped, failed, hidden_tokens, hidden_bytes, seq_sum],
dtype=torch.float64,
)
wall = torch.tensor([elapsed], dtype=torch.float64)
# Export is embarrassingly parallel by rank. During resume, ranks can have
# very different amounts of work left because complete shards are skipped
# per rank. A group-end all_reduce can therefore time out while a slower
# rank is still legitimately filling missing shards. Keep export progress
# rank-local; completeness is determined by shard/index files, not by this
# summary metric.
if dist.is_initialized() and args.mode != "export":
dist.all_reduce(metrics, op=dist.ReduceOp.SUM)
dist.all_reduce(wall, op=dist.ReduceOp.MAX)
total_processed, total_written, total_skipped, total_failed, total_tokens, total_bytes, total_seq_sum = metrics.tolist()
wall_sec = float(wall.item())
avg_seq_len = (total_seq_sum / total_processed) if total_processed > 0 else 0.0
samples_per_sec = (total_processed / wall_sec) if wall_sec > 0 else 0.0
tokens_per_sec = (total_tokens / wall_sec) if wall_sec > 0 else 0.0
avg_file_mb = (total_bytes / max(total_written, 1)) / (1024 ** 2)
group_stats = {
"source_name": preset.name,
"task_name": task_name,
"global_total": (
None if storage_format == "source" else int(global_total)
),
"processed": int(total_processed),
"written": int(total_written),
"skipped": int(total_skipped),
"failed": int(total_failed),
"elapsed_sec": wall_sec,
"samples_per_sec": samples_per_sec,
"tokens_per_sec": tokens_per_sec,
"avg_seq_len": avg_seq_len,
"avg_output_mb": avg_file_mb,
"first_error": first_error or "",
}
return group_stats
def estimate_group(
preset: SourcePreset,
task_name: str,
args: argparse.Namespace,
processor: Qwen2_5OmniProcessor,
hidden_size: int,
) -> dict[str, Any]:
sample_items = sampled_group_items(
preset,
task_name,
Path(args.output_root),
int(args.estimate_samples_per_group),
args,
)
if not sample_items:
return {
"source_name": preset.name,
"task_name": task_name,
"total_items": 0,
"sample_count": 0,
}
seq_lens: list[int] = []
output_bytes: list[int] = []
payloads = load_payloads(sample_items, int(args.num_io_workers))
sample_progress = tqdm(
payloads,
total=len(payloads),
desc=f"Estimate {preset.name}/{task_name}",
disable=not is_rank0(),
leave=False,
unit="sample",
)
if preset.modality == "audio":
for payload in sample_progress:
texts = build_audio_text(processor, [payload["instruction"]])
inputs = processor(
text=texts,
audio=[np.asarray(payload["audio"], dtype=np.float32)],
padding=True,
return_tensors="pt",
)
seq_len = int(inputs["input_ids"].shape[1])
seq_lens.append(seq_len)
output_bytes.append(seq_len * hidden_size * torch.tensor([], dtype=save_dtype_from_arg(args.save_dtype)).element_size())
elif preset.modality == "text":
for payload in sample_progress:
texts = build_text_text(processor, [payload["instruction"]])
tokenized = processor.tokenizer(texts, padding=True, return_tensors="pt")
seq_len = int(tokenized["input_ids"].shape[1])
seq_lens.append(seq_len)
output_bytes.append(seq_len * hidden_size * torch.tensor([], dtype=save_dtype_from_arg(args.save_dtype)).element_size())
else:
for payload in sample_progress:
texts = build_video_text(processor, [payload["instruction"]])
texts = processor.replace_multimodal_special_tokens(
texts,
iter([]),
iter([]),
iter([payload["video_grid_thw"][0]]),
iter([float(np.asarray(payload["video_second_per_grid"]).reshape(-1)[0])]),
False,
25,
2.0,
)
tokenized = processor.tokenizer(texts, padding=True, return_tensors="pt")
seq_len = int(tokenized["input_ids"].shape[1])
seq_lens.append(seq_len)
output_bytes.append(seq_len * hidden_size * torch.tensor([], dtype=save_dtype_from_arg(args.save_dtype)).element_size())
sample_progress.close()
total_items = count_group_items(preset, task_name, Path(args.output_root), args)
avg_seq_len = float(sum(seq_lens) / len(seq_lens))
avg_output_bytes = float(sum(output_bytes) / len(output_bytes))
return {
"source_name": preset.name,
"task_name": task_name,
"total_items": int(total_items),
"sample_count": len(seq_lens),
"avg_seq_len": avg_seq_len,
"min_seq_len": int(min(seq_lens)),
"max_seq_len": int(max(seq_lens)),
"avg_output_mb": avg_output_bytes / (1024 ** 2),
"estimated_total_gib": total_items * avg_output_bytes / (1024 ** 3),
"estimated_total_tib": total_items * avg_output_bytes / (1024 ** 4),
}
def ensure_video_export_is_confirmed(args: argparse.Namespace, pairs: list[tuple[SourcePreset, str]]) -> None:
if args.mode != "export":
return
has_video = any(preset.modality == "video_clip" for preset, _ in pairs)
if has_video and not args.confirm_full_video:
raise ValueError("video_clip export is disabled by default; re-run with --confirm-full-video")
def write_report(output_root: Path, mode: str, report: dict[str, Any]) -> Path:
report_dir = output_root / "_reports"
report_dir.mkdir(parents=True, exist_ok=True)
timestamp = time.strftime("%Y%m%d_%H%M%S")
path = report_dir / f"{mode}.{timestamp}.json"
path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
return path
def run_estimate(args: argparse.Namespace, pairs: list[tuple[SourcePreset, str]]) -> None:
hidden_size = load_hidden_size(args.model_path)
processor = load_processor(args.model_path)
report = {
"mode": args.mode,
"model_path": args.model_path,
"hidden_size": hidden_size,
"save_dtype": args.save_dtype,
"groups": [],
}
group_progress = tqdm(
pairs,
total=len(pairs),
desc="Estimate groups",
disable=not is_rank0(),
leave=True,
unit="group",
)
for preset, task_name in group_progress:
stats = estimate_group(preset, task_name, args, processor, hidden_size)
report["groups"].append(stats)
group_progress.set_postfix(
source=preset.name,
task=task_name,
tib=f"{stats.get('estimated_total_tib', 0):.3f}",
)
log(
f"[ESTIMATE] {preset.name}/{task_name}: total={stats['total_items']:,} "
f"avg_seq={stats.get('avg_seq_len', 0):.2f} avg_mb={stats.get('avg_output_mb', 0):.3f} "
f"total_tib={stats.get('estimated_total_tib', 0):.3f}"
)
group_progress.close()
report_path = write_report(Path(args.output_root), args.mode, report)
log(f"[REPORT] wrote {report_path}")
def run_status(args: argparse.Namespace, pairs: list[tuple[SourcePreset, str]]) -> None:
output_root = Path(args.output_root)
report = {
"mode": args.mode,
"output_root": str(output_root),
"storage_format": args.storage_format,
"groups": [],
}
for preset, task_name in tqdm(pairs, desc="Status groups", disable=not is_rank0(), unit="group"):
group_dir = cache_group_dir(output_root, preset.name, task_name, preset.modality)
total = count_group_items(preset, task_name, output_root, args)
if args.storage_format == "shard":
shard_dir = group_dir / "shards"
existing = 0
shard_count = 0
sqlite_count = 0
if shard_dir.exists():
for index_path in shard_dir.glob("*shard*.index.jsonl"):
existing += count_jsonl_rows(index_path)
shard_count = sum(1 for _ in shard_dir.glob("*shard*.safetensors"))
sqlite_count = sum(1 for _ in shard_dir.glob("index.rank*.sqlite"))
tmp_count = sum(1 for _ in group_dir.rglob("*.tmp")) if group_dir.exists() else 0
elif args.storage_format == "source":
source_dir = group_dir / "sources"
existing = 0
source_count = 0
sqlite_paths = (
sorted(source_dir.glob("index.ws*.rank*.sqlite"))
if source_dir.exists()
else []
)
seen_keys: set[str] = set()
for sqlite_path in sqlite_paths:
conn = sqlite3.connect(
f"file:{sqlite_path}?mode=ro",
uri=True,
)
try:
for key, count in conn.execute(
"SELECT key, count FROM sources"
):
if str(key) in seen_keys:
continue
seen_keys.add(str(key))
source_count += 1
existing += int(count)
finally:
conn.close()
shard_count = source_count
sqlite_count = len(sqlite_paths)
tmp_count = (
sum(1 for _ in source_dir.rglob("*.tmp"))
if source_dir.exists()
else 0
)
else:
existing = 0
for item in iter_group_items(preset, task_name, output_root, args):
if item.output_path.exists():
existing += 1
shard_count = 0
sqlite_count = 0
tmp_count = sum(1 for _ in group_dir.rglob("*.tmp")) if group_dir.exists() else 0
stats = {
"source_name": preset.name,
"task_name": task_name,
"expected": total,
"existing": existing,
"missing": max(0, total - existing),
"tmp": tmp_count,
"shards": shard_count,
"sqlite_indexes": sqlite_count,
}
report["groups"].append(stats)
log(
f"[STATUS] {preset.name}/{task_name}: "
f"existing={existing:,} missing={max(0, total - existing):,} expected={total:,} "
f"shards={shard_count:,} sqlite={sqlite_count:,} tmp={tmp_count:,}"
)
report_path = write_report(output_root, args.mode, report)
log(f"[REPORT] wrote {report_path}")
def run_benchmark_or_export(args: argparse.Namespace, pairs: list[tuple[SourcePreset, str]]) -> None:
maybe_init_distributed()
device = current_device()
save_dtype = save_dtype_from_arg(args.save_dtype)
processor = load_processor(args.model_path)
model = load_model(args.model_path, device, save_dtype)
group_output_root = (
Path(args.output_root) / "_benchmark_outputs"
if args.mode == "benchmark"
else Path(args.output_root)
)
report = {
"mode": args.mode,
"model_path": args.model_path,
"world_size": world_size(),
"rank": rank(),
"save_dtype": args.save_dtype,
"storage_format": args.storage_format,
"shard_size": int(args.shard_size),
"audio_command_full_audio": bool(args.audio_command_full_audio),
"audio_command_window_seconds": float(args.audio_command_window_seconds),
"audio_command_sample_step_frames": int(args.audio_command_sample_step_frames),
"limit_sources": int(args.limit_sources),
"groups": [],
}
group_progress = tqdm(
pairs,
total=len(pairs),
desc=f"{args.mode.title()} groups",
disable=not is_rank0(),
leave=True,
unit="group",
)
for preset, task_name in group_progress:
stats = bench_or_export_group(
preset,
task_name,
args,
processor,
model,
device,
save_dtype,
group_output_root,
)
if is_rank0():
report["groups"].append(stats)
group_progress.set_postfix(
source=preset.name,
task=task_name,
samples_per_sec=f"{stats['samples_per_sec']:.2f}",
avg_mb=f"{stats['avg_output_mb']:.3f}",
)
log(
f"[{args.mode.upper()}] {preset.name}/{task_name}: processed={stats['processed']:,} "
f"written={stats['written']:,} avg_seq={stats['avg_seq_len']:.2f} "
f"samples/s={stats['samples_per_sec']:.2f} tokens/s={stats['tokens_per_sec']:.2f} "
f"avg_mb={stats['avg_output_mb']:.3f}"
)
group_progress.close()
if is_rank0():
report_path = write_report(Path(args.output_root), args.mode, report)
log(f"[REPORT] wrote {report_path}")
cleanup_distributed()
def run_verify_online_hidden(
args: argparse.Namespace,
pair: tuple[SourcePreset, str],
) -> None:
"""Recompute saved audio hidden states through the real B=1 online path."""
preset, task_name = pair
output_root = Path(args.output_root)
contract_path = output_root / "_reports" / "encoder_contract.json"
if not contract_path.is_file():
raise FileNotFoundError(
f"Missing encoder contract before online verification: {contract_path}"
)
recorded_contract = json.loads(contract_path.read_text(encoding="utf-8"))
expected_contract = encoder_contract_payload(args.model_path)
for key, expected in expected_contract.items():
if recorded_contract.get(key) != expected:
raise RuntimeError(
f"Encoder contract changed before verification: {key}="
f"{recorded_contract.get(key)!r}, expected={expected!r}"
)
device = current_device()
processor = load_processor(args.model_path)
model = load_model(args.model_path, device, torch.bfloat16)
verify_limit = max(int(args.verify_sources), 0)
verify_windows_per_source = max(
int(args.verify_windows_per_source),
0,
)
current_source_key: str | None = None
source_audio: np.ndarray | None = None
source_flat: torch.Tensor | None = None
source_offsets: torch.Tensor | None = None
verified_sources = 0
verified_windows = 0
mismatched_windows = 0
max_abs_error = 0.0
first_mismatch = ""
items = iter_group_items(preset, task_name, output_root, args)
for item in items:
source_key = item.source_key or str(item.input_path)
if source_key != current_source_key:
if verify_limit > 0 and verified_sources >= verify_limit:
break
current_source_key = source_key
verified_sources += 1
source_audio = load_audio_array(
item.input_path,
item.audio_sampling_rate,
)
source_flat = None
source_offsets = None
if args.storage_format == "source":
source_path = item.source_output_path
if source_path is None or not source_path.is_file():
raise FileNotFoundError(
f"Missing source hidden file: {source_path}"
)
with safe_open(
str(source_path),
framework="pt",
device="cpu",
) as reader:
metadata = reader.metadata() or {}
for key, value in online_encoder_metadata(task_name).items():
if metadata.get(key) != value:
raise RuntimeError(
f"Source encoder metadata mismatch at "
f"{source_path}: {key}={metadata.get(key)!r}, "
f"expected={value!r}"
)
source_flat = reader.get_tensor("hidden_states")
source_offsets = reader.get_tensor("offsets")
if source_audio is None:
raise RuntimeError(f"Audio was not loaded for {source_key}")
if (
verify_windows_per_source > 0
and item.source_item_index is not None
and int(item.source_item_index) >= verify_windows_per_source
):
continue
audio = source_audio
if not item.full_audio:
audio = audio_segment(
audio,
item.audio_start_sec,
item.audio_duration_sec,
item.audio_sampling_rate,
)
inputs_cpu = collate_audio_batch(
processor,
[
{
"audio": audio,
"instruction": item.instruction,
"full_audio": bool(item.full_audio),
}
],
)
inputs = move_inputs_to_device(inputs_cpu, device)
with torch.inference_mode():
hidden = omni_hidden_states_without_lm_head(model, inputs)
seq_len = int(inputs["attention_mask"].sum().item())
online = (
hidden[0, :seq_len]
.to(dtype=torch.bfloat16)
.cpu()
.contiguous()
)
if args.storage_format == "source":
if source_flat is None or source_offsets is None:
raise RuntimeError(f"Source hidden was not loaded for {source_key}")
source_index = int(item.source_item_index)
cached = source_flat[
int(source_offsets[source_index]) :
int(source_offsets[source_index + 1])
]
cache_path = item.source_output_path
else:
cache_path = item.output_path
if not cache_path.is_file():
raise FileNotFoundError(cache_path)
with safe_open(
str(cache_path),
framework="pt",
device="cpu",
) as reader:
metadata = reader.metadata() or {}
for key, value in online_encoder_metadata(task_name).items():
if metadata.get(key) != value:
raise RuntimeError(
f"Encoder metadata mismatch at {cache_path}: "
f"{key}={metadata.get(key)!r}, expected={value!r}"
)
cached = reader.get_tensor("hidden_states")
verified_windows += 1
if tuple(online.shape) != tuple(cached.shape):
mismatched_windows += 1
if not first_mismatch:
first_mismatch = (
f"{cache_path}: shape {tuple(cached.shape)} != "
f"{tuple(online.shape)}"
)
continue
difference = (online.float() - cached.float()).abs()
window_max = float(difference.max().item())
max_abs_error = max(max_abs_error, window_max)
if not torch.equal(online, cached):
mismatched_windows += 1
if not first_mismatch:
first_mismatch = (
f"{cache_path}: source_index={item.source_item_index}, "
f"max_abs_error={window_max}"
)
report = {
**expected_contract,
"storage_format": args.storage_format,
"verified_sources": verified_sources,
"verified_windows": verified_windows,
"verified_windows_per_source_limit": verify_windows_per_source,
"mismatched_windows": mismatched_windows,
"max_abs_error": max_abs_error,
"first_mismatch": first_mismatch,
"verified_at": time.time(),
}
report_path = output_root / "_reports" / "online_hidden_verify.json"
write_json_atomic(report_path, report)
log(
"[VERIFY] exact B=1 online equivalence: "
f"sources={verified_sources:,} windows={verified_windows:,} "
f"mismatched={mismatched_windows:,} max_abs={max_abs_error:g}"
)
log(f"[REPORT] wrote {report_path}")
if verified_windows <= 0:
raise RuntimeError("Online hidden verification selected no windows")
if mismatched_windows != 0 or max_abs_error != 0.0:
raise RuntimeError(
"Saved hidden states are not bitwise equal to online Qwen output: "
f"{first_mismatch}"
)
def main() -> None:
global ACTIVE_MODEL_FINGERPRINT, SOURCE_PRESETS
warnings.filterwarnings(
"ignore",
message="The image processor of type `Qwen2VLImageProcessor` is now loaded as a fast processor by default.*",
)
warnings.filterwarnings(
"ignore",
message="Unrecognized keys in `rope_scaling` for 'rope_type'='default': .*",
)
args = parse_args()
ACTIVE_MODEL_FINGERPRINT = qwen_model_fingerprint(args.model_path)
if int(args.motion_fps) <= 0 or int(args.audio_sampling_rate) <= 0:
raise ValueError("motion_fps and audio_sampling_rate must be positive")
if not bool(args.audio_command_full_audio):
if float(args.audio_command_window_seconds) <= 0:
raise ValueError("audio_command_window_seconds must be positive in windowed mode")
if int(args.audio_command_sample_step_frames) <= 0:
raise ValueError("audio_command_sample_step_frames must be positive in windowed mode")
SOURCE_PRESETS = load_source_presets(args)
tasks = parse_csv_list(args.tasks)
sources = parse_csv_list(args.sources)
unknown_tasks = [task for task in tasks if task not in SUPPORTED_TASKS]
if unknown_tasks:
raise ValueError(f"Unsupported tasks: {unknown_tasks}")
unknown_sources = [source for source in sources if source not in SOURCE_PRESETS]
if unknown_sources:
raise ValueError(f"Unsupported sources: {unknown_sources}")
pairs = source_task_pairs(tasks, sources)
if not pairs:
raise ValueError("No compatible source/task pairs selected")
ensure_video_export_is_confirmed(args, pairs)
has_audio_command = any(task == "audio_command_text" for _, task in pairs)
if has_audio_command and args.mode in ("export", "verify"):
if int(args.audio_batch_size) != HFD_ONLINE_ENCODER_BATCH_SIZE:
raise ValueError(
"Streaming audio-command hidden states must use "
f"--audio-batch-size {HFD_ONLINE_ENCODER_BATCH_SIZE}; larger "
"batches are not bitwise equal to real one-window inference."
)
if str(args.save_dtype) != "bf16":
raise ValueError(
"Streaming audio-command hidden states must be saved as bf16"
)
if args.storage_format == "source":
if args.mode == "benchmark":
raise ValueError(
"--storage-format source uses complete WAVs and does not support "
"the window-count based benchmark mode; use export with "
"--limit-sources for a representative timed subset."
)
if len(pairs) != 1 or pairs[0][0].modality != "audio" or pairs[0][1] != "audio_command_text":
raise ValueError(
"--storage-format source is intentionally restricted to the "
"HFD audio_command_text source"
)
if bool(args.audio_command_full_audio):
raise ValueError(
"--storage-format source requires windowed audio command export"
)
if int(args.limit) != 0:
raise ValueError(
"Use --limit-sources instead of --limit with source storage"
)
if (
int(args.motion_fps) != 50
or int(args.audio_sampling_rate) != 16000
or not math.isclose(
float(args.audio_command_window_seconds),
0.6,
)
or int(args.audio_command_sample_step_frames) != 20
):
raise ValueError(
"HFD source storage is fixed to 50 fps, 16 kHz, a 0.6 s "
"window and a 20-frame/0.4 s stride"
)
output_root = Path(args.output_root)
output_root.mkdir(parents=True, exist_ok=True)
if args.mode == "export" and is_rank0() and has_audio_command:
write_json_atomic(
output_root / "_reports" / "encoder_contract.json",
encoder_contract_payload(args.model_path),
)
log(f"Mode: {args.mode}")
log(f"Tasks: {','.join(tasks)}")
log(f"Sources: {','.join(sources)}")
log(f"Output root: {output_root}")
if args.storage_format == "source":
log(
"Storage: source (one complete WAV per safetensors, "
f"limit_sources={int(args.limit_sources)})"
)
else:
log(f"Storage: {args.storage_format} shard_size={int(args.shard_size)}")
log(
"Audio command: "
f"full_audio={bool(args.audio_command_full_audio)} "
f"window={float(args.audio_command_window_seconds):g}s "
f"step={int(args.audio_command_sample_step_frames)} motion frames"
)
if args.mode == "estimate":
run_estimate(args, pairs)
return
if args.mode == "status":
run_status(args, pairs)
return
if args.mode == "verify":
if len(pairs) != 1 or pairs[0][0].modality != "audio":
raise ValueError("Online hidden verification requires one audio pair")
run_verify_online_hidden(args, pairs[0])
return
run_benchmark_or_export(args, pairs)
if __name__ == "__main__":
main()
from __future__ import annotations
import argparse
import gc
import hashlib
import json
import math
import os
import random
import sqlite3
import subprocess
import time
import traceback
import warnings
import wave
from collections import deque
from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Iterable, Iterator
import numpy as np
import pandas as pd
import torch
import torch.distributed as dist
from safetensors import safe_open
from safetensors.torch import save_file
from tqdm import tqdm
from transformers import AutoConfig, Qwen2_5OmniProcessor, Qwen2_5OmniThinkerForConditionalGeneration
from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import Qwen2_5OmniProcessorKwargs
from psi.data.seed.full_duplex_audio_contract import (
AUDIO_COMMAND_STREAMING_PROMPT,
HFD_AUDIO_SOURCE_FORMAT,
HFD_ONLINE_ENCODER_ATTENTION,
HFD_ONLINE_ENCODER_BATCH_SIZE,
HFD_ONLINE_ENCODER_CONTRACT,
HFD_ONLINE_ENCODER_DTYPE,
encoder_contract_payload,
qwen_model_fingerprint,
write_json_atomic,
)
from psi.data.seed.seed_audio_motion_dataset import (
_AUDIO_DANCE_PROMPT_POOLS,
ffmpeg_bin,
ffprobe_bin,
)
from psi.data.seed.motion_frame_index import read_motion_frame_count
from psi.data.seed.seed_video_motion_dataset import _VIDEO_MOTION_PROMPT_POOLS
from psi.data.seed.text_prompt import (
DEFAULT_SEED_TEXT_FIELDS,
collect_seed_text_candidates,
compose_seed_motion_prompt_from_candidates,
)
OMNI_DEFAULT_SYSTEM_PROMPT = (
"You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, "
"capable of perceiving auditory and visual inputs, as well as generating text and speech."
)
ACTIVE_MODEL_FINGERPRINT = ""
SUPPORTED_TASKS = (
"text",
"audio_text",
"audio_only",
"audio_command_text",
"audio_command_only",
"video_clip_text",
"video_clip_only",
)
DEFAULT_AUDIO_FILE_EXTENSIONS = (".wav", ".flac", ".mp3", ".m4a", ".aac", ".ogg", ".npy")
class SourcePreset:
name: str
modality: str
dataset_root: Path
manifest_path: Path
allowed_tasks: tuple[str, ...]
instruction_key: str = ""
audio_path_key: str = "audio_path"
video_path_key: str = "video_path"
video_embed_dir: Path | None = None
is_audio_dir_manifest: bool = False
audio_file_extensions: tuple[str, ...] = DEFAULT_AUDIO_FILE_EXTENSIONS
text_fields: tuple[str, ...] = tuple(DEFAULT_SEED_TEXT_FIELDS)
SOURCE_PRESETS: dict[str, SourcePreset] = {
"seed_text": SourcePreset(
name="seed_text",
modality="text",
dataset_root=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all"),
manifest_path=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/metadata/seed_metadata_v004.csv"),
allowed_tasks=("text",),
),
"2264_with_audio": SourcePreset(