#!/usr/bin/env python3 """Generate audited speech pieces for HFD hesitation and intra-command pause. The script never cuts an existing waveform. It regenerates the unfinished prefix and continuation suffix of every splittable bilingual command in the same eight VoiceDesign identities used by the base corpus. Tiny filled-pause and repair-cue inventories are generated separately with disjoint train/val phrase IDs. Runtime data construction overlays these pieces on one continuous office background and inserts an exact 0.4-second-grid pause between them. """ from __future__ import annotations import argparse import concurrent.futures import hashlib import io import json import math import os import re import wave from pathlib import Path from typing import Any, Iterable import numpy as np FORMAT = "hfd_hesitation_pause_tts_v1" PLAN_FORMAT = "hfd_hesitation_pause_plan_v3" TARGET_RATE = 16_000 MAX_INSTRUCTION_SEGMENT_SECONDS = 8.0 # A marker is centred inside a gap of at most 2.8 s and keeps 0.16 s of # surrounding background. Longer TTS output cannot satisfy the formal data # contract and must be regenerated rather than silently stretching the task. MAX_MARKER_SECONDS = 2.64 DEFAULT_MODEL = Path( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen/" "Qwen3-TTS-12Hz-1.7B-VoiceDesign" ) DEFAULT_METADATA = Path( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/" "user_workspace/buwendong/user/code/qwen3-tts/metadata.jsonl" ) DEFAULT_OUTPUT = Path( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/" "user_workspace/yetinghong/videogen/hfd_hesitation_pause_v2" ) DEFAULT_INSTRUCTION_AUDIO_ROOT = Path( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/" "user_workspace/yetinghong/videogen/hfd_hesitation_pause_v1" ) MARKERS: tuple[dict[str, Any], ...] = ( {"split": "train", "category": "hesitation_filler", "delivery": "vocalized_filler", "min_duration_seconds": 0.8, "segment_id": "hes_fill_zh_01", "language": "zh", "text": "嗯……"}, {"split": "train", "category": "hesitation_filler", "delivery": "vocalized_filler", "min_duration_seconds": 0.8, "segment_id": "hes_fill_zh_02", "language": "zh", "text": "呃……"}, {"split": "train", "category": "hesitation_filler", "delivery": "phrase_filler", "min_duration_seconds": 0.4, "segment_id": "hes_fill_zh_03", "language": "zh", "text": "这个……"}, {"split": "val", "category": "hesitation_filler", "delivery": "vocalized_filler", "min_duration_seconds": 0.8, "segment_id": "hes_fill_zh_11", "language": "zh", "text": "唔……"}, {"split": "val", "category": "hesitation_filler", "delivery": "phrase_filler", "min_duration_seconds": 0.4, "segment_id": "hes_fill_zh_12", "language": "zh", "text": "让我想想……"}, {"split": "train", "category": "hesitation_filler", "delivery": "vocalized_filler", "min_duration_seconds": 0.8, "segment_id": "hes_fill_en_01", "language": "en", "text": "Um..."}, {"split": "train", "category": "hesitation_filler", "delivery": "vocalized_filler", "min_duration_seconds": 0.8, "segment_id": "hes_fill_en_02", "language": "en", "text": "Uh..."}, {"split": "train", "category": "hesitation_filler", "delivery": "vocalized_filler", "min_duration_seconds": 0.8, "segment_id": "hes_fill_en_03", "language": "en", "text": "Er..."}, {"split": "val", "category": "hesitation_filler", "delivery": "vocalized_filler", "min_duration_seconds": 0.8, "segment_id": "hes_fill_en_11", "language": "en", "text": "Hmm..."}, {"split": "val", "category": "hesitation_filler", "delivery": "phrase_filler", "min_duration_seconds": 0.4, "segment_id": "hes_fill_en_12", "language": "en", "text": "Let me think..."}, {"split": "train", "category": "repair_cue", "segment_id": "repair_zh_01", "language": "zh", "text": "不对,我是说……"}, {"split": "train", "category": "repair_cue", "segment_id": "repair_zh_02", "language": "zh", "text": "应该是……"}, {"split": "train", "category": "repair_cue", "segment_id": "repair_zh_03", "language": "zh", "text": "不是,改成……"}, {"split": "val", "category": "repair_cue", "segment_id": "repair_zh_11", "language": "zh", "text": "等等,我是说……"}, {"split": "val", "category": "repair_cue", "segment_id": "repair_zh_12", "language": "zh", "text": "抱歉,应该是……"}, {"split": "train", "category": "repair_cue", "segment_id": "repair_en_01", "language": "en", "text": "No, I mean..."}, {"split": "train", "category": "repair_cue", "segment_id": "repair_en_02", "language": "en", "text": "Actually..."}, {"split": "train", "category": "repair_cue", "segment_id": "repair_en_03", "language": "en", "text": "Sorry, make that..."}, {"split": "val", "category": "repair_cue", "segment_id": "repair_en_11", "language": "en", "text": "Wait, I mean..."}, {"split": "val", "category": "repair_cue", "segment_id": "repair_en_12", "language": "en", "text": "Sorry, I meant..."}, ) # Fixed, human-auditable train/validation examples used by both the automatic # closed-loop suite and the pre-training GT review. They use motion sources # already audited by the existing suite, while the composite speech is built # with exactly the same regenerated pieces as training. REVIEW_EXAMPLES: tuple[dict[str, Any], ...] = ( { "name": "pause_grab_zh_female_01_zh", "split": "train", "subtype": "silent-pause", "final_audio_leaf": "000006", "voice_name": "female_01_zh", "gap_steps": 4, }, { "name": "pause_jog_en_female_02_en", "split": "val", "subtype": "silent-pause", "final_audio_leaf": "000175", "voice_name": "female_02_en", "gap_steps": 6, }, { "name": "filled_wave_zh_female_01_zh", "split": "train", "subtype": "filled-pause", "final_audio_leaf": "000155", "voice_name": "female_01_zh", "marker_id": "hes_fill_zh_02", "gap_steps": 6, }, { "name": "filled_pick_en_female_02_en", "split": "val", "subtype": "filled-pause", "final_audio_leaf": "000227", "voice_name": "female_02_en", "marker_id": "hes_fill_en_11", "gap_steps": 5, }, { "name": "repair_grab_to_left_zh_female_01_zh", "split": "train", "subtype": "self-repair", "provisional_audio_leaf": "000006", "final_audio_leaf": "000120", "voice_name": "female_01_zh", "marker_id": "repair_zh_01", "repair_style": "explicit-cue", "gap_steps": 7, }, { "name": "repair_pick_to_jog_en_female_02_en", "split": "val", "subtype": "self-repair", "provisional_audio_leaf": "000227", "final_audio_leaf": "000175", "voice_name": "female_02_en", "repair_style": "implicit-restart", "gap_steps": 7, }, ) def voice_specs() -> tuple[dict[str, str], ...]: english = { "female_01_en": "Use a natural everyday adult female voice. Speak clearly and calmly, with a friendly conversational tone, not theatrical.", "female_02_en": "Use a natural everyday young adult female voice. Speak in a relaxed, neutral, and easy-to-understand style.", "male_01_en": "Use a natural everyday adult male voice. Speak clearly with a calm, friendly, conversational tone, not dramatic.", "male_02_en": "Use a natural everyday young adult male voice. Speak in a relaxed and neutral style, like a normal spoken description.", } chinese = { "female_01_zh": "使用自然日常的成年女性声音,语气清晰平和,像正常描述动作,不要夸张表演。", "female_02_zh": "使用自然日常的年轻女性声音,语速适中,语气放松中性,适合朗读简短动作描述。", "male_01_zh": "使用自然日常的成年男性声音,发音清楚,语气平稳友好,像普通口语描述。", "male_02_zh": "使用自然日常的年轻男性声音,语气放松中性,不要戏剧化,适合动作短语朗读。", } return tuple( { "name": name, "language": name.rsplit("_", 1)[-1], "qwen_language": "English" if name.endswith("_en") else "Chinese", "instruction": instruction, } for name, instruction in (*english.items(), *chinese.items()) ) def normalise_audio_dir(value: str) -> str: parts = Path(str(value).replace("\\", "/")).parts try: start = parts.index("seed_audio") except ValueError: return Path(str(value)).as_posix().strip("/") return Path(*parts[start:]).as_posix() def _strip_terminal_punctuation(text: str) -> str: return re.sub(r"[\s,.;:!?,。;:!?、…]+$", "", text.strip()) def split_english(text: str) -> tuple[str, str] | None: clean = _strip_terminal_punctuation(text) words = clean.split() if len(words) < 2: return None boundary = min(max(int(round(len(words) * 0.5)), 1), len(words) - 1) return " ".join(words[:boundary]), " ".join(words[boundary:]) def split_chinese(text: str) -> tuple[str, str] | None: import jieba clean = re.sub(r"\s+", "", _strip_terminal_punctuation(text)) if len(clean) < 4: return None # Only the one-time planning phase needs jieba. Generation and # finalisation consume the frozen plan, so the Qwen-TTS environment does # not gain an extra runtime dependency. for term in ( "顺时针", "逆时针", "向前", "向后", "向左", "向右", "左前方", "右前方", "左后方", "右后方", "转身", "后退", "前进", "走路", "跑步", "慢跑", "跳跃", "双手", "单手", "双臂", "左臂", "右臂", "左腿", "右腿", ): jieba.add_word(term, freq=1_000_000) tokens = tuple(jieba.lcut(clean, HMM=False)) if "".join(tokens) != clean: raise RuntimeError(f"Chinese tokenizer changed command text: {clean!r}") word_boundaries: set[int] = set() offset = 0 for token in tokens[:-1]: offset += len(token) word_boundaries.add(offset) target = len(clean) / 2.0 # A character midpoint can split one Chinese word (for example # ``双臂交|叉站立``), producing unnatural TTS and teaching an artificial # hesitation. Only admit explicitly grammatical boundaries. It is safer # to omit a command than to manufacture a corrupted utterance; the source # corpus is large enough that this does not reduce any established task. scored: dict[int, int] = {} def offer(boundary: int, priority: int) -> None: if ( 2 <= boundary <= len(clean) - 2 and boundary in word_boundaries ): scored[boundary] = min(scored.get(boundary, priority), priority) for match in re.finditer(r"[,;:、]", clean): offer(match.end(), 0) for phrase in ( "然后", "随后", "接着", "最后", "并且", "同时", "再将", "再把", "再向", ): start = clean.find(phrase) while start >= 0: offer(start, 0) start = clean.find(phrase, start + 1) for phrase in ( "放至", "放到", "放下", "拿起", "举至", "举到", "转身", "转向", "走向", "跑向", "跳向", "起身", "坐下", "站立", "蹲下", "躺下", "挥动", "抬起", "伸出", "扔向", "推向", "拉向", "打开", "关闭", ): start = clean.find(phrase, 2) while start >= 0: offer(start, 1) start = clean.find(phrase, start + 1) for phrase in ( "双手", "单手", "左手", "右手", "双臂", "左臂", "右臂", ): if clean.startswith(phrase): offer(len(phrase), 2) for match in re.finditer(r"(?:前|后|左|右|上|下|外|内|身)[^,;:、]{0,5}(?:方|侧|处|位)", clean): offer(match.end(), 2) for phrase in ( "检查", "停止", "保持", "开始", "继续", "尝试", "试图", ): if clean.startswith(phrase): offer(len(phrase), 2) direction = re.match(r"^(?:向|往)(?:前|后|左|右|上|下)+", clean) if direction is not None: offer(direction.end(), 2) for match in re.finditer(r"[时后]", clean): offer(match.end(), 3) # Any complete-word boundary is a valid natural hesitation point. These # generic candidates have lower priority than punctuation/connectives and # retain broad command coverage without ever splitting one word in half. for boundary in word_boundaries: offer(boundary, 4) if not scored: return None boundary = min( scored, key=lambda index: (scored[index], abs(index - target), index), ) return clean[:boundary], clean[boundary:] def load_command_plans( metadata_path: Path, *, max_commands: int = 0, ) -> tuple[list[dict[str, str]], int]: by_audio_dir: dict[str, dict[str, str]] = {} with metadata_path.open("r", encoding="utf-8") as handle: for line in handle: if not line.strip(): continue raw = json.loads(line) audio_dir = normalise_audio_dir(str(raw["audio_path"])) pair = { "audio_dir": audio_dir, "instruction": str(raw["content_short_description"]).strip(), "instruction_zh": str(raw["content_short_description_zh"]).strip(), } previous = by_audio_dir.setdefault(audio_dir, pair) if previous != pair: raise RuntimeError( f"One audio directory maps to multiple instructions: {audio_dir}" ) plans: list[dict[str, str]] = [] rejected = 0 for audio_dir in sorted(by_audio_dir): row = by_audio_dir[audio_dir] en = split_english(row["instruction"]) zh = split_chinese(row["instruction_zh"]) if en is None or zh is None: rejected += 1 continue plans.append( { **row, "prefix_en": en[0], "suffix_en": en[1], "prefix_zh": zh[0], "suffix_zh": zh[1], } ) if max_commands > 0: plans = plans[: int(max_commands)] if not plans: raise RuntimeError("No bilingual command can be split safely") return plans, rejected def instruction_audio_path( root: Path, plan: dict[str, str], voice: dict[str, str], role: str, ) -> Path: audio_leaf = plan["audio_dir"].rsplit("/", 1)[-1] return root / "audio" / "instruction_segments" / audio_leaf / f"{voice['name']}_{role}.wav" def marker_audio_path( root: Path, marker: dict[str, str], voice: dict[str, str], ) -> Path: return root / "audio" / "markers" / marker["split"] / marker["segment_id"] / f"{voice['name']}.wav" def marker_delivery(marker: dict[str, Any]) -> str: return str(marker.get("delivery", "repair_cue")) def marker_min_duration_seconds(marker: dict[str, Any]) -> float: return float(marker.get("min_duration_seconds", 0.25)) def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def write_pcm16(path: Path, waveform: np.ndarray) -> None: values = np.asarray(waveform, dtype=np.float32).reshape(-1) if values.size <= 0 or not np.isfinite(values).all(): raise RuntimeError(f"TTS produced invalid waveform: {path}") peak = float(np.max(np.abs(values), initial=0.0)) if peak > 1.0: values = values / np.float32(peak) pcm = np.rint(np.clip(values, -1.0, 1.0) * 32767.0).astype(" np.ndarray: values = np.asarray(waveform, dtype=np.float32).reshape(-1) peak = float(np.max(np.abs(values), initial=0.0)) if peak <= 0.0: return values active = np.flatnonzero(np.abs(values) >= max(peak * 0.02, 0.002)) if not active.size: return values start = max(int(active[0]) - int(round(0.08 * TARGET_RATE)), 0) end = min(int(active[-1]) + 1 + int(round(0.16 * TARGET_RATE)), values.size) return np.ascontiguousarray(values[start:end], dtype=np.float32) def valid_pcm16(path: Path) -> bool: if not path.is_file(): return False try: with wave.open(str(path), "rb") as handle: return ( handle.getnchannels() == 1 and handle.getsampwidth() == 2 and handle.getframerate() == TARGET_RATE and 0 < handle.getnframes() < 20 * TARGET_RATE and handle.getcomptype() == "NONE" ) except (EOFError, wave.Error): return False def load_existing_output_index(path: Path, output_root: Path) -> set[str]: """Load a trusted snapshot of atomically completed output WAV paths. A resumed eight-worker generation otherwise performs more than eighty thousand random ``stat``/WAV-header reads against shared storage before a single GPU can resume. The snapshot is only a generation accelerator: ``finalize`` still opens and validates every expected WAV before writing the production marker consumed by training. """ if not path.is_file(): raise FileNotFoundError(f"Missing existing-output index: {path}") indexed: set[str] = set() for line_number, line in enumerate( path.read_text(encoding="utf-8").splitlines(), start=1, ): value = line.strip() if not value: continue relative = Path(value) if ( relative.is_absolute() or ".." in relative.parts or relative.suffix.lower() != ".wav" ): raise ValueError( f"Unsafe existing-output index row {line_number}: {value!r}" ) indexed.add(relative.as_posix()) if not indexed: raise RuntimeError(f"Existing-output index is empty: {path}") print( f"[resume] trusted atomic snapshot entries={len(indexed)} " f"root={output_root}", flush=True, ) return indexed def atomic_json(path: Path, payload: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(path.suffix + f".tmp.{os.getpid()}") temporary.write_text( json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) temporary.replace(path) def atomic_jsonl(path: Path, rows: Iterable[dict[str, Any]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(path.suffix + f".tmp.{os.getpid()}") with temporary.open("w", encoding="utf-8") as handle: for row in rows: handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") handle.flush() os.fsync(handle.fileno()) temporary.replace(path) def plan(args: argparse.Namespace) -> None: plans, rejected = load_command_plans( args.metadata_jsonl, max_commands=args.max_commands, ) payload = { "format": PLAN_FORMAT, "metadata_jsonl": str(args.metadata_jsonl), "metadata_sha256": sha256(args.metadata_jsonl), "instruction_audio_root": str(args.instruction_audio_root.resolve()), "split_algorithm": "english-word-boundary__chinese-jieba-boundary-v2", "splittable_commands": len(plans), "rejected_commands": rejected, "instruction_segment_wavs": len(plans) * 2 * 8, "marker_wavs": sum( 4 for marker in MARKERS ), "pause_grid_seconds": [0.4, 0.8, 1.2, 1.6, 2.0, 2.4, 2.8], "pause_probability_groups": {"0.4-0.8": 0.60, "1.2-1.6": 0.30, "2.0-2.8": 0.10}, "commands": plans, "markers": list(MARKERS), } atomic_json(args.output_root / "reports" / "generation_plan.json", payload) print( f"[ready] splittable={len(plans):,} rejected={rejected:,} " f"segment_wavs={payload['instruction_segment_wavs']:,} " f"marker_wavs={payload['marker_wavs']:,} " f"plan={args.output_root / 'reports' / 'generation_plan.json'}", flush=True, ) def load_frozen_plan(args: argparse.Namespace) -> tuple[list[dict[str, str]], int]: path = args.output_root / "reports" / "generation_plan.json" if not path.is_file(): raise RuntimeError( f"Missing frozen generation plan: {path}. Run --mode plan first." ) payload = json.loads(path.read_text(encoding="utf-8")) if payload.get("format") != PLAN_FORMAT: raise RuntimeError( f"Unsupported generation plan format: {payload.get('format')!r}" ) expected_sha = str(payload.get("metadata_sha256", "")) active_sha = sha256(args.metadata_jsonl) if expected_sha != active_sha: raise RuntimeError( "Metadata changed after the TTS split plan was frozen: " f"expected={expected_sha} active={active_sha}" ) planned_instruction_root = Path( str(payload.get("instruction_audio_root", args.output_root)) ).resolve() if planned_instruction_root != args.instruction_audio_root.resolve(): raise RuntimeError( "Instruction-audio root changed after the split plan was frozen: " f"expected={planned_instruction_root} " f"active={args.instruction_audio_root.resolve()}" ) plans = list(payload.get("commands", ())) if not plans: raise RuntimeError(f"Frozen generation plan contains no commands: {path}") return plans, int(payload.get("rejected_commands", 0)) def generate(args: argparse.Namespace) -> None: import torch from qwen_tts import Qwen3TTSModel from scipy.signal import resample_poly if not 0 <= args.shard_index < args.shard_count: raise ValueError(f"Invalid shard {args.shard_index}/{args.shard_count}") plans, _ = load_frozen_plan(args) existing_outputs = ( load_existing_output_index(args.existing_index, args.output_root) if args.existing_index is not None else None ) def needs_generation(path: Path) -> bool: if args.overwrite: return True if existing_outputs is None: return not valid_pcm16(path) relative = path.relative_to(args.output_root).as_posix() return relative not in existing_outputs voices = tuple( voice for index, voice in enumerate(voice_specs()) if index % args.shard_count == args.shard_index ) torch.backends.cuda.matmul.allow_tf32 = True model = Qwen3TTSModel.from_pretrained( str(args.model_path), device_map=args.device, dtype=torch.bfloat16, attn_implementation="flash_attention_2", ) for voice in voices: jobs: list[tuple[str, str, Path, str, float]] = [] language = voice["language"] if args.instruction_audio_root.resolve() == args.output_root.resolve(): for command in plans: for role in ("prefix", "suffix"): path = instruction_audio_path( args.instruction_audio_root, command, voice, role, ) if needs_generation(path): jobs.append( ( f"{command['audio_dir']}:{role}", command[f"{role}_{language}"], path, role, 0.0, ) ) for marker in MARKERS: if marker["language"] != language: continue path = marker_audio_path(args.output_root, marker, voice) if needs_generation(path): jobs.append( ( marker["segment_id"], marker["text"], path, marker_delivery(marker), marker_min_duration_seconds(marker), ) ) for start in range(0, len(jobs), max(args.batch_size, 1)): batch = jobs[start : start + max(args.batch_size, 1)] instructions = [] for _, _, _, role, _ in batch: suffix = ( " Read this as an unfinished first part of a longer command; do not add words." if role == "prefix" and language == "en" else " 这是较长指令中尚未说完的前半句,不要添加原文之外的词。" if role == "prefix" else " Read this naturally as the continuation of an action command; do not add words." if role == "suffix" and language == "en" else " 这是动作指令自然接续的后半句,不要添加原文之外的词。" if role == "suffix" else " Sustain this single hesitation sound naturally for about 1.0 to 1.6 seconds, as if genuinely thinking; do not make it abrupt and do not add words." if role == "vocalized_filler" and language == "en" else " 把这个单独的犹豫音自然拖长到约1.0至1.6秒,像正在认真思考,不要短促,也不要添加其他词。" if role == "vocalized_filler" else " Say this hesitation phrase naturally at an ordinary conversational pace, without adding words." if role == "phrase_filler" and language == "en" else " 以自然口语速度说出这个犹豫短语,不要刻意加速,也不要添加其他词。" if role == "phrase_filler" else " Say this repair cue naturally and concisely, without adding words." if role == "repair_cue" and language == "en" else " 自然、简洁地说出这个改口提示,不要添加其他词。" if role == "repair_cue" else "" ) instructions.append(voice["instruction"] + suffix) pending = list(zip(batch, instructions)) for attempt in range(1, args.max_tts_attempts + 1): waveforms, source_rate = model.generate_voice_design( text=[job[1] for job, _ in pending], language=[voice["qwen_language"]] * len(pending), instruct=[instruction for _, instruction in pending], ) if len(waveforms) != len(pending): raise RuntimeError("Qwen TTS returned the wrong batch size") retry: list[ tuple[tuple[str, str, Path, str, float], str] ] = [] for pending_job, waveform in zip(pending, waveforms): job, instruction = pending_job values = np.asarray(waveform, dtype=np.float32).reshape(-1) if int(source_rate) != TARGET_RATE: divisor = math.gcd(int(source_rate), TARGET_RATE) values = resample_poly( values, TARGET_RATE // divisor, int(source_rate) // divisor, ).astype(np.float32, copy=False) values = trim_tts_silence(values) max_samples = int( round( ( MAX_MARKER_SECONDS if job[3] in { "vocalized_filler", "phrase_filler", "repair_cue", } else MAX_INSTRUCTION_SEGMENT_SECONDS ) * TARGET_RATE ) ) if ( values.size <= 0 or values.size < int(round(float(job[4]) * TARGET_RATE)) or values.size > max_samples or not np.isfinite(values).all() ): retry.append((job, instruction)) continue write_pcm16(job[2], values) if not retry: break print( f"[tts-retry] shard={args.shard_index}/{args.shard_count} " f"voice={voice['name']} attempt={attempt} invalid={len(retry)}", flush=True, ) pending = retry else: failed = ", ".join(str(job[2]) for job, _ in pending[:8]) raise RuntimeError( "Qwen TTS repeatedly produced empty, non-finite, or " "task-contract-overlong " f"waveforms after {args.max_tts_attempts} attempts: {failed}" ) print( f"[tts] shard={args.shard_index}/{args.shard_count} " f"voice={voice['name']} done={min(start + len(batch), len(jobs))}/{len(jobs)}", flush=True, ) del model torch.cuda.empty_cache() def _manifest_audio_fields( path: Path, *, min_seconds: float = 0.0, max_seconds: float = MAX_INSTRUCTION_SEGMENT_SECONDS, ) -> dict[str, Any]: try: payload = path.read_bytes() with wave.open(io.BytesIO(payload), "rb") as handle: samples = int(handle.getnframes()) if not ( handle.getnchannels() == 1 and handle.getsampwidth() == 2 and handle.getframerate() == TARGET_RATE and int(round(min_seconds * TARGET_RATE)) <= samples <= int(round(max_seconds * TARGET_RATE)) and handle.getcomptype() == "NONE" ): raise RuntimeError( "invalid PCM16/16kHz/mono header or duration " f"(range={min_seconds:g}..{max_seconds:g}s)" ) frame_bytes = handle.readframes(samples) except (EOFError, OSError, wave.Error, RuntimeError) as exc: raise RuntimeError(f"Missing/invalid hesitation-pause WAV: {path}: {exc}") from exc values = np.frombuffer(frame_bytes, dtype=" 0.0 else 1.0 return { "audio_path": str(path.resolve()), "sample_rate": TARGET_RATE, "speech_samples_16k": samples, "duration_seconds": samples / float(TARGET_RATE), "peak": peak, "speech_scale": scale, "sha256": hashlib.sha256(payload).hexdigest(), } def _read_pcm_float(path: Path) -> np.ndarray: from scipy.signal import resample_poly with wave.open(str(path), "rb") as handle: channels = int(handle.getnchannels()) width = int(handle.getsampwidth()) source_rate = int(handle.getframerate()) frames = int(handle.getnframes()) if width != 2 or channels < 1 or frames <= 0: raise RuntimeError(f"Unsupported review audio WAV: {path}") values = np.frombuffer(handle.readframes(frames), dtype=" np.ndarray: values = _read_pcm_float(path) peak = float(np.max(np.abs(values), initial=0.0)) scale = min(1.0, (10.0 ** (-6.0 / 20.0)) / peak) if peak > 0.0 else 1.0 return np.asarray(values * np.float32(scale), dtype=np.float32) def build_review_examples( args: argparse.Namespace, plans: list[dict[str, str]], ) -> list[dict[str, Any]]: by_leaf = {plan["audio_dir"].rsplit("/", 1)[-1]: plan for plan in plans} voices = {voice["name"]: voice for voice in voice_specs()} markers = {marker["segment_id"]: marker for marker in MARKERS} rows: list[dict[str, Any]] = [] for spec in REVIEW_EXAMPLES: final_plan = by_leaf.get(str(spec["final_audio_leaf"])) voice = voices.get(str(spec["voice_name"])) if final_plan is None or voice is None: raise RuntimeError(f"Review example cannot resolve command/voice: {spec}") language = str(voice["language"]) prefix_path = instruction_audio_path( args.instruction_audio_root, final_plan, voice, "prefix" ) suffix_path = instruction_audio_path( args.instruction_audio_root, final_plan, voice, "suffix" ) prefix = _scaled_review_piece(prefix_path) suffix = _scaled_review_piece(suffix_path) gap_samples = int(spec["gap_steps"]) * int(round(0.4 * TARGET_RATE)) subtype = str(spec["subtype"]) component_rows: list[dict[str, Any]] = [] def assemble(parts: list[tuple[str, np.ndarray, str]]) -> np.ndarray: offset = 0 arrays: list[np.ndarray] = [] for role, values, text in parts: array = np.asarray(values, dtype=np.float32).reshape(-1) arrays.append(array) component_rows.append( { "role": role, "text": text, "start_sample": offset, "end_sample": offset + int(array.size), } ) offset += int(array.size) return np.concatenate(arrays).astype(np.float32, copy=False) if subtype == "silent-pause": waveform = assemble( [ ("final-prefix", prefix, final_plan[f"prefix_{language}"]), ("background-only-pause", np.zeros(gap_samples, np.float32), ""), ("final-suffix", suffix, final_plan[f"suffix_{language}"]), ] ) elif subtype == "filled-pause": marker = markers.get(str(spec.get("marker_id", ""))) if marker is None or marker["split"] != spec["split"]: raise RuntimeError(f"Review marker split mismatch: {spec}") marker_path = marker_audio_path(args.output_root, marker, voice) marker_waveform = _scaled_review_piece(marker_path) if marker_waveform.size + round(0.16 * TARGET_RATE) > gap_samples: raise RuntimeError( "Review marker cannot fit its declared uncertainty span: " f"{marker_path} gap_steps={spec['gap_steps']}" ) leading = (gap_samples - int(marker_waveform.size)) // 2 trailing = gap_samples - int(marker_waveform.size) - leading waveform = assemble( [ ("final-prefix", prefix, final_plan[f"prefix_{language}"]), ( "hesitation-background-before", np.zeros(leading, np.float32), "", ), ("hesitation-filler", marker_waveform, marker["text"]), ( "hesitation-background-after", np.zeros(trailing, np.float32), "", ), ("final-suffix", suffix, final_plan[f"suffix_{language}"]), ] ) elif subtype == "self-repair": provisional_plan = by_leaf.get(str(spec["provisional_audio_leaf"])) if provisional_plan is None: raise RuntimeError(f"Missing provisional review command: {spec}") provisional = _scaled_review_piece( instruction_audio_path( args.instruction_audio_root, provisional_plan, voice, "prefix", ) ) final_complete_path = ( args.metadata_jsonl.parent / final_plan["audio_dir"] / f"{voice['name']}.wav" ) # Match the production dataset's per-utterance -6 dB peak policy. final_complete = _scaled_review_piece(final_complete_path) repair_style = str(spec.get("repair_style", "explicit-cue")) if repair_style == "implicit-restart": waveform = assemble( [ ( "provisional-prefix", provisional, provisional_plan[f"prefix_{language}"], ), ( "repair-implicit-background", np.zeros(gap_samples, np.float32), "", ), ( "complete-final-instruction", final_complete, final_plan[ "instruction_zh" if language == "zh" else "instruction" ], ), ] ) else: marker = markers.get(str(spec.get("marker_id", ""))) if marker is None or marker["split"] != spec["split"]: raise RuntimeError(f"Review marker split mismatch: {spec}") marker_path = marker_audio_path(args.output_root, marker, voice) marker_waveform = _scaled_review_piece(marker_path) if marker_waveform.size + round(0.16 * TARGET_RATE) > gap_samples: raise RuntimeError( "Review repair cue cannot fit its uncertainty span: " f"{marker_path} gap_steps={spec['gap_steps']}" ) leading = (gap_samples - int(marker_waveform.size)) // 2 trailing = gap_samples - int(marker_waveform.size) - leading waveform = assemble( [ ( "provisional-prefix", provisional, provisional_plan[f"prefix_{language}"], ), ( "repair-background-before", np.zeros(leading, np.float32), "", ), ("repair-cue", marker_waveform, marker["text"]), ( "repair-background-after", np.zeros(trailing, np.float32), "", ), ( "complete-final-instruction", final_complete, final_plan[ "instruction_zh" if language == "zh" else "instruction" ], ), ] ) else: raise RuntimeError(f"Unknown review subtype: {subtype}") output_path = ( args.output_root / "audio" / "review_examples" / str(spec["split"]) / f"{spec['name']}.wav" ) write_pcm16(output_path, waveform) rows.append( { **spec, "audio_path": str(output_path.resolve()), "final_instruction": final_plan[ "instruction_zh" if language == "zh" else "instruction" ], "duration_seconds": waveform.size / float(TARGET_RATE), "speech_samples_16k": int(waveform.size), "sha256": sha256(output_path), "components": component_rows, } ) atomic_json( args.output_root / "reports" / "review_examples.json", { "format": "hfd_hesitation_pause_review_examples_v1", "examples": rows, }, ) return rows def finalize(args: argparse.Namespace) -> None: plans, rejected = load_frozen_plan(args) jobs: list[tuple[Path, dict[str, Any]]] = [] for command in plans: for voice in voice_specs(): language = voice["language"] for role in ("prefix", "suffix"): path = instruction_audio_path( args.instruction_audio_root, command, voice, role, ) jobs.append( ( path, { "format": FORMAT, "category": "instruction_segment", "segment_id": ( f"{command['audio_dir'].rsplit('/', 1)[-1]}_" f"{voice['name']}_{role}" ), "split": "shared", "voice_name": voice["name"], "language": language, "text": command[f"{role}_{language}"], "audio_dir": command["audio_dir"], "instruction": command["instruction"], "instruction_zh": command["instruction_zh"], "segment_role": role, }, ) ) for marker in MARKERS: for voice in voice_specs(): if voice["language"] != marker["language"]: continue path = marker_audio_path(args.output_root, marker, voice) jobs.append( ( path, { "format": FORMAT, **marker, "voice_name": voice["name"], "segment_role": "marker", "audio_dir": "", "instruction": "", "instruction_zh": "", }, ) ) def inspect( job: tuple[Path, dict[str, Any]], ) -> tuple[dict[str, Any] | None, dict[str, str] | None]: path, base_row = job try: max_seconds = ( MAX_INSTRUCTION_SEGMENT_SECONDS if base_row["category"] == "instruction_segment" else MAX_MARKER_SECONDS ) min_seconds = ( marker_min_duration_seconds(base_row) if base_row["category"] != "instruction_segment" else 0.0 ) return { **base_row, **_manifest_audio_fields( path, min_seconds=min_seconds, max_seconds=max_seconds, ), }, None except RuntimeError as exc: return None, {"path": str(path.resolve()), "error": str(exc)} rows: list[dict[str, Any]] = [] invalid_outputs: list[dict[str, str]] = [] with concurrent.futures.ThreadPoolExecutor( max_workers=args.finalize_workers, thread_name_prefix="hfd-wav-audit", ) as executor: for index, (row, invalid) in enumerate(executor.map(inspect, jobs), start=1): if row is not None: rows.append(row) if invalid is not None: invalid_outputs.append(invalid) if index % 5_000 == 0 or index == len(jobs): print( f"[audit] checked={index:,}/{len(jobs):,} " f"invalid={len(invalid_outputs):,}", flush=True, ) expected_rows = len(plans) * len(voice_specs()) * 2 + sum( 1 for marker in MARKERS for voice in voice_specs() if voice["language"] == marker["language"] ) audit_path = args.output_root / "reports" / "finalize_audit.json" atomic_json( audit_path, { "format": "hfd_hesitation_pause_finalize_audit_v1", "expected_rows": int(expected_rows), "valid_rows": len(rows), "invalid_count": len(invalid_outputs), "invalid_outputs": invalid_outputs, }, ) if invalid_outputs: examples = " | ".join( f"{item['path']}: {item['error']}" for item in invalid_outputs[:20] ) raise RuntimeError( "Hesitation/pause finalize rejected " f"{len(invalid_outputs)}/{expected_rows} WAVs; " f"full report={audit_path}; examples={examples}" ) if len(rows) != expected_rows: raise AssertionError( f"Finalize row accounting drifted: {len(rows)} != {expected_rows}" ) atomic_jsonl(args.output_root / "manifest.jsonl", rows) review_rows = build_review_examples(args, plans) atomic_json( args.output_root / "reports" / "production_complete.json", { "format": "hfd_hesitation_pause_production_v1", "manifest_format": FORMAT, "manifest_sha256": sha256(args.output_root / "manifest.jsonl"), "metadata_jsonl": str(args.metadata_jsonl.resolve()), "instruction_audio_root": str( args.instruction_audio_root.resolve() ), "instruction_audio_reused": ( args.instruction_audio_root.resolve() != args.output_root.resolve() ), "splittable_commands": len(plans), "rejected_commands": rejected, "manifest_rows": len(rows), "instruction_segment_rows": len(plans) * 16, "marker_rows": len(rows) - len(plans) * 16, "voice_count": 8, "finalize_audit_sha256": sha256(audit_path), "review_example_count": len(review_rows), }, ) print( f"[ready] commands={len(plans):,} rows={len(rows):,} rejected={rejected:,} " f"manifest={args.output_root / 'manifest.jsonl'}", flush=True, ) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--mode", choices=("plan", "generate", "finalize"), required=True) parser.add_argument("--metadata-jsonl", type=Path, default=DEFAULT_METADATA) parser.add_argument("--output-root", type=Path, default=DEFAULT_OUTPUT) parser.add_argument( "--instruction-audio-root", type=Path, default=DEFAULT_INSTRUCTION_AUDIO_ROOT, help=( "Root containing the frozen prefix/suffix WAVs. Point this at " "--output-root only when generating every instruction segment; " "the v2 overlay reuses the audited v1 instruction audio." ), ) parser.add_argument("--model-path", type=Path, default=DEFAULT_MODEL) parser.add_argument("--device", default="cuda:0") parser.add_argument("--batch-size", type=int, default=6) parser.add_argument("--max-tts-attempts", type=int, default=3) parser.add_argument("--finalize-workers", type=int, default=32) parser.add_argument("--shard-index", type=int, default=0) parser.add_argument("--shard-count", type=int, default=1) parser.add_argument("--max-commands", type=int, default=0) parser.add_argument("--overwrite", action="store_true") parser.add_argument( "--existing-index", type=Path, default=None, help=( "Optional newline-delimited paths relative to --output-root that " "were produced by prior atomic writes. This avoids reopening all " "existing WAVs during a shared-storage resume; --mode finalize " "still validates every file." ), ) args = parser.parse_args() args.metadata_jsonl = args.metadata_jsonl.expanduser().resolve() args.output_root = args.output_root.expanduser().resolve() args.instruction_audio_root = ( args.instruction_audio_root.expanduser().resolve() ) args.model_path = args.model_path.expanduser().resolve() if args.existing_index is not None: args.existing_index = args.existing_index.expanduser().resolve() if args.overwrite and args.existing_index is not None: parser.error("--overwrite and --existing-index are mutually exclusive") if ( args.batch_size < 1 or args.max_tts_attempts < 1 or args.finalize_workers < 1 or args.max_commands < 0 ): parser.error( "batch-size, max-tts-attempts, and finalize-workers must be " "positive; max-commands must be non-negative" ) return args def main() -> None: args = parse_args() if args.mode == "plan": plan(args) elif args.mode == "generate": generate(args) else: finalize(args) if __name__ == "__main__": main()