#!/usr/bin/env python3 """Generate and audit the small HFD emergency-stop/safe-hold TTS corpus. Generation is shardable by voice so four local GPUs can run independently. The final manifest is written only by ``--mode finalize`` after every expected file passes the exact PCM16/16 kHz contract consumed by HFD training. """ from __future__ import annotations import argparse import hashlib import json import math import os import wave from pathlib import Path from typing import Any import numpy as np FORMAT = "hfd_control_command_tts_v1" TARGET_RATE = 16_000 DEFAULT_MODEL = Path( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen/" "Qwen3-TTS-12Hz-1.7B-VoiceDesign" ) DEFAULT_OUTPUT = Path( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_control_commands_v1" ) # Each phrase ID represents one bilingual meaning. Every meaning is rendered # by female/male × speaker 01/02 × Chinese/English = exactly eight WAVs. PHRASES: tuple[dict[str, str], ...] = ( { "intent": "emergency-stop", "phrase_id": "stop_01", "zh": "停。", "en": "Stop.", }, { "intent": "emergency-stop", "phrase_id": "stop_02", "zh": "马上停下来。", "en": "Stop now.", }, { "intent": "emergency-stop", "phrase_id": "stop_03", "zh": "先停一下。", "en": "Pause for a moment.", }, { "intent": "emergency-stop", "phrase_id": "stop_04", "zh": "停止动作。", "en": "Stop the movement.", }, { "intent": "emergency-stop", "phrase_id": "stop_05", "zh": "立刻停止。", "en": "Stop immediately.", }, { "intent": "emergency-stop", "phrase_id": "stop_06", "zh": "不要再继续了。", "en": "Do not continue.", }, { "intent": "safe-hold", "phrase_id": "hold_01", "zh": "别动。", "en": "Don't move.", }, { "intent": "safe-hold", "phrase_id": "hold_02", "zh": "不要动。", "en": "Stay still.", }, { "intent": "safe-hold", "phrase_id": "hold_03", "zh": "不对,先别动。", "en": "That's not right. Hold still.", }, { "intent": "safe-hold", "phrase_id": "hold_04", "zh": "保持现在的姿势。", "en": "Hold your current pose.", }, { "intent": "safe-hold", "phrase_id": "hold_05", "zh": "先保持不动。", "en": "Keep still for now.", }, { "intent": "safe-hold", "phrase_id": "hold_06", "zh": "站稳,别动。", "en": "Stay balanced and don't move.", }, ) def _voice_specs() -> tuple[dict[str, str], ...]: return ( { "name": "female_01_en", "language": "en", "qwen_language": "English", "instruction": ( "Use a natural everyday adult female voice. Speak the short " "urgent command clearly and calmly, without theatrical acting." ), }, { "name": "female_02_en", "language": "en", "qwen_language": "English", "instruction": ( "Use a natural young adult female voice. Speak clearly in a " "concise, neutral, everyday tone." ), }, { "name": "male_01_en", "language": "en", "qwen_language": "English", "instruction": ( "Use a natural everyday adult male voice. Speak the short " "command clearly and firmly, but do not shout." ), }, { "name": "male_02_en", "language": "en", "qwen_language": "English", "instruction": ( "Use a natural young adult male voice. Speak concisely in a " "relaxed, neutral conversational tone." ), }, { "name": "female_01_zh", "language": "zh", "qwen_language": "Chinese", "instruction": "使用自然日常的成年女性声音,清晰简短地说出指令,语气坚定但不要喊叫。", }, { "name": "female_02_zh", "language": "zh", "qwen_language": "Chinese", "instruction": "使用自然日常的年轻女性声音,语速适中,语气中性简洁,不要戏剧化。", }, { "name": "male_01_zh", "language": "zh", "qwen_language": "Chinese", "instruction": "使用自然日常的成年男性声音,清晰简短地说出指令,坚定但不要大喊。", }, { "name": "male_02_zh", "language": "zh", "qwen_language": "Chinese", "instruction": "使用自然日常的年轻男性声音,语气放松中性,简洁自然地说出指令。", }, ) def _audio_path(root: Path, phrase: dict[str, str], voice: dict[str, str]) -> Path: return root / "audio" / phrase["intent"] / phrase["phrase_id"] / f"{voice['name']}.wav" def _sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def _atomic_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(path.suffix + f".tmp.{os.getpid()}") with temporary.open("w", encoding="utf-8") as handle: for row in rows: handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") handle.flush() os.fsync(handle.fileno()) temporary.replace(path) def _write_pcm16(path: Path, waveform: np.ndarray) -> None: values = np.asarray(waveform, dtype=np.float32).reshape(-1) if values.size <= 0 or not np.isfinite(values).all(): raise RuntimeError(f"TTS produced invalid waveform for {path}") peak = float(np.max(np.abs(values), initial=0.0)) if peak > 1.0: values = values / np.float32(peak) pcm = np.rint(np.clip(values, -1.0, 1.0) * 32767.0).astype(" bool: if not path.is_file(): return False try: with wave.open(str(path), "rb") as handle: return ( handle.getnchannels() == 1 and handle.getsampwidth() == 2 and handle.getframerate() == TARGET_RATE and handle.getnframes() > 0 and handle.getcomptype() == "NONE" ) except (EOFError, wave.Error): return False def generate(args: argparse.Namespace) -> None: import torch from scipy.signal import resample_poly from qwen_tts import Qwen3TTSModel if not 0 <= args.shard_index < args.shard_count: raise ValueError( f"Invalid shard {args.shard_index}/{args.shard_count}" ) voices = tuple( voice for index, voice in enumerate(_voice_specs()) if index % args.shard_count == args.shard_index ) if not voices: print(f"[ready] shard={args.shard_index} has no assigned voices") return torch.backends.cuda.enable_cudnn_sdp(False) torch.backends.cuda.enable_flash_sdp(True) torch.backends.cuda.enable_mem_efficient_sdp(True) torch.backends.cuda.enable_math_sdp(True) torch.backends.cuda.matmul.allow_tf32 = True model = Qwen3TTSModel.from_pretrained( str(args.model_path), device_map=args.device, dtype=torch.bfloat16, attn_implementation="flash_attention_2", ) for voice in voices: pending = [ phrase for phrase in PHRASES if not _valid_pcm16(_audio_path(args.output_root, phrase, voice)) ] for start in range(0, len(pending), max(args.batch_size, 1)): batch = pending[start : start + max(args.batch_size, 1)] texts = [phrase[voice["language"]] for phrase in batch] waveforms, source_rate = model.generate_voice_design( text=texts, language=[voice["qwen_language"]] * len(batch), instruct=[voice["instruction"]] * len(batch), ) if len(waveforms) != len(batch): raise RuntimeError("Qwen TTS returned the wrong batch size") for phrase, waveform in zip(batch, waveforms): values = np.asarray(waveform, dtype=np.float32).reshape(-1) if int(source_rate) != TARGET_RATE: divisor = math.gcd(int(source_rate), TARGET_RATE) values = resample_poly( values, TARGET_RATE // divisor, int(source_rate) // divisor, ).astype(np.float32, copy=False) _write_pcm16(_audio_path(args.output_root, phrase, voice), values) print( f"[tts] shard={args.shard_index}/{args.shard_count} " f"voice={voice['name']} done={min(start + len(batch), len(pending))}/" f"{len(pending)}", flush=True, ) del model torch.cuda.empty_cache() def finalize(args: argparse.Namespace) -> None: rows: list[dict[str, Any]] = [] for phrase in PHRASES: for voice in _voice_specs(): path = _audio_path(args.output_root, phrase, voice).resolve() if not _valid_pcm16(path): raise RuntimeError(f"Missing/invalid expected control WAV: {path}") with wave.open(str(path), "rb") as handle: samples = int(handle.getnframes()) raw = handle.readframes(samples) values = np.frombuffer(raw, dtype=" 0.0 else 1.0 ) rows.append( { "format": FORMAT, "intent": phrase["intent"], "phrase_id": phrase["phrase_id"], "voice_name": voice["name"], "language": voice["language"], "text": phrase[voice["language"]], "audio_path": str(path), "sample_rate": TARGET_RATE, "speech_samples_16k": samples, "duration_seconds": samples / float(TARGET_RATE), "peak": peak, "speech_scale": speech_scale, "sha256": _sha256(path), } ) expected = len(PHRASES) * len(_voice_specs()) if len(rows) != expected: raise AssertionError(f"Manifest row count drifted: {len(rows)} != {expected}") _atomic_jsonl(args.output_root / "manifest.jsonl", rows) durations = [float(row["duration_seconds"]) for row in rows] print( f"[ready] rows={len(rows)} meanings={len(PHRASES)} voices=8 " f"duration={min(durations):.2f}-{max(durations):.2f}s " f"manifest={args.output_root / 'manifest.jsonl'}", flush=True, ) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--mode", choices=("generate", "finalize"), required=True) parser.add_argument("--output-root", type=Path, default=DEFAULT_OUTPUT) parser.add_argument("--model-path", type=Path, default=DEFAULT_MODEL) parser.add_argument("--device", default="cuda:0") parser.add_argument("--batch-size", type=int, default=6) parser.add_argument("--shard-index", type=int, default=0) parser.add_argument("--shard-count", type=int, default=1) return parser.parse_args() def main() -> None: args = parse_args() args.output_root = args.output_root.expanduser().resolve() args.model_path = args.model_path.expanduser().resolve() if args.mode == "generate": generate(args) else: finalize(args) if __name__ == "__main__": main()