code-HFD_lzk / scripts /data /generate_hfd_backchannels.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
8562e9b verified
Raw History Blame Contribute Delete
14 kB
#!/usr/bin/env python3
"""Generate and audit short HFD backchannel utterances in eight voices.
The inventory follows the two recurrent dialogue functions used in
backchannel research: continuers (keep the floor with the current speaker) and
assessments/acknowledgements (briefly signal understanding or evaluation).
Train and validation use disjoint phrase IDs. Sentence-like acknowledgements
are deliberately excluded: a backchannel must be a brief listener cue that
does not need a full ``listen`` turn to understand. Generation is shardable by
voice; ``--mode finalize`` writes the manifest only after every expected WAV
passes the mono PCM16/16 kHz and sub-two-second contracts.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import os
import wave
from pathlib import Path
from typing import Any
import numpy as np
FORMAT = "hfd_backchannel_tts_v1"
TARGET_RATE = 16_000
DEFAULT_MODEL = Path(
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen/"
"Qwen3-TTS-12Hz-1.7B-VoiceDesign"
)
DEFAULT_OUTPUT = Path(
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_backchannel_commands_v2_short"
)
# Short, non-directive listener feedback only. The acknowledgements 对/Right
# and Yeah are deliberate hard negatives: context, not the mere presence of
# speech, must distinguish them from a new floor-taking command. Compound or
# sentence-like validation phrases (Yeah, right / Mm, exactly / That makes
# sense) are excluded because understanding them reasonably requires a listen
# turn; treating them as "ignore and keep moving" creates an ambiguous target.
PHRASES: tuple[dict[str, str], ...] = (
{"split": "train", "category": "continuer", "phrase_id": "bc_cont_01", "zh": "嗯嗯。", "en": "Mm-hmm."},
{"split": "train", "category": "continuer", "phrase_id": "bc_cont_02", "zh": "嗯。", "en": "Yeah."},
{"split": "train", "category": "continuer", "phrase_id": "bc_cont_03", "zh": "啊哈。", "en": "Uh-huh."},
{"split": "train", "category": "continuer", "phrase_id": "bc_cont_04", "zh": "对。", "en": "Right."},
{"split": "train", "category": "continuer", "phrase_id": "bc_cont_05", "zh": "是啊。", "en": "Yep."},
{"split": "train", "category": "continuer", "phrase_id": "bc_cont_06", "zh": "好。", "en": "Okay."},
{"split": "train", "category": "continuer", "phrase_id": "bc_cont_07", "zh": "嗯哼。", "en": "Mhm."},
{"split": "train", "category": "continuer", "phrase_id": "bc_cont_08", "zh": "是的。", "en": "Yes."},
{"split": "train", "category": "assessment", "phrase_id": "bc_assess_01", "zh": "明白。", "en": "I see."},
{"split": "train", "category": "assessment", "phrase_id": "bc_assess_02", "zh": "好的。", "en": "Got it."},
{"split": "train", "category": "assessment", "phrase_id": "bc_assess_03", "zh": "没错。", "en": "Exactly."},
{"split": "train", "category": "assessment", "phrase_id": "bc_assess_04", "zh": "哦。", "en": "Oh."},
{"split": "val", "category": "continuer", "phrase_id": "bc_cont_10", "zh": "对的。", "en": "That's right."},
{"split": "val", "category": "continuer", "phrase_id": "bc_cont_11", "zh": "确实。", "en": "Indeed."},
{"split": "val", "category": "assessment", "phrase_id": "bc_assess_06", "zh": "哇。", "en": "Wow."},
)
def _voice_specs() -> tuple[dict[str, str], ...]:
# These strings are intentionally byte-for-byte identical to the eight
# prompts in qwen3-tts/synthesize_seed_audio.py. A backchannel and the real
# command carrying the same ``voice_name`` must not acquire a new speaker
# description halfway through one session.
english = {
"female_01_en": "Use a natural everyday adult female voice. Speak clearly and calmly, with a friendly conversational tone, not theatrical.",
"female_02_en": "Use a natural everyday young adult female voice. Speak in a relaxed, neutral, and easy-to-understand style.",
"male_01_en": "Use a natural everyday adult male voice. Speak clearly with a calm, friendly, conversational tone, not dramatic.",
"male_02_en": "Use a natural everyday young adult male voice. Speak in a relaxed and neutral style, like a normal spoken description.",
}
chinese = {
"female_01_zh": "使用自然日常的成年女性声音,语气清晰平和,像正常描述动作,不要夸张表演。",
"female_02_zh": "使用自然日常的年轻女性声音,语速适中,语气放松中性,适合朗读简短动作描述。",
"male_01_zh": "使用自然日常的成年男性声音,发音清楚,语气平稳友好,像普通口语描述。",
"male_02_zh": "使用自然日常的年轻男性声音,语气放松中性,不要戏剧化,适合动作短语朗读。",
}
rows = []
for name, instruction in (*english.items(), *chinese.items()):
language = name.rsplit("_", 1)[-1]
rows.append(
{
"name": name,
"language": language,
"qwen_language": "English" if language == "en" else "Chinese",
"instruction": instruction,
}
)
return tuple(rows)
def _audio_path(root: Path, phrase: dict[str, str], voice: dict[str, str]) -> Path:
return root / "audio" / phrase["split"] / phrase["phrase_id"] / f"{voice['name']}.wav"
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _atomic_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(path.suffix + f".tmp.{os.getpid()}")
with temporary.open("w", encoding="utf-8") as handle:
for row in rows:
handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
handle.flush()
os.fsync(handle.fileno())
temporary.replace(path)
def _write_pcm16(path: Path, waveform: np.ndarray) -> None:
values = np.asarray(waveform, dtype=np.float32).reshape(-1)
if values.size <= 0 or not np.isfinite(values).all():
raise RuntimeError(f"TTS produced invalid waveform for {path}")
peak = float(np.max(np.abs(values), initial=0.0))
if peak > 1.0:
values = values / np.float32(peak)
pcm = np.rint(np.clip(values, -1.0, 1.0) * 32767.0).astype("<i2")
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(f".tmp.{os.getpid()}.wav")
with wave.open(str(temporary), "wb") as handle:
handle.setnchannels(1)
handle.setsampwidth(2)
handle.setframerate(TARGET_RATE)
handle.writeframes(pcm.tobytes())
temporary.replace(path)
def _trim_tts_silence(waveform: np.ndarray) -> np.ndarray:
"""Remove only TTS padding while retaining natural onset/offset margins."""
values = np.asarray(waveform, dtype=np.float32).reshape(-1)
peak = float(np.max(np.abs(values), initial=0.0))
if peak <= 0.0:
return values
threshold = max(peak * 0.02, 0.002)
active = np.flatnonzero(np.abs(values) >= threshold)
if active.size == 0:
return values
start = max(int(active[0]) - int(round(0.08 * TARGET_RATE)), 0)
end = min(int(active[-1]) + 1 + int(round(0.16 * TARGET_RATE)), values.size)
return np.ascontiguousarray(values[start:end], dtype=np.float32)
def _valid_pcm16(path: Path) -> bool:
if not path.is_file():
return False
try:
with wave.open(str(path), "rb") as handle:
return (
handle.getnchannels() == 1
and handle.getsampwidth() == 2
and handle.getframerate() == TARGET_RATE
and 0 < handle.getnframes() < 2 * TARGET_RATE
and handle.getcomptype() == "NONE"
)
except (EOFError, wave.Error):
return False
def generate(args: argparse.Namespace) -> None:
import torch
from qwen_tts import Qwen3TTSModel
from scipy.signal import resample_poly
if not 0 <= args.shard_index < args.shard_count:
raise ValueError(f"Invalid shard {args.shard_index}/{args.shard_count}")
voices = tuple(
voice
for index, voice in enumerate(_voice_specs())
if index % args.shard_count == args.shard_index
)
if not voices:
print(f"[ready] shard={args.shard_index} has no assigned voices")
return
torch.backends.cuda.matmul.allow_tf32 = True
model = Qwen3TTSModel.from_pretrained(
str(args.model_path),
device_map=args.device,
dtype=torch.bfloat16,
attn_implementation="flash_attention_2",
)
for voice in voices:
pending = [
phrase
for phrase in PHRASES
if args.overwrite
or not _valid_pcm16(_audio_path(args.output_root, phrase, voice))
]
for start in range(0, len(pending), max(args.batch_size, 1)):
batch = pending[start : start + max(args.batch_size, 1)]
waveforms, source_rate = model.generate_voice_design(
text=[phrase[voice["language"]] for phrase in batch],
language=[voice["qwen_language"]] * len(batch),
instruct=[voice["instruction"]] * len(batch),
)
if len(waveforms) != len(batch):
raise RuntimeError("Qwen TTS returned the wrong batch size")
for phrase, waveform in zip(batch, waveforms):
values = np.asarray(waveform, dtype=np.float32).reshape(-1)
if int(source_rate) != TARGET_RATE:
divisor = math.gcd(int(source_rate), TARGET_RATE)
values = resample_poly(
values,
TARGET_RATE // divisor,
int(source_rate) // divisor,
).astype(np.float32, copy=False)
values = _trim_tts_silence(values)
if values.size >= 2 * TARGET_RATE:
raise RuntimeError(
"Backchannel TTS exceeded two seconds; shorten the "
f"phrase or prosody: {phrase['phrase_id']} "
f"voice={voice['name']} duration="
f"{values.size / TARGET_RATE:.3f}s"
)
_write_pcm16(_audio_path(args.output_root, phrase, voice), values)
print(
f"[tts] shard={args.shard_index}/{args.shard_count} "
f"voice={voice['name']} done="
f"{min(start + len(batch), len(pending))}/{len(pending)}",
flush=True,
)
del model
torch.cuda.empty_cache()
def finalize(args: argparse.Namespace) -> None:
rows: list[dict[str, Any]] = []
for phrase in PHRASES:
for voice in _voice_specs():
path = _audio_path(args.output_root, phrase, voice).resolve()
if not _valid_pcm16(path):
raise RuntimeError(f"Missing/invalid backchannel WAV: {path}")
with wave.open(str(path), "rb") as handle:
samples = int(handle.getnframes())
raw = handle.readframes(samples)
values = np.frombuffer(raw, dtype="<i2").astype(np.float32) / 32768.0
peak = float(np.max(np.abs(values), initial=0.0))
speech_scale = min(1.0, (10.0 ** (-6.0 / 20.0)) / peak) if peak > 0 else 1.0
rows.append(
{
"format": FORMAT,
"split": phrase["split"],
"category": phrase["category"],
"phrase_id": phrase["phrase_id"],
"voice_name": voice["name"],
"language": voice["language"],
"text": phrase[voice["language"]],
"audio_path": str(path),
"sample_rate": TARGET_RATE,
"speech_samples_16k": samples,
"duration_seconds": samples / float(TARGET_RATE),
"peak": peak,
"speech_scale": speech_scale,
"sha256": _sha256(path),
}
)
expected = len(PHRASES) * len(_voice_specs())
if len(rows) != expected:
raise AssertionError(f"Manifest row count drifted: {len(rows)} != {expected}")
_atomic_jsonl(args.output_root / "manifest.jsonl", rows)
durations = [float(row["duration_seconds"]) for row in rows]
print(
f"[ready] rows={len(rows)} meanings={len(PHRASES)} voices=8 "
f"duration={min(durations):.2f}-{max(durations):.2f}s "
f"manifest={args.output_root / 'manifest.jsonl'}",
flush=True,
)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--mode", choices=("generate", "finalize"), required=True)
parser.add_argument("--output-root", type=Path, default=DEFAULT_OUTPUT)
parser.add_argument("--model-path", type=Path, default=DEFAULT_MODEL)
parser.add_argument("--device", default="cuda:0")
parser.add_argument("--batch-size", type=int, default=6)
parser.add_argument("--shard-index", type=int, default=0)
parser.add_argument("--shard-count", type=int, default=1)
parser.add_argument(
"--overwrite",
action="store_true",
help="Regenerate assigned WAVs even when a valid file already exists.",
)
return parser.parse_args()
def main() -> None:
args = parse_args()
args.output_root = args.output_root.expanduser().resolve()
args.model_path = args.model_path.expanduser().resolve()
if args.mode == "generate":
generate(args)
else:
finalize(args)
if __name__ == "__main__":
main()