code-HFD_lzk / scripts /data /hfd_third_party_speech_v3.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
8562e9b verified
Raw History Blame Contribute Delete
9.28 kB
"""Deterministic speech-only interference policies; no target timeline changes."""
from __future__ import annotations
import math
import random
from collections import defaultdict
from pathlib import Path
from typing import Any, Sequence
import numpy as np
from scripts.data.hfd_third_party_common import (
SAMPLE_RATE,
active_rms,
read_pcm16_wav,
stable_u64,
)
SPEECH_PROFILES = ("foreground_non_command", "babble", "babble_continuous")
SPEECH_FOCUS_PROFILE_WEIGHTS = (
("foreground_non_command", 0.40),
("babble", 0.20),
("babble_continuous", 0.40),
)
CONTINUOUS_SYNTHESIS_POLICY = "speaker_tracks_complete_utterances_v1"
SPEAKER_SELECTION_POLICY = "prefer_distinct_content_capacity_v1"
CAPACITY_TIERS = ("full", "near", "limited")
def speaker_capacity_tier(available_unique_samples: int, event_samples: int) -> str:
if available_unique_samples >= event_samples:
return "full"
if available_unique_samples * 5 >= event_samples * 4:
return "near"
return "limited"
def speech_language(profile: str, profile_index: int, profile_policy: str = "legacy_v2") -> str:
"""Opposite babble parity balances both odd validation profile counts."""
offset = int(profile_policy == "speech_focus_v3" and profile == "babble")
return "zh" if (profile_index + offset) % 2 == 0 else "en"
def continuous_event_geometry(
total_samples: int, profile_index: int, session_seed: int
) -> dict[str, Any]:
"""Alternate two full/two partial events independently of language parity."""
if total_samples <= 0 or profile_index < 0:
raise ValueError("Invalid continuous speech event geometry")
full = (profile_index // 2) % 2 == 0
rng = random.Random(stable_u64("continuous-geometry-v3", session_seed))
length = total_samples if full else rng.randint(
max(1, math.ceil(0.40 * total_samples)),
max(1, math.floor(0.80 * total_samples)),
)
start = 0 if full else rng.randrange(total_samples - length + 1)
return {
"placement": "full_session" if full else "uniform_session",
"coverage_mode": "full" if full else "partial",
"start_sample": start,
"end_sample": start + length,
"uniform_start_max_sample": total_samples - length,
"coverage_ratio": length / total_samples,
}
def continuous_babble_waveform(
assets: Sequence[dict[str, Any]], *, sample_count: int, session_seed: int
) -> tuple[np.ndarray, list[dict[str, Any]], dict[str, Any]]:
"""Arrange 2–4 speaker tracks from intact clips, not a looped short mix.
A track exhausts its unused fitting clips before reusing any, never repeats
the previous clip immediately, and never cuts a final utterance to fit.
Coverage describes the union of complete utterance intervals, including
natural pauses inside source recordings; it is not a voice-activity score.
"""
requested_tracks = 2 + stable_u64("continuous-track-count-v3", session_seed) % 3
for attempt in range(16):
seed = session_seed if attempt == 0 else stable_u64("continuous-coverage-retry-v3", session_seed, attempt)
try:
waveform, chosen, metadata = _continuous_babble_attempt(
assets, sample_count=sample_count, session_seed=seed,
requested_track_count=requested_tracks,
)
except _InsufficientCoverage:
continue
metadata["synthesis_attempt"] = attempt
return waveform, chosen, metadata
raise ValueError("No complete-utterance continuous mix reaches 70% interval coverage after 16 deterministic attempts")
class _InsufficientCoverage(ValueError):
pass
def _continuous_babble_attempt(
assets: Sequence[dict[str, Any]], *, sample_count: int, session_seed: int,
requested_track_count: int,
) -> tuple[np.ndarray, list[dict[str, Any]], dict[str, Any]]:
if sample_count <= 0:
raise ValueError("Continuous babble needs a positive event duration")
if len({row["split"] for row in assets}) != 1:
raise ValueError("Continuous babble assets must share one split")
if len({row["language"] for row in assets}) != 1:
raise ValueError("Continuous babble assets must share one language")
by_speaker: dict[str, list[dict[str, Any]]] = defaultdict(list)
for asset in assets:
if asset["role"] != "non_command_speech":
raise ValueError("Continuous babble accepts only non-command speech")
if 0 < int(asset["sample_count"]) <= sample_count:
by_speaker[str(asset["speaker_id"])].append(asset)
# Two distinct utterances per track prevent a single sentence from looping.
by_speaker = {speaker: sorted({row["asset_id"]: row for row in rows}.values(),
key=lambda row: str(row["asset_id"]))
for speaker, rows in by_speaker.items()
if speaker and len({row["asset_id"] for row in rows}) >= 2}
capacities = {speaker: sum(int(row["sample_count"]) for row in rows)
for speaker, rows in by_speaker.items()}
tiers = {speaker: speaker_capacity_tier(samples, sample_count)
for speaker, samples in capacities.items()}
speaker_ids = sorted(by_speaker, key=lambda speaker: (
CAPACITY_TIERS.index(tiers[speaker]),
stable_u64("continuous-speaker-v3", session_seed, speaker)))
if len(speaker_ids) < 2:
raise ValueError("Continuous babble needs two speakers with two fitting clips each")
track_count = min(requested_track_count, len(speaker_ids))
mix = np.zeros(sample_count, dtype=np.float32)
occupied = np.zeros(sample_count, dtype=np.bool_)
chosen: list[dict[str, Any]] = []
tracks = []
repeated = 0
for track_index, speaker in enumerate(speaker_ids[:track_count]):
rng = random.Random(stable_u64("continuous-track-v3", session_seed, speaker))
pool = by_speaker[speaker]
shortest = min(int(row["sample_count"]) for row in pool)
offset = rng.randint(0, min(round(0.6 * SAMPLE_RATE), sample_count - shortest))
position = offset
used: set[str] = set()
previous = None
segments = []
while position < sample_count:
fitting = [row for row in pool if int(row["sample_count"]) <= sample_count - position
and row["asset_id"] != previous]
if not fitting:
break
unused = [row for row in fitting if row["asset_id"] not in used]
candidates = unused or fitting
asset = candidates[rng.randrange(len(candidates))]
identifier = str(asset["asset_id"])
waveform = read_pcm16_wav(Path(asset["normalized_path"]))
if len(waveform) != int(asset["sample_count"]):
raise ValueError(f"Continuous source length changed: {identifier}")
rms = active_rms(waveform)
if rms <= 0.0:
raise ValueError(f"Silent continuous babble source: {identifier}")
gain = 1.0 / rms / track_count
end = position + len(waveform)
mix[position:end] += waveform * np.float32(gain)
occupied[position:end] = True
is_repeat = identifier in used
repeated += int(is_repeat)
segments.append({
"asset_id": identifier,
"asset_sha256": asset["normalized_sha256"],
"source_group": asset["source_group"],
"source_start_sample": 0,
"source_end_sample": len(waveform),
"start_sample": position,
"end_sample": end,
"normalization_gain": gain,
"transcript": asset["transcript"],
"reused_asset": is_repeat,
})
chosen.append(asset)
used.add(identifier)
previous = identifier
position = end + rng.randint(round(0.1 * SAMPLE_RATE), round(0.6 * SAMPLE_RATE))
if not segments:
raise RuntimeError(f"Continuous babble track is empty: {speaker}")
tracks.append({"track_index": track_index, "speaker_id": speaker,
"language": pool[0]["language"], "start_offset_samples": offset,
"available_unique_samples": capacities[speaker],
"capacity_tier": tiers[speaker],
"segments": segments})
coverage = float(np.count_nonzero(occupied)) / sample_count
if coverage < 0.70:
raise _InsufficientCoverage(f"Continuous utterance-interval coverage is too low: {coverage:.3f}")
return mix, chosen, {
"synthesis_policy": CONTINUOUS_SYNTHESIS_POLICY,
"speaker_selection_policy": SPEAKER_SELECTION_POLICY,
"speaker_capacity_tier_counts": {
tier: sum(value == tier for value in tiers.values()) for tier in CAPACITY_TIERS
},
"synthetic_babble": True,
"track_count": track_count,
"tracks": tracks,
"actual_union_coverage": coverage,
"coverage_measure": "complete_utterance_interval_union_including_internal_pauses",
"reused_asset_segments": repeated,
"segment_offsets_relative_to": "interference_event_start",
}