code-HFD_lzk / scripts /data /plan_hfd_third_party_multi_v5.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
8562e9b verified
Raw History Blame Contribute Delete
24.1 kB
#!/usr/bin/env python3
"""Prepare a CPU-only multi-event refresh plan; never replace production audio."""
from __future__ import annotations
import argparse
from collections import Counter, defaultdict
import json
from pathlib import Path
import sys
import wave
ROOT = Path(__file__).resolve().parents[2]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from scripts.data.build_hfd_diverse_catalog import filter_text
from scripts.data.build_hfd_cross_catalog import new_source_filter_reason
from scripts.data.hfd_third_party_common import (
canonical_json_sha256, read_jsonl, sha256_file, stable_u64,
write_json_atomic, write_jsonl_atomic,
)
from scripts.data.hfd_third_party_multi_v5 import CLASS_WEIGHTS, quota_counts
POLICY = "multi_event_speech_v5"
SEED = 26092801
DEFAULT_REFERENCE = ROOT / "data/hfd_third_party_cross_v4"
DEFAULT_OUTPUT = ROOT / "data/hfd_third_party_multi_v5/work/plan_qualified"
DEFAULT_GENERATION_PLAN = ROOT / "data/hfd_third_party_multi_v5/work/plan_reuse"
KNOWN_PENDING_ASSETS = {"alimeeting:R1021_M1959_F_SPK1953:00157"}
MAX_ASIDE_REFERENCE_USES = 32
def class_counts(size: int) -> dict[str, int]:
return quota_counts(size)
def spread_take(rows: list[dict], count: int, seed: int) -> list[dict]:
"""Round-robin speakers, retaining diverse real voice identities."""
if not 0 <= count <= len(rows):
raise ValueError("Insufficient catalog population")
speakers = defaultdict(list)
for row in rows:
speakers[row["speaker_id"]].append(row)
ranked = []
for speaker, group in speakers.items():
ordered = sorted(group, key=lambda r: stable_u64(seed, r["asset_id"]))
for index, row in enumerate(ordered):
ranked.append((index, stable_u64(seed, speaker), row["asset_id"], row))
return [item[-1] for item in sorted(ranked)[:count]]
def balanced_catalog(rows: list[dict], semantics: dict, seed: int) -> tuple[list[dict], dict]:
accepted, rejected = [], Counter()
for row in rows:
if row.get("role") != "non_command_speech":
continue
reason = ("pending_human_semantic_decision" if row["asset_id"] in KNOWN_PENDING_ASSETS else
filter_text(row["transcript"], row["language"],
set(semantics["target_phrases"][row["language"]]), semantics=semantics) or
new_source_filter_reason(row, semantics))
if reason:
rejected[reason] += 1
else:
accepted.append(row)
selected = []
for split in ("train", "validation"):
pools = {lang: [r for r in accepted if (r["split"], r["language"]) == (split, lang)]
for lang in ("zh", "en")}
count = min(map(len, pools.values()))
if not count:
raise ValueError(f"Empty bilingual catalog: {split}")
for lang, pool in pools.items():
if len(pool) == count:
selected.extend(pool)
continue
domains = {domain: [r for r in pool if r.get("speech_domain") == domain]
for domain in ("conversation", "read")}
targets = {"conversation": count // 2, "read": count - count // 2}
for domain, number in targets.items():
selected.extend(spread_take(domains[domain], number, stable_u64(seed, split, lang, domain)))
if len({r["asset_id"] for r in selected}) != len(selected):
raise ValueError("Duplicate source asset IDs")
return sorted(selected, key=lambda r: r["asset_id"]), dict(rejected)
def anchor_candidates(recipe: dict) -> list[int]:
total = recipe["audio"]["sample_count"]
return [index for index, event in enumerate(recipe["target_speech_events"])
if event["sample_count"] >= 3 * 16000 and total - event["end_sample"] >= 2 * 16000]
def reusable_aside_anchors(population: list[dict], need: int, seed: int,
eligible_reference_sha256: set[str] | None = None) -> list[tuple[dict, int]]:
"""Reuse exact reference WAVs, not voice labels; cap repeated asides at 32.
This changes only which existing sessions receive the aside condition.
No original target session is dropped, rebuilt or relabelled. Reference
selection favors reuse; ties and sessions within each reference use hashes.
"""
by_reference = defaultdict(dict)
for row in population:
for index in anchor_candidates(row):
event = row["target_speech_events"][index]
if eligible_reference_sha256 is not None and event["original_audio_sha256"] not in eligible_reference_sha256:
continue
candidates = by_reference[event["original_audio_sha256"]]
previous = candidates.get(row["session_id"])
if previous is None or stable_u64(seed, row["session_id"], index) < stable_u64(seed, row["session_id"], previous[1]):
candidates[row["session_id"]] = (row, index)
chosen, used = [], set()
while len(chosen) < need:
if not by_reference:
raise ValueError("Not enough real >=3s reference voices")
key = min(by_reference, key=lambda ref: (
-min(MAX_ASIDE_REFERENCE_USES, len(by_reference[ref].keys() - used)),
stable_u64(seed, "reference", ref)))
available = [entry for sid, entry in by_reference.pop(key).items() if sid not in used]
if not available:
raise ValueError("Not enough real >=3s reference voices")
available.sort(key=lambda entry: stable_u64(seed, "aside", entry[0]["session_id"]))
selected = available[:min(MAX_ASIDE_REFERENCE_USES, need-len(chosen))]
chosen.extend(selected)
used.update(row["session_id"] for row, _ in selected)
return chosen
def assign_sessions(rows: list[dict], seed: int,
eligible_reference_sha256: set[str] | None = None, *,
eligible_reference_keys: set[tuple[str, str, str]] | None = None) -> dict[str, dict]:
if len({r["session_id"] for r in rows}) != len(rows):
raise ValueError("Duplicate reference session IDs")
if eligible_reference_sha256 is not None and eligible_reference_keys is not None:
raise ValueError("Choose scoped reference eligibility, not both eligibility APIs")
result = {}
validation_references = set()
splits = ("validation", "train") if eligible_reference_keys is not None else ("train", "validation")
for split in splits:
population = [r for r in rows if r["split"] == split]
counts = class_counts(len(population))
used = set()
for lang in ("zh", "en"):
need = counts["same_voice_aside"] // 2
cohort_references = eligible_reference_sha256
if eligible_reference_keys is not None:
cohort_references = {sha for key_split, key_lang, sha in eligible_reference_keys
if (key_split, key_lang) == (split, lang)}
if split == "train":
cohort_references -= validation_references
eligible = [r for r in population if r["session_language"] == lang and any(
cohort_references is None or r["target_speech_events"][i]["original_audio_sha256"] in cohort_references
for i in anchor_candidates(r))]
if len(eligible) < need:
raise ValueError(f"Not enough real >=3s reference voices: {split}/{lang}")
for row, anchor in reusable_aside_anchors(eligible, need, stable_u64(seed, split, lang),
cohort_references):
result[row["session_id"]] = dict(session_type="same_voice_aside", language=lang,
anchor_target_turn_index=anchor)
used.add(row["session_id"])
if split == "validation":
validation_references.add(row["target_speech_events"][anchor]["original_audio_sha256"])
remaining = sorted([r for r in population if r["session_id"] not in used],
key=lambda r: stable_u64(seed, "class", split, r["session_id"]))
offset = 0
for name, _ in CLASS_WEIGHTS:
if name == "same_voice_aside":
continue
for lang in ("zh", "en"):
n = counts[name] // 2
for row in remaining[offset:offset+n]:
result[row["session_id"]] = dict(session_type=name, language=lang)
offset += n
if offset != len(remaining):
raise AssertionError("Class quotas failed to partition sessions")
reference_uses = Counter(row["target_speech_events"][result[row["session_id"]]["anchor_target_turn_index"]]["original_audio_sha256"]
for row in rows if result[row["session_id"]]["session_type"] == "same_voice_aside")
if any(count > MAX_ASIDE_REFERENCE_USES for count in reference_uses.values()):
raise ValueError("Exact reference reuse exceeds the global cap across split/language groups")
return result
def plan_tts_jobs(rows: list[dict], assignments: dict, assets: list[dict], seed: int):
pools = defaultdict(list)
for asset in assets:
# Short, existing screened statements. Do not synthesize target commands as noise.
if asset.get("speech_domain") == "conversation" and 2 * 16000 <= asset["sample_count"] <= 6 * 16000:
pools[asset["split"], asset["language"]].append(asset)
jobs = {}
for row in rows:
assignment = assignments[row["session_id"]]
if assignment["session_type"] != "same_voice_aside":
continue
index = assignment["anchor_target_turn_index"]
turn, event = row["turns"][index], row["target_speech_events"][index]
candidates = sorted(pools[row["split"], assignment["language"]], key=lambda r: r["asset_id"])
if not candidates:
raise ValueError("No screened conversational TTS texts")
# Reuse the same generated aside when the actual reference waveform repeats.
text_source = candidates[stable_u64(seed, "tts-text", event["original_audio_sha256"]) % len(candidates)]
identity = canonical_json_sha256(dict(reference_sha=event["original_audio_sha256"],
text_asset=text_source["asset_id"], policy=POLICY))
job = dict(job_id=identity, split=row["split"], language=assignment["language"],
reference_audio=event["original_audio_path"], reference_sha256=event["original_audio_sha256"],
reference_text=turn["instruction_text"], reference_source_key=turn["tts_source_key"],
text=text_source["transcript"], text_source_asset_id=text_source["asset_id"],
text_source_normalized_sha256=text_source["normalized_sha256"],
voice_relation="reference_clone", semantic_review="screened_text_audio_pending",
synthesis_status="not_started", same_speaker_verified=False)
if identity in jobs and jobs[identity] != job:
raise ValueError("A TTS reference crosses split/language contracts")
jobs[identity] = job
assignment["tts_job_id"] = identity
return sorted(jobs.values(), key=lambda r: r["job_id"])
def validate_qualified_asset(asset: dict, frozen_job: dict) -> dict[str, str]:
"""Verify one reusable generated waveform and its job-bound QA evidence.
The canonical *individual job* is the reuse identity. Its original whole
generation manifest may differ from the new selected jobs manifest.
Returns actual evidence-file hashes for freezing and later revalidation.
"""
if asset.get("eligible_for_mixing") is not True:
raise ValueError("Qualified candidate is not eligible_for_mixing")
if asset.get("role") != "non_command_speech" or asset.get("voice_relation") != "reference_clone":
raise ValueError("Qualified candidate is not a non-command reference clone")
job_hash = canonical_json_sha256(frozen_job)
if asset.get("job_sha256") != job_hash:
raise ValueError("Qualified candidate individual job hash mismatch")
for key in ("job_id", "split", "language", "reference_audio", "reference_sha256", "reference_text"):
if asset.get(key) != frozen_job.get(key):
raise ValueError(f"Qualified candidate/frozen job binding differs: {key}")
if asset.get("transcript") != frozen_job["text"]:
raise ValueError("Qualified candidate transcript differs from frozen job")
for key in ("asset_id", "speaker_id", "source_group"):
if not asset.get(key):
raise ValueError(f"Missing qualified asset provenance: {key}")
verified = {}
for path_key, hash_key in (("normalized_path", "normalized_sha256"),
("qa_path", "qa_sha256"),
("generation_provenance_path", "generation_provenance_sha256")):
path = Path(asset[path_key]).absolute()
actual = sha256_file(path)
if actual != asset[hash_key]:
raise ValueError(f"Qualified candidate evidence hash mismatch: {path_key}")
verified[str(path)] = actual
with wave.open(str(asset["normalized_path"]), "rb") as handle:
if (handle.getframerate(), handle.getnchannels(), handle.getsampwidth(), handle.getcomptype(), handle.getnframes()) != (
16000, 1, 2, "NONE", int(asset["sample_count"])) or handle.getnframes() <= 0:
raise ValueError("Qualified candidate WAV geometry mismatch")
qa = json.loads(Path(asset["qa_path"]).read_text())
if (qa.get("engineering_eligible") is not True or qa.get("job_id") != frozen_job["job_id"]
or qa.get("audio_sha256") != asset["normalized_sha256"] or qa.get("job_sha256") != job_hash):
raise ValueError("Qualified candidate QA does not approve this exact audio/job")
provenance = json.loads(Path(asset["generation_provenance_path"]).read_text())
if (provenance.get("job_id") != frozen_job["job_id"] or provenance.get("job_sha256") != job_hash
or canonical_json_sha256(provenance.get("job")) != job_hash):
raise ValueError("Generated provenance individual job differs from frozen job")
normalized = provenance["normalized"]
if (normalized["sha256"] != asset["normalized_sha256"]
or Path(normalized["path"]).resolve() != Path(asset["normalized_path"]).resolve()
or int(normalized["sample_count"]) != int(asset["sample_count"])):
raise ValueError("Generated provenance points to different normalized audio")
if any(sha256_file(Path(path)) != expected for path, expected in verified.items()):
raise ValueError("Qualified evidence changed during verification")
return verified
def _qualified_inputs(candidate_registry: Path, generation_plan: Path, seed: int,
reference_hash: str, assets: list[dict]) -> tuple[dict, dict, dict]:
"""Read-only qualification; never regenerate jobs or change old provenance."""
report_path = generation_plan / "preparation_report.json"
report = json.loads(report_path.read_text())
if report["policy"] != POLICY or int(report["seed"]) != seed or report["reference_recipe_sha256"] != reference_hash:
raise ValueError("Parent generation plan policy/seed/reference mismatch")
generation_hashes = {str(report_path.absolute()): sha256_file(report_path)}
for name, expected in report["output_hashes"].items():
path = generation_plan / name
if sha256_file(path) != expected:
raise ValueError(f"Parent generation plan changed: {name}")
generation_hashes[str(path.absolute())] = expected
old_assets = read_jsonl(generation_plan / "selected_real_assets.jsonl")
if canonical_json_sha256(old_assets) != canonical_json_sha256(assets):
raise ValueError("Qualified refresh must retain the frozen real-text/source pool")
old_jobs = read_jsonl(generation_plan / "tts_jobs.jsonl")
by_job = {job["job_id"]: job for job in old_jobs}
if len(by_job) != len(old_jobs):
raise ValueError("Duplicate frozen generation job IDs")
candidates = read_jsonl(candidate_registry)
qualified = {}
evidence = {str(candidate_registry.absolute()): sha256_file(candidate_registry)}
for asset in candidates:
identifier = asset["job_id"]
if identifier in qualified:
raise ValueError("Duplicate qualified candidate job ID")
if identifier not in by_job:
raise ValueError("Qualified candidate absent from parent generation plan")
evidence.update(validate_qualified_asset(asset, by_job[identifier]))
qualified[identifier] = asset
if not qualified:
raise ValueError("No eligible reference-clone candidates")
return qualified, by_job, {"generation_hashes": generation_hashes, "evidence_hashes": evidence}
def prepare(reference: Path, output: Path, seed: int, qualified_registry: Path | None = None,
generation_plan_root: Path | None = None) -> dict:
reference, output = reference.absolute(), output.absolute()
if output.resolve() != output or not output.is_relative_to(ROOT / "data/hfd_third_party_multi_v5/work"):
raise ValueError("Plan output must remain in the new LZK-owned work directory")
if output.exists():
raise FileExistsError("Do not overwrite an existing plan")
recipe_path = reference / "production/manifests/session_recipes.jsonl"
registry_path = reference / "work/catalog/manifests/third_party_source_registry.jsonl"
semantics_path = reference / "work/catalog/manifests/command_semantics.json"
if not json.loads((reference / "production/reports/strict_audit.json").read_text()).get("passed"):
raise ValueError("Reference production has not passed its audit")
frozen_inputs = {str(path): sha256_file(path) for path in (recipe_path, registry_path, semantics_path)}
rows = read_jsonl(recipe_path)
assets, rejected = balanced_catalog(read_jsonl(registry_path), json.loads(semantics_path.read_text()), seed)
qualified = None
qualification = dict(qualified_registry_path=None, qualified_registry_sha256=None,
generation_plan_root=None, generation_plan_hashes={})
if qualified_registry is not None:
qualified_registry = qualified_registry.absolute()
generation_plan_root = (generation_plan_root or DEFAULT_GENERATION_PLAN).absolute()
qualified, frozen_jobs, evidence = _qualified_inputs(
qualified_registry, generation_plan_root, seed, frozen_inputs[str(recipe_path)], assets)
frozen_inputs.update(evidence["generation_hashes"])
frozen_inputs.update(evidence["evidence_hashes"])
qualification = dict(qualified_registry_path=str(qualified_registry),
qualified_registry_sha256=frozen_inputs[str(qualified_registry)],
generation_plan_root=str(generation_plan_root),
generation_plan_hashes=evidence["generation_hashes"],
qualified_evidence_hashes=evidence["evidence_hashes"],
qualified_candidate_jobs=len(qualified),
individual_job_reuse_contract="canonical_job_equal_to_frozen_generation_job_not_whole_manifest_identity")
reference_keys = {(asset["split"], asset["language"], asset["reference_sha256"])
for asset in qualified.values()} if qualified is not None else None
assignments = assign_sessions(rows, seed, eligible_reference_keys=reference_keys)
jobs = plan_tts_jobs(rows, assignments, assets, seed)
if qualified is not None:
for job in jobs:
if job["job_id"] not in qualified or canonical_json_sha256(job) != canonical_json_sha256(frozen_jobs[job["job_id"]]):
raise ValueError("Qualified refresh changed an individual frozen TTS job")
qualification.update(selected_tts_job_ids=[job["job_id"] for job in jobs],
selected_tts_job_sha256={job["job_id"]: canonical_json_sha256(job) for job in jobs},
qualified_existing_tts_jobs=len(jobs), additional_tts_generation_required=0,
qualified_reference_eligibility_scope="split_language_reference_sha256")
selected_by_split = {split: sorted({job["reference_sha256"] for job in jobs if job["split"] == split})
for split in ("train", "validation")}
if set(selected_by_split["train"]) & set(selected_by_split["validation"]):
raise ValueError("Qualified clone references overlap train and validation")
qualification["selected_reference_sha256_by_split"] = selected_by_split
planned = [dict(session_id=r["session_id"], session_signature=r["session_signature"], split=r["split"],
parent_recipe_sha256=canonical_json_sha256(r), **assignments[r["session_id"]]) for r in rows]
if any(sha256_file(Path(path)) != expected for path, expected in frozen_inputs.items()):
raise RuntimeError("Reference inputs changed during planning")
report = dict(format="hfd_multi_event_preparation_v1", policy=POLICY, seed=seed,
status="plan_only_not_a_generated_dataset", production_modified=False,
reference_root=str(reference), reference_recipe_sha256=sha256_file(recipe_path),
reference_registry_sha256=sha256_file(registry_path), semantics_sha256=sha256_file(semantics_path),
sessions=len(rows), class_counts=dict(Counter(r["session_type"] for r in planned)),
background_languages=dict(Counter(r["language"] for r in planned)),
class_language_split_counts=dict(Counter('/'.join((r['split'],r['session_type'],r['language'])) for r in planned)),
real_catalog_languages=dict(Counter(r["language"] for r in assets)),
catalog_rejected_reasons=rejected, tts_jobs=len(jobs),
tts_job_languages=dict(Counter(r["language"] for r in jobs)),
tts_audio_generated=0, gt_generated=0, training_started=False,
aside_reference_selection="exact_reference_reuse_capped_then_hash_random_sessions",
max_aside_uses_per_reference=MAX_ASIDE_REFERENCE_USES,
sir_distribution=dict(scope="combined_background_active_RMS_vs_complete_target_active_RMS",
family="truncated_normal_db", mean=2.0, standard_deviation=3.0,
lower=-4.0, upper=8.0, final_peak_policy="uniform_scale_whole_mix"),
language_ratio_note="Sessions and selected real-audio candidates are balanced; cached unique TTS jobs need not have equal language counts.")
report.update(qualification)
output.mkdir(parents=True, exist_ok=False)
write_jsonl_atomic(output / "session_assignments.jsonl", planned)
write_jsonl_atomic(output / "selected_real_assets.jsonl", assets)
write_jsonl_atomic(output / "tts_jobs.jsonl", jobs)
report["output_hashes"] = {name: sha256_file(output / name) for name in (
"session_assignments.jsonl", "selected_real_assets.jsonl", "tts_jobs.jsonl")}
write_json_atomic(output / "preparation_report.json", report)
return report
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--reference-root", type=Path, default=DEFAULT_REFERENCE)
parser.add_argument("--output-root", type=Path, default=DEFAULT_OUTPUT)
parser.add_argument("--seed", type=int, default=SEED)
parser.add_argument("--qualified-registry", type=Path)
parser.add_argument("--generation-plan-root", type=Path, default=DEFAULT_GENERATION_PLAN)
args = parser.parse_args()
print(json.dumps(prepare(args.reference_root, args.output_root, args.seed,
args.qualified_registry, args.generation_plan_root), ensure_ascii=False, indent=2))