#!/usr/bin/env python3 """Prepare a CPU-only multi-event refresh plan; never replace production audio.""" from __future__ import annotations import argparse from collections import Counter, defaultdict import json from pathlib import Path import sys import wave ROOT = Path(__file__).resolve().parents[2] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from scripts.data.build_hfd_diverse_catalog import filter_text from scripts.data.build_hfd_cross_catalog import new_source_filter_reason from scripts.data.hfd_third_party_common import ( canonical_json_sha256, read_jsonl, sha256_file, stable_u64, write_json_atomic, write_jsonl_atomic, ) from scripts.data.hfd_third_party_multi_v5 import CLASS_WEIGHTS, quota_counts POLICY = "multi_event_speech_v5" SEED = 26092801 DEFAULT_REFERENCE = ROOT / "data/hfd_third_party_cross_v4" DEFAULT_OUTPUT = ROOT / "data/hfd_third_party_multi_v5/work/plan_qualified" DEFAULT_GENERATION_PLAN = ROOT / "data/hfd_third_party_multi_v5/work/plan_reuse" KNOWN_PENDING_ASSETS = {"alimeeting:R1021_M1959_F_SPK1953:00157"} MAX_ASIDE_REFERENCE_USES = 32 def class_counts(size: int) -> dict[str, int]: return quota_counts(size) def spread_take(rows: list[dict], count: int, seed: int) -> list[dict]: """Round-robin speakers, retaining diverse real voice identities.""" if not 0 <= count <= len(rows): raise ValueError("Insufficient catalog population") speakers = defaultdict(list) for row in rows: speakers[row["speaker_id"]].append(row) ranked = [] for speaker, group in speakers.items(): ordered = sorted(group, key=lambda r: stable_u64(seed, r["asset_id"])) for index, row in enumerate(ordered): ranked.append((index, stable_u64(seed, speaker), row["asset_id"], row)) return [item[-1] for item in sorted(ranked)[:count]] def balanced_catalog(rows: list[dict], semantics: dict, seed: int) -> tuple[list[dict], dict]: accepted, rejected = [], Counter() for row in rows: if row.get("role") != "non_command_speech": continue reason = ("pending_human_semantic_decision" if row["asset_id"] in KNOWN_PENDING_ASSETS else filter_text(row["transcript"], row["language"], set(semantics["target_phrases"][row["language"]]), semantics=semantics) or new_source_filter_reason(row, semantics)) if reason: rejected[reason] += 1 else: accepted.append(row) selected = [] for split in ("train", "validation"): pools = {lang: [r for r in accepted if (r["split"], r["language"]) == (split, lang)] for lang in ("zh", "en")} count = min(map(len, pools.values())) if not count: raise ValueError(f"Empty bilingual catalog: {split}") for lang, pool in pools.items(): if len(pool) == count: selected.extend(pool) continue domains = {domain: [r for r in pool if r.get("speech_domain") == domain] for domain in ("conversation", "read")} targets = {"conversation": count // 2, "read": count - count // 2} for domain, number in targets.items(): selected.extend(spread_take(domains[domain], number, stable_u64(seed, split, lang, domain))) if len({r["asset_id"] for r in selected}) != len(selected): raise ValueError("Duplicate source asset IDs") return sorted(selected, key=lambda r: r["asset_id"]), dict(rejected) def anchor_candidates(recipe: dict) -> list[int]: total = recipe["audio"]["sample_count"] return [index for index, event in enumerate(recipe["target_speech_events"]) if event["sample_count"] >= 3 * 16000 and total - event["end_sample"] >= 2 * 16000] def reusable_aside_anchors(population: list[dict], need: int, seed: int, eligible_reference_sha256: set[str] | None = None) -> list[tuple[dict, int]]: """Reuse exact reference WAVs, not voice labels; cap repeated asides at 32. This changes only which existing sessions receive the aside condition. No original target session is dropped, rebuilt or relabelled. Reference selection favors reuse; ties and sessions within each reference use hashes. """ by_reference = defaultdict(dict) for row in population: for index in anchor_candidates(row): event = row["target_speech_events"][index] if eligible_reference_sha256 is not None and event["original_audio_sha256"] not in eligible_reference_sha256: continue candidates = by_reference[event["original_audio_sha256"]] previous = candidates.get(row["session_id"]) if previous is None or stable_u64(seed, row["session_id"], index) < stable_u64(seed, row["session_id"], previous[1]): candidates[row["session_id"]] = (row, index) chosen, used = [], set() while len(chosen) < need: if not by_reference: raise ValueError("Not enough real >=3s reference voices") key = min(by_reference, key=lambda ref: ( -min(MAX_ASIDE_REFERENCE_USES, len(by_reference[ref].keys() - used)), stable_u64(seed, "reference", ref))) available = [entry for sid, entry in by_reference.pop(key).items() if sid not in used] if not available: raise ValueError("Not enough real >=3s reference voices") available.sort(key=lambda entry: stable_u64(seed, "aside", entry[0]["session_id"])) selected = available[:min(MAX_ASIDE_REFERENCE_USES, need-len(chosen))] chosen.extend(selected) used.update(row["session_id"] for row, _ in selected) return chosen def assign_sessions(rows: list[dict], seed: int, eligible_reference_sha256: set[str] | None = None, *, eligible_reference_keys: set[tuple[str, str, str]] | None = None) -> dict[str, dict]: if len({r["session_id"] for r in rows}) != len(rows): raise ValueError("Duplicate reference session IDs") if eligible_reference_sha256 is not None and eligible_reference_keys is not None: raise ValueError("Choose scoped reference eligibility, not both eligibility APIs") result = {} validation_references = set() splits = ("validation", "train") if eligible_reference_keys is not None else ("train", "validation") for split in splits: population = [r for r in rows if r["split"] == split] counts = class_counts(len(population)) used = set() for lang in ("zh", "en"): need = counts["same_voice_aside"] // 2 cohort_references = eligible_reference_sha256 if eligible_reference_keys is not None: cohort_references = {sha for key_split, key_lang, sha in eligible_reference_keys if (key_split, key_lang) == (split, lang)} if split == "train": cohort_references -= validation_references eligible = [r for r in population if r["session_language"] == lang and any( cohort_references is None or r["target_speech_events"][i]["original_audio_sha256"] in cohort_references for i in anchor_candidates(r))] if len(eligible) < need: raise ValueError(f"Not enough real >=3s reference voices: {split}/{lang}") for row, anchor in reusable_aside_anchors(eligible, need, stable_u64(seed, split, lang), cohort_references): result[row["session_id"]] = dict(session_type="same_voice_aside", language=lang, anchor_target_turn_index=anchor) used.add(row["session_id"]) if split == "validation": validation_references.add(row["target_speech_events"][anchor]["original_audio_sha256"]) remaining = sorted([r for r in population if r["session_id"] not in used], key=lambda r: stable_u64(seed, "class", split, r["session_id"])) offset = 0 for name, _ in CLASS_WEIGHTS: if name == "same_voice_aside": continue for lang in ("zh", "en"): n = counts[name] // 2 for row in remaining[offset:offset+n]: result[row["session_id"]] = dict(session_type=name, language=lang) offset += n if offset != len(remaining): raise AssertionError("Class quotas failed to partition sessions") reference_uses = Counter(row["target_speech_events"][result[row["session_id"]]["anchor_target_turn_index"]]["original_audio_sha256"] for row in rows if result[row["session_id"]]["session_type"] == "same_voice_aside") if any(count > MAX_ASIDE_REFERENCE_USES for count in reference_uses.values()): raise ValueError("Exact reference reuse exceeds the global cap across split/language groups") return result def plan_tts_jobs(rows: list[dict], assignments: dict, assets: list[dict], seed: int): pools = defaultdict(list) for asset in assets: # Short, existing screened statements. Do not synthesize target commands as noise. if asset.get("speech_domain") == "conversation" and 2 * 16000 <= asset["sample_count"] <= 6 * 16000: pools[asset["split"], asset["language"]].append(asset) jobs = {} for row in rows: assignment = assignments[row["session_id"]] if assignment["session_type"] != "same_voice_aside": continue index = assignment["anchor_target_turn_index"] turn, event = row["turns"][index], row["target_speech_events"][index] candidates = sorted(pools[row["split"], assignment["language"]], key=lambda r: r["asset_id"]) if not candidates: raise ValueError("No screened conversational TTS texts") # Reuse the same generated aside when the actual reference waveform repeats. text_source = candidates[stable_u64(seed, "tts-text", event["original_audio_sha256"]) % len(candidates)] identity = canonical_json_sha256(dict(reference_sha=event["original_audio_sha256"], text_asset=text_source["asset_id"], policy=POLICY)) job = dict(job_id=identity, split=row["split"], language=assignment["language"], reference_audio=event["original_audio_path"], reference_sha256=event["original_audio_sha256"], reference_text=turn["instruction_text"], reference_source_key=turn["tts_source_key"], text=text_source["transcript"], text_source_asset_id=text_source["asset_id"], text_source_normalized_sha256=text_source["normalized_sha256"], voice_relation="reference_clone", semantic_review="screened_text_audio_pending", synthesis_status="not_started", same_speaker_verified=False) if identity in jobs and jobs[identity] != job: raise ValueError("A TTS reference crosses split/language contracts") jobs[identity] = job assignment["tts_job_id"] = identity return sorted(jobs.values(), key=lambda r: r["job_id"]) def validate_qualified_asset(asset: dict, frozen_job: dict) -> dict[str, str]: """Verify one reusable generated waveform and its job-bound QA evidence. The canonical *individual job* is the reuse identity. Its original whole generation manifest may differ from the new selected jobs manifest. Returns actual evidence-file hashes for freezing and later revalidation. """ if asset.get("eligible_for_mixing") is not True: raise ValueError("Qualified candidate is not eligible_for_mixing") if asset.get("role") != "non_command_speech" or asset.get("voice_relation") != "reference_clone": raise ValueError("Qualified candidate is not a non-command reference clone") job_hash = canonical_json_sha256(frozen_job) if asset.get("job_sha256") != job_hash: raise ValueError("Qualified candidate individual job hash mismatch") for key in ("job_id", "split", "language", "reference_audio", "reference_sha256", "reference_text"): if asset.get(key) != frozen_job.get(key): raise ValueError(f"Qualified candidate/frozen job binding differs: {key}") if asset.get("transcript") != frozen_job["text"]: raise ValueError("Qualified candidate transcript differs from frozen job") for key in ("asset_id", "speaker_id", "source_group"): if not asset.get(key): raise ValueError(f"Missing qualified asset provenance: {key}") verified = {} for path_key, hash_key in (("normalized_path", "normalized_sha256"), ("qa_path", "qa_sha256"), ("generation_provenance_path", "generation_provenance_sha256")): path = Path(asset[path_key]).absolute() actual = sha256_file(path) if actual != asset[hash_key]: raise ValueError(f"Qualified candidate evidence hash mismatch: {path_key}") verified[str(path)] = actual with wave.open(str(asset["normalized_path"]), "rb") as handle: if (handle.getframerate(), handle.getnchannels(), handle.getsampwidth(), handle.getcomptype(), handle.getnframes()) != ( 16000, 1, 2, "NONE", int(asset["sample_count"])) or handle.getnframes() <= 0: raise ValueError("Qualified candidate WAV geometry mismatch") qa = json.loads(Path(asset["qa_path"]).read_text()) if (qa.get("engineering_eligible") is not True or qa.get("job_id") != frozen_job["job_id"] or qa.get("audio_sha256") != asset["normalized_sha256"] or qa.get("job_sha256") != job_hash): raise ValueError("Qualified candidate QA does not approve this exact audio/job") provenance = json.loads(Path(asset["generation_provenance_path"]).read_text()) if (provenance.get("job_id") != frozen_job["job_id"] or provenance.get("job_sha256") != job_hash or canonical_json_sha256(provenance.get("job")) != job_hash): raise ValueError("Generated provenance individual job differs from frozen job") normalized = provenance["normalized"] if (normalized["sha256"] != asset["normalized_sha256"] or Path(normalized["path"]).resolve() != Path(asset["normalized_path"]).resolve() or int(normalized["sample_count"]) != int(asset["sample_count"])): raise ValueError("Generated provenance points to different normalized audio") if any(sha256_file(Path(path)) != expected for path, expected in verified.items()): raise ValueError("Qualified evidence changed during verification") return verified def _qualified_inputs(candidate_registry: Path, generation_plan: Path, seed: int, reference_hash: str, assets: list[dict]) -> tuple[dict, dict, dict]: """Read-only qualification; never regenerate jobs or change old provenance.""" report_path = generation_plan / "preparation_report.json" report = json.loads(report_path.read_text()) if report["policy"] != POLICY or int(report["seed"]) != seed or report["reference_recipe_sha256"] != reference_hash: raise ValueError("Parent generation plan policy/seed/reference mismatch") generation_hashes = {str(report_path.absolute()): sha256_file(report_path)} for name, expected in report["output_hashes"].items(): path = generation_plan / name if sha256_file(path) != expected: raise ValueError(f"Parent generation plan changed: {name}") generation_hashes[str(path.absolute())] = expected old_assets = read_jsonl(generation_plan / "selected_real_assets.jsonl") if canonical_json_sha256(old_assets) != canonical_json_sha256(assets): raise ValueError("Qualified refresh must retain the frozen real-text/source pool") old_jobs = read_jsonl(generation_plan / "tts_jobs.jsonl") by_job = {job["job_id"]: job for job in old_jobs} if len(by_job) != len(old_jobs): raise ValueError("Duplicate frozen generation job IDs") candidates = read_jsonl(candidate_registry) qualified = {} evidence = {str(candidate_registry.absolute()): sha256_file(candidate_registry)} for asset in candidates: identifier = asset["job_id"] if identifier in qualified: raise ValueError("Duplicate qualified candidate job ID") if identifier not in by_job: raise ValueError("Qualified candidate absent from parent generation plan") evidence.update(validate_qualified_asset(asset, by_job[identifier])) qualified[identifier] = asset if not qualified: raise ValueError("No eligible reference-clone candidates") return qualified, by_job, {"generation_hashes": generation_hashes, "evidence_hashes": evidence} def prepare(reference: Path, output: Path, seed: int, qualified_registry: Path | None = None, generation_plan_root: Path | None = None) -> dict: reference, output = reference.absolute(), output.absolute() if output.resolve() != output or not output.is_relative_to(ROOT / "data/hfd_third_party_multi_v5/work"): raise ValueError("Plan output must remain in the new LZK-owned work directory") if output.exists(): raise FileExistsError("Do not overwrite an existing plan") recipe_path = reference / "production/manifests/session_recipes.jsonl" registry_path = reference / "work/catalog/manifests/third_party_source_registry.jsonl" semantics_path = reference / "work/catalog/manifests/command_semantics.json" if not json.loads((reference / "production/reports/strict_audit.json").read_text()).get("passed"): raise ValueError("Reference production has not passed its audit") frozen_inputs = {str(path): sha256_file(path) for path in (recipe_path, registry_path, semantics_path)} rows = read_jsonl(recipe_path) assets, rejected = balanced_catalog(read_jsonl(registry_path), json.loads(semantics_path.read_text()), seed) qualified = None qualification = dict(qualified_registry_path=None, qualified_registry_sha256=None, generation_plan_root=None, generation_plan_hashes={}) if qualified_registry is not None: qualified_registry = qualified_registry.absolute() generation_plan_root = (generation_plan_root or DEFAULT_GENERATION_PLAN).absolute() qualified, frozen_jobs, evidence = _qualified_inputs( qualified_registry, generation_plan_root, seed, frozen_inputs[str(recipe_path)], assets) frozen_inputs.update(evidence["generation_hashes"]) frozen_inputs.update(evidence["evidence_hashes"]) qualification = dict(qualified_registry_path=str(qualified_registry), qualified_registry_sha256=frozen_inputs[str(qualified_registry)], generation_plan_root=str(generation_plan_root), generation_plan_hashes=evidence["generation_hashes"], qualified_evidence_hashes=evidence["evidence_hashes"], qualified_candidate_jobs=len(qualified), individual_job_reuse_contract="canonical_job_equal_to_frozen_generation_job_not_whole_manifest_identity") reference_keys = {(asset["split"], asset["language"], asset["reference_sha256"]) for asset in qualified.values()} if qualified is not None else None assignments = assign_sessions(rows, seed, eligible_reference_keys=reference_keys) jobs = plan_tts_jobs(rows, assignments, assets, seed) if qualified is not None: for job in jobs: if job["job_id"] not in qualified or canonical_json_sha256(job) != canonical_json_sha256(frozen_jobs[job["job_id"]]): raise ValueError("Qualified refresh changed an individual frozen TTS job") qualification.update(selected_tts_job_ids=[job["job_id"] for job in jobs], selected_tts_job_sha256={job["job_id"]: canonical_json_sha256(job) for job in jobs}, qualified_existing_tts_jobs=len(jobs), additional_tts_generation_required=0, qualified_reference_eligibility_scope="split_language_reference_sha256") selected_by_split = {split: sorted({job["reference_sha256"] for job in jobs if job["split"] == split}) for split in ("train", "validation")} if set(selected_by_split["train"]) & set(selected_by_split["validation"]): raise ValueError("Qualified clone references overlap train and validation") qualification["selected_reference_sha256_by_split"] = selected_by_split planned = [dict(session_id=r["session_id"], session_signature=r["session_signature"], split=r["split"], parent_recipe_sha256=canonical_json_sha256(r), **assignments[r["session_id"]]) for r in rows] if any(sha256_file(Path(path)) != expected for path, expected in frozen_inputs.items()): raise RuntimeError("Reference inputs changed during planning") report = dict(format="hfd_multi_event_preparation_v1", policy=POLICY, seed=seed, status="plan_only_not_a_generated_dataset", production_modified=False, reference_root=str(reference), reference_recipe_sha256=sha256_file(recipe_path), reference_registry_sha256=sha256_file(registry_path), semantics_sha256=sha256_file(semantics_path), sessions=len(rows), class_counts=dict(Counter(r["session_type"] for r in planned)), background_languages=dict(Counter(r["language"] for r in planned)), class_language_split_counts=dict(Counter('/'.join((r['split'],r['session_type'],r['language'])) for r in planned)), real_catalog_languages=dict(Counter(r["language"] for r in assets)), catalog_rejected_reasons=rejected, tts_jobs=len(jobs), tts_job_languages=dict(Counter(r["language"] for r in jobs)), tts_audio_generated=0, gt_generated=0, training_started=False, aside_reference_selection="exact_reference_reuse_capped_then_hash_random_sessions", max_aside_uses_per_reference=MAX_ASIDE_REFERENCE_USES, sir_distribution=dict(scope="combined_background_active_RMS_vs_complete_target_active_RMS", family="truncated_normal_db", mean=2.0, standard_deviation=3.0, lower=-4.0, upper=8.0, final_peak_policy="uniform_scale_whole_mix"), language_ratio_note="Sessions and selected real-audio candidates are balanced; cached unique TTS jobs need not have equal language counts.") report.update(qualification) output.mkdir(parents=True, exist_ok=False) write_jsonl_atomic(output / "session_assignments.jsonl", planned) write_jsonl_atomic(output / "selected_real_assets.jsonl", assets) write_jsonl_atomic(output / "tts_jobs.jsonl", jobs) report["output_hashes"] = {name: sha256_file(output / name) for name in ( "session_assignments.jsonl", "selected_real_assets.jsonl", "tts_jobs.jsonl")} write_json_atomic(output / "preparation_report.json", report) return report if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--reference-root", type=Path, default=DEFAULT_REFERENCE) parser.add_argument("--output-root", type=Path, default=DEFAULT_OUTPUT) parser.add_argument("--seed", type=int, default=SEED) parser.add_argument("--qualified-registry", type=Path) parser.add_argument("--generation-plan-root", type=Path, default=DEFAULT_GENERATION_PLAN) args = parser.parse_args() print(json.dumps(prepare(args.reference_root, args.output_root, args.seed, args.qualified_registry, args.generation_plan_root), ensure_ascii=False, indent=2))