Download scripts/data/plan_hfd_third_party_multi_v5.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 24.1 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/plan_hfd_third_party_multi_v5.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/data/plan_hfd_third_party_multi_v5.py
-
curl -L -o plan_hfd_third_party_multi_v5.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/plan_hfd_third_party_multi_v5.py
24.1 kB
| #!/usr/bin/env python3 | |
| """Prepare a CPU-only multi-event refresh plan; never replace production audio.""" | |
| from __future__ import annotations | |
| import argparse | |
| from collections import Counter, defaultdict | |
| import json | |
| from pathlib import Path | |
| import sys | |
| import wave | |
| ROOT = Path(__file__).resolve().parents[2] | |
| if str(ROOT) not in sys.path: | |
| sys.path.insert(0, str(ROOT)) | |
| from scripts.data.build_hfd_diverse_catalog import filter_text | |
| from scripts.data.build_hfd_cross_catalog import new_source_filter_reason | |
| from scripts.data.hfd_third_party_common import ( | |
| canonical_json_sha256, read_jsonl, sha256_file, stable_u64, | |
| write_json_atomic, write_jsonl_atomic, | |
| ) | |
| from scripts.data.hfd_third_party_multi_v5 import CLASS_WEIGHTS, quota_counts | |
| POLICY = "multi_event_speech_v5" | |
| SEED = 26092801 | |
| DEFAULT_REFERENCE = ROOT / "data/hfd_third_party_cross_v4" | |
| DEFAULT_OUTPUT = ROOT / "data/hfd_third_party_multi_v5/work/plan_qualified" | |
| DEFAULT_GENERATION_PLAN = ROOT / "data/hfd_third_party_multi_v5/work/plan_reuse" | |
| KNOWN_PENDING_ASSETS = {"alimeeting:R1021_M1959_F_SPK1953:00157"} | |
| MAX_ASIDE_REFERENCE_USES = 32 | |
| def class_counts(size: int) -> dict[str, int]: | |
| return quota_counts(size) | |
| def spread_take(rows: list[dict], count: int, seed: int) -> list[dict]: | |
| """Round-robin speakers, retaining diverse real voice identities.""" | |
| if not 0 <= count <= len(rows): | |
| raise ValueError("Insufficient catalog population") | |
| speakers = defaultdict(list) | |
| for row in rows: | |
| speakers[row["speaker_id"]].append(row) | |
| ranked = [] | |
| for speaker, group in speakers.items(): | |
| ordered = sorted(group, key=lambda r: stable_u64(seed, r["asset_id"])) | |
| for index, row in enumerate(ordered): | |
| ranked.append((index, stable_u64(seed, speaker), row["asset_id"], row)) | |
| return [item[-1] for item in sorted(ranked)[:count]] | |
| def balanced_catalog(rows: list[dict], semantics: dict, seed: int) -> tuple[list[dict], dict]: | |
| accepted, rejected = [], Counter() | |
| for row in rows: | |
| if row.get("role") != "non_command_speech": | |
| continue | |
| reason = ("pending_human_semantic_decision" if row["asset_id"] in KNOWN_PENDING_ASSETS else | |
| filter_text(row["transcript"], row["language"], | |
| set(semantics["target_phrases"][row["language"]]), semantics=semantics) or | |
| new_source_filter_reason(row, semantics)) | |
| if reason: | |
| rejected[reason] += 1 | |
| else: | |
| accepted.append(row) | |
| selected = [] | |
| for split in ("train", "validation"): | |
| pools = {lang: [r for r in accepted if (r["split"], r["language"]) == (split, lang)] | |
| for lang in ("zh", "en")} | |
| count = min(map(len, pools.values())) | |
| if not count: | |
| raise ValueError(f"Empty bilingual catalog: {split}") | |
| for lang, pool in pools.items(): | |
| if len(pool) == count: | |
| selected.extend(pool) | |
| continue | |
| domains = {domain: [r for r in pool if r.get("speech_domain") == domain] | |
| for domain in ("conversation", "read")} | |
| targets = {"conversation": count // 2, "read": count - count // 2} | |
| for domain, number in targets.items(): | |
| selected.extend(spread_take(domains[domain], number, stable_u64(seed, split, lang, domain))) | |
| if len({r["asset_id"] for r in selected}) != len(selected): | |
| raise ValueError("Duplicate source asset IDs") | |
| return sorted(selected, key=lambda r: r["asset_id"]), dict(rejected) | |
| def anchor_candidates(recipe: dict) -> list[int]: | |
| total = recipe["audio"]["sample_count"] | |
| return [index for index, event in enumerate(recipe["target_speech_events"]) | |
| if event["sample_count"] >= 3 * 16000 and total - event["end_sample"] >= 2 * 16000] | |
| def reusable_aside_anchors(population: list[dict], need: int, seed: int, | |
| eligible_reference_sha256: set[str] | None = None) -> list[tuple[dict, int]]: | |
| """Reuse exact reference WAVs, not voice labels; cap repeated asides at 32. | |
| This changes only which existing sessions receive the aside condition. | |
| No original target session is dropped, rebuilt or relabelled. Reference | |
| selection favors reuse; ties and sessions within each reference use hashes. | |
| """ | |
| by_reference = defaultdict(dict) | |
| for row in population: | |
| for index in anchor_candidates(row): | |
| event = row["target_speech_events"][index] | |
| if eligible_reference_sha256 is not None and event["original_audio_sha256"] not in eligible_reference_sha256: | |
| continue | |
| candidates = by_reference[event["original_audio_sha256"]] | |
| previous = candidates.get(row["session_id"]) | |
| if previous is None or stable_u64(seed, row["session_id"], index) < stable_u64(seed, row["session_id"], previous[1]): | |
| candidates[row["session_id"]] = (row, index) | |
| chosen, used = [], set() | |
| while len(chosen) < need: | |
| if not by_reference: | |
| raise ValueError("Not enough real >=3s reference voices") | |
| key = min(by_reference, key=lambda ref: ( | |
| -min(MAX_ASIDE_REFERENCE_USES, len(by_reference[ref].keys() - used)), | |
| stable_u64(seed, "reference", ref))) | |
| available = [entry for sid, entry in by_reference.pop(key).items() if sid not in used] | |
| if not available: | |
| raise ValueError("Not enough real >=3s reference voices") | |
| available.sort(key=lambda entry: stable_u64(seed, "aside", entry[0]["session_id"])) | |
| selected = available[:min(MAX_ASIDE_REFERENCE_USES, need-len(chosen))] | |
| chosen.extend(selected) | |
| used.update(row["session_id"] for row, _ in selected) | |
| return chosen | |
| def assign_sessions(rows: list[dict], seed: int, | |
| eligible_reference_sha256: set[str] | None = None, *, | |
| eligible_reference_keys: set[tuple[str, str, str]] | None = None) -> dict[str, dict]: | |
| if len({r["session_id"] for r in rows}) != len(rows): | |
| raise ValueError("Duplicate reference session IDs") | |
| if eligible_reference_sha256 is not None and eligible_reference_keys is not None: | |
| raise ValueError("Choose scoped reference eligibility, not both eligibility APIs") | |
| result = {} | |
| validation_references = set() | |
| splits = ("validation", "train") if eligible_reference_keys is not None else ("train", "validation") | |
| for split in splits: | |
| population = [r for r in rows if r["split"] == split] | |
| counts = class_counts(len(population)) | |
| used = set() | |
| for lang in ("zh", "en"): | |
| need = counts["same_voice_aside"] // 2 | |
| cohort_references = eligible_reference_sha256 | |
| if eligible_reference_keys is not None: | |
| cohort_references = {sha for key_split, key_lang, sha in eligible_reference_keys | |
| if (key_split, key_lang) == (split, lang)} | |
| if split == "train": | |
| cohort_references -= validation_references | |
| eligible = [r for r in population if r["session_language"] == lang and any( | |
| cohort_references is None or r["target_speech_events"][i]["original_audio_sha256"] in cohort_references | |
| for i in anchor_candidates(r))] | |
| if len(eligible) < need: | |
| raise ValueError(f"Not enough real >=3s reference voices: {split}/{lang}") | |
| for row, anchor in reusable_aside_anchors(eligible, need, stable_u64(seed, split, lang), | |
| cohort_references): | |
| result[row["session_id"]] = dict(session_type="same_voice_aside", language=lang, | |
| anchor_target_turn_index=anchor) | |
| used.add(row["session_id"]) | |
| if split == "validation": | |
| validation_references.add(row["target_speech_events"][anchor]["original_audio_sha256"]) | |
| remaining = sorted([r for r in population if r["session_id"] not in used], | |
| key=lambda r: stable_u64(seed, "class", split, r["session_id"])) | |
| offset = 0 | |
| for name, _ in CLASS_WEIGHTS: | |
| if name == "same_voice_aside": | |
| continue | |
| for lang in ("zh", "en"): | |
| n = counts[name] // 2 | |
| for row in remaining[offset:offset+n]: | |
| result[row["session_id"]] = dict(session_type=name, language=lang) | |
| offset += n | |
| if offset != len(remaining): | |
| raise AssertionError("Class quotas failed to partition sessions") | |
| reference_uses = Counter(row["target_speech_events"][result[row["session_id"]]["anchor_target_turn_index"]]["original_audio_sha256"] | |
| for row in rows if result[row["session_id"]]["session_type"] == "same_voice_aside") | |
| if any(count > MAX_ASIDE_REFERENCE_USES for count in reference_uses.values()): | |
| raise ValueError("Exact reference reuse exceeds the global cap across split/language groups") | |
| return result | |
| def plan_tts_jobs(rows: list[dict], assignments: dict, assets: list[dict], seed: int): | |
| pools = defaultdict(list) | |
| for asset in assets: | |
| # Short, existing screened statements. Do not synthesize target commands as noise. | |
| if asset.get("speech_domain") == "conversation" and 2 * 16000 <= asset["sample_count"] <= 6 * 16000: | |
| pools[asset["split"], asset["language"]].append(asset) | |
| jobs = {} | |
| for row in rows: | |
| assignment = assignments[row["session_id"]] | |
| if assignment["session_type"] != "same_voice_aside": | |
| continue | |
| index = assignment["anchor_target_turn_index"] | |
| turn, event = row["turns"][index], row["target_speech_events"][index] | |
| candidates = sorted(pools[row["split"], assignment["language"]], key=lambda r: r["asset_id"]) | |
| if not candidates: | |
| raise ValueError("No screened conversational TTS texts") | |
| # Reuse the same generated aside when the actual reference waveform repeats. | |
| text_source = candidates[stable_u64(seed, "tts-text", event["original_audio_sha256"]) % len(candidates)] | |
| identity = canonical_json_sha256(dict(reference_sha=event["original_audio_sha256"], | |
| text_asset=text_source["asset_id"], policy=POLICY)) | |
| job = dict(job_id=identity, split=row["split"], language=assignment["language"], | |
| reference_audio=event["original_audio_path"], reference_sha256=event["original_audio_sha256"], | |
| reference_text=turn["instruction_text"], reference_source_key=turn["tts_source_key"], | |
| text=text_source["transcript"], text_source_asset_id=text_source["asset_id"], | |
| text_source_normalized_sha256=text_source["normalized_sha256"], | |
| voice_relation="reference_clone", semantic_review="screened_text_audio_pending", | |
| synthesis_status="not_started", same_speaker_verified=False) | |
| if identity in jobs and jobs[identity] != job: | |
| raise ValueError("A TTS reference crosses split/language contracts") | |
| jobs[identity] = job | |
| assignment["tts_job_id"] = identity | |
| return sorted(jobs.values(), key=lambda r: r["job_id"]) | |
| def validate_qualified_asset(asset: dict, frozen_job: dict) -> dict[str, str]: | |
| """Verify one reusable generated waveform and its job-bound QA evidence. | |
| The canonical *individual job* is the reuse identity. Its original whole | |
| generation manifest may differ from the new selected jobs manifest. | |
| Returns actual evidence-file hashes for freezing and later revalidation. | |
| """ | |
| if asset.get("eligible_for_mixing") is not True: | |
| raise ValueError("Qualified candidate is not eligible_for_mixing") | |
| if asset.get("role") != "non_command_speech" or asset.get("voice_relation") != "reference_clone": | |
| raise ValueError("Qualified candidate is not a non-command reference clone") | |
| job_hash = canonical_json_sha256(frozen_job) | |
| if asset.get("job_sha256") != job_hash: | |
| raise ValueError("Qualified candidate individual job hash mismatch") | |
| for key in ("job_id", "split", "language", "reference_audio", "reference_sha256", "reference_text"): | |
| if asset.get(key) != frozen_job.get(key): | |
| raise ValueError(f"Qualified candidate/frozen job binding differs: {key}") | |
| if asset.get("transcript") != frozen_job["text"]: | |
| raise ValueError("Qualified candidate transcript differs from frozen job") | |
| for key in ("asset_id", "speaker_id", "source_group"): | |
| if not asset.get(key): | |
| raise ValueError(f"Missing qualified asset provenance: {key}") | |
| verified = {} | |
| for path_key, hash_key in (("normalized_path", "normalized_sha256"), | |
| ("qa_path", "qa_sha256"), | |
| ("generation_provenance_path", "generation_provenance_sha256")): | |
| path = Path(asset[path_key]).absolute() | |
| actual = sha256_file(path) | |
| if actual != asset[hash_key]: | |
| raise ValueError(f"Qualified candidate evidence hash mismatch: {path_key}") | |
| verified[str(path)] = actual | |
| with wave.open(str(asset["normalized_path"]), "rb") as handle: | |
| if (handle.getframerate(), handle.getnchannels(), handle.getsampwidth(), handle.getcomptype(), handle.getnframes()) != ( | |
| 16000, 1, 2, "NONE", int(asset["sample_count"])) or handle.getnframes() <= 0: | |
| raise ValueError("Qualified candidate WAV geometry mismatch") | |
| qa = json.loads(Path(asset["qa_path"]).read_text()) | |
| if (qa.get("engineering_eligible") is not True or qa.get("job_id") != frozen_job["job_id"] | |
| or qa.get("audio_sha256") != asset["normalized_sha256"] or qa.get("job_sha256") != job_hash): | |
| raise ValueError("Qualified candidate QA does not approve this exact audio/job") | |
| provenance = json.loads(Path(asset["generation_provenance_path"]).read_text()) | |
| if (provenance.get("job_id") != frozen_job["job_id"] or provenance.get("job_sha256") != job_hash | |
| or canonical_json_sha256(provenance.get("job")) != job_hash): | |
| raise ValueError("Generated provenance individual job differs from frozen job") | |
| normalized = provenance["normalized"] | |
| if (normalized["sha256"] != asset["normalized_sha256"] | |
| or Path(normalized["path"]).resolve() != Path(asset["normalized_path"]).resolve() | |
| or int(normalized["sample_count"]) != int(asset["sample_count"])): | |
| raise ValueError("Generated provenance points to different normalized audio") | |
| if any(sha256_file(Path(path)) != expected for path, expected in verified.items()): | |
| raise ValueError("Qualified evidence changed during verification") | |
| return verified | |
| def _qualified_inputs(candidate_registry: Path, generation_plan: Path, seed: int, | |
| reference_hash: str, assets: list[dict]) -> tuple[dict, dict, dict]: | |
| """Read-only qualification; never regenerate jobs or change old provenance.""" | |
| report_path = generation_plan / "preparation_report.json" | |
| report = json.loads(report_path.read_text()) | |
| if report["policy"] != POLICY or int(report["seed"]) != seed or report["reference_recipe_sha256"] != reference_hash: | |
| raise ValueError("Parent generation plan policy/seed/reference mismatch") | |
| generation_hashes = {str(report_path.absolute()): sha256_file(report_path)} | |
| for name, expected in report["output_hashes"].items(): | |
| path = generation_plan / name | |
| if sha256_file(path) != expected: | |
| raise ValueError(f"Parent generation plan changed: {name}") | |
| generation_hashes[str(path.absolute())] = expected | |
| old_assets = read_jsonl(generation_plan / "selected_real_assets.jsonl") | |
| if canonical_json_sha256(old_assets) != canonical_json_sha256(assets): | |
| raise ValueError("Qualified refresh must retain the frozen real-text/source pool") | |
| old_jobs = read_jsonl(generation_plan / "tts_jobs.jsonl") | |
| by_job = {job["job_id"]: job for job in old_jobs} | |
| if len(by_job) != len(old_jobs): | |
| raise ValueError("Duplicate frozen generation job IDs") | |
| candidates = read_jsonl(candidate_registry) | |
| qualified = {} | |
| evidence = {str(candidate_registry.absolute()): sha256_file(candidate_registry)} | |
| for asset in candidates: | |
| identifier = asset["job_id"] | |
| if identifier in qualified: | |
| raise ValueError("Duplicate qualified candidate job ID") | |
| if identifier not in by_job: | |
| raise ValueError("Qualified candidate absent from parent generation plan") | |
| evidence.update(validate_qualified_asset(asset, by_job[identifier])) | |
| qualified[identifier] = asset | |
| if not qualified: | |
| raise ValueError("No eligible reference-clone candidates") | |
| return qualified, by_job, {"generation_hashes": generation_hashes, "evidence_hashes": evidence} | |
| def prepare(reference: Path, output: Path, seed: int, qualified_registry: Path | None = None, | |
| generation_plan_root: Path | None = None) -> dict: | |
| reference, output = reference.absolute(), output.absolute() | |
| if output.resolve() != output or not output.is_relative_to(ROOT / "data/hfd_third_party_multi_v5/work"): | |
| raise ValueError("Plan output must remain in the new LZK-owned work directory") | |
| if output.exists(): | |
| raise FileExistsError("Do not overwrite an existing plan") | |
| recipe_path = reference / "production/manifests/session_recipes.jsonl" | |
| registry_path = reference / "work/catalog/manifests/third_party_source_registry.jsonl" | |
| semantics_path = reference / "work/catalog/manifests/command_semantics.json" | |
| if not json.loads((reference / "production/reports/strict_audit.json").read_text()).get("passed"): | |
| raise ValueError("Reference production has not passed its audit") | |
| frozen_inputs = {str(path): sha256_file(path) for path in (recipe_path, registry_path, semantics_path)} | |
| rows = read_jsonl(recipe_path) | |
| assets, rejected = balanced_catalog(read_jsonl(registry_path), json.loads(semantics_path.read_text()), seed) | |
| qualified = None | |
| qualification = dict(qualified_registry_path=None, qualified_registry_sha256=None, | |
| generation_plan_root=None, generation_plan_hashes={}) | |
| if qualified_registry is not None: | |
| qualified_registry = qualified_registry.absolute() | |
| generation_plan_root = (generation_plan_root or DEFAULT_GENERATION_PLAN).absolute() | |
| qualified, frozen_jobs, evidence = _qualified_inputs( | |
| qualified_registry, generation_plan_root, seed, frozen_inputs[str(recipe_path)], assets) | |
| frozen_inputs.update(evidence["generation_hashes"]) | |
| frozen_inputs.update(evidence["evidence_hashes"]) | |
| qualification = dict(qualified_registry_path=str(qualified_registry), | |
| qualified_registry_sha256=frozen_inputs[str(qualified_registry)], | |
| generation_plan_root=str(generation_plan_root), | |
| generation_plan_hashes=evidence["generation_hashes"], | |
| qualified_evidence_hashes=evidence["evidence_hashes"], | |
| qualified_candidate_jobs=len(qualified), | |
| individual_job_reuse_contract="canonical_job_equal_to_frozen_generation_job_not_whole_manifest_identity") | |
| reference_keys = {(asset["split"], asset["language"], asset["reference_sha256"]) | |
| for asset in qualified.values()} if qualified is not None else None | |
| assignments = assign_sessions(rows, seed, eligible_reference_keys=reference_keys) | |
| jobs = plan_tts_jobs(rows, assignments, assets, seed) | |
| if qualified is not None: | |
| for job in jobs: | |
| if job["job_id"] not in qualified or canonical_json_sha256(job) != canonical_json_sha256(frozen_jobs[job["job_id"]]): | |
| raise ValueError("Qualified refresh changed an individual frozen TTS job") | |
| qualification.update(selected_tts_job_ids=[job["job_id"] for job in jobs], | |
| selected_tts_job_sha256={job["job_id"]: canonical_json_sha256(job) for job in jobs}, | |
| qualified_existing_tts_jobs=len(jobs), additional_tts_generation_required=0, | |
| qualified_reference_eligibility_scope="split_language_reference_sha256") | |
| selected_by_split = {split: sorted({job["reference_sha256"] for job in jobs if job["split"] == split}) | |
| for split in ("train", "validation")} | |
| if set(selected_by_split["train"]) & set(selected_by_split["validation"]): | |
| raise ValueError("Qualified clone references overlap train and validation") | |
| qualification["selected_reference_sha256_by_split"] = selected_by_split | |
| planned = [dict(session_id=r["session_id"], session_signature=r["session_signature"], split=r["split"], | |
| parent_recipe_sha256=canonical_json_sha256(r), **assignments[r["session_id"]]) for r in rows] | |
| if any(sha256_file(Path(path)) != expected for path, expected in frozen_inputs.items()): | |
| raise RuntimeError("Reference inputs changed during planning") | |
| report = dict(format="hfd_multi_event_preparation_v1", policy=POLICY, seed=seed, | |
| status="plan_only_not_a_generated_dataset", production_modified=False, | |
| reference_root=str(reference), reference_recipe_sha256=sha256_file(recipe_path), | |
| reference_registry_sha256=sha256_file(registry_path), semantics_sha256=sha256_file(semantics_path), | |
| sessions=len(rows), class_counts=dict(Counter(r["session_type"] for r in planned)), | |
| background_languages=dict(Counter(r["language"] for r in planned)), | |
| class_language_split_counts=dict(Counter('/'.join((r['split'],r['session_type'],r['language'])) for r in planned)), | |
| real_catalog_languages=dict(Counter(r["language"] for r in assets)), | |
| catalog_rejected_reasons=rejected, tts_jobs=len(jobs), | |
| tts_job_languages=dict(Counter(r["language"] for r in jobs)), | |
| tts_audio_generated=0, gt_generated=0, training_started=False, | |
| aside_reference_selection="exact_reference_reuse_capped_then_hash_random_sessions", | |
| max_aside_uses_per_reference=MAX_ASIDE_REFERENCE_USES, | |
| sir_distribution=dict(scope="combined_background_active_RMS_vs_complete_target_active_RMS", | |
| family="truncated_normal_db", mean=2.0, standard_deviation=3.0, | |
| lower=-4.0, upper=8.0, final_peak_policy="uniform_scale_whole_mix"), | |
| language_ratio_note="Sessions and selected real-audio candidates are balanced; cached unique TTS jobs need not have equal language counts.") | |
| report.update(qualification) | |
| output.mkdir(parents=True, exist_ok=False) | |
| write_jsonl_atomic(output / "session_assignments.jsonl", planned) | |
| write_jsonl_atomic(output / "selected_real_assets.jsonl", assets) | |
| write_jsonl_atomic(output / "tts_jobs.jsonl", jobs) | |
| report["output_hashes"] = {name: sha256_file(output / name) for name in ( | |
| "session_assignments.jsonl", "selected_real_assets.jsonl", "tts_jobs.jsonl")} | |
| write_json_atomic(output / "preparation_report.json", report) | |
| return report | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--reference-root", type=Path, default=DEFAULT_REFERENCE) | |
| parser.add_argument("--output-root", type=Path, default=DEFAULT_OUTPUT) | |
| parser.add_argument("--seed", type=int, default=SEED) | |
| parser.add_argument("--qualified-registry", type=Path) | |
| parser.add_argument("--generation-plan-root", type=Path, default=DEFAULT_GENERATION_PLAN) | |
| args = parser.parse_args() | |
| print(json.dumps(prepare(args.reference_root, args.output_root, args.seed, | |
| args.qualified_registry, args.generation_plan_root), ensure_ascii=False, indent=2)) | |