Download scripts/data/build_hfd_third_party_multi_v5.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 37 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/build_hfd_third_party_multi_v5.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/data/build_hfd_third_party_multi_v5.py
-
curl -L -o build_hfd_third_party_multi_v5.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/build_hfd_third_party_multi_v5.py
37 kB
| #!/usr/bin/env python3 | |
| """Build the isolated multi-event refresh without changing any reference data. | |
| This is a construction tool, not a strict auditor. A completed build remains | |
| pending an independent audit and human listening. No training/GPU imports occur. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| from collections import Counter, deque | |
| from concurrent.futures import ThreadPoolExecutor | |
| import copy | |
| from datetime import datetime, timezone | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import random | |
| import shutil | |
| import sys | |
| import tempfile | |
| from typing import Any, Iterable | |
| import wave | |
| os.environ["CUDA_VISIBLE_DEVICES"] = "" | |
| os.environ["HIP_VISIBLE_DEVICES"] = "" | |
| for _thread_variable in ("OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS", "MKL_NUM_THREADS"): | |
| os.environ[_thread_variable] = "1" | |
| import numpy as np | |
| ROOT = Path(__file__).resolve().parents[2] | |
| if str(ROOT) not in sys.path: | |
| sys.path.insert(0, str(ROOT)) | |
| from scripts.data import hfd_third_party_multi_v5 as core | |
| from scripts.data.hfd_third_party_common import ( | |
| SAMPLE_RATE, STRIDE_SAMPLES, audio_stats, canonical_json_sha256, | |
| quantize_pcm16, read_jsonl, read_pcm16_wav, sha256_file, stable_u64, | |
| write_json_atomic, write_jsonl_atomic, write_pcm16_wav_atomic, | |
| ) | |
| from scripts.data.scan_hfd_third_party_loudness import read_background, target_track | |
| DEFAULT_REFERENCE = ROOT / "data/hfd_third_party_cross_v4/production" | |
| DEFAULT_PLAN = ROOT / "data/hfd_third_party_multi_v5/work/plan_qualified" | |
| DEFAULT_TTS = ROOT / "data/hfd_third_party_multi_v5/work/tts_assets_qualified/eligible_registry.jsonl" | |
| DEFAULT_OUTPUT = ROOT / "data/hfd_third_party_multi_v5/production" | |
| DEFAULT_PILOT = ROOT / "data/hfd_third_party_multi_v5/work/pilot" | |
| BACKGROUND_SEED = 26092801 | |
| COPY_MANIFESTS = ("primitive_split_snapshot.jsonl", "baseline_session_signatures.jsonl", "source_contract.json") | |
| PLAN_FILES = ("session_assignments.jsonl", "selected_real_assets.jsonl", "tts_jobs.jsonl") | |
| CLASSES = tuple(name for name, _ in core.CLASS_WEIGHTS) | |
| PROTECTED_FIELDS = ("session_id", "session_signature", "split", "turns", "turn_count", "timeline", | |
| "target_speech_events", "supervision_targets", "session_language", "session_voice_name", | |
| "base_background", "interaction_mode", "composition_salt", "composition_seed", | |
| "composition_variant_id", "timeline_seed", "timeline_seed_attempt", "task") | |
| OBSOLETE_FIELDS = ("loudness_revision", "speech_domain_assignment", "profile_index", | |
| "cross_domain_revision", "retained_audio", "background_gain_revision") | |
| def _json(path: Path) -> dict: | |
| return json.loads(path.read_text(encoding="utf-8")) | |
| def _owned(path: Path, root: Path) -> Path: | |
| path, root = path.absolute(), root.absolute() | |
| if path.resolve() != path or not path.is_relative_to(root): | |
| raise ValueError(f"Output is not an unaliased LZK-owned path: {path}") | |
| if path.is_file() and path.stat().st_nlink != 1: | |
| raise ValueError(f"Refusing hard-linked output: {path}") | |
| return path | |
| def _output_root(path: Path, *, pilot: bool) -> Path: | |
| expected = ROOT / ("data/hfd_third_party_multi_v5/work/pilot" if pilot else | |
| "data/hfd_third_party_multi_v5/production") | |
| path = _owned(path, ROOT) | |
| if path != expected: | |
| raise ValueError(f"Output must be the fixed isolated {'pilot' if pilot else 'production'} root: {expected}") | |
| return path | |
| def _relative_payload(root: Path, relative: str, expected: str) -> Path: | |
| if relative != expected: | |
| raise ValueError(f"Unexpected session payload path: {relative}") | |
| return _owned(root / relative, root) | |
| def _unique(rows: Iterable[dict], key: str) -> dict[str, dict]: | |
| result = {} | |
| for row in rows: | |
| identity = str(row[key]) | |
| if identity in result: | |
| raise ValueError(f"Duplicate {key}: {identity}") | |
| result[identity] = row | |
| return result | |
| def _frozen_hashes(paths: Iterable[Path]) -> dict[str, str]: | |
| return {str(path.absolute()): sha256_file(path) for path in paths} | |
| def _assert_frozen(hashes: dict[str, str]) -> None: | |
| for path, expected in hashes.items(): | |
| if sha256_file(Path(path)) != expected: | |
| raise RuntimeError(f"Read-only input changed during construction: {path}") | |
| def select_population(parents: list[dict], assignments: dict[str, dict], *, | |
| limit: int | None, session_types: Iterable[str] | None, seed: int) -> list[dict]: | |
| classes = tuple(session_types) if session_types is not None else CLASSES | |
| if len(set(classes)) != len(classes) or not classes or set(classes) - set(CLASSES): | |
| raise ValueError("Select distinct known session classes") | |
| if limit is None: | |
| if set(classes) != set(CLASSES): | |
| raise ValueError("Class filtering is permitted only for an explicit limited pilot") | |
| return parents | |
| if isinstance(limit, bool) or not isinstance(limit, int) or limit < len(classes): | |
| raise ValueError("Pilot limit must include at least one session of every selected class") | |
| groups = {name: sorted((row for row in parents if assignments[row["session_id"]]["session_type"] == name), | |
| key=lambda row: (stable_u64("multi-v5-pilot", seed, name, row["session_id"]), row["session_id"])) | |
| for name in classes} | |
| if any(not group for group in groups.values()) or limit > sum(map(len, groups.values())): | |
| raise ValueError("Pilot population cannot satisfy the requested classes and limit") | |
| selected, offset = [], 0 | |
| while len(selected) < limit: | |
| for name in classes: | |
| if offset < len(groups[name]) and len(selected) < limit: | |
| selected.append(groups[name][offset]) | |
| offset += 1 | |
| identities = {row["session_id"] for row in selected} | |
| return [row for row in parents if row["session_id"] in identities] | |
| def _validate_clone(parent: dict, assignment: dict, asset: dict, job: dict) -> None: | |
| index = int(assignment["anchor_target_turn_index"]) | |
| event, turn = parent["target_speech_events"][index], parent["turns"][index] | |
| expected = {"job_id": assignment["tts_job_id"], "split": parent["split"], | |
| "language": assignment["language"], "role": "non_command_speech", | |
| "reference_audio": event["original_audio_path"], | |
| "reference_sha256": event["original_audio_sha256"], "reference_text": turn["instruction_text"]} | |
| if asset.get("eligible_for_mixing") is not True: | |
| raise ValueError("Clone asset has not passed production eligibility checks") | |
| for key, value in expected.items(): | |
| if asset.get(key) != value: | |
| raise ValueError(f"Clone does not bind its actual target reference: {key}") | |
| if asset.get("transcript") != job["text"] or job["reference_sha256"] != expected["reference_sha256"]: | |
| raise ValueError("Clone transcript/reference differs from the frozen TTS job") | |
| if parent["session_language"] != assignment["language"]: | |
| raise ValueError("Same-voice aside language differs from its target reference") | |
| for field in ("asset_id", "normalized_path", "normalized_sha256", "speaker_id", "source_group"): | |
| if not asset.get(field): | |
| raise ValueError(f"Missing clone provenance: {field}") | |
| if int(asset.get("sample_count", 0)) <= 0: | |
| raise ValueError("Invalid clone audio length") | |
| def load_inputs(reference_root: Path, plan_root: Path, tts_registry: Path, *, | |
| limit: int | None, session_types: Iterable[str] | None, seed: int) -> dict: | |
| reference_root, plan_root, tts_registry = (path.absolute() for path in (reference_root, plan_root, tts_registry)) | |
| report = _json(plan_root / "preparation_report.json") | |
| if report.get("policy") != core.POLICY or int(report["seed"]) != seed: | |
| raise ValueError("Frozen plan policy/seed mismatch") | |
| if limit is None and not report.get("qualified_registry_path"): | |
| raise ValueError("Full production requires a plan selected from quality-screened clone references") | |
| parent_build = _json(reference_root / "reports/pilot_build.json") | |
| if _json(reference_root / "reports/strict_audit.json").get("passed") is not True: | |
| raise ValueError("Reference production must have passed its independent audit") | |
| recipes_path = reference_root / "manifests/session_recipes.jsonl" | |
| reference_sha = sha256_file(recipes_path) | |
| if reference_sha != report["reference_recipe_sha256"] or reference_sha != parent_build["output_hashes"]["session_recipes"]: | |
| raise ValueError("Frozen reference recipe manifest changed") | |
| plan_hashes = report["output_hashes"] | |
| for name in PLAN_FILES: | |
| if sha256_file(plan_root / name) != plan_hashes[name]: | |
| raise ValueError(f"Frozen plan hash mismatch: {name}") | |
| parents = read_jsonl(recipes_path) | |
| parent_by_id = _unique(parents, "session_id") | |
| assignments = _unique(read_jsonl(plan_root / "session_assignments.jsonl"), "session_id") | |
| if set(parent_by_id) != set(assignments) or len(parents) != int(report["sessions"]): | |
| raise ValueError("Plan does not exactly partition the reference population") | |
| for parent in parents: | |
| assignment = assignments[parent["session_id"]] | |
| if (assignment.get("parent_recipe_sha256") != canonical_json_sha256(parent) | |
| or assignment.get("split") != parent["split"] | |
| or assignment.get("session_signature") != parent["session_signature"] | |
| or assignment.get("session_type") not in CLASSES or assignment.get("language") not in {"zh", "en"}): | |
| raise ValueError("Assignment parent identity/class/language mismatch") | |
| if dict(Counter(row["session_type"] for row in assignments.values())) != report["class_counts"]: | |
| raise ValueError("Plan class counts disagree with assignments") | |
| if dict(Counter(row["language"] for row in assignments.values())) != report["background_languages"]: | |
| raise ValueError("Plan language counts disagree with assignments") | |
| selected = select_population(parents, assignments, limit=limit, session_types=session_types, seed=seed) | |
| assets = _unique(read_jsonl(plan_root / "selected_real_assets.jsonl"), "asset_id") | |
| jobs = _unique(read_jsonl(plan_root / "tts_jobs.jsonl"), "job_id") | |
| clone_rows = read_jsonl(tts_registry) if tts_registry.is_file() else [] | |
| if any(row.get("eligible_for_mixing") is not True for row in clone_rows): | |
| raise ValueError("Eligible clone registry contains an unapproved asset") | |
| clones = _unique(clone_rows, "job_id") | |
| train_references = {a["reference_sha256"] for a in clone_rows if a["split"] == "train"} | |
| validation_references = {a["reference_sha256"] for a in clone_rows if a["split"] == "validation"} | |
| if train_references & validation_references: | |
| raise ValueError("Clone reference audio overlaps train and validation") | |
| for parent in selected: | |
| assignment = assignments[parent["session_id"]] | |
| if assignment["session_type"] != "same_voice_aside": | |
| continue | |
| job_id = assignment["tts_job_id"] | |
| if job_id not in clones or job_id not in jobs: | |
| raise ValueError(f"Missing eligible same-voice TTS asset for selected session: {job_id}") | |
| _validate_clone(parent, assignment, clones[job_id], jobs[job_id]) | |
| combined = _unique([*assets.values(), *clone_rows], "asset_id") | |
| paths = [recipes_path, reference_root / "reports/pilot_build.json", reference_root / "reports/strict_audit.json", | |
| plan_root / "preparation_report.json", *(plan_root / name for name in PLAN_FILES), | |
| *(reference_root / "manifests" / name for name in COPY_MANIFESTS)] | |
| if tts_registry.is_file(): | |
| paths.append(tts_registry) | |
| if report.get("qualified_registry_path"): | |
| from scripts.data.plan_hfd_third_party_multi_v5 import validate_qualified_asset | |
| candidate_path = Path(report["qualified_registry_path"]) | |
| if sha256_file(candidate_path) != report["qualified_registry_sha256"]: | |
| raise ValueError("Quality-screened candidate registry changed") | |
| paths.append(candidate_path) | |
| for asset in clone_rows: | |
| validate_qualified_asset(asset, jobs[asset["job_id"]]) | |
| paths.extend(Path(asset[key]) for key in ("qa_path", "generation_provenance_path", "normalized_path")) | |
| frozen = _frozen_hashes(paths) | |
| return {"parents": selected, "assignments": assignments, "assets": assets, "clones": clones, | |
| "combined_assets": combined, "parent_build": parent_build, "plan_report": report, | |
| "reference_root": reference_root, "reference_recipe_sha256": reference_sha, | |
| "plan_root": plan_root, "plan_hashes": {name: plan_hashes[name] for name in PLAN_FILES}, | |
| "eligible_tts_registry_sha256": sha256_file(tts_registry) if tts_registry.is_file() else None, | |
| "frozen_inputs": frozen} | |
| def boundary_base_fallback(parent: dict, requested_sir_db: float, error_events: list[dict]) -> tuple[np.ndarray, list[dict], dict]: | |
| """Keep the exact boundary draw; do not amplify or resample unusable speech.""" | |
| for event in parent["target_speech_events"]: | |
| if sha256_file(Path(event["original_audio_path"])) != event["original_audio_sha256"]: | |
| raise ValueError("Target source changed before boundary fallback") | |
| target, _ = target_track(parent) | |
| background = parent["base_background"] | |
| if sha256_file(Path(background["path"])) != background["sha256"]: | |
| raise ValueError("Base background changed before boundary fallback") | |
| noise = read_background(background["path"], int(background["start_window"]) * STRIDE_SAMPLES, len(target)) | |
| base = quantize_pcm16(noise + target).astype(np.float32) / np.float32(32768.) | |
| peak = float(np.max(np.abs(base), initial=0.)) | |
| scale = min(1., core.PEAK_LIMIT / max(peak, 1e-12)) | |
| audio = base * np.float32(scale) | |
| events = copy.deepcopy(error_events) | |
| for event in events: | |
| event.update(gain=0., background_gain=0., applied_to_audio=False, | |
| gain_scope="boundary_unmeasurable_no_background_applied", | |
| requested_session_sir_db=requested_sir_db, measured_event_sir_db=None) | |
| metadata = {"policy": core.POLICY, "status": "boundary_unmeasurable", | |
| "requested_background_sir_db": requested_sir_db, "measured_background_sir_db": None, | |
| "background_gain": 0., "background_applied": False, | |
| "reason": "insufficient_background_active_support", | |
| "minimum_background_active_samples": core.MIN_ACTIVE_SAMPLES, | |
| "placement_resampled": False, "source_events_and_seeds_preserved": True, | |
| "sir_scope": "aggregate_all_third_party_speech_active_rms", | |
| "peak_before_global_scale": peak, "global_scale": scale, | |
| "global_scale_scope": "entire_mixture_including_target_and_base_noise", | |
| "target_content_and_timing_preserved": True, "target_amplitude_preserved": scale == 1., | |
| "base_audio_stats": audio_stats(base), "audio_stats": audio_stats(audio)} | |
| return audio, events, metadata | |
| def assert_target_identity(parent: dict, recipe: dict) -> None: | |
| for field in PROTECTED_FIELDS: | |
| if parent.get(field) != recipe.get(field): | |
| raise ValueError(f"Protected target/timeline identity changed: {field}") | |
| for field in ("sample_count", "sample_rate", "channels", "duration_seconds", "path", | |
| "window_samples", "stride_samples", "overlap_samples", "window_count"): | |
| if parent["audio"].get(field) != recipe["audio"].get(field): | |
| raise ValueError(f"Protected audio timeline geometry changed: {field}") | |
| if recipe["parent_recipe_sha256"] != canonical_json_sha256(parent): | |
| raise ValueError("Parent recipe provenance mismatch") | |
| def _copy_verified(source: Path, destination: Path, expected: str, output_root: Path) -> None: | |
| _owned(destination, output_root) | |
| if destination.exists(): | |
| if sha256_file(destination) != expected: | |
| raise ValueError(f"Existing copied metadata differs from its frozen input: {destination}") | |
| return | |
| if sha256_file(source) != expected: | |
| raise ValueError(f"Copy source changed: {source}") | |
| destination.parent.mkdir(parents=True, exist_ok=True) | |
| fd, name = tempfile.mkstemp(prefix=f".{destination.name}.", suffix=".copy", dir=destination.parent) | |
| os.close(fd) | |
| temporary = Path(name) | |
| try: | |
| shutil.copyfile(source, temporary) | |
| if sha256_file(temporary) != expected: | |
| raise ValueError(f"Copied payload checksum mismatch: {destination}") | |
| os.replace(temporary, destination) | |
| finally: | |
| temporary.unlink(missing_ok=True) | |
| def _resume_record(path: Path, parent: dict, assignment: dict, output_root: Path, config_sha: str) -> dict: | |
| payload = _json(_owned(path, output_root)) | |
| recipe = payload["recipe"] | |
| if (payload.get("build_config_sha256") != config_sha | |
| or payload.get("recipe_sha256") != canonical_json_sha256(recipe) | |
| or recipe.get("background_assignment") != assignment): | |
| raise ValueError("Existing session completion record is not from this exact build") | |
| assert_target_identity(parent, recipe) | |
| for kind, extension, hash_key in (("audio", "wav", "sha256"), ("timeline", "npz", "file_sha256")): | |
| folder = "audio" if kind == "audio" else "timelines" | |
| expected = f"{folder}/{parent['split']}/{parent['session_id']}.{extension}" | |
| actual = _relative_payload(output_root, recipe[kind]["path"], expected) | |
| if sha256_file(actual) != recipe[kind][hash_key]: | |
| raise ValueError(f"Existing completed {kind} changed: {actual}") | |
| with wave.open(str(output_root / recipe["audio"]["path"]), "rb") as handle: | |
| if (handle.getnframes(), handle.getframerate(), handle.getnchannels(), handle.getsampwidth()) != ( | |
| recipe["audio"]["sample_count"], SAMPLE_RATE, 1, 2): | |
| raise ValueError("Existing completed WAV geometry differs") | |
| return recipe | |
| def build_one(parent: dict, assignment: dict, inputs: dict, output_root: Path, config_sha: str, | |
| catalog_sha: str, background_seed: int, *, resume: bool) -> dict: | |
| split, identity = parent["split"], parent["session_id"] | |
| if split not in {"train", "validation"} or not identity or any(c not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-" for c in identity): | |
| raise ValueError("Unsafe session split/identity") | |
| record_path = _owned(output_root / ".progress/recipes" / split / f"{identity}.json", output_root) | |
| audio_path = _relative_payload(output_root, parent["audio"]["path"], f"audio/{split}/{identity}.wav") | |
| timeline_path = _relative_payload(output_root, parent["timeline"]["path"], f"timelines/{split}/{identity}.npz") | |
| source_timeline = inputs["reference_root"] / parent["timeline"]["path"] | |
| if sha256_file(source_timeline) != parent["timeline"]["file_sha256"]: | |
| raise ValueError("Read-only parent timeline changed") | |
| if record_path.exists(): | |
| if not resume: | |
| raise FileExistsError(f"Unexpected existing session record: {record_path}") | |
| return _resume_record(record_path, parent, assignment, output_root, config_sha) | |
| if audio_path.exists() or timeline_path.exists(): | |
| raise FileExistsError(f"Unfinished/unknown payload without a validated completion record: {identity}") | |
| sir_seed = stable_u64("multi-v5-sir", background_seed, parent["session_signature"]) | |
| requested = core.truncated_sir(random.Random(sir_seed)) | |
| events = core.schedule_events(parent, assignment, inputs["assets"], background_seed, tts_assets=inputs["clones"]) | |
| try: | |
| audio, events, mix = core.mix_scheduled_events( | |
| {**parent, "background_assignment": assignment}, events, inputs["combined_assets"], requested) | |
| mix["status"] = "calibrated" | |
| mix["background_applied"] = True | |
| except core.UnusableBackgroundError as error: | |
| audio, events, mix = boundary_base_fallback(parent, requested, error.events) | |
| mix["background_active_sample_count"] = error.active_sample_count | |
| recipe = copy.deepcopy(parent) | |
| for field in OBSOLETE_FIELDS: | |
| recipe.pop(field, None) | |
| recipe.update(format="hfd_third_party_session_recipe_v3", profile_policy=core.POLICY, | |
| session_type=assignment["session_type"], audio_corruption_profile=assignment["session_type"], | |
| corruption_difficulty="gaussian", background_assignment=copy.deepcopy(assignment), | |
| parent_recipe_sha256=canonical_json_sha256(parent), | |
| parent_recipe_manifest_sha256=inputs["reference_recipe_sha256"], | |
| parent_production_root=str(inputs["reference_root"]), | |
| background_seed=background_seed, background_sir_seed=sir_seed, | |
| interference_seed=stable_u64("multi-v5-interference", background_seed, parent["session_signature"]), | |
| interference_events=events, mix_metadata=mix, base_audio_stats=mix["base_audio_stats"], | |
| session_id_semantics="stable_parent_lineage_not_audio_content_hash") | |
| recipe["contracts"] = {**parent.get("contracts", {}), "builder_config_sha256": config_sha, | |
| "catalog_manifest_sha256": catalog_sha, "parent_recipe_sha256": recipe["parent_recipe_sha256"]} | |
| pcm = quantize_pcm16(audio) | |
| written = pcm.astype(np.float32) / np.float32(32768.) | |
| recipe["audio"].update(audio_stats(written), pcm16_sha256=hashlib.sha256(pcm.tobytes()).hexdigest(), | |
| sample_rate=SAMPLE_RATE, channels=1, sample_width_bytes=2) | |
| assert_target_identity(parent, recipe) | |
| if not np.isfinite(written).all() or float(np.max(np.abs(written), initial=0.)) > core.PEAK_LIMIT + 1 / 32768: | |
| raise ValueError("Final PCM fails finite-value/peak protection") | |
| audio_path.parent.mkdir(parents=True, exist_ok=True) | |
| write_pcm16_wav_atomic(audio_path, written) | |
| recipe["audio"]["sha256"] = sha256_file(audio_path) | |
| if not np.array_equal(read_pcm16_wav(audio_path), written): | |
| raise ValueError("Written audio PCM differs from the deterministic mix") | |
| _copy_verified(source_timeline, timeline_path, parent["timeline"]["file_sha256"], output_root) | |
| record_path.parent.mkdir(parents=True, exist_ok=True) | |
| write_json_atomic(record_path, {"format": "hfd_multi_v5_session_completion_v1", | |
| "build_config_sha256": config_sha, "recipe_sha256": canonical_json_sha256(recipe), "recipe": recipe}) | |
| return recipe | |
| def _distribution(values: Iterable[float | None]) -> dict: | |
| array = np.asarray([value for value in values if value is not None], dtype=np.float64) | |
| return {"count": int(array.size), "mean": float(array.mean()) if array.size else None, | |
| "standard_deviation": float(array.std()) if array.size else None, | |
| "minimum": float(array.min()) if array.size else None, "maximum": float(array.max()) if array.size else None, | |
| "fraction_background_louder_than_target": float(np.mean(array < 0)) if array.size else None} | |
| def summarize(recipes: list[dict]) -> dict: | |
| events = [event for recipe in recipes for event in recipe["interference_events"]] | |
| return {"sessions": len(recipes), "sessions_by_split": dict(Counter(row["split"] for row in recipes)), | |
| "class_counts": dict(Counter(row["session_type"] for row in recipes)), | |
| "background_languages": dict(Counter(row["background_assignment"]["language"] for row in recipes)), | |
| "class_language_split_counts": dict(Counter("/".join((row["split"], row["session_type"], row["background_assignment"]["language"])) for row in recipes)), | |
| "events": len(events), "events_by_type": dict(Counter(event["type"] for event in events)), | |
| "events_per_session": dict(Counter(str(len(row["interference_events"])) for row in recipes)), | |
| "right_truncated_events": sum(bool(event.get("right_truncated")) for event in events), | |
| "event_truncated_events": sum(bool(event.get("event_truncated", event.get("right_truncated"))) for event in events), | |
| "crop_reasons": dict(Counter(event["crop_reason"] for event in events if event.get("crop_reason"))), | |
| "clone_events": sum(event.get("voice_relation") == "reference_clone" for event in events), | |
| "unique_clone_reference_sha256": len({row["target_speech_events"][row["background_assignment"]["anchor_target_turn_index"]]["original_audio_sha256"] | |
| for row in recipes if row["session_type"] == "same_voice_aside"}), | |
| "boundary_unmeasurable_sessions": sum(row["mix_metadata"]["status"] == "boundary_unmeasurable" for row in recipes), | |
| "requested_sir_db": _distribution(row["mix_metadata"]["requested_background_sir_db"] for row in recipes), | |
| "measured_sir_db": _distribution(row["mix_metadata"]["measured_background_sir_db"] for row in recipes), | |
| "peak_scaled_sessions": sum(row["mix_metadata"]["global_scale"] < 1 for row in recipes)} | |
| def _assignments(recipes: list[dict]) -> list[dict]: | |
| return [{"format": "hfd_third_party_composed_assignment_v3", "session_id": row["session_id"], | |
| "split": row["split"], "session_type": row["session_type"], "audio_corruption_profile": row["session_type"], | |
| "audio_path": row["audio"]["path"], "audio_sha256": row["audio"]["sha256"], | |
| "audio_sample_count": row["audio"]["sample_count"], "timeline": row["timeline"], | |
| "target_speech_events": row["target_speech_events"], "interference_events": row["interference_events"], | |
| "background_assignment": row["background_assignment"], "parent_recipe_sha256": row["parent_recipe_sha256"]} | |
| for row in recipes] | |
| def _bounded_process(process: Any, parents: list[dict], workers: int) -> Iterable[dict]: | |
| """Keep at most two tasks per worker in flight and stop promptly on errors.""" | |
| pending = deque() | |
| iterator = iter(parents) | |
| with ThreadPoolExecutor(max_workers=workers) as executor: | |
| for _ in range(workers * 2): | |
| parent = next(iterator, None) | |
| if parent is not None: | |
| pending.append(executor.submit(process, parent)) | |
| try: | |
| while pending: | |
| yield pending.popleft().result() | |
| parent = next(iterator, None) | |
| if parent is not None: | |
| pending.append(executor.submit(process, parent)) | |
| except BaseException: | |
| for future in pending: | |
| future.cancel() | |
| raise | |
| def build(reference_root: Path = DEFAULT_REFERENCE, plan_root: Path = DEFAULT_PLAN, | |
| tts_registry: Path = DEFAULT_TTS, output_root: Path | None = None, *, | |
| background_seed: int = BACKGROUND_SEED, workers: int = 4, limit: int | None = None, | |
| session_types: Iterable[str] | None = None, resume: bool = False) -> dict: | |
| if isinstance(workers, bool) or not isinstance(workers, int) or not 1 <= workers <= 8: | |
| raise ValueError("Use 1..8 CPU workers") | |
| pilot = limit is not None | |
| output_root = _output_root(output_root or (ROOT / "data/hfd_third_party_multi_v5/work/pilot" if pilot else | |
| ROOT / "data/hfd_third_party_multi_v5/production"), pilot=pilot) | |
| inputs = load_inputs(reference_root, plan_root, tts_registry, limit=limit, session_types=session_types, seed=background_seed) | |
| config = {"format": "hfd_multi_v5_build_config_v1", "profile_policy": core.POLICY, | |
| "background_seed": background_seed, "pilot": pilot, "limit": limit, | |
| "selected_session_ids": [row["session_id"] for row in inputs["parents"]], | |
| "output_root": str(output_root), "frozen_inputs": inputs["frozen_inputs"], | |
| "builder_script_sha256": sha256_file(Path(__file__)), "policy_script_sha256": sha256_file(Path(core.__file__))} | |
| config_sha = canonical_json_sha256(config) | |
| state_path = output_root / ".progress/build_state.json" | |
| completed_resume = False | |
| if output_root.exists(): | |
| if not resume or not state_path.is_file(): | |
| raise FileExistsError("Output exists; --resume requires an exact build-state record") | |
| state = _json(_owned(state_path, output_root)) | |
| if state.get("config_sha256") != config_sha or state.get("config") != config: | |
| raise ValueError("Resume config/input hashes differ from the original build") | |
| completed_resume = state.get("status") == "built_pending_strict_audit" | |
| else: | |
| output_root.mkdir(parents=True, exist_ok=False) | |
| write_json_atomic(state_path, {"config_sha256": config_sha, "config": config, "status": "building"}) | |
| manifest_root, report_root = output_root / "manifests", output_root / "reports" | |
| manifest_root.mkdir(exist_ok=True) | |
| report_root.mkdir(exist_ok=True) | |
| final_paths = (manifest_root / "session_recipes.jsonl", manifest_root / "composed_session_assignments.jsonl", | |
| report_root / "multi_event_summary.json", report_root / "pilot_build.json", report_root / "pilot_complete.json") | |
| if not completed_resume and any(path.exists() for path in final_paths): | |
| raise FileExistsError("Unknown final outputs without a completed build-state record; refusing overwrite") | |
| for name in COPY_MANIFESTS: | |
| source = inputs["reference_root"] / "manifests" / name | |
| _copy_verified(source, manifest_root / name, inputs["frozen_inputs"][str(source.absolute())], output_root) | |
| registry_path = _owned(manifest_root / "third_party_source_registry.jsonl", output_root) | |
| registry_rows = sorted(inputs["combined_assets"].values(), key=lambda row: row["asset_id"]) | |
| if registry_path.exists(): | |
| if read_jsonl(registry_path) != registry_rows: | |
| raise ValueError("Existing source registry differs from the frozen build") | |
| else: | |
| write_jsonl_atomic(registry_path, registry_rows) | |
| catalog_sha = sha256_file(registry_path) | |
| def process(parent: dict) -> dict: | |
| return build_one(parent, inputs["assignments"][parent["session_id"]], inputs, output_root, | |
| config_sha, catalog_sha, background_seed, resume=resume) | |
| recipes = [] | |
| for index, recipe in enumerate(_bounded_process(process, inputs["parents"], workers), 1): | |
| recipes.append(recipe) | |
| if index == 1 or index % 100 == 0 or index == len(inputs["parents"]): | |
| print(f"[built {index}/{len(inputs['parents'])}] {recipe['session_id']} {recipe['mix_metadata']['status']}", flush=True) | |
| _assert_frozen(inputs["frozen_inputs"]) | |
| for parent, recipe in zip(inputs["parents"], recipes): | |
| assert_target_identity(parent, recipe) | |
| if completed_resume: | |
| if not state.get("final_payload_hashes"): | |
| raise ValueError("Completed resume has no frozen final payload hashes") | |
| for relative, expected in state["final_payload_hashes"].items(): | |
| path = _owned(output_root / relative, output_root) | |
| if sha256_file(path) != expected: | |
| raise ValueError(f"Completed build payload changed: {relative}") | |
| existing_report = _json(report_root / "pilot_build.json") | |
| if existing_report.get("config_sha256") != config_sha or existing_report.get("counts", {}).get("sessions") != len(recipes): | |
| raise ValueError("Completed build report changed its identity") | |
| return existing_report | |
| write_jsonl_atomic(_owned(manifest_root / "session_recipes.jsonl", output_root), recipes) | |
| write_jsonl_atomic(_owned(manifest_root / "composed_session_assignments.jsonl", output_root), _assignments(recipes)) | |
| summary = summarize(recipes) | |
| write_json_atomic(_owned(report_root / "multi_event_summary.json", output_root), summary) | |
| parent_build = inputs["parent_build"] | |
| counts = {"sessions": len(recipes), "sessions_by_split": summary["sessions_by_split"], | |
| "audio_samples": sum(row["audio"]["sample_count"] for row in recipes), | |
| "audio_hours": sum(row["audio"]["sample_count"] for row in recipes) / SAMPLE_RATE / 3600, | |
| "turns": sum(row["turn_count"] for row in recipes), | |
| "semantic_windows": sum(row["timeline"].get("semantic_window_count", 0) for row in recipes), | |
| "dense_action_windows": sum(row["timeline"].get("action_steps", 0) for row in recipes)} | |
| build_report = {key: copy.deepcopy(parent_build[key]) for key in ( | |
| "composition_salt", "composition_variant_id", "seed", "baseline_semantics_version", | |
| "baseline_manifest_sha256", "baseline_collision_count") if key in parent_build} | |
| build_report.update(format="hfd_third_party_pilot_build_v3", profile_policy=core.POLICY, | |
| complete=True, status="built_pending_strict_audit", production_ready=False, | |
| strict_audit_passed=False, human_listening_review="pending", training_started=False, pilot=pilot, | |
| counts=counts, composition_reference_sessions_by_split=parent_build["counts"]["sessions_by_split"], | |
| background_seed=background_seed, reference_root=str(inputs["reference_root"]), | |
| reference_recipe_sha256=inputs["reference_recipe_sha256"], plan_hashes=inputs["plan_hashes"], | |
| eligible_tts_registry_sha256=inputs["eligible_tts_registry_sha256"], | |
| selected_session_ids=config["selected_session_ids"], | |
| input_paths={**parent_build.get("input_paths", {}), "catalog_registry": str(registry_path), | |
| "reference_production_root": str(inputs["reference_root"]), "plan_root": str(inputs["plan_root"])}, | |
| input_hashes=inputs["frozen_inputs"], config_sha256=config_sha, | |
| builder_script_sha256=config["builder_script_sha256"], policy_script_sha256=config["policy_script_sha256"], | |
| output_hashes={"session_recipes": sha256_file(manifest_root / "session_recipes.jsonl"), | |
| "composed_session_assignments": sha256_file(manifest_root / "composed_session_assignments.jsonl"), | |
| "catalog_registry": catalog_sha, "multi_event_summary": sha256_file(report_root / "multi_event_summary.json")}, | |
| sir_distribution=inputs["plan_report"]["sir_distribution"], | |
| boundary_unmeasurable_policy="keep_event_position_and_seed_emit_peak_safe_base_without_interference", | |
| boundary_unmeasurable_sessions=summary["boundary_unmeasurable_sessions"], | |
| generated_utc=datetime.now(timezone.utc).isoformat(), environment={"device": "cpu", "workers": workers}) | |
| # The previous catalog verification concerns the parent catalog, not this one. | |
| build_report["input_paths"].pop("catalog_verify_report", None) | |
| write_json_atomic(_owned(report_root / "pilot_build.json", output_root), build_report) | |
| write_json_atomic(_owned(report_root / "pilot_complete.json", output_root), { | |
| "format": "hfd_multi_v5_construction_status_v1", "complete": True, | |
| "status": "built_pending_strict_audit", "strict_audit_passed": False, "production_ready": False, | |
| "training_started": False, "sessions": len(recipes), "pilot": pilot}) | |
| write_json_atomic(_owned(state_path, output_root), {"config_sha256": config_sha, "config": config, | |
| "status": "built_pending_strict_audit", "completed_sessions": len(recipes), | |
| "final_payload_hashes": {str(path.relative_to(output_root)): sha256_file(path) for path in ( | |
| manifest_root / "session_recipes.jsonl", manifest_root / "composed_session_assignments.jsonl", | |
| manifest_root / "third_party_source_registry.jsonl", report_root / "multi_event_summary.json")}}) | |
| return build_report | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--reference-root", type=Path, default=DEFAULT_REFERENCE) | |
| parser.add_argument("--plan-root", type=Path, default=DEFAULT_PLAN) | |
| parser.add_argument("--tts-registry", type=Path, default=DEFAULT_TTS) | |
| parser.add_argument("--output-root", type=Path) | |
| parser.add_argument("--background-seed", type=int, default=BACKGROUND_SEED) | |
| parser.add_argument("--workers", type=int, default=4) | |
| parser.add_argument("--limit", type=int, help="Explicit pilot size; default pilot covers all six classes") | |
| parser.add_argument("--session-types", help="Comma-separated class filter, permitted only with --limit") | |
| parser.add_argument("--resume", action="store_true") | |
| args = parser.parse_args() | |
| report = build(args.reference_root, args.plan_root, args.tts_registry, args.output_root, | |
| background_seed=args.background_seed, workers=args.workers, limit=args.limit, | |
| session_types=args.session_types.split(",") if args.session_types else None, resume=args.resume) | |
| print(json.dumps({key: report[key] for key in ("status", "counts", "pilot", "production_ready")}, ensure_ascii=False, indent=2)) | |
| if __name__ == "__main__": | |
| main() | |