code-HFD_lzk / scripts /data /build_hfd_third_party_multi_v5.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
8562e9b verified
Raw History Blame Contribute Delete
37 kB
#!/usr/bin/env python3
"""Build the isolated multi-event refresh without changing any reference data.
This is a construction tool, not a strict auditor. A completed build remains
pending an independent audit and human listening. No training/GPU imports occur.
"""
from __future__ import annotations
import argparse
from collections import Counter, deque
from concurrent.futures import ThreadPoolExecutor
import copy
from datetime import datetime, timezone
import hashlib
import json
import os
from pathlib import Path
import random
import shutil
import sys
import tempfile
from typing import Any, Iterable
import wave
os.environ["CUDA_VISIBLE_DEVICES"] = ""
os.environ["HIP_VISIBLE_DEVICES"] = ""
for _thread_variable in ("OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS", "MKL_NUM_THREADS"):
os.environ[_thread_variable] = "1"
import numpy as np
ROOT = Path(__file__).resolve().parents[2]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from scripts.data import hfd_third_party_multi_v5 as core
from scripts.data.hfd_third_party_common import (
SAMPLE_RATE, STRIDE_SAMPLES, audio_stats, canonical_json_sha256,
quantize_pcm16, read_jsonl, read_pcm16_wav, sha256_file, stable_u64,
write_json_atomic, write_jsonl_atomic, write_pcm16_wav_atomic,
)
from scripts.data.scan_hfd_third_party_loudness import read_background, target_track
DEFAULT_REFERENCE = ROOT / "data/hfd_third_party_cross_v4/production"
DEFAULT_PLAN = ROOT / "data/hfd_third_party_multi_v5/work/plan_qualified"
DEFAULT_TTS = ROOT / "data/hfd_third_party_multi_v5/work/tts_assets_qualified/eligible_registry.jsonl"
DEFAULT_OUTPUT = ROOT / "data/hfd_third_party_multi_v5/production"
DEFAULT_PILOT = ROOT / "data/hfd_third_party_multi_v5/work/pilot"
BACKGROUND_SEED = 26092801
COPY_MANIFESTS = ("primitive_split_snapshot.jsonl", "baseline_session_signatures.jsonl", "source_contract.json")
PLAN_FILES = ("session_assignments.jsonl", "selected_real_assets.jsonl", "tts_jobs.jsonl")
CLASSES = tuple(name for name, _ in core.CLASS_WEIGHTS)
PROTECTED_FIELDS = ("session_id", "session_signature", "split", "turns", "turn_count", "timeline",
"target_speech_events", "supervision_targets", "session_language", "session_voice_name",
"base_background", "interaction_mode", "composition_salt", "composition_seed",
"composition_variant_id", "timeline_seed", "timeline_seed_attempt", "task")
OBSOLETE_FIELDS = ("loudness_revision", "speech_domain_assignment", "profile_index",
"cross_domain_revision", "retained_audio", "background_gain_revision")
def _json(path: Path) -> dict:
return json.loads(path.read_text(encoding="utf-8"))
def _owned(path: Path, root: Path) -> Path:
path, root = path.absolute(), root.absolute()
if path.resolve() != path or not path.is_relative_to(root):
raise ValueError(f"Output is not an unaliased LZK-owned path: {path}")
if path.is_file() and path.stat().st_nlink != 1:
raise ValueError(f"Refusing hard-linked output: {path}")
return path
def _output_root(path: Path, *, pilot: bool) -> Path:
expected = ROOT / ("data/hfd_third_party_multi_v5/work/pilot" if pilot else
"data/hfd_third_party_multi_v5/production")
path = _owned(path, ROOT)
if path != expected:
raise ValueError(f"Output must be the fixed isolated {'pilot' if pilot else 'production'} root: {expected}")
return path
def _relative_payload(root: Path, relative: str, expected: str) -> Path:
if relative != expected:
raise ValueError(f"Unexpected session payload path: {relative}")
return _owned(root / relative, root)
def _unique(rows: Iterable[dict], key: str) -> dict[str, dict]:
result = {}
for row in rows:
identity = str(row[key])
if identity in result:
raise ValueError(f"Duplicate {key}: {identity}")
result[identity] = row
return result
def _frozen_hashes(paths: Iterable[Path]) -> dict[str, str]:
return {str(path.absolute()): sha256_file(path) for path in paths}
def _assert_frozen(hashes: dict[str, str]) -> None:
for path, expected in hashes.items():
if sha256_file(Path(path)) != expected:
raise RuntimeError(f"Read-only input changed during construction: {path}")
def select_population(parents: list[dict], assignments: dict[str, dict], *,
limit: int | None, session_types: Iterable[str] | None, seed: int) -> list[dict]:
classes = tuple(session_types) if session_types is not None else CLASSES
if len(set(classes)) != len(classes) or not classes or set(classes) - set(CLASSES):
raise ValueError("Select distinct known session classes")
if limit is None:
if set(classes) != set(CLASSES):
raise ValueError("Class filtering is permitted only for an explicit limited pilot")
return parents
if isinstance(limit, bool) or not isinstance(limit, int) or limit < len(classes):
raise ValueError("Pilot limit must include at least one session of every selected class")
groups = {name: sorted((row for row in parents if assignments[row["session_id"]]["session_type"] == name),
key=lambda row: (stable_u64("multi-v5-pilot", seed, name, row["session_id"]), row["session_id"]))
for name in classes}
if any(not group for group in groups.values()) or limit > sum(map(len, groups.values())):
raise ValueError("Pilot population cannot satisfy the requested classes and limit")
selected, offset = [], 0
while len(selected) < limit:
for name in classes:
if offset < len(groups[name]) and len(selected) < limit:
selected.append(groups[name][offset])
offset += 1
identities = {row["session_id"] for row in selected}
return [row for row in parents if row["session_id"] in identities]
def _validate_clone(parent: dict, assignment: dict, asset: dict, job: dict) -> None:
index = int(assignment["anchor_target_turn_index"])
event, turn = parent["target_speech_events"][index], parent["turns"][index]
expected = {"job_id": assignment["tts_job_id"], "split": parent["split"],
"language": assignment["language"], "role": "non_command_speech",
"reference_audio": event["original_audio_path"],
"reference_sha256": event["original_audio_sha256"], "reference_text": turn["instruction_text"]}
if asset.get("eligible_for_mixing") is not True:
raise ValueError("Clone asset has not passed production eligibility checks")
for key, value in expected.items():
if asset.get(key) != value:
raise ValueError(f"Clone does not bind its actual target reference: {key}")
if asset.get("transcript") != job["text"] or job["reference_sha256"] != expected["reference_sha256"]:
raise ValueError("Clone transcript/reference differs from the frozen TTS job")
if parent["session_language"] != assignment["language"]:
raise ValueError("Same-voice aside language differs from its target reference")
for field in ("asset_id", "normalized_path", "normalized_sha256", "speaker_id", "source_group"):
if not asset.get(field):
raise ValueError(f"Missing clone provenance: {field}")
if int(asset.get("sample_count", 0)) <= 0:
raise ValueError("Invalid clone audio length")
def load_inputs(reference_root: Path, plan_root: Path, tts_registry: Path, *,
limit: int | None, session_types: Iterable[str] | None, seed: int) -> dict:
reference_root, plan_root, tts_registry = (path.absolute() for path in (reference_root, plan_root, tts_registry))
report = _json(plan_root / "preparation_report.json")
if report.get("policy") != core.POLICY or int(report["seed"]) != seed:
raise ValueError("Frozen plan policy/seed mismatch")
if limit is None and not report.get("qualified_registry_path"):
raise ValueError("Full production requires a plan selected from quality-screened clone references")
parent_build = _json(reference_root / "reports/pilot_build.json")
if _json(reference_root / "reports/strict_audit.json").get("passed") is not True:
raise ValueError("Reference production must have passed its independent audit")
recipes_path = reference_root / "manifests/session_recipes.jsonl"
reference_sha = sha256_file(recipes_path)
if reference_sha != report["reference_recipe_sha256"] or reference_sha != parent_build["output_hashes"]["session_recipes"]:
raise ValueError("Frozen reference recipe manifest changed")
plan_hashes = report["output_hashes"]
for name in PLAN_FILES:
if sha256_file(plan_root / name) != plan_hashes[name]:
raise ValueError(f"Frozen plan hash mismatch: {name}")
parents = read_jsonl(recipes_path)
parent_by_id = _unique(parents, "session_id")
assignments = _unique(read_jsonl(plan_root / "session_assignments.jsonl"), "session_id")
if set(parent_by_id) != set(assignments) or len(parents) != int(report["sessions"]):
raise ValueError("Plan does not exactly partition the reference population")
for parent in parents:
assignment = assignments[parent["session_id"]]
if (assignment.get("parent_recipe_sha256") != canonical_json_sha256(parent)
or assignment.get("split") != parent["split"]
or assignment.get("session_signature") != parent["session_signature"]
or assignment.get("session_type") not in CLASSES or assignment.get("language") not in {"zh", "en"}):
raise ValueError("Assignment parent identity/class/language mismatch")
if dict(Counter(row["session_type"] for row in assignments.values())) != report["class_counts"]:
raise ValueError("Plan class counts disagree with assignments")
if dict(Counter(row["language"] for row in assignments.values())) != report["background_languages"]:
raise ValueError("Plan language counts disagree with assignments")
selected = select_population(parents, assignments, limit=limit, session_types=session_types, seed=seed)
assets = _unique(read_jsonl(plan_root / "selected_real_assets.jsonl"), "asset_id")
jobs = _unique(read_jsonl(plan_root / "tts_jobs.jsonl"), "job_id")
clone_rows = read_jsonl(tts_registry) if tts_registry.is_file() else []
if any(row.get("eligible_for_mixing") is not True for row in clone_rows):
raise ValueError("Eligible clone registry contains an unapproved asset")
clones = _unique(clone_rows, "job_id")
train_references = {a["reference_sha256"] for a in clone_rows if a["split"] == "train"}
validation_references = {a["reference_sha256"] for a in clone_rows if a["split"] == "validation"}
if train_references & validation_references:
raise ValueError("Clone reference audio overlaps train and validation")
for parent in selected:
assignment = assignments[parent["session_id"]]
if assignment["session_type"] != "same_voice_aside":
continue
job_id = assignment["tts_job_id"]
if job_id not in clones or job_id not in jobs:
raise ValueError(f"Missing eligible same-voice TTS asset for selected session: {job_id}")
_validate_clone(parent, assignment, clones[job_id], jobs[job_id])
combined = _unique([*assets.values(), *clone_rows], "asset_id")
paths = [recipes_path, reference_root / "reports/pilot_build.json", reference_root / "reports/strict_audit.json",
plan_root / "preparation_report.json", *(plan_root / name for name in PLAN_FILES),
*(reference_root / "manifests" / name for name in COPY_MANIFESTS)]
if tts_registry.is_file():
paths.append(tts_registry)
if report.get("qualified_registry_path"):
from scripts.data.plan_hfd_third_party_multi_v5 import validate_qualified_asset
candidate_path = Path(report["qualified_registry_path"])
if sha256_file(candidate_path) != report["qualified_registry_sha256"]:
raise ValueError("Quality-screened candidate registry changed")
paths.append(candidate_path)
for asset in clone_rows:
validate_qualified_asset(asset, jobs[asset["job_id"]])
paths.extend(Path(asset[key]) for key in ("qa_path", "generation_provenance_path", "normalized_path"))
frozen = _frozen_hashes(paths)
return {"parents": selected, "assignments": assignments, "assets": assets, "clones": clones,
"combined_assets": combined, "parent_build": parent_build, "plan_report": report,
"reference_root": reference_root, "reference_recipe_sha256": reference_sha,
"plan_root": plan_root, "plan_hashes": {name: plan_hashes[name] for name in PLAN_FILES},
"eligible_tts_registry_sha256": sha256_file(tts_registry) if tts_registry.is_file() else None,
"frozen_inputs": frozen}
def boundary_base_fallback(parent: dict, requested_sir_db: float, error_events: list[dict]) -> tuple[np.ndarray, list[dict], dict]:
"""Keep the exact boundary draw; do not amplify or resample unusable speech."""
for event in parent["target_speech_events"]:
if sha256_file(Path(event["original_audio_path"])) != event["original_audio_sha256"]:
raise ValueError("Target source changed before boundary fallback")
target, _ = target_track(parent)
background = parent["base_background"]
if sha256_file(Path(background["path"])) != background["sha256"]:
raise ValueError("Base background changed before boundary fallback")
noise = read_background(background["path"], int(background["start_window"]) * STRIDE_SAMPLES, len(target))
base = quantize_pcm16(noise + target).astype(np.float32) / np.float32(32768.)
peak = float(np.max(np.abs(base), initial=0.))
scale = min(1., core.PEAK_LIMIT / max(peak, 1e-12))
audio = base * np.float32(scale)
events = copy.deepcopy(error_events)
for event in events:
event.update(gain=0., background_gain=0., applied_to_audio=False,
gain_scope="boundary_unmeasurable_no_background_applied",
requested_session_sir_db=requested_sir_db, measured_event_sir_db=None)
metadata = {"policy": core.POLICY, "status": "boundary_unmeasurable",
"requested_background_sir_db": requested_sir_db, "measured_background_sir_db": None,
"background_gain": 0., "background_applied": False,
"reason": "insufficient_background_active_support",
"minimum_background_active_samples": core.MIN_ACTIVE_SAMPLES,
"placement_resampled": False, "source_events_and_seeds_preserved": True,
"sir_scope": "aggregate_all_third_party_speech_active_rms",
"peak_before_global_scale": peak, "global_scale": scale,
"global_scale_scope": "entire_mixture_including_target_and_base_noise",
"target_content_and_timing_preserved": True, "target_amplitude_preserved": scale == 1.,
"base_audio_stats": audio_stats(base), "audio_stats": audio_stats(audio)}
return audio, events, metadata
def assert_target_identity(parent: dict, recipe: dict) -> None:
for field in PROTECTED_FIELDS:
if parent.get(field) != recipe.get(field):
raise ValueError(f"Protected target/timeline identity changed: {field}")
for field in ("sample_count", "sample_rate", "channels", "duration_seconds", "path",
"window_samples", "stride_samples", "overlap_samples", "window_count"):
if parent["audio"].get(field) != recipe["audio"].get(field):
raise ValueError(f"Protected audio timeline geometry changed: {field}")
if recipe["parent_recipe_sha256"] != canonical_json_sha256(parent):
raise ValueError("Parent recipe provenance mismatch")
def _copy_verified(source: Path, destination: Path, expected: str, output_root: Path) -> None:
_owned(destination, output_root)
if destination.exists():
if sha256_file(destination) != expected:
raise ValueError(f"Existing copied metadata differs from its frozen input: {destination}")
return
if sha256_file(source) != expected:
raise ValueError(f"Copy source changed: {source}")
destination.parent.mkdir(parents=True, exist_ok=True)
fd, name = tempfile.mkstemp(prefix=f".{destination.name}.", suffix=".copy", dir=destination.parent)
os.close(fd)
temporary = Path(name)
try:
shutil.copyfile(source, temporary)
if sha256_file(temporary) != expected:
raise ValueError(f"Copied payload checksum mismatch: {destination}")
os.replace(temporary, destination)
finally:
temporary.unlink(missing_ok=True)
def _resume_record(path: Path, parent: dict, assignment: dict, output_root: Path, config_sha: str) -> dict:
payload = _json(_owned(path, output_root))
recipe = payload["recipe"]
if (payload.get("build_config_sha256") != config_sha
or payload.get("recipe_sha256") != canonical_json_sha256(recipe)
or recipe.get("background_assignment") != assignment):
raise ValueError("Existing session completion record is not from this exact build")
assert_target_identity(parent, recipe)
for kind, extension, hash_key in (("audio", "wav", "sha256"), ("timeline", "npz", "file_sha256")):
folder = "audio" if kind == "audio" else "timelines"
expected = f"{folder}/{parent['split']}/{parent['session_id']}.{extension}"
actual = _relative_payload(output_root, recipe[kind]["path"], expected)
if sha256_file(actual) != recipe[kind][hash_key]:
raise ValueError(f"Existing completed {kind} changed: {actual}")
with wave.open(str(output_root / recipe["audio"]["path"]), "rb") as handle:
if (handle.getnframes(), handle.getframerate(), handle.getnchannels(), handle.getsampwidth()) != (
recipe["audio"]["sample_count"], SAMPLE_RATE, 1, 2):
raise ValueError("Existing completed WAV geometry differs")
return recipe
def build_one(parent: dict, assignment: dict, inputs: dict, output_root: Path, config_sha: str,
catalog_sha: str, background_seed: int, *, resume: bool) -> dict:
split, identity = parent["split"], parent["session_id"]
if split not in {"train", "validation"} or not identity or any(c not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-" for c in identity):
raise ValueError("Unsafe session split/identity")
record_path = _owned(output_root / ".progress/recipes" / split / f"{identity}.json", output_root)
audio_path = _relative_payload(output_root, parent["audio"]["path"], f"audio/{split}/{identity}.wav")
timeline_path = _relative_payload(output_root, parent["timeline"]["path"], f"timelines/{split}/{identity}.npz")
source_timeline = inputs["reference_root"] / parent["timeline"]["path"]
if sha256_file(source_timeline) != parent["timeline"]["file_sha256"]:
raise ValueError("Read-only parent timeline changed")
if record_path.exists():
if not resume:
raise FileExistsError(f"Unexpected existing session record: {record_path}")
return _resume_record(record_path, parent, assignment, output_root, config_sha)
if audio_path.exists() or timeline_path.exists():
raise FileExistsError(f"Unfinished/unknown payload without a validated completion record: {identity}")
sir_seed = stable_u64("multi-v5-sir", background_seed, parent["session_signature"])
requested = core.truncated_sir(random.Random(sir_seed))
events = core.schedule_events(parent, assignment, inputs["assets"], background_seed, tts_assets=inputs["clones"])
try:
audio, events, mix = core.mix_scheduled_events(
{**parent, "background_assignment": assignment}, events, inputs["combined_assets"], requested)
mix["status"] = "calibrated"
mix["background_applied"] = True
except core.UnusableBackgroundError as error:
audio, events, mix = boundary_base_fallback(parent, requested, error.events)
mix["background_active_sample_count"] = error.active_sample_count
recipe = copy.deepcopy(parent)
for field in OBSOLETE_FIELDS:
recipe.pop(field, None)
recipe.update(format="hfd_third_party_session_recipe_v3", profile_policy=core.POLICY,
session_type=assignment["session_type"], audio_corruption_profile=assignment["session_type"],
corruption_difficulty="gaussian", background_assignment=copy.deepcopy(assignment),
parent_recipe_sha256=canonical_json_sha256(parent),
parent_recipe_manifest_sha256=inputs["reference_recipe_sha256"],
parent_production_root=str(inputs["reference_root"]),
background_seed=background_seed, background_sir_seed=sir_seed,
interference_seed=stable_u64("multi-v5-interference", background_seed, parent["session_signature"]),
interference_events=events, mix_metadata=mix, base_audio_stats=mix["base_audio_stats"],
session_id_semantics="stable_parent_lineage_not_audio_content_hash")
recipe["contracts"] = {**parent.get("contracts", {}), "builder_config_sha256": config_sha,
"catalog_manifest_sha256": catalog_sha, "parent_recipe_sha256": recipe["parent_recipe_sha256"]}
pcm = quantize_pcm16(audio)
written = pcm.astype(np.float32) / np.float32(32768.)
recipe["audio"].update(audio_stats(written), pcm16_sha256=hashlib.sha256(pcm.tobytes()).hexdigest(),
sample_rate=SAMPLE_RATE, channels=1, sample_width_bytes=2)
assert_target_identity(parent, recipe)
if not np.isfinite(written).all() or float(np.max(np.abs(written), initial=0.)) > core.PEAK_LIMIT + 1 / 32768:
raise ValueError("Final PCM fails finite-value/peak protection")
audio_path.parent.mkdir(parents=True, exist_ok=True)
write_pcm16_wav_atomic(audio_path, written)
recipe["audio"]["sha256"] = sha256_file(audio_path)
if not np.array_equal(read_pcm16_wav(audio_path), written):
raise ValueError("Written audio PCM differs from the deterministic mix")
_copy_verified(source_timeline, timeline_path, parent["timeline"]["file_sha256"], output_root)
record_path.parent.mkdir(parents=True, exist_ok=True)
write_json_atomic(record_path, {"format": "hfd_multi_v5_session_completion_v1",
"build_config_sha256": config_sha, "recipe_sha256": canonical_json_sha256(recipe), "recipe": recipe})
return recipe
def _distribution(values: Iterable[float | None]) -> dict:
array = np.asarray([value for value in values if value is not None], dtype=np.float64)
return {"count": int(array.size), "mean": float(array.mean()) if array.size else None,
"standard_deviation": float(array.std()) if array.size else None,
"minimum": float(array.min()) if array.size else None, "maximum": float(array.max()) if array.size else None,
"fraction_background_louder_than_target": float(np.mean(array < 0)) if array.size else None}
def summarize(recipes: list[dict]) -> dict:
events = [event for recipe in recipes for event in recipe["interference_events"]]
return {"sessions": len(recipes), "sessions_by_split": dict(Counter(row["split"] for row in recipes)),
"class_counts": dict(Counter(row["session_type"] for row in recipes)),
"background_languages": dict(Counter(row["background_assignment"]["language"] for row in recipes)),
"class_language_split_counts": dict(Counter("/".join((row["split"], row["session_type"], row["background_assignment"]["language"])) for row in recipes)),
"events": len(events), "events_by_type": dict(Counter(event["type"] for event in events)),
"events_per_session": dict(Counter(str(len(row["interference_events"])) for row in recipes)),
"right_truncated_events": sum(bool(event.get("right_truncated")) for event in events),
"event_truncated_events": sum(bool(event.get("event_truncated", event.get("right_truncated"))) for event in events),
"crop_reasons": dict(Counter(event["crop_reason"] for event in events if event.get("crop_reason"))),
"clone_events": sum(event.get("voice_relation") == "reference_clone" for event in events),
"unique_clone_reference_sha256": len({row["target_speech_events"][row["background_assignment"]["anchor_target_turn_index"]]["original_audio_sha256"]
for row in recipes if row["session_type"] == "same_voice_aside"}),
"boundary_unmeasurable_sessions": sum(row["mix_metadata"]["status"] == "boundary_unmeasurable" for row in recipes),
"requested_sir_db": _distribution(row["mix_metadata"]["requested_background_sir_db"] for row in recipes),
"measured_sir_db": _distribution(row["mix_metadata"]["measured_background_sir_db"] for row in recipes),
"peak_scaled_sessions": sum(row["mix_metadata"]["global_scale"] < 1 for row in recipes)}
def _assignments(recipes: list[dict]) -> list[dict]:
return [{"format": "hfd_third_party_composed_assignment_v3", "session_id": row["session_id"],
"split": row["split"], "session_type": row["session_type"], "audio_corruption_profile": row["session_type"],
"audio_path": row["audio"]["path"], "audio_sha256": row["audio"]["sha256"],
"audio_sample_count": row["audio"]["sample_count"], "timeline": row["timeline"],
"target_speech_events": row["target_speech_events"], "interference_events": row["interference_events"],
"background_assignment": row["background_assignment"], "parent_recipe_sha256": row["parent_recipe_sha256"]}
for row in recipes]
def _bounded_process(process: Any, parents: list[dict], workers: int) -> Iterable[dict]:
"""Keep at most two tasks per worker in flight and stop promptly on errors."""
pending = deque()
iterator = iter(parents)
with ThreadPoolExecutor(max_workers=workers) as executor:
for _ in range(workers * 2):
parent = next(iterator, None)
if parent is not None:
pending.append(executor.submit(process, parent))
try:
while pending:
yield pending.popleft().result()
parent = next(iterator, None)
if parent is not None:
pending.append(executor.submit(process, parent))
except BaseException:
for future in pending:
future.cancel()
raise
def build(reference_root: Path = DEFAULT_REFERENCE, plan_root: Path = DEFAULT_PLAN,
tts_registry: Path = DEFAULT_TTS, output_root: Path | None = None, *,
background_seed: int = BACKGROUND_SEED, workers: int = 4, limit: int | None = None,
session_types: Iterable[str] | None = None, resume: bool = False) -> dict:
if isinstance(workers, bool) or not isinstance(workers, int) or not 1 <= workers <= 8:
raise ValueError("Use 1..8 CPU workers")
pilot = limit is not None
output_root = _output_root(output_root or (ROOT / "data/hfd_third_party_multi_v5/work/pilot" if pilot else
ROOT / "data/hfd_third_party_multi_v5/production"), pilot=pilot)
inputs = load_inputs(reference_root, plan_root, tts_registry, limit=limit, session_types=session_types, seed=background_seed)
config = {"format": "hfd_multi_v5_build_config_v1", "profile_policy": core.POLICY,
"background_seed": background_seed, "pilot": pilot, "limit": limit,
"selected_session_ids": [row["session_id"] for row in inputs["parents"]],
"output_root": str(output_root), "frozen_inputs": inputs["frozen_inputs"],
"builder_script_sha256": sha256_file(Path(__file__)), "policy_script_sha256": sha256_file(Path(core.__file__))}
config_sha = canonical_json_sha256(config)
state_path = output_root / ".progress/build_state.json"
completed_resume = False
if output_root.exists():
if not resume or not state_path.is_file():
raise FileExistsError("Output exists; --resume requires an exact build-state record")
state = _json(_owned(state_path, output_root))
if state.get("config_sha256") != config_sha or state.get("config") != config:
raise ValueError("Resume config/input hashes differ from the original build")
completed_resume = state.get("status") == "built_pending_strict_audit"
else:
output_root.mkdir(parents=True, exist_ok=False)
write_json_atomic(state_path, {"config_sha256": config_sha, "config": config, "status": "building"})
manifest_root, report_root = output_root / "manifests", output_root / "reports"
manifest_root.mkdir(exist_ok=True)
report_root.mkdir(exist_ok=True)
final_paths = (manifest_root / "session_recipes.jsonl", manifest_root / "composed_session_assignments.jsonl",
report_root / "multi_event_summary.json", report_root / "pilot_build.json", report_root / "pilot_complete.json")
if not completed_resume and any(path.exists() for path in final_paths):
raise FileExistsError("Unknown final outputs without a completed build-state record; refusing overwrite")
for name in COPY_MANIFESTS:
source = inputs["reference_root"] / "manifests" / name
_copy_verified(source, manifest_root / name, inputs["frozen_inputs"][str(source.absolute())], output_root)
registry_path = _owned(manifest_root / "third_party_source_registry.jsonl", output_root)
registry_rows = sorted(inputs["combined_assets"].values(), key=lambda row: row["asset_id"])
if registry_path.exists():
if read_jsonl(registry_path) != registry_rows:
raise ValueError("Existing source registry differs from the frozen build")
else:
write_jsonl_atomic(registry_path, registry_rows)
catalog_sha = sha256_file(registry_path)
def process(parent: dict) -> dict:
return build_one(parent, inputs["assignments"][parent["session_id"]], inputs, output_root,
config_sha, catalog_sha, background_seed, resume=resume)
recipes = []
for index, recipe in enumerate(_bounded_process(process, inputs["parents"], workers), 1):
recipes.append(recipe)
if index == 1 or index % 100 == 0 or index == len(inputs["parents"]):
print(f"[built {index}/{len(inputs['parents'])}] {recipe['session_id']} {recipe['mix_metadata']['status']}", flush=True)
_assert_frozen(inputs["frozen_inputs"])
for parent, recipe in zip(inputs["parents"], recipes):
assert_target_identity(parent, recipe)
if completed_resume:
if not state.get("final_payload_hashes"):
raise ValueError("Completed resume has no frozen final payload hashes")
for relative, expected in state["final_payload_hashes"].items():
path = _owned(output_root / relative, output_root)
if sha256_file(path) != expected:
raise ValueError(f"Completed build payload changed: {relative}")
existing_report = _json(report_root / "pilot_build.json")
if existing_report.get("config_sha256") != config_sha or existing_report.get("counts", {}).get("sessions") != len(recipes):
raise ValueError("Completed build report changed its identity")
return existing_report
write_jsonl_atomic(_owned(manifest_root / "session_recipes.jsonl", output_root), recipes)
write_jsonl_atomic(_owned(manifest_root / "composed_session_assignments.jsonl", output_root), _assignments(recipes))
summary = summarize(recipes)
write_json_atomic(_owned(report_root / "multi_event_summary.json", output_root), summary)
parent_build = inputs["parent_build"]
counts = {"sessions": len(recipes), "sessions_by_split": summary["sessions_by_split"],
"audio_samples": sum(row["audio"]["sample_count"] for row in recipes),
"audio_hours": sum(row["audio"]["sample_count"] for row in recipes) / SAMPLE_RATE / 3600,
"turns": sum(row["turn_count"] for row in recipes),
"semantic_windows": sum(row["timeline"].get("semantic_window_count", 0) for row in recipes),
"dense_action_windows": sum(row["timeline"].get("action_steps", 0) for row in recipes)}
build_report = {key: copy.deepcopy(parent_build[key]) for key in (
"composition_salt", "composition_variant_id", "seed", "baseline_semantics_version",
"baseline_manifest_sha256", "baseline_collision_count") if key in parent_build}
build_report.update(format="hfd_third_party_pilot_build_v3", profile_policy=core.POLICY,
complete=True, status="built_pending_strict_audit", production_ready=False,
strict_audit_passed=False, human_listening_review="pending", training_started=False, pilot=pilot,
counts=counts, composition_reference_sessions_by_split=parent_build["counts"]["sessions_by_split"],
background_seed=background_seed, reference_root=str(inputs["reference_root"]),
reference_recipe_sha256=inputs["reference_recipe_sha256"], plan_hashes=inputs["plan_hashes"],
eligible_tts_registry_sha256=inputs["eligible_tts_registry_sha256"],
selected_session_ids=config["selected_session_ids"],
input_paths={**parent_build.get("input_paths", {}), "catalog_registry": str(registry_path),
"reference_production_root": str(inputs["reference_root"]), "plan_root": str(inputs["plan_root"])},
input_hashes=inputs["frozen_inputs"], config_sha256=config_sha,
builder_script_sha256=config["builder_script_sha256"], policy_script_sha256=config["policy_script_sha256"],
output_hashes={"session_recipes": sha256_file(manifest_root / "session_recipes.jsonl"),
"composed_session_assignments": sha256_file(manifest_root / "composed_session_assignments.jsonl"),
"catalog_registry": catalog_sha, "multi_event_summary": sha256_file(report_root / "multi_event_summary.json")},
sir_distribution=inputs["plan_report"]["sir_distribution"],
boundary_unmeasurable_policy="keep_event_position_and_seed_emit_peak_safe_base_without_interference",
boundary_unmeasurable_sessions=summary["boundary_unmeasurable_sessions"],
generated_utc=datetime.now(timezone.utc).isoformat(), environment={"device": "cpu", "workers": workers})
# The previous catalog verification concerns the parent catalog, not this one.
build_report["input_paths"].pop("catalog_verify_report", None)
write_json_atomic(_owned(report_root / "pilot_build.json", output_root), build_report)
write_json_atomic(_owned(report_root / "pilot_complete.json", output_root), {
"format": "hfd_multi_v5_construction_status_v1", "complete": True,
"status": "built_pending_strict_audit", "strict_audit_passed": False, "production_ready": False,
"training_started": False, "sessions": len(recipes), "pilot": pilot})
write_json_atomic(_owned(state_path, output_root), {"config_sha256": config_sha, "config": config,
"status": "built_pending_strict_audit", "completed_sessions": len(recipes),
"final_payload_hashes": {str(path.relative_to(output_root)): sha256_file(path) for path in (
manifest_root / "session_recipes.jsonl", manifest_root / "composed_session_assignments.jsonl",
manifest_root / "third_party_source_registry.jsonl", report_root / "multi_event_summary.json")}})
return build_report
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--reference-root", type=Path, default=DEFAULT_REFERENCE)
parser.add_argument("--plan-root", type=Path, default=DEFAULT_PLAN)
parser.add_argument("--tts-registry", type=Path, default=DEFAULT_TTS)
parser.add_argument("--output-root", type=Path)
parser.add_argument("--background-seed", type=int, default=BACKGROUND_SEED)
parser.add_argument("--workers", type=int, default=4)
parser.add_argument("--limit", type=int, help="Explicit pilot size; default pilot covers all six classes")
parser.add_argument("--session-types", help="Comma-separated class filter, permitted only with --limit")
parser.add_argument("--resume", action="store_true")
args = parser.parse_args()
report = build(args.reference_root, args.plan_root, args.tts_registry, args.output_root,
background_seed=args.background_seed, workers=args.workers, limit=args.limit,
session_types=args.session_types.split(",") if args.session_types else None, resume=args.resume)
print(json.dumps({key: report[key] for key in ("status", "counts", "pilot", "production_ready")}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()