code-HFD_lzk / scripts /data /audit_hfd_third_party_multi_v5.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
8562e9b verified
Raw History Blame Contribute Delete
24.9 kB
#!/usr/bin/env python3
"""Parallel CPU audit for v5: lineage, schedule, exact PCM and real input hashes.
Unchanged target supervision inherits its original audit only after the copied
timeline bytes and complete target identity match. This is not the legacy
single-event auditor and does not label a pilot as a complete production set.
"""
from __future__ import annotations
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import hashlib
import json
import math
from pathlib import Path
import random
import sys
import wave
import numpy as np
ROOT = Path(__file__).resolve().parents[2]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from scripts.data import hfd_third_party_multi_v5 as policy
from scripts.data.hfd_third_party_common import (
STRIDE_SAMPLES, canonical_json_sha256, quantize_pcm16, read_jsonl,
sha256_file, stable_u64, write_json_atomic,
)
from scripts.data.report_hfd_third_party_refresh import _target_identity
from scripts.data.scan_hfd_third_party_loudness import read_background, target_track
DATA = ROOT / "data/hfd_third_party_multi_v5"
REFERENCE = ROOT / "data/hfd_third_party_cross_v4/production"
def boundary_waveform(parent: dict) -> tuple[np.ndarray, float]:
target, _ = target_track(parent)
bg = parent["base_background"]
noise = read_background(bg["path"], bg["start_window"] * STRIDE_SAMPLES, len(target))
base = quantize_pcm16(target + noise).astype(np.float32) / np.float32(32768.)
peak = float(np.max(np.abs(base), initial=0.))
scale = min(1., .95 / max(peak, 1e-12))
return base * np.float32(scale), scale
def compare_one(parent: dict, row: dict, assignment: dict, assets: dict, tts_assets: dict,
production: Path, reference: Path, seed: int) -> dict:
errors = []
identity = row.get("session_id", "missing")
try:
if row["parent_recipe_sha256"] != canonical_json_sha256(parent):
raise ValueError("Original parent recipe changed")
if identity != parent["session_id"] or row["session_signature"] != parent["session_signature"]:
raise ValueError("Stable session identity changed")
if _target_identity(row) != _target_identity(parent):
raise ValueError("Target speech, motion supervision or timing changed")
if row["session_type"] != assignment["session_type"] or row["audio_corruption_profile"] != assignment["session_type"]:
raise ValueError("Session class differs from frozen assignment")
if row["background_assignment"] != assignment:
raise ValueError("Complete background assignment differs from frozen assignment")
if row.get("profile_policy") != policy.POLICY or row.get("background_seed") != seed:
raise ValueError("Background policy/seed metadata mismatch")
if row.get("background_sir_seed") != stable_u64("multi-v5-sir", seed, parent["session_signature"]):
raise ValueError("Background SIR seed metadata mismatch")
if row.get("interference_seed") != stable_u64("multi-v5-interference", seed, parent["session_signature"]):
raise ValueError("Interference seed metadata mismatch")
for kind, folder, extension in (("audio", "audio", "wav"), ("timeline", "timelines", "npz")):
expected_path = f"{folder}/{parent['split']}/{identity}.{extension}"
actual_path = production / row[kind]["path"]
if row[kind]["path"] != expected_path or actual_path.resolve() != actual_path:
raise ValueError(f"Unexpected or aliased {kind} path")
if row["timeline"] != parent["timeline"]:
raise ValueError("Timeline metadata differs from frozen parent")
timeline = production / row["timeline"]["path"]
if (sha256_file(timeline) != parent["timeline"]["file_sha256"] or
sha256_file(reference / parent["timeline"]["path"]) != parent["timeline"]["file_sha256"]):
raise ValueError("Actual motion timeline bytes changed")
requested = policy.truncated_sir(random.Random(stable_u64("multi-v5-sir", seed, parent["session_signature"])))
if not math.isclose(requested, row["mix_metadata"]["requested_background_sir_db"], abs_tol=1e-10):
raise ValueError("Requested SIR differs from independent deterministic draw")
clone_ids = {a["asset_id"] for a in tts_assets.values()}
real_assets = {key: value for key, value in assets.items() if key not in clone_ids}
scheduled = policy.schedule_events(parent, assignment, real_assets, seed, tts_assets=tts_assets)
# The mixer enforces the complete target source SHA and language/clone binding.
replay_parent = dict(parent, background_assignment=assignment)
try:
expected, events, metadata = policy.mix_scheduled_events(replay_parent, scheduled, assets, requested)
if row["interference_events"] != events:
raise ValueError("Event schedule/provenance/gain replay differs")
for key, value in metadata.items():
if row["mix_metadata"].get(key) != value:
raise ValueError(f"Mix metadata replay differs: {key}")
if row["mix_metadata"].get("status") != "calibrated" or row["mix_metadata"].get("background_applied") is not True:
raise ValueError("Measurable background lacks calibrated/applied state")
boundary = False
except policy.UnusableBackgroundError as exc:
expected, scale = boundary_waveform(parent)
if row["mix_metadata"].get("status") != "boundary_unmeasurable":
raise ValueError("Unusable boundary draw lacks explicit exception")
if row["mix_metadata"].get("measured_background_sir_db") is not None:
raise ValueError("Silent boundary cannot claim a measured SIR")
if not math.isclose(row["mix_metadata"]["global_scale"], scale, abs_tol=1e-12):
raise ValueError("Boundary peak scale mismatch")
for key, value in dict(background_gain=0., background_applied=False, placement_resampled=False,
background_active_sample_count=exc.active_sample_count).items():
if row["mix_metadata"].get(key) != value:
raise ValueError(f"Boundary metadata mismatch: {key}")
# Gain fields may be enriched with zero; geometry/provenance must survive.
if len(row["interference_events"]) != len(exc.events):
raise ValueError("Boundary exception changed event count")
for original, actual in zip(exc.events, row["interference_events"]):
if any(actual.get(k) != v for k, v in original.items()):
raise ValueError("Boundary event schedule was resampled or altered")
if actual.get("gain") != 0 or actual.get("background_gain") != 0 or actual.get("applied_to_audio") is not False:
raise ValueError("Boundary event falsely claims applied speech")
boundary = True
audio_path = production / row["audio"]["path"]
if sha256_file(audio_path) != row["audio"]["sha256"]:
raise ValueError("Stored audio file hash mismatch")
with wave.open(str(audio_path), "rb") as handle:
if (handle.getframerate(), handle.getnchannels(), handle.getsampwidth(), handle.getnframes()) != (
16000, 1, 2, parent["audio"]["sample_count"]):
raise ValueError("Stored audio geometry mismatch")
actual_pcm = np.frombuffer(handle.readframes(handle.getnframes()), dtype="<i2")
if hashlib.sha256(actual_pcm.tobytes()).hexdigest() != row["audio"]["pcm16_sha256"]:
raise ValueError("Stored PCM hash metadata mismatch")
if not np.array_equal(actual_pcm, quantize_pcm16(expected)):
raise ValueError("Actual PCM differs from independent scheduled replay")
if np.max(np.abs(actual_pcm.astype(np.int32)), initial=0) > math.ceil(.9501 * 32768):
raise ValueError("Peak safety limit exceeded")
event_count = len(scheduled)
class_name = assignment["session_type"]
ranges = {"single_short": (1, 1), "multi_single": (2, 4), "multi_babble": (2, 3),
"mixed_short": (3, 5), "continuous": (1, 1), "same_voice_aside": (2, 3)}
low, high = ranges[class_name]
if not low <= event_count <= high:
raise ValueError("Event count outside class contract")
for event in row["interference_events"]:
if set(event["languages"]) != {assignment["language"]}:
raise ValueError("Actual event language mismatch")
if event.get("voice_relation") == "reference_clone":
if any(event["start_sample"] < t["end_sample"] and event["end_sample"] > t["start_sample"]
for t in parent["target_speech_events"]):
raise ValueError("Same voice speaks simultaneously with a target command")
return dict(session_id=identity, errors=[], split=row["split"], session_type=class_name,
language=assignment["language"], requested_sir_db=requested,
measured_sir_db=row["mix_metadata"]["measured_background_sir_db"],
boundary_unmeasurable=boundary, event_count=event_count,
right_truncated_events=sum(e.get("right_truncated", False) for e in scheduled),
target_boundary_truncated_events=sum(e.get("crop_reason") == "next_target_speech_start" for e in scheduled),
target_overlapping_events=sum(bool(e.get("overlapped_target_turn_indices")) for e in row["interference_events"]),
uniform_start_values=[e["start_sample"] / parent["audio"]["sample_count"] for e in scheduled
if e.get("placement_policy") == "uniform_full_session_right_crop_v1"],
globally_scaled=row["mix_metadata"]["global_scale"] < 1,
source_assets=[a for e in row["interference_events"] for a in e["asset_ids"]])
except (KeyError, ValueError, OSError, RuntimeError, AssertionError) as exc:
errors.append(str(exc))
return dict(session_id=identity, errors=errors)
def sir_statistics(values: list[float]) -> dict:
if not values:
return dict(count=0)
data = np.sort(np.asarray(values, dtype=np.float64))
cdf = lambda x: .5 * (1 + math.erf((x - 2.) / (3. * math.sqrt(2.))))
low, high = cdf(-4.), cdf(8.)
expected = np.asarray([(cdf(v) - low) / (high - low) for v in data])
n = len(data)
ks = max(float(np.max(np.arange(1, n+1) / n - expected)),
float(np.max(expected - np.arange(n) / n)))
return dict(count=n, mean=float(data.mean()), standard_deviation=float(data.std()),
minimum=float(data[0]), maximum=float(data[-1]),
background_louder_fraction=float(np.mean(data < 0)),
theoretical_background_louder_fraction=(cdf(0.)-low)/(high-low),
truncated_normal_ks_distance=ks)
def audit(production: Path, reference: Path, plan_root: Path, tts_registry: Path,
workers: int = 8, allow_pilot: bool = False) -> dict:
if not 1 <= workers <= 8:
raise ValueError("CPU workers must be 1..8")
production, reference, plan_root = production.absolute(), reference.absolute(), plan_root.absolute()
if production.resolve() != production or not production.is_relative_to(DATA):
raise ValueError("Audit report output must remain inside the owned v5 dataset")
build = json.loads((production / "reports/pilot_build.json").read_text())
if build.get("profile_policy") != policy.POLICY or not build.get("complete"):
raise ValueError("Not a completed v5 construction")
pilot = bool(build.get("pilot"))
if pilot and not allow_pilot:
raise ValueError("Explicit --allow-pilot required for a limited build")
plan = json.loads((plan_root / "preparation_report.json").read_text())
if not pilot and not plan.get("qualified_registry_path"):
raise ValueError("Full production lacks quality-screened reference selection")
if plan.get("qualified_registry_path"):
if sha256_file(Path(plan["qualified_registry_path"])) != plan["qualified_registry_sha256"]:
raise ValueError("Frozen quality-screened candidate registry changed")
if build["background_seed"] != plan["seed"]:
raise ValueError("Plan/background construction seeds differ")
state = json.loads((production / ".progress/build_state.json").read_text())
if state["config_sha256"] != canonical_json_sha256(state["config"]) or state["config_sha256"] != build["config_sha256"]:
raise ValueError("Frozen build configuration changed")
for field, path in (("builder_script_sha256", ROOT / "scripts/data/build_hfd_third_party_multi_v5.py"),
("policy_script_sha256", Path(policy.__file__))):
if sha256_file(path) != build[field] or build[field] != state["config"][field]:
raise ValueError(f"Construction implementation changed: {field}")
for path, expected in build["input_hashes"].items():
if sha256_file(Path(path)) != expected:
raise ValueError(f"Frozen build input changed: {path}")
for name, digest in plan["output_hashes"].items():
if sha256_file(plan_root / name) != digest or build["plan_hashes"].get(name) != digest:
raise ValueError("Frozen construction plan changed")
parent_path = reference / "manifests/session_recipes.jsonl"
reference_hash = sha256_file(parent_path)
if reference_hash != plan["reference_recipe_sha256"] or reference_hash != build["reference_recipe_sha256"]:
raise ValueError("Parent recipe manifest changed")
parent_audit = json.loads((reference / "reports/strict_audit.json").read_text())
if not parent_audit.get("passed"):
raise ValueError("Parent dataset audit must remain passed")
parent_build = json.loads((reference / "reports/pilot_build.json").read_text())
if any(build.get(key) != parent_build.get(key) for key in ("seed", "composition_salt", "composition_variant_id")):
raise ValueError("Original target composition parameters changed")
for name in ("source_contract.json", "primitive_split_snapshot.jsonl", "baseline_session_signatures.jsonl"):
if sha256_file(production / "manifests" / name) != sha256_file(reference / "manifests" / name):
raise ValueError(f"Target composition input changed: {name}")
parent = {r["session_id"]: r for r in read_jsonl(parent_path)}
assignments = {r["session_id"]: r for r in read_jsonl(plan_root / "session_assignments.jsonl")}
source_rows = read_jsonl(plan_root / "selected_real_assets.jsonl")
clones = read_jsonl(tts_registry) if tts_registry.exists() else []
if (sha256_file(tts_registry) if tts_registry.exists() else None) != build.get("eligible_tts_registry_sha256"):
raise ValueError("Eligible clone catalog changed")
if any(not a.get("eligible_for_mixing") for a in clones):
raise ValueError("Unapproved generated audio present in eligible clone catalog")
assets = {a["asset_id"]: a for a in [*source_rows, *clones]}
if len(assets) != len(source_rows) + len(clones):
raise ValueError("Duplicate merged catalog asset ID")
tts_assets = {a["job_id"]: a for a in clones}
if len(tts_assets) != len(clones):
raise ValueError("Duplicate clone job ID")
if ({a["reference_sha256"] for a in clones if a["split"] == "train"}
& {a["reference_sha256"] for a in clones if a["split"] == "validation"}):
raise ValueError("Actual clone reference audio overlaps train and validation")
jobs = {j["job_id"]: j for j in read_jsonl(plan_root / "tts_jobs.jsonl")}
for asset in clones:
job = jobs[asset["job_id"]]
if plan.get("qualified_registry_path"):
from scripts.data.plan_hfd_third_party_multi_v5 import validate_qualified_asset
validate_qualified_asset(asset, job)
for key in ("reference_audio", "reference_sha256", "reference_text", "split", "language"):
if asset[key] != job[key]:
raise ValueError(f"Clone provenance differs from its frozen TTS job: {key}")
if asset["transcript"] != job["text"]:
raise ValueError("Clone transcript differs from its frozen TTS job")
for field in ("source_group", "speaker_id", "normalized_sha256"):
a = {r[field] for r in assets.values() if r["split"] == "train"}
b = {r[field] for r in assets.values() if r["split"] == "validation"}
if a & b:
raise ValueError(f"Actual catalog train/validation overlap: {field}")
manifest = production / "manifests/session_recipes.jsonl"
output_paths = {"session_recipes": manifest,
"composed_session_assignments": production / "manifests/composed_session_assignments.jsonl",
"catalog_registry": production / "manifests/third_party_source_registry.jsonl",
"multi_event_summary": production / "reports/multi_event_summary.json"}
for key, path in output_paths.items():
if sha256_file(path) != build["output_hashes"][key]:
raise ValueError(f"Generated payload changed: {key}")
catalog = read_jsonl(output_paths["catalog_registry"])
if catalog != sorted(assets.values(), key=lambda row: row["asset_id"]):
raise ValueError("Production source catalog differs from frozen inputs")
manifest_hash = sha256_file(manifest)
if manifest_hash != build["output_hashes"]["session_recipes"]:
raise ValueError("Output recipe manifest changed")
rows = read_jsonl(manifest)
composed = read_jsonl(output_paths["composed_session_assignments"])
if len(composed) != len(rows):
raise ValueError("Composed assignment population differs from recipe population")
for row, assignment in zip(rows, composed):
for key in ("session_id", "split", "session_type", "audio_corruption_profile", "timeline",
"target_speech_events", "interference_events", "background_assignment", "parent_recipe_sha256"):
if row[key] != assignment[key]:
raise ValueError(f"Composed assignment differs from recipe: {key}")
if (row["audio"]["path"], row["audio"]["sha256"], row["audio"]["sample_count"]) != (
assignment["audio_path"], assignment["audio_sha256"], assignment["audio_sample_count"]):
raise ValueError("Composed assignment audio differs from recipe")
if row["contracts"]["builder_config_sha256"] != build["config_sha256"] or row["contracts"]["catalog_manifest_sha256"] != build["output_hashes"]["catalog_registry"]:
raise ValueError("Recipe construction contract differs from frozen build")
del composed
identities = {r["session_id"] for r in rows}
if len(identities) != len(rows) or not identities <= parent.keys():
raise ValueError("Duplicate or unexpected output identity")
if not pilot and identities != parent.keys():
raise ValueError("Full dataset is not a one-for-one replacement")
if not pilot and len(rows) != 27584:
raise ValueError("Production population must remain 27,584")
seed = int(build["background_seed"])
def check(row):
return compare_one(parent[row["session_id"]], row, assignments[row["session_id"]],
assets, tts_assets, production, reference, seed)
results = []
with ThreadPoolExecutor(max_workers=workers) as pool:
for index, result in enumerate(pool.map(check, rows), 1):
results.append(result)
if index % 1000 == 0 or index == len(rows):
print(f"v5 exact audit {index}/{len(rows)}", flush=True)
failures = [r for r in results if r["errors"]]
passed = [r for r in results if not r["errors"]]
counts = Counter('/'.join((r['split'],r['session_type'],r['language'])) for r in passed)
errors = [f"{r['session_id']}:{';'.join(r['errors'])}" for r in failures]
if not pilot and dict(counts) != plan["class_language_split_counts"]:
errors.append("Production class/language/split quotas differ from plan")
stats = sir_statistics([r["requested_sir_db"] for r in passed])
if not pilot and stats.get("truncated_normal_ks_distance", 1) > .025:
errors.append("Requested SIR population does not match declared truncated normal")
if sha256_file(manifest) != manifest_hash or sha256_file(parent_path) != reference_hash:
errors.append("Recipe inputs changed during audit")
starts = [v for result in passed for v in result["uniform_start_values"]]
start_bins = Counter(str(min(9, int(value * 10))) for value in starts)
if not pilot and starts:
ordered = np.sort(starts)
uniform_ks = max(float(np.max(np.arange(1, len(ordered)+1) / len(ordered) - ordered)),
float(np.max(ordered - np.arange(len(ordered)) / len(ordered))))
if uniform_ks > .02:
errors.append("Background event start population is not uniform over full sessions")
else:
uniform_ks = None
report = dict(format="hfd_multi_event_strict_audit_v1", passed=not errors,
automated_checks_complete=not errors, pilot=pilot,
completed_utc=datetime.now(timezone.utc).isoformat(),
recipe_sha256=manifest_hash, parent_recipe_sha256=reference_hash,
sessions=len(rows), checked_sessions=len(passed), errors=errors,
class_language_split_counts=dict(counts), requested_sir_statistics=stats,
measured_sir_statistics=sir_statistics([r["measured_sir_db"] for r in passed if r["measured_sir_db"] is not None]),
boundary_unmeasurable_sessions=sum(r["boundary_unmeasurable"] for r in passed),
total_events=sum(r["event_count"] for r in passed),
right_truncated_events=sum(r["right_truncated_events"] for r in passed),
target_boundary_truncated_events=sum(r["target_boundary_truncated_events"] for r in passed),
target_overlapping_events=sum(r["target_overlapping_events"] for r in passed),
full_session_uniform_start_deciles={str(i): start_bins[str(i)] for i in range(10)},
full_session_uniform_start_ks_distance=uniform_ks,
same_voice_start_note="Reference-clone starts are uniform over non-target intervals after their anchor, separately from full-session event starts.",
globally_scaled_sessions=sum(r["globally_scaled"] for r in passed),
requested_sir_by_class={name: sir_statistics([r["requested_sir_db"] for r in passed if r["session_type"] == name])
for name, _ in policy.CLASS_WEIGHTS},
used_assets=len({a for r in passed for a in r["source_assets"]}),
validation_scope="Exact schedule/PCM replay, real source hashes, copied timeline equality and unchanged complete target supervision. Original timeline semantics inherited from passed parent audit, not reinterpreted.",
human_listening_review="pending", production_ready=False, training_started=False)
write_json_atomic(production / "reports/strict_audit.json", report)
write_json_atomic(production / "reports/pilot_complete.json", dict(
format="hfd_multi_event_completion_v1", automated_checks_complete=not errors,
full_dataset=not pilot, automated_audit_sha256=sha256_file(production / "reports/strict_audit.json"),
human_listening_review="pending", production_ready=False, training_started=False))
return report
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--production-root", type=Path, default=DATA / "production")
parser.add_argument("--reference-root", type=Path, default=REFERENCE)
parser.add_argument("--plan-root", type=Path, default=DATA / "work/plan_qualified")
parser.add_argument("--tts-registry", type=Path, default=DATA / "work/tts_assets_qualified/eligible_registry.jsonl")
parser.add_argument("--workers", type=int, default=8)
parser.add_argument("--allow-pilot", action="store_true")
args = parser.parse_args()
result = audit(args.production_root, args.reference_root, args.plan_root,
args.tts_registry, args.workers, args.allow_pilot)
print(json.dumps(result, ensure_ascii=False, indent=2))
raise SystemExit(0 if result["passed"] else 1)