Download scripts/data/audit_hfd_third_party_multi_v5.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 24.9 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/audit_hfd_third_party_multi_v5.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/data/audit_hfd_third_party_multi_v5.py
-
curl -L -o audit_hfd_third_party_multi_v5.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/audit_hfd_third_party_multi_v5.py
24.9 kB
| #!/usr/bin/env python3 | |
| """Parallel CPU audit for v5: lineage, schedule, exact PCM and real input hashes. | |
| Unchanged target supervision inherits its original audit only after the copied | |
| timeline bytes and complete target identity match. This is not the legacy | |
| single-event auditor and does not label a pilot as a complete production set. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| from collections import Counter | |
| from concurrent.futures import ThreadPoolExecutor | |
| from datetime import datetime, timezone | |
| import hashlib | |
| import json | |
| import math | |
| from pathlib import Path | |
| import random | |
| import sys | |
| import wave | |
| import numpy as np | |
| ROOT = Path(__file__).resolve().parents[2] | |
| if str(ROOT) not in sys.path: | |
| sys.path.insert(0, str(ROOT)) | |
| from scripts.data import hfd_third_party_multi_v5 as policy | |
| from scripts.data.hfd_third_party_common import ( | |
| STRIDE_SAMPLES, canonical_json_sha256, quantize_pcm16, read_jsonl, | |
| sha256_file, stable_u64, write_json_atomic, | |
| ) | |
| from scripts.data.report_hfd_third_party_refresh import _target_identity | |
| from scripts.data.scan_hfd_third_party_loudness import read_background, target_track | |
| DATA = ROOT / "data/hfd_third_party_multi_v5" | |
| REFERENCE = ROOT / "data/hfd_third_party_cross_v4/production" | |
| def boundary_waveform(parent: dict) -> tuple[np.ndarray, float]: | |
| target, _ = target_track(parent) | |
| bg = parent["base_background"] | |
| noise = read_background(bg["path"], bg["start_window"] * STRIDE_SAMPLES, len(target)) | |
| base = quantize_pcm16(target + noise).astype(np.float32) / np.float32(32768.) | |
| peak = float(np.max(np.abs(base), initial=0.)) | |
| scale = min(1., .95 / max(peak, 1e-12)) | |
| return base * np.float32(scale), scale | |
| def compare_one(parent: dict, row: dict, assignment: dict, assets: dict, tts_assets: dict, | |
| production: Path, reference: Path, seed: int) -> dict: | |
| errors = [] | |
| identity = row.get("session_id", "missing") | |
| try: | |
| if row["parent_recipe_sha256"] != canonical_json_sha256(parent): | |
| raise ValueError("Original parent recipe changed") | |
| if identity != parent["session_id"] or row["session_signature"] != parent["session_signature"]: | |
| raise ValueError("Stable session identity changed") | |
| if _target_identity(row) != _target_identity(parent): | |
| raise ValueError("Target speech, motion supervision or timing changed") | |
| if row["session_type"] != assignment["session_type"] or row["audio_corruption_profile"] != assignment["session_type"]: | |
| raise ValueError("Session class differs from frozen assignment") | |
| if row["background_assignment"] != assignment: | |
| raise ValueError("Complete background assignment differs from frozen assignment") | |
| if row.get("profile_policy") != policy.POLICY or row.get("background_seed") != seed: | |
| raise ValueError("Background policy/seed metadata mismatch") | |
| if row.get("background_sir_seed") != stable_u64("multi-v5-sir", seed, parent["session_signature"]): | |
| raise ValueError("Background SIR seed metadata mismatch") | |
| if row.get("interference_seed") != stable_u64("multi-v5-interference", seed, parent["session_signature"]): | |
| raise ValueError("Interference seed metadata mismatch") | |
| for kind, folder, extension in (("audio", "audio", "wav"), ("timeline", "timelines", "npz")): | |
| expected_path = f"{folder}/{parent['split']}/{identity}.{extension}" | |
| actual_path = production / row[kind]["path"] | |
| if row[kind]["path"] != expected_path or actual_path.resolve() != actual_path: | |
| raise ValueError(f"Unexpected or aliased {kind} path") | |
| if row["timeline"] != parent["timeline"]: | |
| raise ValueError("Timeline metadata differs from frozen parent") | |
| timeline = production / row["timeline"]["path"] | |
| if (sha256_file(timeline) != parent["timeline"]["file_sha256"] or | |
| sha256_file(reference / parent["timeline"]["path"]) != parent["timeline"]["file_sha256"]): | |
| raise ValueError("Actual motion timeline bytes changed") | |
| requested = policy.truncated_sir(random.Random(stable_u64("multi-v5-sir", seed, parent["session_signature"]))) | |
| if not math.isclose(requested, row["mix_metadata"]["requested_background_sir_db"], abs_tol=1e-10): | |
| raise ValueError("Requested SIR differs from independent deterministic draw") | |
| clone_ids = {a["asset_id"] for a in tts_assets.values()} | |
| real_assets = {key: value for key, value in assets.items() if key not in clone_ids} | |
| scheduled = policy.schedule_events(parent, assignment, real_assets, seed, tts_assets=tts_assets) | |
| # The mixer enforces the complete target source SHA and language/clone binding. | |
| replay_parent = dict(parent, background_assignment=assignment) | |
| try: | |
| expected, events, metadata = policy.mix_scheduled_events(replay_parent, scheduled, assets, requested) | |
| if row["interference_events"] != events: | |
| raise ValueError("Event schedule/provenance/gain replay differs") | |
| for key, value in metadata.items(): | |
| if row["mix_metadata"].get(key) != value: | |
| raise ValueError(f"Mix metadata replay differs: {key}") | |
| if row["mix_metadata"].get("status") != "calibrated" or row["mix_metadata"].get("background_applied") is not True: | |
| raise ValueError("Measurable background lacks calibrated/applied state") | |
| boundary = False | |
| except policy.UnusableBackgroundError as exc: | |
| expected, scale = boundary_waveform(parent) | |
| if row["mix_metadata"].get("status") != "boundary_unmeasurable": | |
| raise ValueError("Unusable boundary draw lacks explicit exception") | |
| if row["mix_metadata"].get("measured_background_sir_db") is not None: | |
| raise ValueError("Silent boundary cannot claim a measured SIR") | |
| if not math.isclose(row["mix_metadata"]["global_scale"], scale, abs_tol=1e-12): | |
| raise ValueError("Boundary peak scale mismatch") | |
| for key, value in dict(background_gain=0., background_applied=False, placement_resampled=False, | |
| background_active_sample_count=exc.active_sample_count).items(): | |
| if row["mix_metadata"].get(key) != value: | |
| raise ValueError(f"Boundary metadata mismatch: {key}") | |
| # Gain fields may be enriched with zero; geometry/provenance must survive. | |
| if len(row["interference_events"]) != len(exc.events): | |
| raise ValueError("Boundary exception changed event count") | |
| for original, actual in zip(exc.events, row["interference_events"]): | |
| if any(actual.get(k) != v for k, v in original.items()): | |
| raise ValueError("Boundary event schedule was resampled or altered") | |
| if actual.get("gain") != 0 or actual.get("background_gain") != 0 or actual.get("applied_to_audio") is not False: | |
| raise ValueError("Boundary event falsely claims applied speech") | |
| boundary = True | |
| audio_path = production / row["audio"]["path"] | |
| if sha256_file(audio_path) != row["audio"]["sha256"]: | |
| raise ValueError("Stored audio file hash mismatch") | |
| with wave.open(str(audio_path), "rb") as handle: | |
| if (handle.getframerate(), handle.getnchannels(), handle.getsampwidth(), handle.getnframes()) != ( | |
| 16000, 1, 2, parent["audio"]["sample_count"]): | |
| raise ValueError("Stored audio geometry mismatch") | |
| actual_pcm = np.frombuffer(handle.readframes(handle.getnframes()), dtype="<i2") | |
| if hashlib.sha256(actual_pcm.tobytes()).hexdigest() != row["audio"]["pcm16_sha256"]: | |
| raise ValueError("Stored PCM hash metadata mismatch") | |
| if not np.array_equal(actual_pcm, quantize_pcm16(expected)): | |
| raise ValueError("Actual PCM differs from independent scheduled replay") | |
| if np.max(np.abs(actual_pcm.astype(np.int32)), initial=0) > math.ceil(.9501 * 32768): | |
| raise ValueError("Peak safety limit exceeded") | |
| event_count = len(scheduled) | |
| class_name = assignment["session_type"] | |
| ranges = {"single_short": (1, 1), "multi_single": (2, 4), "multi_babble": (2, 3), | |
| "mixed_short": (3, 5), "continuous": (1, 1), "same_voice_aside": (2, 3)} | |
| low, high = ranges[class_name] | |
| if not low <= event_count <= high: | |
| raise ValueError("Event count outside class contract") | |
| for event in row["interference_events"]: | |
| if set(event["languages"]) != {assignment["language"]}: | |
| raise ValueError("Actual event language mismatch") | |
| if event.get("voice_relation") == "reference_clone": | |
| if any(event["start_sample"] < t["end_sample"] and event["end_sample"] > t["start_sample"] | |
| for t in parent["target_speech_events"]): | |
| raise ValueError("Same voice speaks simultaneously with a target command") | |
| return dict(session_id=identity, errors=[], split=row["split"], session_type=class_name, | |
| language=assignment["language"], requested_sir_db=requested, | |
| measured_sir_db=row["mix_metadata"]["measured_background_sir_db"], | |
| boundary_unmeasurable=boundary, event_count=event_count, | |
| right_truncated_events=sum(e.get("right_truncated", False) for e in scheduled), | |
| target_boundary_truncated_events=sum(e.get("crop_reason") == "next_target_speech_start" for e in scheduled), | |
| target_overlapping_events=sum(bool(e.get("overlapped_target_turn_indices")) for e in row["interference_events"]), | |
| uniform_start_values=[e["start_sample"] / parent["audio"]["sample_count"] for e in scheduled | |
| if e.get("placement_policy") == "uniform_full_session_right_crop_v1"], | |
| globally_scaled=row["mix_metadata"]["global_scale"] < 1, | |
| source_assets=[a for e in row["interference_events"] for a in e["asset_ids"]]) | |
| except (KeyError, ValueError, OSError, RuntimeError, AssertionError) as exc: | |
| errors.append(str(exc)) | |
| return dict(session_id=identity, errors=errors) | |
| def sir_statistics(values: list[float]) -> dict: | |
| if not values: | |
| return dict(count=0) | |
| data = np.sort(np.asarray(values, dtype=np.float64)) | |
| cdf = lambda x: .5 * (1 + math.erf((x - 2.) / (3. * math.sqrt(2.)))) | |
| low, high = cdf(-4.), cdf(8.) | |
| expected = np.asarray([(cdf(v) - low) / (high - low) for v in data]) | |
| n = len(data) | |
| ks = max(float(np.max(np.arange(1, n+1) / n - expected)), | |
| float(np.max(expected - np.arange(n) / n))) | |
| return dict(count=n, mean=float(data.mean()), standard_deviation=float(data.std()), | |
| minimum=float(data[0]), maximum=float(data[-1]), | |
| background_louder_fraction=float(np.mean(data < 0)), | |
| theoretical_background_louder_fraction=(cdf(0.)-low)/(high-low), | |
| truncated_normal_ks_distance=ks) | |
| def audit(production: Path, reference: Path, plan_root: Path, tts_registry: Path, | |
| workers: int = 8, allow_pilot: bool = False) -> dict: | |
| if not 1 <= workers <= 8: | |
| raise ValueError("CPU workers must be 1..8") | |
| production, reference, plan_root = production.absolute(), reference.absolute(), plan_root.absolute() | |
| if production.resolve() != production or not production.is_relative_to(DATA): | |
| raise ValueError("Audit report output must remain inside the owned v5 dataset") | |
| build = json.loads((production / "reports/pilot_build.json").read_text()) | |
| if build.get("profile_policy") != policy.POLICY or not build.get("complete"): | |
| raise ValueError("Not a completed v5 construction") | |
| pilot = bool(build.get("pilot")) | |
| if pilot and not allow_pilot: | |
| raise ValueError("Explicit --allow-pilot required for a limited build") | |
| plan = json.loads((plan_root / "preparation_report.json").read_text()) | |
| if not pilot and not plan.get("qualified_registry_path"): | |
| raise ValueError("Full production lacks quality-screened reference selection") | |
| if plan.get("qualified_registry_path"): | |
| if sha256_file(Path(plan["qualified_registry_path"])) != plan["qualified_registry_sha256"]: | |
| raise ValueError("Frozen quality-screened candidate registry changed") | |
| if build["background_seed"] != plan["seed"]: | |
| raise ValueError("Plan/background construction seeds differ") | |
| state = json.loads((production / ".progress/build_state.json").read_text()) | |
| if state["config_sha256"] != canonical_json_sha256(state["config"]) or state["config_sha256"] != build["config_sha256"]: | |
| raise ValueError("Frozen build configuration changed") | |
| for field, path in (("builder_script_sha256", ROOT / "scripts/data/build_hfd_third_party_multi_v5.py"), | |
| ("policy_script_sha256", Path(policy.__file__))): | |
| if sha256_file(path) != build[field] or build[field] != state["config"][field]: | |
| raise ValueError(f"Construction implementation changed: {field}") | |
| for path, expected in build["input_hashes"].items(): | |
| if sha256_file(Path(path)) != expected: | |
| raise ValueError(f"Frozen build input changed: {path}") | |
| for name, digest in plan["output_hashes"].items(): | |
| if sha256_file(plan_root / name) != digest or build["plan_hashes"].get(name) != digest: | |
| raise ValueError("Frozen construction plan changed") | |
| parent_path = reference / "manifests/session_recipes.jsonl" | |
| reference_hash = sha256_file(parent_path) | |
| if reference_hash != plan["reference_recipe_sha256"] or reference_hash != build["reference_recipe_sha256"]: | |
| raise ValueError("Parent recipe manifest changed") | |
| parent_audit = json.loads((reference / "reports/strict_audit.json").read_text()) | |
| if not parent_audit.get("passed"): | |
| raise ValueError("Parent dataset audit must remain passed") | |
| parent_build = json.loads((reference / "reports/pilot_build.json").read_text()) | |
| if any(build.get(key) != parent_build.get(key) for key in ("seed", "composition_salt", "composition_variant_id")): | |
| raise ValueError("Original target composition parameters changed") | |
| for name in ("source_contract.json", "primitive_split_snapshot.jsonl", "baseline_session_signatures.jsonl"): | |
| if sha256_file(production / "manifests" / name) != sha256_file(reference / "manifests" / name): | |
| raise ValueError(f"Target composition input changed: {name}") | |
| parent = {r["session_id"]: r for r in read_jsonl(parent_path)} | |
| assignments = {r["session_id"]: r for r in read_jsonl(plan_root / "session_assignments.jsonl")} | |
| source_rows = read_jsonl(plan_root / "selected_real_assets.jsonl") | |
| clones = read_jsonl(tts_registry) if tts_registry.exists() else [] | |
| if (sha256_file(tts_registry) if tts_registry.exists() else None) != build.get("eligible_tts_registry_sha256"): | |
| raise ValueError("Eligible clone catalog changed") | |
| if any(not a.get("eligible_for_mixing") for a in clones): | |
| raise ValueError("Unapproved generated audio present in eligible clone catalog") | |
| assets = {a["asset_id"]: a for a in [*source_rows, *clones]} | |
| if len(assets) != len(source_rows) + len(clones): | |
| raise ValueError("Duplicate merged catalog asset ID") | |
| tts_assets = {a["job_id"]: a for a in clones} | |
| if len(tts_assets) != len(clones): | |
| raise ValueError("Duplicate clone job ID") | |
| if ({a["reference_sha256"] for a in clones if a["split"] == "train"} | |
| & {a["reference_sha256"] for a in clones if a["split"] == "validation"}): | |
| raise ValueError("Actual clone reference audio overlaps train and validation") | |
| jobs = {j["job_id"]: j for j in read_jsonl(plan_root / "tts_jobs.jsonl")} | |
| for asset in clones: | |
| job = jobs[asset["job_id"]] | |
| if plan.get("qualified_registry_path"): | |
| from scripts.data.plan_hfd_third_party_multi_v5 import validate_qualified_asset | |
| validate_qualified_asset(asset, job) | |
| for key in ("reference_audio", "reference_sha256", "reference_text", "split", "language"): | |
| if asset[key] != job[key]: | |
| raise ValueError(f"Clone provenance differs from its frozen TTS job: {key}") | |
| if asset["transcript"] != job["text"]: | |
| raise ValueError("Clone transcript differs from its frozen TTS job") | |
| for field in ("source_group", "speaker_id", "normalized_sha256"): | |
| a = {r[field] for r in assets.values() if r["split"] == "train"} | |
| b = {r[field] for r in assets.values() if r["split"] == "validation"} | |
| if a & b: | |
| raise ValueError(f"Actual catalog train/validation overlap: {field}") | |
| manifest = production / "manifests/session_recipes.jsonl" | |
| output_paths = {"session_recipes": manifest, | |
| "composed_session_assignments": production / "manifests/composed_session_assignments.jsonl", | |
| "catalog_registry": production / "manifests/third_party_source_registry.jsonl", | |
| "multi_event_summary": production / "reports/multi_event_summary.json"} | |
| for key, path in output_paths.items(): | |
| if sha256_file(path) != build["output_hashes"][key]: | |
| raise ValueError(f"Generated payload changed: {key}") | |
| catalog = read_jsonl(output_paths["catalog_registry"]) | |
| if catalog != sorted(assets.values(), key=lambda row: row["asset_id"]): | |
| raise ValueError("Production source catalog differs from frozen inputs") | |
| manifest_hash = sha256_file(manifest) | |
| if manifest_hash != build["output_hashes"]["session_recipes"]: | |
| raise ValueError("Output recipe manifest changed") | |
| rows = read_jsonl(manifest) | |
| composed = read_jsonl(output_paths["composed_session_assignments"]) | |
| if len(composed) != len(rows): | |
| raise ValueError("Composed assignment population differs from recipe population") | |
| for row, assignment in zip(rows, composed): | |
| for key in ("session_id", "split", "session_type", "audio_corruption_profile", "timeline", | |
| "target_speech_events", "interference_events", "background_assignment", "parent_recipe_sha256"): | |
| if row[key] != assignment[key]: | |
| raise ValueError(f"Composed assignment differs from recipe: {key}") | |
| if (row["audio"]["path"], row["audio"]["sha256"], row["audio"]["sample_count"]) != ( | |
| assignment["audio_path"], assignment["audio_sha256"], assignment["audio_sample_count"]): | |
| raise ValueError("Composed assignment audio differs from recipe") | |
| if row["contracts"]["builder_config_sha256"] != build["config_sha256"] or row["contracts"]["catalog_manifest_sha256"] != build["output_hashes"]["catalog_registry"]: | |
| raise ValueError("Recipe construction contract differs from frozen build") | |
| del composed | |
| identities = {r["session_id"] for r in rows} | |
| if len(identities) != len(rows) or not identities <= parent.keys(): | |
| raise ValueError("Duplicate or unexpected output identity") | |
| if not pilot and identities != parent.keys(): | |
| raise ValueError("Full dataset is not a one-for-one replacement") | |
| if not pilot and len(rows) != 27584: | |
| raise ValueError("Production population must remain 27,584") | |
| seed = int(build["background_seed"]) | |
| def check(row): | |
| return compare_one(parent[row["session_id"]], row, assignments[row["session_id"]], | |
| assets, tts_assets, production, reference, seed) | |
| results = [] | |
| with ThreadPoolExecutor(max_workers=workers) as pool: | |
| for index, result in enumerate(pool.map(check, rows), 1): | |
| results.append(result) | |
| if index % 1000 == 0 or index == len(rows): | |
| print(f"v5 exact audit {index}/{len(rows)}", flush=True) | |
| failures = [r for r in results if r["errors"]] | |
| passed = [r for r in results if not r["errors"]] | |
| counts = Counter('/'.join((r['split'],r['session_type'],r['language'])) for r in passed) | |
| errors = [f"{r['session_id']}:{';'.join(r['errors'])}" for r in failures] | |
| if not pilot and dict(counts) != plan["class_language_split_counts"]: | |
| errors.append("Production class/language/split quotas differ from plan") | |
| stats = sir_statistics([r["requested_sir_db"] for r in passed]) | |
| if not pilot and stats.get("truncated_normal_ks_distance", 1) > .025: | |
| errors.append("Requested SIR population does not match declared truncated normal") | |
| if sha256_file(manifest) != manifest_hash or sha256_file(parent_path) != reference_hash: | |
| errors.append("Recipe inputs changed during audit") | |
| starts = [v for result in passed for v in result["uniform_start_values"]] | |
| start_bins = Counter(str(min(9, int(value * 10))) for value in starts) | |
| if not pilot and starts: | |
| ordered = np.sort(starts) | |
| uniform_ks = max(float(np.max(np.arange(1, len(ordered)+1) / len(ordered) - ordered)), | |
| float(np.max(ordered - np.arange(len(ordered)) / len(ordered)))) | |
| if uniform_ks > .02: | |
| errors.append("Background event start population is not uniform over full sessions") | |
| else: | |
| uniform_ks = None | |
| report = dict(format="hfd_multi_event_strict_audit_v1", passed=not errors, | |
| automated_checks_complete=not errors, pilot=pilot, | |
| completed_utc=datetime.now(timezone.utc).isoformat(), | |
| recipe_sha256=manifest_hash, parent_recipe_sha256=reference_hash, | |
| sessions=len(rows), checked_sessions=len(passed), errors=errors, | |
| class_language_split_counts=dict(counts), requested_sir_statistics=stats, | |
| measured_sir_statistics=sir_statistics([r["measured_sir_db"] for r in passed if r["measured_sir_db"] is not None]), | |
| boundary_unmeasurable_sessions=sum(r["boundary_unmeasurable"] for r in passed), | |
| total_events=sum(r["event_count"] for r in passed), | |
| right_truncated_events=sum(r["right_truncated_events"] for r in passed), | |
| target_boundary_truncated_events=sum(r["target_boundary_truncated_events"] for r in passed), | |
| target_overlapping_events=sum(r["target_overlapping_events"] for r in passed), | |
| full_session_uniform_start_deciles={str(i): start_bins[str(i)] for i in range(10)}, | |
| full_session_uniform_start_ks_distance=uniform_ks, | |
| same_voice_start_note="Reference-clone starts are uniform over non-target intervals after their anchor, separately from full-session event starts.", | |
| globally_scaled_sessions=sum(r["globally_scaled"] for r in passed), | |
| requested_sir_by_class={name: sir_statistics([r["requested_sir_db"] for r in passed if r["session_type"] == name]) | |
| for name, _ in policy.CLASS_WEIGHTS}, | |
| used_assets=len({a for r in passed for a in r["source_assets"]}), | |
| validation_scope="Exact schedule/PCM replay, real source hashes, copied timeline equality and unchanged complete target supervision. Original timeline semantics inherited from passed parent audit, not reinterpreted.", | |
| human_listening_review="pending", production_ready=False, training_started=False) | |
| write_json_atomic(production / "reports/strict_audit.json", report) | |
| write_json_atomic(production / "reports/pilot_complete.json", dict( | |
| format="hfd_multi_event_completion_v1", automated_checks_complete=not errors, | |
| full_dataset=not pilot, automated_audit_sha256=sha256_file(production / "reports/strict_audit.json"), | |
| human_listening_review="pending", production_ready=False, training_started=False)) | |
| return report | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--production-root", type=Path, default=DATA / "production") | |
| parser.add_argument("--reference-root", type=Path, default=REFERENCE) | |
| parser.add_argument("--plan-root", type=Path, default=DATA / "work/plan_qualified") | |
| parser.add_argument("--tts-registry", type=Path, default=DATA / "work/tts_assets_qualified/eligible_registry.jsonl") | |
| parser.add_argument("--workers", type=int, default=8) | |
| parser.add_argument("--allow-pilot", action="store_true") | |
| args = parser.parse_args() | |
| result = audit(args.production_root, args.reference_root, args.plan_root, | |
| args.tts_registry, args.workers, args.allow_pilot) | |
| print(json.dumps(result, ensure_ascii=False, indent=2)) | |
| raise SystemExit(0 if result["passed"] else 1) | |