"""Append 18 stratified exact-GT exports, without rewriting the original six. Rendering and publication are separate: the current review manifest remains authoritative until every new export has been rendered and checked. """ from __future__ import annotations import argparse from collections import Counter import gc import hashlib import json import math import os from pathlib import Path import sys sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from scripts.data import render_hfd_third_party_gt as gt CLASSES = tuple(spec["session_type"] for spec in gt.multi_event_review_specs()) CELLS = tuple((kind, lang, band) for kind in CLASSES for lang in ("zh", "en") for band in ("high", "low")) SEED = 26092802 NAMESPACE = "gt-multi-v5-balanced-24-v1" def write_json(path: Path, value: object) -> None: """Verify persisted bytes before and after an atomic publication.""" payload = (json.dumps(value, ensure_ascii=False, indent=2) + "\n").encode() path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(path.name + ".pending") with temporary.open("wb") as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) if temporary.read_bytes() != payload: raise RuntimeError(f"Short write: {temporary}") os.replace(temporary, path) if path.read_bytes() != payload: raise RuntimeError(f"Publication mismatch: {path}") def cell(recipe: dict) -> tuple[str, str, str] | None: mix = recipe.get("mix_metadata", {}) sir = mix.get("measured_background_sir_db") if (mix.get("status") != "calibrated" or mix.get("background_applied") is not True or isinstance(sir, bool) or not isinstance(sir, (int, float)) or not math.isfinite(sir)): return None kind = recipe.get("session_type") language = recipe.get("background_assignment", {}).get("language") result = (kind, language, "high" if sir < 0 else "low") return result if result in CELLS else None def eligible(recipe: dict) -> bool: if cell(recipe) is None: return False if recipe["session_type"] == "same_voice_aside": clones = [event for event in recipe["interference_events"] if event.get("type") == "reference_clone"] return bool(clones) and all(event.get("boundary_unmeasurable") is False and event.get("active_sample_count", 0) >= 160 for event in clones) return True def select(recipes, originals: list[dict], seed: int = SEED) -> list[dict]: occupied = [cell(recipe) for recipe in originals] if len(originals) != 6 or None in occupied or len(set(occupied)) != 6: raise ValueError("Need six distinct valid retained cells") retained_ids = {recipe["session_id"] for recipe in originals} if len(retained_ids) != 6: raise ValueError("Duplicate retained sessions") missing = [key for key in CELLS if key not in occupied] counts: Counter = Counter() best: dict = {} for recipe in recipes: key = cell(recipe) if key not in missing or not eligible(recipe) or recipe["session_id"] in retained_ids: continue counts[key] += 1 rank = hashlib.sha256(json.dumps( [NAMESPACE, seed, *key, recipe["session_id"]], separators=(",", ":") ).encode()).hexdigest() candidate = (rank, recipe["session_id"]) if key not in best or candidate < best[key][0]: best[key] = (candidate, recipe) if set(best) != set(missing): raise ValueError(f"Empty review strata: {set(missing) - set(best)}") selected = [] for number, key in enumerate(missing, 7): kind, lang, band = key recipe = best[key][1] spec = next(dict(row) for row in gt.multi_event_review_specs() if row["session_type"] == kind) spec.update(key=f"{number:02d}_{kind}_{lang}_{band}", interference_language=lang, language=recipe["session_language"], split=recipe["split"], difficulty=recipe["corruption_difficulty"], background_loudness_band=band, sampling_seed=seed, sampling_policy="seeded_hash_conditional_append_v1", sampling_namespace=NAMESPACE, eligible_sessions=counts[key], selection_conditions={"session_type": kind, "interference_language": lang, "background_loudness_band": band, "threshold_sir_db": 0, "aggregate_background_calibrated_and_applied": True, "clone_active_samples_minimum": 160 if kind == "same_voice_aside" else None, "split": None, "duration_filter": None}, event_count=len(recipe["interference_events"]), right_truncated_events=sum(bool(e.get("right_truncated")) for e in recipe["interference_events"]), command_overlap=any(e.get("overlapped_target_turn_indices") for e in recipe["interference_events"]), requested_background_sir_db=recipe["mix_metadata"]["requested_background_sir_db"], measured_background_sir_db=recipe["mix_metadata"]["measured_background_sir_db"], purpose_zh=f"审核{'中文' if lang == 'zh' else '英文'}{'高' if band == 'high' else '低'}背景;保留随机位置及尾截") selected.append({"spec": spec, "recipe": recipe}) if len({row["recipe"]["session_id"] for row in selected}) != 18: raise ValueError("Duplicate selected sessions") return selected def fingerprint(root: Path, keys: list[str]) -> dict: return {str(path.relative_to(root)): {"bytes": path.stat().st_size, "sha256": gt.sha256_file(path)} for key in keys for path in sorted((root / key).rglob("*")) if path.is_file()} def verify_export(root: Path, spec: dict, recipe: dict) -> dict: directory = root / spec["key"] meta = gt._read_json(directory / "gt_meta.json") if (meta["review_spec"] != spec or meta["session_id"] != recipe["session_id"] or meta["sample_key"] != spec["key"] or gt._read_json(directory / "session_recipe.json") != recipe): raise RuntimeError(f"Export identity mismatch: {directory}") expected = {"joint_pos.csv", "body_pos.csv", "body_quat.csv", "control_labels.csv", "session_recipe.json"} if set(meta["export_hashes"]) != expected: raise RuntimeError(f"Incomplete export hashes: {directory}") if not (directory / "events.ass").is_file() or not (directory / "events.ass").stat().st_size: raise RuntimeError(f"Missing subtitles: {directory}") for name, digest in meta["export_hashes"].items(): if gt.sha256_file(directory / name) != digest: raise RuntimeError(f"Export hash mismatch: {directory / name}") for path in (Path(meta["audio"]["source_path"]), directory / "input_audio_augmented_16k_mono.wav"): if gt.sha256_file(path) != recipe["audio"]["sha256"]: raise RuntimeError(f"Audio mismatch: {path}") return meta def prepare(production: Path, output: Path, work: Path, seed: int = SEED) -> dict: build = gt._read_json(production / "reports/pilot_build.json") if not gt._read_json(production / "reports/strict_audit.json").get("passed"): raise RuntimeError("Production strict audit must pass") hold = production / "reports/semantic_hold.json" if hold.exists() and gt._read_json(hold).get("eligible_for_final_handoff") is not True: raise RuntimeError("Production is held") recipe_path = production / "manifests/session_recipes.jsonl" digest = gt.sha256_file(recipe_path) if digest != build["output_hashes"]["session_recipes"]: raise RuntimeError("Production manifest drift") plan_path = work / "append_plan.json" if plan_path.exists(): plan = gt._read_json(plan_path) if (plan["seed"] != seed or plan["production_recipe_sha256"] != digest or plan["output_root"] != str(output) or plan["production_root"] != str(production)): raise RuntimeError("Append plan cannot be reinterpreted on resume") if fingerprint(output, plan["retained_keys"]) != plan["original_files"]: raise RuntimeError("Original six changed") return plan manifest = gt._read_json(output / "review_manifest.json") if manifest.get("sample_count") != 6 or manifest.get("complete") is not True: raise RuntimeError("Expected complete original six-sample review") keys = [row["sample_key"] for row in manifest["samples"]] originals = [gt._read_json(output / key / "session_recipe.json") for key in keys] for row, recipe in zip(manifest["samples"], originals): meta = gt._read_json(output / row["sample_key"] / "gt_meta.json") if row["session_id"] != recipe["session_id"]: raise RuntimeError("Original manifest identity mismatch") verify_export(output, meta["review_spec"], recipe) if meta.get("video_validation", {}).get("passed") is not True: raise RuntimeError("Original video is not validated") for name, artifact in meta["artifacts"].items(): path = output / row["sample_key"] / name if Path(artifact["path"]).resolve() != path.resolve() or gt.sha256_file(path) != artifact["sha256"]: raise RuntimeError("Original artifact mismatch") with recipe_path.open() as handle: selected = select((json.loads(line) for line in handle if line.strip()), originals, seed) if any((output / row["spec"]["key"]).exists() for row in selected): raise FileExistsError("Unplanned new review directory exists") work.mkdir(parents=True, exist_ok=True) for name in ("review_manifest.json", "handoff_audit.json", "review_findings.json", "README.md"): source = output / name destination = work / ("original_" + name) if destination.exists(): if destination.read_bytes() != source.read_bytes(): raise RuntimeError("Archived original differs") else: destination.write_bytes(source.read_bytes()) if destination.read_bytes() != source.read_bytes(): raise RuntimeError("Archive write mismatch") plan = {"format": "hfd_gt_append_24_v1", "seed": seed, "namespace": NAMESPACE, "production_root": str(production), "production_recipe_sha256": digest, "output_root": str(output), "retained_keys": keys, "original_files": fingerprint(output, keys), "selected": selected, "note": "Retain six original draws; conditionally draw only 18 missing language/SIR cells. No duration filter or split quota. Review balance is not production prevalence."} write_json(plan_path, plan) return plan def export(production: Path, output: Path, plan: dict) -> None: pending = [] for row in plan["selected"]: if (output / row["spec"]["key"]).exists(): verify_export(output, row["spec"], row["recipe"]) else: pending.append(row) args = gt._dataset_args(production) counts = gt._read_json(production / "reports/pilot_build.json")["counts"]["sessions_by_split"] for split in ("train", "validation"): selected = [row for row in pending if row["recipe"]["split"] == split] if not selected: continue requested = int(counts[split]) dataset = gt.builder.PilotSessionDataset(**gt.builder._dataset_kwargs( args, "train" if split == "train" else "val", max(requested * 8, requested + 256))) baseline = gt._load_baseline_signatures(production / "manifests/baseline_session_signatures.jsonl", split) sessions = gt.builder._select_sessions(dataset, count=requested, baseline_signatures=baseline, seed=int(args.seed)) dataset.sessions = sessions indexes = {gt.builder._session_signature(session): i for i, session in enumerate(sessions)} for row in selected: spec, recipe = row["spec"], row["recipe"] item = gt.builder._scoped_build_item(dataset, indexes[recipe["session_signature"]], int(recipe["timeline_seed"])) gt._export_one(production_root=production, output_root=output, spec=spec, recipe=recipe, item=item) verify_export(output, spec, recipe) print(f"[exported] {spec['key']} SIR={spec['measured_background_sir_db']:.3f}", flush=True) del dataset, baseline, sessions, indexes gc.collect() if fingerprint(output, plan["retained_keys"]) != plan["original_files"]: raise RuntimeError("Original six changed during append") def publish(work: Path, validation: dict) -> dict: """Publish a complete 24-row index only after independent media validation. Never write sample directories, including the original six metadata files. The caller separately updates the human-facing report and handoff selector. """ plan = gt._read_json(work / "append_plan.json") output, production = Path(plan["output_root"]), Path(plan["production_root"]) expected_checks = {"passed": True, "sample_count": 24, "videos_full_decoded": 72, "wavs_full_decoded": 24, "retained_original_files_unchanged": len(plan["original_files"])} if any(validation.get(key) != value for key, value in expected_checks.items()): raise RuntimeError("Independent full-media validation is incomplete") if (validation.get("append_plan_sha256") != gt.sha256_file(work / "append_plan.json") or validation.get("output_root") != str(output) or validation.get("production_recipe_sha256") != plan["production_recipe_sha256"]): raise RuntimeError("Independent validation belongs to a different plan or production") if gt.sha256_file(production / "manifests/session_recipes.jsonl") != plan["production_recipe_sha256"]: raise RuntimeError("Production recipe drift before publication") if fingerprint(output, plan["retained_keys"]) != plan["original_files"]: raise RuntimeError("Original six changed before publication") original = gt._read_json(work / "original_review_manifest.json") keys = plan["retained_keys"] + [row["spec"]["key"] for row in plan["selected"]] rows, recipes = [], [] for key in keys: directory = output / key meta = gt._read_json(directory / "gt_meta.json") recipe = gt._read_json(directory / "session_recipe.json") verify_export(output, meta["review_spec"], recipe) if (meta["production_root"] != str(production) or meta["production_recipe_sha256"] != plan["production_recipe_sha256"] or meta.get("video_validation", {}).get("full_decode") is not True): raise RuntimeError(f"Invalid rendered metadata: {key}") expected_artifacts = {"input_audio_augmented_16k_mono.wav", "reference_motion.mp4", "gt_with_audio.mp4", "gt_with_audio_annotated.mp4", "events.ass"} if set(meta["artifacts"]) != expected_artifacts: raise RuntimeError(f"Incomplete rendered artifacts: {key}") for name, artifact in meta["artifacts"].items(): path = directory / name if Path(artifact["path"]).resolve() != path.resolve() or gt.sha256_file(path) != artifact["sha256"]: raise RuntimeError(f"Rendered artifact drift: {path}") rows.append(meta) recipes.append(recipe) for selected in plan["selected"]: verify_export(output, selected["spec"], selected["recipe"]) if (Counter(cell(recipe) for recipe in recipes) != Counter(CELLS) or len({recipe["session_id"] for recipe in recipes}) != 24): raise RuntimeError("Review does not cover exactly 24 distinct cells") staging = work / "publication" staging.mkdir(exist_ok=True) gt._write_root_manifest(staging, rows, rendered=True) manifest = gt._read_json(staging / "review_manifest.json") # Keep the original six sampling descriptions and manifest rows verbatim. manifest["samples"][:6] = original["samples"] manifest["output_root"] = str(output) design = {"policy": "retain_six_append_18_conditional_hash", "retained_seed": 26092801, "new_seed": plan["seed"], "classes": 6, "examples_per_class": 4, "background_language_counts": dict(Counter(cell(r)[1] for r in recipes)), "background_loudness_counts": dict(Counter(cell(r)[2] for r in recipes)), "split_counts": dict(Counter(r["split"] for r in recipes)), "target_language_counts": dict(Counter(r["session_language"] for r in recipes)), "definition": "High background: measured aggregate SIR < 0 dB; low: SIR >= 0 dB. Active RMS relative to complete target utterances, not instantaneous peak.", "note": plan["note"], "new_clone_minimum_active_samples": 160, "plan_path": str(work / "append_plan.json"), "plan_sha256": gt.sha256_file(work / "append_plan.json"), "extension_script_sha256": gt.sha256_file(Path(__file__))} manifest["balanced_review_design"] = design manifest["review_cells"] = [{"sample_key": key, "session_type": cell(r)[0], "background_language": cell(r)[1], "background_loudness_band": cell(r)[2], "measured_background_sir_db": r["mix_metadata"]["measured_background_sir_db"]} for key, r in zip(keys, recipes)] audit = dict(validation, format="hfd_gt_handoff_audit_v2", status="published_and_validated", handoff_complete=True, production_recipe_sha256=plan["production_recipe_sha256"], sampling=design, human_listening_review="pending", training_started=False, root_review_documents_modified=True, independent_validation_path=str(work / "extension_validation.json"), duration_seconds=sum(r["audio"]["sample_count"] for r in recipes) / 16000, frames=sum(r["timeline_verification"]["rendered_frames"] for r in rows)) findings = gt._read_json(work / "original_review_findings.json") findings["review_scope"] = "24 stratified examples; automated media checks and sparse extracted-frame review, not human listening approval" findings["resampling_performed"] = False findings["append_sampling"] = design for selected in plan["selected"]: key, recipe = selected["spec"]["key"], selected["recipe"] for index, turn in enumerate(recipe["turns"]): instruction = turn["instruction_text"] if instruction.startswith("停止") or instruction.lower().startswith("stop "): findings["findings"].append({"status": "needs_human_target_semantic_decision", "sample_key": key, "target_turn_index": index, "instruction": instruction, "reason_zh": "继承原source的停止类表达,可能与动作标签有歧义;只标记,不修改配对。"}) for event in recipe["interference_events"]: seconds = (event["end_sample"] - event["start_sample"]) / 16000 if seconds < 1 or event.get("type") == "reference_clone": findings["findings"].append({"status": "reference_clone_requires_human_listening" if event.get("type") == "reference_clone" else "expected_random_right_boundary_case", "sample_key": key, "event_index": event["event_index"], "anchor_target_turn_index": event.get("anchor_target_turn_index"), "event_interval_seconds": [event["start_sample"] / 16000, event["end_sample"] / 16000], "event_duration_seconds": seconds, "reason_zh": "保留固定抽样;短尾及参考音色相似度仍需人工听审,不保证真人身份。"}) findings["extension_visual_review"] = validation.get("visual_review", {}) write_json(output / "handoff_audit.json", audit) write_json(output / "review_findings.json", findings) write_json(output / "review_manifest.json", manifest) return manifest def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--production-root", type=Path, required=True) parser.add_argument("--output-root", type=Path, required=True) parser.add_argument("--work-root", type=Path, required=True) parser.add_argument("--plan-only", action="store_true") args = parser.parse_args() production, output, work = (p.resolve() for p in (args.production_root, args.output_root, args.work_root)) plan = prepare(production, output, work) print(f"Plan fixed: {len(plan['selected'])} new samples", flush=True) if not args.plan_only: export(production, output, plan) if __name__ == "__main__": main()