Download scripts/data/extend_hfd_multi_v5_gt.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 21 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/extend_hfd_multi_v5_gt.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/data/extend_hfd_multi_v5_gt.py
-
curl -L -o extend_hfd_multi_v5_gt.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/extend_hfd_multi_v5_gt.py
21 kB
| """Append 18 stratified exact-GT exports, without rewriting the original six. | |
| Rendering and publication are separate: the current review manifest remains | |
| authoritative until every new export has been rendered and checked. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| from collections import Counter | |
| import gc | |
| import hashlib | |
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| import sys | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[2])) | |
| from scripts.data import render_hfd_third_party_gt as gt | |
| CLASSES = tuple(spec["session_type"] for spec in gt.multi_event_review_specs()) | |
| CELLS = tuple((kind, lang, band) for kind in CLASSES | |
| for lang in ("zh", "en") for band in ("high", "low")) | |
| SEED = 26092802 | |
| NAMESPACE = "gt-multi-v5-balanced-24-v1" | |
| def write_json(path: Path, value: object) -> None: | |
| """Verify persisted bytes before and after an atomic publication.""" | |
| payload = (json.dumps(value, ensure_ascii=False, indent=2) + "\n").encode() | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = path.with_name(path.name + ".pending") | |
| with temporary.open("wb") as handle: | |
| handle.write(payload) | |
| handle.flush() | |
| os.fsync(handle.fileno()) | |
| if temporary.read_bytes() != payload: | |
| raise RuntimeError(f"Short write: {temporary}") | |
| os.replace(temporary, path) | |
| if path.read_bytes() != payload: | |
| raise RuntimeError(f"Publication mismatch: {path}") | |
| def cell(recipe: dict) -> tuple[str, str, str] | None: | |
| mix = recipe.get("mix_metadata", {}) | |
| sir = mix.get("measured_background_sir_db") | |
| if (mix.get("status") != "calibrated" or mix.get("background_applied") is not True | |
| or isinstance(sir, bool) or not isinstance(sir, (int, float)) or not math.isfinite(sir)): | |
| return None | |
| kind = recipe.get("session_type") | |
| language = recipe.get("background_assignment", {}).get("language") | |
| result = (kind, language, "high" if sir < 0 else "low") | |
| return result if result in CELLS else None | |
| def eligible(recipe: dict) -> bool: | |
| if cell(recipe) is None: | |
| return False | |
| if recipe["session_type"] == "same_voice_aside": | |
| clones = [event for event in recipe["interference_events"] | |
| if event.get("type") == "reference_clone"] | |
| return bool(clones) and all(event.get("boundary_unmeasurable") is False | |
| and event.get("active_sample_count", 0) >= 160 | |
| for event in clones) | |
| return True | |
| def select(recipes, originals: list[dict], seed: int = SEED) -> list[dict]: | |
| occupied = [cell(recipe) for recipe in originals] | |
| if len(originals) != 6 or None in occupied or len(set(occupied)) != 6: | |
| raise ValueError("Need six distinct valid retained cells") | |
| retained_ids = {recipe["session_id"] for recipe in originals} | |
| if len(retained_ids) != 6: | |
| raise ValueError("Duplicate retained sessions") | |
| missing = [key for key in CELLS if key not in occupied] | |
| counts: Counter = Counter() | |
| best: dict = {} | |
| for recipe in recipes: | |
| key = cell(recipe) | |
| if key not in missing or not eligible(recipe) or recipe["session_id"] in retained_ids: | |
| continue | |
| counts[key] += 1 | |
| rank = hashlib.sha256(json.dumps( | |
| [NAMESPACE, seed, *key, recipe["session_id"]], separators=(",", ":") | |
| ).encode()).hexdigest() | |
| candidate = (rank, recipe["session_id"]) | |
| if key not in best or candidate < best[key][0]: | |
| best[key] = (candidate, recipe) | |
| if set(best) != set(missing): | |
| raise ValueError(f"Empty review strata: {set(missing) - set(best)}") | |
| selected = [] | |
| for number, key in enumerate(missing, 7): | |
| kind, lang, band = key | |
| recipe = best[key][1] | |
| spec = next(dict(row) for row in gt.multi_event_review_specs() if row["session_type"] == kind) | |
| spec.update(key=f"{number:02d}_{kind}_{lang}_{band}", | |
| interference_language=lang, language=recipe["session_language"], | |
| split=recipe["split"], difficulty=recipe["corruption_difficulty"], | |
| background_loudness_band=band, sampling_seed=seed, | |
| sampling_policy="seeded_hash_conditional_append_v1", | |
| sampling_namespace=NAMESPACE, eligible_sessions=counts[key], | |
| selection_conditions={"session_type": kind, "interference_language": lang, | |
| "background_loudness_band": band, "threshold_sir_db": 0, | |
| "aggregate_background_calibrated_and_applied": True, | |
| "clone_active_samples_minimum": 160 if kind == "same_voice_aside" else None, | |
| "split": None, "duration_filter": None}, | |
| event_count=len(recipe["interference_events"]), | |
| right_truncated_events=sum(bool(e.get("right_truncated")) for e in recipe["interference_events"]), | |
| command_overlap=any(e.get("overlapped_target_turn_indices") for e in recipe["interference_events"]), | |
| requested_background_sir_db=recipe["mix_metadata"]["requested_background_sir_db"], | |
| measured_background_sir_db=recipe["mix_metadata"]["measured_background_sir_db"], | |
| purpose_zh=f"审核{'中文' if lang == 'zh' else '英文'}{'高' if band == 'high' else '低'}背景;保留随机位置及尾截") | |
| selected.append({"spec": spec, "recipe": recipe}) | |
| if len({row["recipe"]["session_id"] for row in selected}) != 18: | |
| raise ValueError("Duplicate selected sessions") | |
| return selected | |
| def fingerprint(root: Path, keys: list[str]) -> dict: | |
| return {str(path.relative_to(root)): {"bytes": path.stat().st_size, "sha256": gt.sha256_file(path)} | |
| for key in keys for path in sorted((root / key).rglob("*")) if path.is_file()} | |
| def verify_export(root: Path, spec: dict, recipe: dict) -> dict: | |
| directory = root / spec["key"] | |
| meta = gt._read_json(directory / "gt_meta.json") | |
| if (meta["review_spec"] != spec or meta["session_id"] != recipe["session_id"] | |
| or meta["sample_key"] != spec["key"] | |
| or gt._read_json(directory / "session_recipe.json") != recipe): | |
| raise RuntimeError(f"Export identity mismatch: {directory}") | |
| expected = {"joint_pos.csv", "body_pos.csv", "body_quat.csv", "control_labels.csv", "session_recipe.json"} | |
| if set(meta["export_hashes"]) != expected: | |
| raise RuntimeError(f"Incomplete export hashes: {directory}") | |
| if not (directory / "events.ass").is_file() or not (directory / "events.ass").stat().st_size: | |
| raise RuntimeError(f"Missing subtitles: {directory}") | |
| for name, digest in meta["export_hashes"].items(): | |
| if gt.sha256_file(directory / name) != digest: | |
| raise RuntimeError(f"Export hash mismatch: {directory / name}") | |
| for path in (Path(meta["audio"]["source_path"]), directory / "input_audio_augmented_16k_mono.wav"): | |
| if gt.sha256_file(path) != recipe["audio"]["sha256"]: | |
| raise RuntimeError(f"Audio mismatch: {path}") | |
| return meta | |
| def prepare(production: Path, output: Path, work: Path, seed: int = SEED) -> dict: | |
| build = gt._read_json(production / "reports/pilot_build.json") | |
| if not gt._read_json(production / "reports/strict_audit.json").get("passed"): | |
| raise RuntimeError("Production strict audit must pass") | |
| hold = production / "reports/semantic_hold.json" | |
| if hold.exists() and gt._read_json(hold).get("eligible_for_final_handoff") is not True: | |
| raise RuntimeError("Production is held") | |
| recipe_path = production / "manifests/session_recipes.jsonl" | |
| digest = gt.sha256_file(recipe_path) | |
| if digest != build["output_hashes"]["session_recipes"]: | |
| raise RuntimeError("Production manifest drift") | |
| plan_path = work / "append_plan.json" | |
| if plan_path.exists(): | |
| plan = gt._read_json(plan_path) | |
| if (plan["seed"] != seed or plan["production_recipe_sha256"] != digest | |
| or plan["output_root"] != str(output) or plan["production_root"] != str(production)): | |
| raise RuntimeError("Append plan cannot be reinterpreted on resume") | |
| if fingerprint(output, plan["retained_keys"]) != plan["original_files"]: | |
| raise RuntimeError("Original six changed") | |
| return plan | |
| manifest = gt._read_json(output / "review_manifest.json") | |
| if manifest.get("sample_count") != 6 or manifest.get("complete") is not True: | |
| raise RuntimeError("Expected complete original six-sample review") | |
| keys = [row["sample_key"] for row in manifest["samples"]] | |
| originals = [gt._read_json(output / key / "session_recipe.json") for key in keys] | |
| for row, recipe in zip(manifest["samples"], originals): | |
| meta = gt._read_json(output / row["sample_key"] / "gt_meta.json") | |
| if row["session_id"] != recipe["session_id"]: | |
| raise RuntimeError("Original manifest identity mismatch") | |
| verify_export(output, meta["review_spec"], recipe) | |
| if meta.get("video_validation", {}).get("passed") is not True: | |
| raise RuntimeError("Original video is not validated") | |
| for name, artifact in meta["artifacts"].items(): | |
| path = output / row["sample_key"] / name | |
| if Path(artifact["path"]).resolve() != path.resolve() or gt.sha256_file(path) != artifact["sha256"]: | |
| raise RuntimeError("Original artifact mismatch") | |
| with recipe_path.open() as handle: | |
| selected = select((json.loads(line) for line in handle if line.strip()), originals, seed) | |
| if any((output / row["spec"]["key"]).exists() for row in selected): | |
| raise FileExistsError("Unplanned new review directory exists") | |
| work.mkdir(parents=True, exist_ok=True) | |
| for name in ("review_manifest.json", "handoff_audit.json", "review_findings.json", "README.md"): | |
| source = output / name | |
| destination = work / ("original_" + name) | |
| if destination.exists(): | |
| if destination.read_bytes() != source.read_bytes(): | |
| raise RuntimeError("Archived original differs") | |
| else: | |
| destination.write_bytes(source.read_bytes()) | |
| if destination.read_bytes() != source.read_bytes(): | |
| raise RuntimeError("Archive write mismatch") | |
| plan = {"format": "hfd_gt_append_24_v1", "seed": seed, "namespace": NAMESPACE, | |
| "production_root": str(production), "production_recipe_sha256": digest, | |
| "output_root": str(output), "retained_keys": keys, | |
| "original_files": fingerprint(output, keys), "selected": selected, | |
| "note": "Retain six original draws; conditionally draw only 18 missing language/SIR cells. No duration filter or split quota. Review balance is not production prevalence."} | |
| write_json(plan_path, plan) | |
| return plan | |
| def export(production: Path, output: Path, plan: dict) -> None: | |
| pending = [] | |
| for row in plan["selected"]: | |
| if (output / row["spec"]["key"]).exists(): | |
| verify_export(output, row["spec"], row["recipe"]) | |
| else: | |
| pending.append(row) | |
| args = gt._dataset_args(production) | |
| counts = gt._read_json(production / "reports/pilot_build.json")["counts"]["sessions_by_split"] | |
| for split in ("train", "validation"): | |
| selected = [row for row in pending if row["recipe"]["split"] == split] | |
| if not selected: | |
| continue | |
| requested = int(counts[split]) | |
| dataset = gt.builder.PilotSessionDataset(**gt.builder._dataset_kwargs( | |
| args, "train" if split == "train" else "val", max(requested * 8, requested + 256))) | |
| baseline = gt._load_baseline_signatures(production / "manifests/baseline_session_signatures.jsonl", split) | |
| sessions = gt.builder._select_sessions(dataset, count=requested, baseline_signatures=baseline, seed=int(args.seed)) | |
| dataset.sessions = sessions | |
| indexes = {gt.builder._session_signature(session): i for i, session in enumerate(sessions)} | |
| for row in selected: | |
| spec, recipe = row["spec"], row["recipe"] | |
| item = gt.builder._scoped_build_item(dataset, indexes[recipe["session_signature"]], int(recipe["timeline_seed"])) | |
| gt._export_one(production_root=production, output_root=output, spec=spec, recipe=recipe, item=item) | |
| verify_export(output, spec, recipe) | |
| print(f"[exported] {spec['key']} SIR={spec['measured_background_sir_db']:.3f}", flush=True) | |
| del dataset, baseline, sessions, indexes | |
| gc.collect() | |
| if fingerprint(output, plan["retained_keys"]) != plan["original_files"]: | |
| raise RuntimeError("Original six changed during append") | |
| def publish(work: Path, validation: dict) -> dict: | |
| """Publish a complete 24-row index only after independent media validation. | |
| Never write sample directories, including the original six metadata files. | |
| The caller separately updates the human-facing report and handoff selector. | |
| """ | |
| plan = gt._read_json(work / "append_plan.json") | |
| output, production = Path(plan["output_root"]), Path(plan["production_root"]) | |
| expected_checks = {"passed": True, "sample_count": 24, | |
| "videos_full_decoded": 72, "wavs_full_decoded": 24, | |
| "retained_original_files_unchanged": len(plan["original_files"])} | |
| if any(validation.get(key) != value for key, value in expected_checks.items()): | |
| raise RuntimeError("Independent full-media validation is incomplete") | |
| if (validation.get("append_plan_sha256") != gt.sha256_file(work / "append_plan.json") | |
| or validation.get("output_root") != str(output) | |
| or validation.get("production_recipe_sha256") != plan["production_recipe_sha256"]): | |
| raise RuntimeError("Independent validation belongs to a different plan or production") | |
| if gt.sha256_file(production / "manifests/session_recipes.jsonl") != plan["production_recipe_sha256"]: | |
| raise RuntimeError("Production recipe drift before publication") | |
| if fingerprint(output, plan["retained_keys"]) != plan["original_files"]: | |
| raise RuntimeError("Original six changed before publication") | |
| original = gt._read_json(work / "original_review_manifest.json") | |
| keys = plan["retained_keys"] + [row["spec"]["key"] for row in plan["selected"]] | |
| rows, recipes = [], [] | |
| for key in keys: | |
| directory = output / key | |
| meta = gt._read_json(directory / "gt_meta.json") | |
| recipe = gt._read_json(directory / "session_recipe.json") | |
| verify_export(output, meta["review_spec"], recipe) | |
| if (meta["production_root"] != str(production) | |
| or meta["production_recipe_sha256"] != plan["production_recipe_sha256"] | |
| or meta.get("video_validation", {}).get("full_decode") is not True): | |
| raise RuntimeError(f"Invalid rendered metadata: {key}") | |
| expected_artifacts = {"input_audio_augmented_16k_mono.wav", "reference_motion.mp4", | |
| "gt_with_audio.mp4", "gt_with_audio_annotated.mp4", "events.ass"} | |
| if set(meta["artifacts"]) != expected_artifacts: | |
| raise RuntimeError(f"Incomplete rendered artifacts: {key}") | |
| for name, artifact in meta["artifacts"].items(): | |
| path = directory / name | |
| if Path(artifact["path"]).resolve() != path.resolve() or gt.sha256_file(path) != artifact["sha256"]: | |
| raise RuntimeError(f"Rendered artifact drift: {path}") | |
| rows.append(meta) | |
| recipes.append(recipe) | |
| for selected in plan["selected"]: | |
| verify_export(output, selected["spec"], selected["recipe"]) | |
| if (Counter(cell(recipe) for recipe in recipes) != Counter(CELLS) | |
| or len({recipe["session_id"] for recipe in recipes}) != 24): | |
| raise RuntimeError("Review does not cover exactly 24 distinct cells") | |
| staging = work / "publication" | |
| staging.mkdir(exist_ok=True) | |
| gt._write_root_manifest(staging, rows, rendered=True) | |
| manifest = gt._read_json(staging / "review_manifest.json") | |
| # Keep the original six sampling descriptions and manifest rows verbatim. | |
| manifest["samples"][:6] = original["samples"] | |
| manifest["output_root"] = str(output) | |
| design = {"policy": "retain_six_append_18_conditional_hash", | |
| "retained_seed": 26092801, "new_seed": plan["seed"], | |
| "classes": 6, "examples_per_class": 4, | |
| "background_language_counts": dict(Counter(cell(r)[1] for r in recipes)), | |
| "background_loudness_counts": dict(Counter(cell(r)[2] for r in recipes)), | |
| "split_counts": dict(Counter(r["split"] for r in recipes)), | |
| "target_language_counts": dict(Counter(r["session_language"] for r in recipes)), | |
| "definition": "High background: measured aggregate SIR < 0 dB; low: SIR >= 0 dB. Active RMS relative to complete target utterances, not instantaneous peak.", | |
| "note": plan["note"], "new_clone_minimum_active_samples": 160, | |
| "plan_path": str(work / "append_plan.json"), | |
| "plan_sha256": gt.sha256_file(work / "append_plan.json"), | |
| "extension_script_sha256": gt.sha256_file(Path(__file__))} | |
| manifest["balanced_review_design"] = design | |
| manifest["review_cells"] = [{"sample_key": key, "session_type": cell(r)[0], | |
| "background_language": cell(r)[1], "background_loudness_band": cell(r)[2], | |
| "measured_background_sir_db": r["mix_metadata"]["measured_background_sir_db"]} | |
| for key, r in zip(keys, recipes)] | |
| audit = dict(validation, format="hfd_gt_handoff_audit_v2", status="published_and_validated", | |
| handoff_complete=True, production_recipe_sha256=plan["production_recipe_sha256"], | |
| sampling=design, human_listening_review="pending", training_started=False, | |
| root_review_documents_modified=True, | |
| independent_validation_path=str(work / "extension_validation.json"), | |
| duration_seconds=sum(r["audio"]["sample_count"] for r in recipes) / 16000, | |
| frames=sum(r["timeline_verification"]["rendered_frames"] for r in rows)) | |
| findings = gt._read_json(work / "original_review_findings.json") | |
| findings["review_scope"] = "24 stratified examples; automated media checks and sparse extracted-frame review, not human listening approval" | |
| findings["resampling_performed"] = False | |
| findings["append_sampling"] = design | |
| for selected in plan["selected"]: | |
| key, recipe = selected["spec"]["key"], selected["recipe"] | |
| for index, turn in enumerate(recipe["turns"]): | |
| instruction = turn["instruction_text"] | |
| if instruction.startswith("停止") or instruction.lower().startswith("stop "): | |
| findings["findings"].append({"status": "needs_human_target_semantic_decision", | |
| "sample_key": key, "target_turn_index": index, "instruction": instruction, | |
| "reason_zh": "继承原source的停止类表达,可能与动作标签有歧义;只标记,不修改配对。"}) | |
| for event in recipe["interference_events"]: | |
| seconds = (event["end_sample"] - event["start_sample"]) / 16000 | |
| if seconds < 1 or event.get("type") == "reference_clone": | |
| findings["findings"].append({"status": "reference_clone_requires_human_listening" if event.get("type") == "reference_clone" else "expected_random_right_boundary_case", | |
| "sample_key": key, "event_index": event["event_index"], | |
| "anchor_target_turn_index": event.get("anchor_target_turn_index"), | |
| "event_interval_seconds": [event["start_sample"] / 16000, event["end_sample"] / 16000], | |
| "event_duration_seconds": seconds, | |
| "reason_zh": "保留固定抽样;短尾及参考音色相似度仍需人工听审,不保证真人身份。"}) | |
| findings["extension_visual_review"] = validation.get("visual_review", {}) | |
| write_json(output / "handoff_audit.json", audit) | |
| write_json(output / "review_findings.json", findings) | |
| write_json(output / "review_manifest.json", manifest) | |
| return manifest | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--production-root", type=Path, required=True) | |
| parser.add_argument("--output-root", type=Path, required=True) | |
| parser.add_argument("--work-root", type=Path, required=True) | |
| parser.add_argument("--plan-only", action="store_true") | |
| args = parser.parse_args() | |
| production, output, work = (p.resolve() for p in (args.production_root, args.output_root, args.work_root)) | |
| plan = prepare(production, output, work) | |
| print(f"Plan fixed: {len(plan['selected'])} new samples", flush=True) | |
| if not args.plan_only: | |
| export(production, output, plan) | |
| if __name__ == "__main__": | |
| main() | |