code-HFD_lzk / scripts /data /extend_hfd_multi_v5_gt.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
8562e9b verified
Raw History Blame Contribute Delete
21 kB
"""Append 18 stratified exact-GT exports, without rewriting the original six.
Rendering and publication are separate: the current review manifest remains
authoritative until every new export has been rendered and checked.
"""
from __future__ import annotations
import argparse
from collections import Counter
import gc
import hashlib
import json
import math
import os
from pathlib import Path
import sys
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from scripts.data import render_hfd_third_party_gt as gt
CLASSES = tuple(spec["session_type"] for spec in gt.multi_event_review_specs())
CELLS = tuple((kind, lang, band) for kind in CLASSES
for lang in ("zh", "en") for band in ("high", "low"))
SEED = 26092802
NAMESPACE = "gt-multi-v5-balanced-24-v1"
def write_json(path: Path, value: object) -> None:
"""Verify persisted bytes before and after an atomic publication."""
payload = (json.dumps(value, ensure_ascii=False, indent=2) + "\n").encode()
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(path.name + ".pending")
with temporary.open("wb") as handle:
handle.write(payload)
handle.flush()
os.fsync(handle.fileno())
if temporary.read_bytes() != payload:
raise RuntimeError(f"Short write: {temporary}")
os.replace(temporary, path)
if path.read_bytes() != payload:
raise RuntimeError(f"Publication mismatch: {path}")
def cell(recipe: dict) -> tuple[str, str, str] | None:
mix = recipe.get("mix_metadata", {})
sir = mix.get("measured_background_sir_db")
if (mix.get("status") != "calibrated" or mix.get("background_applied") is not True
or isinstance(sir, bool) or not isinstance(sir, (int, float)) or not math.isfinite(sir)):
return None
kind = recipe.get("session_type")
language = recipe.get("background_assignment", {}).get("language")
result = (kind, language, "high" if sir < 0 else "low")
return result if result in CELLS else None
def eligible(recipe: dict) -> bool:
if cell(recipe) is None:
return False
if recipe["session_type"] == "same_voice_aside":
clones = [event for event in recipe["interference_events"]
if event.get("type") == "reference_clone"]
return bool(clones) and all(event.get("boundary_unmeasurable") is False
and event.get("active_sample_count", 0) >= 160
for event in clones)
return True
def select(recipes, originals: list[dict], seed: int = SEED) -> list[dict]:
occupied = [cell(recipe) for recipe in originals]
if len(originals) != 6 or None in occupied or len(set(occupied)) != 6:
raise ValueError("Need six distinct valid retained cells")
retained_ids = {recipe["session_id"] for recipe in originals}
if len(retained_ids) != 6:
raise ValueError("Duplicate retained sessions")
missing = [key for key in CELLS if key not in occupied]
counts: Counter = Counter()
best: dict = {}
for recipe in recipes:
key = cell(recipe)
if key not in missing or not eligible(recipe) or recipe["session_id"] in retained_ids:
continue
counts[key] += 1
rank = hashlib.sha256(json.dumps(
[NAMESPACE, seed, *key, recipe["session_id"]], separators=(",", ":")
).encode()).hexdigest()
candidate = (rank, recipe["session_id"])
if key not in best or candidate < best[key][0]:
best[key] = (candidate, recipe)
if set(best) != set(missing):
raise ValueError(f"Empty review strata: {set(missing) - set(best)}")
selected = []
for number, key in enumerate(missing, 7):
kind, lang, band = key
recipe = best[key][1]
spec = next(dict(row) for row in gt.multi_event_review_specs() if row["session_type"] == kind)
spec.update(key=f"{number:02d}_{kind}_{lang}_{band}",
interference_language=lang, language=recipe["session_language"],
split=recipe["split"], difficulty=recipe["corruption_difficulty"],
background_loudness_band=band, sampling_seed=seed,
sampling_policy="seeded_hash_conditional_append_v1",
sampling_namespace=NAMESPACE, eligible_sessions=counts[key],
selection_conditions={"session_type": kind, "interference_language": lang,
"background_loudness_band": band, "threshold_sir_db": 0,
"aggregate_background_calibrated_and_applied": True,
"clone_active_samples_minimum": 160 if kind == "same_voice_aside" else None,
"split": None, "duration_filter": None},
event_count=len(recipe["interference_events"]),
right_truncated_events=sum(bool(e.get("right_truncated")) for e in recipe["interference_events"]),
command_overlap=any(e.get("overlapped_target_turn_indices") for e in recipe["interference_events"]),
requested_background_sir_db=recipe["mix_metadata"]["requested_background_sir_db"],
measured_background_sir_db=recipe["mix_metadata"]["measured_background_sir_db"],
purpose_zh=f"审核{'中文' if lang == 'zh' else '英文'}{'高' if band == 'high' else '低'}背景;保留随机位置及尾截")
selected.append({"spec": spec, "recipe": recipe})
if len({row["recipe"]["session_id"] for row in selected}) != 18:
raise ValueError("Duplicate selected sessions")
return selected
def fingerprint(root: Path, keys: list[str]) -> dict:
return {str(path.relative_to(root)): {"bytes": path.stat().st_size, "sha256": gt.sha256_file(path)}
for key in keys for path in sorted((root / key).rglob("*")) if path.is_file()}
def verify_export(root: Path, spec: dict, recipe: dict) -> dict:
directory = root / spec["key"]
meta = gt._read_json(directory / "gt_meta.json")
if (meta["review_spec"] != spec or meta["session_id"] != recipe["session_id"]
or meta["sample_key"] != spec["key"]
or gt._read_json(directory / "session_recipe.json") != recipe):
raise RuntimeError(f"Export identity mismatch: {directory}")
expected = {"joint_pos.csv", "body_pos.csv", "body_quat.csv", "control_labels.csv", "session_recipe.json"}
if set(meta["export_hashes"]) != expected:
raise RuntimeError(f"Incomplete export hashes: {directory}")
if not (directory / "events.ass").is_file() or not (directory / "events.ass").stat().st_size:
raise RuntimeError(f"Missing subtitles: {directory}")
for name, digest in meta["export_hashes"].items():
if gt.sha256_file(directory / name) != digest:
raise RuntimeError(f"Export hash mismatch: {directory / name}")
for path in (Path(meta["audio"]["source_path"]), directory / "input_audio_augmented_16k_mono.wav"):
if gt.sha256_file(path) != recipe["audio"]["sha256"]:
raise RuntimeError(f"Audio mismatch: {path}")
return meta
def prepare(production: Path, output: Path, work: Path, seed: int = SEED) -> dict:
build = gt._read_json(production / "reports/pilot_build.json")
if not gt._read_json(production / "reports/strict_audit.json").get("passed"):
raise RuntimeError("Production strict audit must pass")
hold = production / "reports/semantic_hold.json"
if hold.exists() and gt._read_json(hold).get("eligible_for_final_handoff") is not True:
raise RuntimeError("Production is held")
recipe_path = production / "manifests/session_recipes.jsonl"
digest = gt.sha256_file(recipe_path)
if digest != build["output_hashes"]["session_recipes"]:
raise RuntimeError("Production manifest drift")
plan_path = work / "append_plan.json"
if plan_path.exists():
plan = gt._read_json(plan_path)
if (plan["seed"] != seed or plan["production_recipe_sha256"] != digest
or plan["output_root"] != str(output) or plan["production_root"] != str(production)):
raise RuntimeError("Append plan cannot be reinterpreted on resume")
if fingerprint(output, plan["retained_keys"]) != plan["original_files"]:
raise RuntimeError("Original six changed")
return plan
manifest = gt._read_json(output / "review_manifest.json")
if manifest.get("sample_count") != 6 or manifest.get("complete") is not True:
raise RuntimeError("Expected complete original six-sample review")
keys = [row["sample_key"] for row in manifest["samples"]]
originals = [gt._read_json(output / key / "session_recipe.json") for key in keys]
for row, recipe in zip(manifest["samples"], originals):
meta = gt._read_json(output / row["sample_key"] / "gt_meta.json")
if row["session_id"] != recipe["session_id"]:
raise RuntimeError("Original manifest identity mismatch")
verify_export(output, meta["review_spec"], recipe)
if meta.get("video_validation", {}).get("passed") is not True:
raise RuntimeError("Original video is not validated")
for name, artifact in meta["artifacts"].items():
path = output / row["sample_key"] / name
if Path(artifact["path"]).resolve() != path.resolve() or gt.sha256_file(path) != artifact["sha256"]:
raise RuntimeError("Original artifact mismatch")
with recipe_path.open() as handle:
selected = select((json.loads(line) for line in handle if line.strip()), originals, seed)
if any((output / row["spec"]["key"]).exists() for row in selected):
raise FileExistsError("Unplanned new review directory exists")
work.mkdir(parents=True, exist_ok=True)
for name in ("review_manifest.json", "handoff_audit.json", "review_findings.json", "README.md"):
source = output / name
destination = work / ("original_" + name)
if destination.exists():
if destination.read_bytes() != source.read_bytes():
raise RuntimeError("Archived original differs")
else:
destination.write_bytes(source.read_bytes())
if destination.read_bytes() != source.read_bytes():
raise RuntimeError("Archive write mismatch")
plan = {"format": "hfd_gt_append_24_v1", "seed": seed, "namespace": NAMESPACE,
"production_root": str(production), "production_recipe_sha256": digest,
"output_root": str(output), "retained_keys": keys,
"original_files": fingerprint(output, keys), "selected": selected,
"note": "Retain six original draws; conditionally draw only 18 missing language/SIR cells. No duration filter or split quota. Review balance is not production prevalence."}
write_json(plan_path, plan)
return plan
def export(production: Path, output: Path, plan: dict) -> None:
pending = []
for row in plan["selected"]:
if (output / row["spec"]["key"]).exists():
verify_export(output, row["spec"], row["recipe"])
else:
pending.append(row)
args = gt._dataset_args(production)
counts = gt._read_json(production / "reports/pilot_build.json")["counts"]["sessions_by_split"]
for split in ("train", "validation"):
selected = [row for row in pending if row["recipe"]["split"] == split]
if not selected:
continue
requested = int(counts[split])
dataset = gt.builder.PilotSessionDataset(**gt.builder._dataset_kwargs(
args, "train" if split == "train" else "val", max(requested * 8, requested + 256)))
baseline = gt._load_baseline_signatures(production / "manifests/baseline_session_signatures.jsonl", split)
sessions = gt.builder._select_sessions(dataset, count=requested, baseline_signatures=baseline, seed=int(args.seed))
dataset.sessions = sessions
indexes = {gt.builder._session_signature(session): i for i, session in enumerate(sessions)}
for row in selected:
spec, recipe = row["spec"], row["recipe"]
item = gt.builder._scoped_build_item(dataset, indexes[recipe["session_signature"]], int(recipe["timeline_seed"]))
gt._export_one(production_root=production, output_root=output, spec=spec, recipe=recipe, item=item)
verify_export(output, spec, recipe)
print(f"[exported] {spec['key']} SIR={spec['measured_background_sir_db']:.3f}", flush=True)
del dataset, baseline, sessions, indexes
gc.collect()
if fingerprint(output, plan["retained_keys"]) != plan["original_files"]:
raise RuntimeError("Original six changed during append")
def publish(work: Path, validation: dict) -> dict:
"""Publish a complete 24-row index only after independent media validation.
Never write sample directories, including the original six metadata files.
The caller separately updates the human-facing report and handoff selector.
"""
plan = gt._read_json(work / "append_plan.json")
output, production = Path(plan["output_root"]), Path(plan["production_root"])
expected_checks = {"passed": True, "sample_count": 24,
"videos_full_decoded": 72, "wavs_full_decoded": 24,
"retained_original_files_unchanged": len(plan["original_files"])}
if any(validation.get(key) != value for key, value in expected_checks.items()):
raise RuntimeError("Independent full-media validation is incomplete")
if (validation.get("append_plan_sha256") != gt.sha256_file(work / "append_plan.json")
or validation.get("output_root") != str(output)
or validation.get("production_recipe_sha256") != plan["production_recipe_sha256"]):
raise RuntimeError("Independent validation belongs to a different plan or production")
if gt.sha256_file(production / "manifests/session_recipes.jsonl") != plan["production_recipe_sha256"]:
raise RuntimeError("Production recipe drift before publication")
if fingerprint(output, plan["retained_keys"]) != plan["original_files"]:
raise RuntimeError("Original six changed before publication")
original = gt._read_json(work / "original_review_manifest.json")
keys = plan["retained_keys"] + [row["spec"]["key"] for row in plan["selected"]]
rows, recipes = [], []
for key in keys:
directory = output / key
meta = gt._read_json(directory / "gt_meta.json")
recipe = gt._read_json(directory / "session_recipe.json")
verify_export(output, meta["review_spec"], recipe)
if (meta["production_root"] != str(production)
or meta["production_recipe_sha256"] != plan["production_recipe_sha256"]
or meta.get("video_validation", {}).get("full_decode") is not True):
raise RuntimeError(f"Invalid rendered metadata: {key}")
expected_artifacts = {"input_audio_augmented_16k_mono.wav", "reference_motion.mp4",
"gt_with_audio.mp4", "gt_with_audio_annotated.mp4", "events.ass"}
if set(meta["artifacts"]) != expected_artifacts:
raise RuntimeError(f"Incomplete rendered artifacts: {key}")
for name, artifact in meta["artifacts"].items():
path = directory / name
if Path(artifact["path"]).resolve() != path.resolve() or gt.sha256_file(path) != artifact["sha256"]:
raise RuntimeError(f"Rendered artifact drift: {path}")
rows.append(meta)
recipes.append(recipe)
for selected in plan["selected"]:
verify_export(output, selected["spec"], selected["recipe"])
if (Counter(cell(recipe) for recipe in recipes) != Counter(CELLS)
or len({recipe["session_id"] for recipe in recipes}) != 24):
raise RuntimeError("Review does not cover exactly 24 distinct cells")
staging = work / "publication"
staging.mkdir(exist_ok=True)
gt._write_root_manifest(staging, rows, rendered=True)
manifest = gt._read_json(staging / "review_manifest.json")
# Keep the original six sampling descriptions and manifest rows verbatim.
manifest["samples"][:6] = original["samples"]
manifest["output_root"] = str(output)
design = {"policy": "retain_six_append_18_conditional_hash",
"retained_seed": 26092801, "new_seed": plan["seed"],
"classes": 6, "examples_per_class": 4,
"background_language_counts": dict(Counter(cell(r)[1] for r in recipes)),
"background_loudness_counts": dict(Counter(cell(r)[2] for r in recipes)),
"split_counts": dict(Counter(r["split"] for r in recipes)),
"target_language_counts": dict(Counter(r["session_language"] for r in recipes)),
"definition": "High background: measured aggregate SIR < 0 dB; low: SIR >= 0 dB. Active RMS relative to complete target utterances, not instantaneous peak.",
"note": plan["note"], "new_clone_minimum_active_samples": 160,
"plan_path": str(work / "append_plan.json"),
"plan_sha256": gt.sha256_file(work / "append_plan.json"),
"extension_script_sha256": gt.sha256_file(Path(__file__))}
manifest["balanced_review_design"] = design
manifest["review_cells"] = [{"sample_key": key, "session_type": cell(r)[0],
"background_language": cell(r)[1], "background_loudness_band": cell(r)[2],
"measured_background_sir_db": r["mix_metadata"]["measured_background_sir_db"]}
for key, r in zip(keys, recipes)]
audit = dict(validation, format="hfd_gt_handoff_audit_v2", status="published_and_validated",
handoff_complete=True, production_recipe_sha256=plan["production_recipe_sha256"],
sampling=design, human_listening_review="pending", training_started=False,
root_review_documents_modified=True,
independent_validation_path=str(work / "extension_validation.json"),
duration_seconds=sum(r["audio"]["sample_count"] for r in recipes) / 16000,
frames=sum(r["timeline_verification"]["rendered_frames"] for r in rows))
findings = gt._read_json(work / "original_review_findings.json")
findings["review_scope"] = "24 stratified examples; automated media checks and sparse extracted-frame review, not human listening approval"
findings["resampling_performed"] = False
findings["append_sampling"] = design
for selected in plan["selected"]:
key, recipe = selected["spec"]["key"], selected["recipe"]
for index, turn in enumerate(recipe["turns"]):
instruction = turn["instruction_text"]
if instruction.startswith("停止") or instruction.lower().startswith("stop "):
findings["findings"].append({"status": "needs_human_target_semantic_decision",
"sample_key": key, "target_turn_index": index, "instruction": instruction,
"reason_zh": "继承原source的停止类表达,可能与动作标签有歧义;只标记,不修改配对。"})
for event in recipe["interference_events"]:
seconds = (event["end_sample"] - event["start_sample"]) / 16000
if seconds < 1 or event.get("type") == "reference_clone":
findings["findings"].append({"status": "reference_clone_requires_human_listening" if event.get("type") == "reference_clone" else "expected_random_right_boundary_case",
"sample_key": key, "event_index": event["event_index"],
"anchor_target_turn_index": event.get("anchor_target_turn_index"),
"event_interval_seconds": [event["start_sample"] / 16000, event["end_sample"] / 16000],
"event_duration_seconds": seconds,
"reason_zh": "保留固定抽样;短尾及参考音色相似度仍需人工听审,不保证真人身份。"})
findings["extension_visual_review"] = validation.get("visual_review", {})
write_json(output / "handoff_audit.json", audit)
write_json(output / "review_findings.json", findings)
write_json(output / "review_manifest.json", manifest)
return manifest
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--production-root", type=Path, required=True)
parser.add_argument("--output-root", type=Path, required=True)
parser.add_argument("--work-root", type=Path, required=True)
parser.add_argument("--plan-only", action="store_true")
args = parser.parse_args()
production, output, work = (p.resolve() for p in (args.production_root, args.output_root, args.work_root))
plan = prepare(production, output, work)
print(f"Plan fixed: {len(plan['selected'])} new samples", flush=True)
if not args.plan_only:
export(production, output, plan)
if __name__ == "__main__":
main()