Download scripts/data/prepare_third_party_training.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 10.7 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/prepare_third_party_training.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/data/prepare_third_party_training.py
-
curl -L -o prepare_third_party_training.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/prepare_third_party_training.py
10.7 kB
| #!/usr/bin/env python3 | |
| """Freeze the train-only 6 x 50 pilot and CPU-accept every complete example.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import sys | |
| from collections import Counter | |
| from pathlib import Path | |
| PROJECT_ROOT = Path(__file__).resolve().parents[2] | |
| sys.path[:0] = [str(PROJECT_ROOT), str(PROJECT_ROOT / "src")] | |
| from psi.data.seed.third_party_frozen_dataset import ( # noqa: E402 | |
| PROFILES, SELECTION_FORMAT, ThirdPartyFrozenDataset, read_jsonl, | |
| select_balanced_recipes, sha256_file, | |
| ) | |
| from scripts.data.hfd_third_party_common import write_json_atomic # noqa: E402 | |
| def validate_acceptance(selection_path: str | Path, production_root: str | Path, | |
| acceptance_path: str | Path | None = None, *, | |
| deep_payload: bool = True) -> dict: | |
| """Read-only launch gate; no model, CUDA allocation, or keeper changes.""" | |
| selection_path, production_root = Path(selection_path).resolve(), Path(production_root).resolve() | |
| acceptance_path = Path(acceptance_path).resolve() if acceptance_path else selection_path.parent / "acceptance.json" | |
| selection = json.loads(selection_path.read_text()) | |
| report = json.loads(acceptance_path.read_text()) | |
| if selection.get("format") != SELECTION_FORMAT or selection.get("split") != "train": | |
| raise RuntimeError("Launch requires the frozen train-only selection") | |
| expected_counts = {profile: 50 for profile in PROFILES} | |
| ids = selection.get("session_ids", []) | |
| if (len(ids) != 300 or len(set(ids)) != 300 or selection.get("counts") != expected_counts | |
| or selection.get("background_language_counts") != {"zh": 150, "en": 150}): | |
| raise RuntimeError("Frozen 300-session quota changed") | |
| probes = selection.get("probe_session_ids", []) | |
| if len(probes) != 12 or len(set(probes)) != 12 or not set(probes).issubset(ids): | |
| raise RuntimeError("Invalid fixed training probe IDs") | |
| if (report.get("passed") is not True or report.get("checked_sessions") != 300 | |
| or report.get("train_count") != 300 | |
| or any(report.get(key) != 0 for key in ("visual_inputs", "other_task_inputs", "heldout_inputs"))): | |
| raise RuntimeError("Full CPU acceptance has not passed") | |
| if (Path(selection["production_root"]).resolve() != production_root | |
| or Path(report["production_root"]).resolve() != production_root | |
| or Path(report["selection_path"]).resolve() != selection_path): | |
| raise RuntimeError("CPU acceptance refers to another selection/root") | |
| checks = { | |
| selection_path: report["manifest_sha256"], | |
| production_root / "manifests/session_recipes.jsonl": selection["recipe_manifest_sha256"], | |
| production_root / "manifests/source_contract.json": selection["source_contract_sha256"], | |
| PROJECT_ROOT / "src/psi/data/seed/third_party_frozen_dataset.py": report["loader_sha256"], | |
| PROJECT_ROOT / "src/psi/data/seed/full_duplex_audio_command_dataset.py": report["timeline_builder_sha256"], | |
| Path(report["runtime_lengths_path"]): report["runtime_lengths_sha256"], | |
| } | |
| for path, expected in checks.items(): | |
| if sha256_file(path) != expected: | |
| raise RuntimeError(f"Accepted input/code hash changed; rerun CPU acceptance: {path}") | |
| contract = json.loads((production_root / "manifests/source_contract.json").read_text()) | |
| for name in ("target_metadata", "motion_metadata", "source_assignment", "source_production_marker"): | |
| if sha256_file(Path(contract[f"{name}_path"])) != contract[f"{name}_sha256"]: | |
| raise RuntimeError(f"Original source contract changed: {name}") | |
| if deep_payload: | |
| wanted, seen = set(ids), set() | |
| for recipe in read_jsonl(production_root / "manifests/session_recipes.jsonl"): | |
| session_id = recipe["session_id"] | |
| if session_id not in wanted: | |
| continue | |
| if session_id in seen or recipe.get("split") != "train": | |
| raise RuntimeError("Duplicate or heldout accepted session") | |
| seen.add(session_id) | |
| for name, hash_key in (("audio", "sha256"), ("timeline", "file_sha256")): | |
| path = production_root / recipe[name]["path"] | |
| if sha256_file(path) != recipe[name][hash_key]: | |
| raise RuntimeError(f"Frozen {name} payload changed: {session_id}") | |
| if seen != wanted: | |
| raise RuntimeError("Accepted sessions missing from production") | |
| return report | |
| def prepare(production_root: Path, output_root: Path, *, seed: int = 26100401) -> dict: | |
| production_root, output_root = production_root.resolve(), output_root.resolve() | |
| output_root.relative_to(PROJECT_ROOT) | |
| if output_root == PROJECT_ROOT or production_root == output_root: | |
| raise ValueError("Preparation output must be a dedicated LZK work directory") | |
| output_root.mkdir(parents=True, exist_ok=True) | |
| selection_path = output_root / "selection.json" | |
| acceptance_path = output_root / "acceptance.json" | |
| manifest = production_root / "manifests/session_recipes.jsonl" | |
| selected, probe_ids, exclusions = select_balanced_recipes(read_jsonl(manifest), seed=seed) | |
| selection = { | |
| "format": SELECTION_FORMAT, "production_root": str(production_root), | |
| "split": "train", "seed": seed, "baseline_semantics_version": 24, | |
| "recipe_manifest_sha256": sha256_file(manifest), | |
| "source_contract_sha256": sha256_file(production_root / "manifests/source_contract.json"), | |
| "session_ids": [row["session_id"] for row in selected], | |
| "probe_session_ids": probe_ids, "sessions": selected, | |
| "counts": dict(Counter(row["session_type"] for row in selected)), | |
| "background_language_counts": dict(Counter(row["background_language"] for row in selected)), | |
| "background_high_count": sum(row["background_high"] for row in selected), | |
| "exclusion_policy": "train_only_calibrated_background_no_stop_ambiguity_clone_active_ge160_v1", | |
| "evaluation_policy": "fixed_train_probe_not_heldout_generalization", | |
| "exclusion_reason_counts": dict(Counter(reason for row in exclusions for reason in row["reasons"])), | |
| } | |
| if selection_path.exists(): | |
| if json.loads(selection_path.read_text()) != selection: | |
| raise RuntimeError("Refusing to overwrite a different frozen training selection") | |
| else: | |
| write_json_atomic(selection_path, selection) | |
| write_json_atomic(output_root / "excluded_sessions.json", exclusions) | |
| manifest_hash = sha256_file(selection_path) | |
| status = { | |
| "format": "hfd_third_party_training_acceptance_v1", "passed": False, | |
| "selection_path": str(selection_path), "manifest_sha256": manifest_hash, | |
| "recipe_manifest_sha256": selection["recipe_manifest_sha256"], | |
| "production_root": str(production_root), "train_count": 300, | |
| "visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0, | |
| "checked_sessions": 0, "device": "cpu", | |
| "baseline_semantics_version": 24, | |
| "loader_sha256": sha256_file(PROJECT_ROOT / "src/psi/data/seed/third_party_frozen_dataset.py"), | |
| "timeline_builder_sha256": sha256_file(PROJECT_ROOT / "src/psi/data/seed/full_duplex_audio_command_dataset.py"), | |
| } | |
| write_json_atomic(acceptance_path, status) | |
| try: | |
| dataset = ThirdPartyFrozenDataset(production_root=production_root, selection_path=selection_path) | |
| lengths = [] | |
| for index, recipe in enumerate(dataset.recipes): | |
| item = dataset[index] | |
| if len(item["steps"]) != dataset.estimated_stream_steps(index): | |
| raise RuntimeError("Exact runtime length disagrees with frozen recipe") | |
| if item["control_command_events"] or item["backchannel_events"] or item["disfluency_events"]: | |
| raise RuntimeError("Unexpected other-task event in third-party pilot") | |
| if any(any(key in step for key in ("images", "image", "video", "pixel_values")) | |
| for step in item["steps"]): | |
| raise RuntimeError("Unexpected visual input in audio-only pilot") | |
| lengths.append({"session_id": recipe["session_id"], "dense_steps": len(item["steps"]), | |
| "semantic_steps": int(item["semantic_steps"]), | |
| "audio_sha256": recipe["audio"]["sha256"], | |
| "timeline_content_sha256": recipe["timeline"]["content_sha256"]}) | |
| status["checked_sessions"] = index + 1 | |
| if (index + 1) % 10 == 0 or index == 0: | |
| write_json_atomic(acceptance_path, status) | |
| print(f"[third-party CPU acceptance] {index + 1}/{len(dataset)}", flush=True) | |
| del item | |
| if len(dataset) != 300 or selection["counts"] != {profile: 50 for profile in PROFILES}: | |
| raise RuntimeError("Pilot quota mismatch") | |
| write_json_atomic(output_root / "runtime_lengths.json", lengths) | |
| status.update(passed=True, checked_sessions=len(dataset), | |
| dense_steps=sum(row["dense_steps"] for row in lengths), | |
| semantic_steps=sum(row["semantic_steps"] for row in lengths), | |
| runtime_lengths_path=str(output_root / "runtime_lengths.json"), | |
| runtime_lengths_sha256=sha256_file(output_root / "runtime_lengths.json"), | |
| profile_counts=selection["counts"], probe_session_ids=probe_ids, | |
| verification="all compact arrays + sparse targets + target source hashes + exact mixed WAV windows") | |
| except Exception as error: | |
| status["error"] = f"{type(error).__name__}: {error}" | |
| write_json_atomic(acceptance_path, status) | |
| raise | |
| write_json_atomic(acceptance_path, status) | |
| print(json.dumps(status, ensure_ascii=False, indent=2), flush=True) | |
| return status | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--production-root", type=Path, | |
| default=PROJECT_ROOT / "data/hfd_third_party_multi_v5/production") | |
| parser.add_argument("--output-root", type=Path, | |
| default=PROJECT_ROOT / "data/hfd_third_party_multi_v5/training_pilot_300") | |
| parser.add_argument("--seed", type=int, default=26100401) | |
| args = parser.parse_args() | |
| # This script never initializes a CUDA model, including when inherited from | |
| # a GPU shell. Source/GT reconstruction uses numpy/scipy and CPU torch. | |
| os.environ["CUDA_VISIBLE_DEVICES"] = "" | |
| prepare(args.production_root, args.output_root, seed=args.seed) | |
| if __name__ == "__main__": | |
| main() | |