code-HFD_lzk / scripts /data /prepare_third_party_training.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
8562e9b verified
Raw History Blame Contribute Delete
10.7 kB
#!/usr/bin/env python3
"""Freeze the train-only 6 x 50 pilot and CPU-accept every complete example."""
from __future__ import annotations
import argparse
import json
import os
import sys
from collections import Counter
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[2]
sys.path[:0] = [str(PROJECT_ROOT), str(PROJECT_ROOT / "src")]
from psi.data.seed.third_party_frozen_dataset import ( # noqa: E402
PROFILES, SELECTION_FORMAT, ThirdPartyFrozenDataset, read_jsonl,
select_balanced_recipes, sha256_file,
)
from scripts.data.hfd_third_party_common import write_json_atomic # noqa: E402
def validate_acceptance(selection_path: str | Path, production_root: str | Path,
acceptance_path: str | Path | None = None, *,
deep_payload: bool = True) -> dict:
"""Read-only launch gate; no model, CUDA allocation, or keeper changes."""
selection_path, production_root = Path(selection_path).resolve(), Path(production_root).resolve()
acceptance_path = Path(acceptance_path).resolve() if acceptance_path else selection_path.parent / "acceptance.json"
selection = json.loads(selection_path.read_text())
report = json.loads(acceptance_path.read_text())
if selection.get("format") != SELECTION_FORMAT or selection.get("split") != "train":
raise RuntimeError("Launch requires the frozen train-only selection")
expected_counts = {profile: 50 for profile in PROFILES}
ids = selection.get("session_ids", [])
if (len(ids) != 300 or len(set(ids)) != 300 or selection.get("counts") != expected_counts
or selection.get("background_language_counts") != {"zh": 150, "en": 150}):
raise RuntimeError("Frozen 300-session quota changed")
probes = selection.get("probe_session_ids", [])
if len(probes) != 12 or len(set(probes)) != 12 or not set(probes).issubset(ids):
raise RuntimeError("Invalid fixed training probe IDs")
if (report.get("passed") is not True or report.get("checked_sessions") != 300
or report.get("train_count") != 300
or any(report.get(key) != 0 for key in ("visual_inputs", "other_task_inputs", "heldout_inputs"))):
raise RuntimeError("Full CPU acceptance has not passed")
if (Path(selection["production_root"]).resolve() != production_root
or Path(report["production_root"]).resolve() != production_root
or Path(report["selection_path"]).resolve() != selection_path):
raise RuntimeError("CPU acceptance refers to another selection/root")
checks = {
selection_path: report["manifest_sha256"],
production_root / "manifests/session_recipes.jsonl": selection["recipe_manifest_sha256"],
production_root / "manifests/source_contract.json": selection["source_contract_sha256"],
PROJECT_ROOT / "src/psi/data/seed/third_party_frozen_dataset.py": report["loader_sha256"],
PROJECT_ROOT / "src/psi/data/seed/full_duplex_audio_command_dataset.py": report["timeline_builder_sha256"],
Path(report["runtime_lengths_path"]): report["runtime_lengths_sha256"],
}
for path, expected in checks.items():
if sha256_file(path) != expected:
raise RuntimeError(f"Accepted input/code hash changed; rerun CPU acceptance: {path}")
contract = json.loads((production_root / "manifests/source_contract.json").read_text())
for name in ("target_metadata", "motion_metadata", "source_assignment", "source_production_marker"):
if sha256_file(Path(contract[f"{name}_path"])) != contract[f"{name}_sha256"]:
raise RuntimeError(f"Original source contract changed: {name}")
if deep_payload:
wanted, seen = set(ids), set()
for recipe in read_jsonl(production_root / "manifests/session_recipes.jsonl"):
session_id = recipe["session_id"]
if session_id not in wanted:
continue
if session_id in seen or recipe.get("split") != "train":
raise RuntimeError("Duplicate or heldout accepted session")
seen.add(session_id)
for name, hash_key in (("audio", "sha256"), ("timeline", "file_sha256")):
path = production_root / recipe[name]["path"]
if sha256_file(path) != recipe[name][hash_key]:
raise RuntimeError(f"Frozen {name} payload changed: {session_id}")
if seen != wanted:
raise RuntimeError("Accepted sessions missing from production")
return report
def prepare(production_root: Path, output_root: Path, *, seed: int = 26100401) -> dict:
production_root, output_root = production_root.resolve(), output_root.resolve()
output_root.relative_to(PROJECT_ROOT)
if output_root == PROJECT_ROOT or production_root == output_root:
raise ValueError("Preparation output must be a dedicated LZK work directory")
output_root.mkdir(parents=True, exist_ok=True)
selection_path = output_root / "selection.json"
acceptance_path = output_root / "acceptance.json"
manifest = production_root / "manifests/session_recipes.jsonl"
selected, probe_ids, exclusions = select_balanced_recipes(read_jsonl(manifest), seed=seed)
selection = {
"format": SELECTION_FORMAT, "production_root": str(production_root),
"split": "train", "seed": seed, "baseline_semantics_version": 24,
"recipe_manifest_sha256": sha256_file(manifest),
"source_contract_sha256": sha256_file(production_root / "manifests/source_contract.json"),
"session_ids": [row["session_id"] for row in selected],
"probe_session_ids": probe_ids, "sessions": selected,
"counts": dict(Counter(row["session_type"] for row in selected)),
"background_language_counts": dict(Counter(row["background_language"] for row in selected)),
"background_high_count": sum(row["background_high"] for row in selected),
"exclusion_policy": "train_only_calibrated_background_no_stop_ambiguity_clone_active_ge160_v1",
"evaluation_policy": "fixed_train_probe_not_heldout_generalization",
"exclusion_reason_counts": dict(Counter(reason for row in exclusions for reason in row["reasons"])),
}
if selection_path.exists():
if json.loads(selection_path.read_text()) != selection:
raise RuntimeError("Refusing to overwrite a different frozen training selection")
else:
write_json_atomic(selection_path, selection)
write_json_atomic(output_root / "excluded_sessions.json", exclusions)
manifest_hash = sha256_file(selection_path)
status = {
"format": "hfd_third_party_training_acceptance_v1", "passed": False,
"selection_path": str(selection_path), "manifest_sha256": manifest_hash,
"recipe_manifest_sha256": selection["recipe_manifest_sha256"],
"production_root": str(production_root), "train_count": 300,
"visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0,
"checked_sessions": 0, "device": "cpu",
"baseline_semantics_version": 24,
"loader_sha256": sha256_file(PROJECT_ROOT / "src/psi/data/seed/third_party_frozen_dataset.py"),
"timeline_builder_sha256": sha256_file(PROJECT_ROOT / "src/psi/data/seed/full_duplex_audio_command_dataset.py"),
}
write_json_atomic(acceptance_path, status)
try:
dataset = ThirdPartyFrozenDataset(production_root=production_root, selection_path=selection_path)
lengths = []
for index, recipe in enumerate(dataset.recipes):
item = dataset[index]
if len(item["steps"]) != dataset.estimated_stream_steps(index):
raise RuntimeError("Exact runtime length disagrees with frozen recipe")
if item["control_command_events"] or item["backchannel_events"] or item["disfluency_events"]:
raise RuntimeError("Unexpected other-task event in third-party pilot")
if any(any(key in step for key in ("images", "image", "video", "pixel_values"))
for step in item["steps"]):
raise RuntimeError("Unexpected visual input in audio-only pilot")
lengths.append({"session_id": recipe["session_id"], "dense_steps": len(item["steps"]),
"semantic_steps": int(item["semantic_steps"]),
"audio_sha256": recipe["audio"]["sha256"],
"timeline_content_sha256": recipe["timeline"]["content_sha256"]})
status["checked_sessions"] = index + 1
if (index + 1) % 10 == 0 or index == 0:
write_json_atomic(acceptance_path, status)
print(f"[third-party CPU acceptance] {index + 1}/{len(dataset)}", flush=True)
del item
if len(dataset) != 300 or selection["counts"] != {profile: 50 for profile in PROFILES}:
raise RuntimeError("Pilot quota mismatch")
write_json_atomic(output_root / "runtime_lengths.json", lengths)
status.update(passed=True, checked_sessions=len(dataset),
dense_steps=sum(row["dense_steps"] for row in lengths),
semantic_steps=sum(row["semantic_steps"] for row in lengths),
runtime_lengths_path=str(output_root / "runtime_lengths.json"),
runtime_lengths_sha256=sha256_file(output_root / "runtime_lengths.json"),
profile_counts=selection["counts"], probe_session_ids=probe_ids,
verification="all compact arrays + sparse targets + target source hashes + exact mixed WAV windows")
except Exception as error:
status["error"] = f"{type(error).__name__}: {error}"
write_json_atomic(acceptance_path, status)
raise
write_json_atomic(acceptance_path, status)
print(json.dumps(status, ensure_ascii=False, indent=2), flush=True)
return status
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--production-root", type=Path,
default=PROJECT_ROOT / "data/hfd_third_party_multi_v5/production")
parser.add_argument("--output-root", type=Path,
default=PROJECT_ROOT / "data/hfd_third_party_multi_v5/training_pilot_300")
parser.add_argument("--seed", type=int, default=26100401)
args = parser.parse_args()
# This script never initializes a CUDA model, including when inherited from
# a GPU shell. Source/GT reconstruction uses numpy/scipy and CPU torch.
os.environ["CUDA_VISIBLE_DEVICES"] = ""
prepare(args.production_root, args.output_root, seed=args.seed)
if __name__ == "__main__":
main()