#!/usr/bin/env python3 """Freeze all eligible production train/validation IDs without changing data.""" from __future__ import annotations import argparse from collections import Counter from concurrent.futures import ThreadPoolExecutor import json import math import os from pathlib import Path import sys PROJECT = Path(__file__).resolve().parents[2] sys.path[:0] = [str(PROJECT), str(PROJECT / "src")] from scripts.third_party_strategy_plan import write_json from scripts.third_party_scale_plan import SEED, PROFILES, fixed_panel, review_ids, safe_relative_path def read(path): return json.loads(Path(path).read_text()) def prepare(root, output): from psi.data.seed.third_party_frozen_dataset import ( selection_exclusion_reasons, ALL_TARGET_INSTRUCTIONS, read_jsonl, sha256_file, SELECTION_FORMAT) policy = {"scaleup": True, "target_instruction_policy": ALL_TARGET_INSTRUCTIONS} root, output = root.resolve(), output.resolve() if not output.is_relative_to(PROJECT / "data") or output == root or output.exists(): raise ValueError("Use a fresh preparation directory under HFD_lzk/data") audit = read(root / "reports/strict_audit.json") manifest = root / "manifests/session_recipes.jsonl" manifest_hash = sha256_file(manifest) if audit.get("passed") is not True or audit.get("recipe_sha256") != manifest_hash: raise ValueError("Production audit/hash mismatch") rows = {"train": [], "validation": []} sources = {"train": set(), "validation": set()} raw_counts, raw_lang, raw_hours = Counter(), Counter(), Counter() excluded, checks, seen = [], [], set() for recipe in read_jsonl(manifest): sid, split = recipe["session_id"], recipe["split"] if sid in seen or split not in rows: raise ValueError("Duplicate/unknown production session") seen.add(sid) raw_counts[f"{split}/{recipe['session_type']}"] += 1 raw_lang[f"{split}/{recipe['background_assignment']['language']}"] += 1 raw_hours[split] += recipe["audio"]["duration_seconds"] / 3600 # Keep every target instruction, including stop commands. Technical # background checks remain identical for both original data splits. reasons = [r for r in selection_exclusion_reasons(recipe, policy) if r != "heldout_split"] if reasons: excluded.append({"session_id": sid, "split": split, "reasons": reasons}) continue row = {"session_id": sid, "session_type": recipe["session_type"], "split": split, "background_language": recipe["background_assignment"]["language"], "background_high": recipe["mix_metadata"]["measured_background_sir_db"] < 0, "dense_steps": int(recipe["timeline"]["action_steps"]), "semantic_steps": int(recipe["timeline"]["semantic_window_count"]), "duration_seconds": recipe["audio"]["duration_seconds"]} rows[split].append(row) sources[split].update(turn["motion_filename"] for turn in recipe["turns"]) for key, hash_key in (("audio", "sha256"), ("timeline", "file_sha256")): checks.append((safe_relative_path(root, recipe[key]["path"]), recipe[key][hash_key])) if sources["train"] & sources["validation"]: raise ValueError("Train/validation target motion primitives overlap") if len(seen) != audit["checked_sessions"]: raise ValueError("Production count changed") output.mkdir(parents=True) report = {"format": "hfd_third_party_scale_readiness_v1", "passed": False, "target_instruction_policy": ALL_TARGET_INSTRUCTIONS, "train_count": len(rows["train"]), "validation_count": len(rows["validation"]), "visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0, "heldout_inputs_scope": "training loader; separate heldout evaluation is permitted", "production_root": str(root), "source_motion_split_disjoint": True, "raw_class_split_counts": dict(raw_counts), "raw_background_languages": dict(raw_lang), "raw_hours": dict(raw_hours), "excluded_sessions": len(excluded), "exclusion_reason_counts": dict(Counter(f"{r['split']}/{why}" for r in excluded for why in r["reasons"])), "recipe_manifest_sha256": manifest_hash, "source_contract_sha256": sha256_file(root / "manifests/source_contract.json")} write_json(output / "acceptance.json", report) write_json(output / "excluded_sessions.json", excluded) for split, selected in rows.items(): selected.sort(key=lambda r: r["session_id"]) selection = {"format": SELECTION_FORMAT if split == "train" else "hfd_third_party_scale_validation_v1", **policy, "split": split, "seed": SEED, "baseline_semantics_version": 24, "production_root": str(root), "recipe_manifest_sha256": manifest_hash, "source_contract_sha256": report["source_contract_sha256"], "session_ids": [r["session_id"] for r in selected], "sessions": selected, "counts": dict(Counter(r["session_type"] for r in selected)), "background_language_counts": dict(Counter(r["background_language"] for r in selected)), "background_high_count": sum(r["background_high"] for r in selected), "hours": sum(r["duration_seconds"] for r in selected) / 3600, "probe_session_ids": fixed_panel(selected, 1), "evaluation_session_ids": fixed_panel(selected, 10), "video_session_ids": review_ids(selected), "exclusion_policy": "include all target commands regardless of stop keywords; retain background integrity checks and natural class proportions"} write_json(output / ("selection.json" if split == "train" else "validation.json"), selection) def verify(pair): path, expected = pair if sha256_file(path) != expected: raise ValueError(f"Frozen payload changed: {path}") try: with ThreadPoolExecutor(max_workers=16) as pool: for index, _ in enumerate(pool.map(verify, checks), 1): if index % 2000 == 0: print(f"[scale payload hashes] {index}/{len(checks)}", flush=True) contract = read(root / "manifests/source_contract.json") for name in ("target_metadata", "motion_metadata", "source_assignment", "source_production_marker"): if sha256_file(Path(contract[f"{name}_path"])) != contract[f"{name}_sha256"]: raise ValueError(f"Source contract changed: {name}") report.update(passed=True, checked_payloads=len(checks), manifest_sha256=sha256_file(output / "selection.json"), validation_sha256=sha256_file(output / "validation.json"), loader_sha256=sha256_file(PROJECT / "src/psi/data/seed/third_party_frozen_dataset.py"), timeline_builder_sha256=sha256_file(PROJECT / "src/psi/data/seed/full_duplex_audio_command_dataset.py"), verification="all selected WAV/NPZ hashes; existing complete audit; runtime full-target replay per training item") except Exception as error: report["error"] = repr(error) raise finally: write_json(output / "acceptance.json", report) print(json.dumps(report, ensure_ascii=False, indent=2), flush=True) def validate(manifest): from psi.data.seed.third_party_frozen_dataset import sha256_file, ALL_TARGET_INSTRUCTIONS manifest = Path(manifest).resolve() if not manifest.is_relative_to(PROJECT / "data"): raise ValueError("Scale manifest must be local") selection = read(manifest); report = read(manifest.parent / "acceptance.json") validation = read(manifest.parent / "validation.json") if (report.get("passed") is not True or report.get("format") != "hfd_third_party_scale_readiness_v1" or selection.get("scaleup") is not True or selection.get("split") != "train" or any(item.get("target_instruction_policy") != ALL_TARGET_INSTRUCTIONS for item in (selection, validation, report)) or validation.get("split") != "validation" or report["train_count"] != len(selection["session_ids"]) or report["validation_count"] != len(validation["session_ids"]) or set(selection["session_ids"]) & set(validation["session_ids"]) or any(report.get(key) != 0 for key in ("visual_inputs", "other_task_inputs", "heldout_inputs"))): raise ValueError("Invalid full-scale train/validation readiness") root = Path(selection["production_root"]).resolve() hashes = {manifest: report["manifest_sha256"], manifest.parent / "validation.json": report["validation_sha256"], root / "manifests/session_recipes.jsonl": report["recipe_manifest_sha256"], root / "manifests/source_contract.json": report["source_contract_sha256"], PROJECT / "src/psi/data/seed/third_party_frozen_dataset.py": report["loader_sha256"], PROJECT / "src/psi/data/seed/full_duplex_audio_command_dataset.py": report["timeline_builder_sha256"]} for path, expected in hashes.items(): if sha256_file(path) != expected: raise ValueError(f"Scale preparation identity changed: {path}") return selection, validation, report if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--production-root", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/production") parser.add_argument("--output-root", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/training_scale_all_commands") args = parser.parse_args() os.environ["CUDA_VISIBLE_DEVICES"] = "" prepare(args.production_root, args.output_root)