File size: 9,969 Bytes
8562e9b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 | #!/usr/bin/env python3
"""Freeze all eligible production train/validation IDs without changing data."""
from __future__ import annotations
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import json
import math
import os
from pathlib import Path
import sys
PROJECT = Path(__file__).resolve().parents[2]
sys.path[:0] = [str(PROJECT), str(PROJECT / "src")]
from scripts.third_party_strategy_plan import write_json
from scripts.third_party_scale_plan import SEED, PROFILES, fixed_panel, review_ids, safe_relative_path
def read(path):
return json.loads(Path(path).read_text())
def prepare(root, output):
from psi.data.seed.third_party_frozen_dataset import (
selection_exclusion_reasons, ALL_TARGET_INSTRUCTIONS, read_jsonl, sha256_file, SELECTION_FORMAT)
policy = {"scaleup": True, "target_instruction_policy": ALL_TARGET_INSTRUCTIONS}
root, output = root.resolve(), output.resolve()
if not output.is_relative_to(PROJECT / "data") or output == root or output.exists():
raise ValueError("Use a fresh preparation directory under HFD_lzk/data")
audit = read(root / "reports/strict_audit.json")
manifest = root / "manifests/session_recipes.jsonl"
manifest_hash = sha256_file(manifest)
if audit.get("passed") is not True or audit.get("recipe_sha256") != manifest_hash:
raise ValueError("Production audit/hash mismatch")
rows = {"train": [], "validation": []}
sources = {"train": set(), "validation": set()}
raw_counts, raw_lang, raw_hours = Counter(), Counter(), Counter()
excluded, checks, seen = [], [], set()
for recipe in read_jsonl(manifest):
sid, split = recipe["session_id"], recipe["split"]
if sid in seen or split not in rows:
raise ValueError("Duplicate/unknown production session")
seen.add(sid)
raw_counts[f"{split}/{recipe['session_type']}"] += 1
raw_lang[f"{split}/{recipe['background_assignment']['language']}"] += 1
raw_hours[split] += recipe["audio"]["duration_seconds"] / 3600
# Keep every target instruction, including stop commands. Technical
# background checks remain identical for both original data splits.
reasons = [r for r in selection_exclusion_reasons(recipe, policy) if r != "heldout_split"]
if reasons:
excluded.append({"session_id": sid, "split": split, "reasons": reasons})
continue
row = {"session_id": sid, "session_type": recipe["session_type"], "split": split,
"background_language": recipe["background_assignment"]["language"],
"background_high": recipe["mix_metadata"]["measured_background_sir_db"] < 0,
"dense_steps": int(recipe["timeline"]["action_steps"]),
"semantic_steps": int(recipe["timeline"]["semantic_window_count"]),
"duration_seconds": recipe["audio"]["duration_seconds"]}
rows[split].append(row)
sources[split].update(turn["motion_filename"] for turn in recipe["turns"])
for key, hash_key in (("audio", "sha256"), ("timeline", "file_sha256")):
checks.append((safe_relative_path(root, recipe[key]["path"]), recipe[key][hash_key]))
if sources["train"] & sources["validation"]:
raise ValueError("Train/validation target motion primitives overlap")
if len(seen) != audit["checked_sessions"]:
raise ValueError("Production count changed")
output.mkdir(parents=True)
report = {"format": "hfd_third_party_scale_readiness_v1", "passed": False,
"target_instruction_policy": ALL_TARGET_INSTRUCTIONS,
"train_count": len(rows["train"]), "validation_count": len(rows["validation"]),
"visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0,
"heldout_inputs_scope": "training loader; separate heldout evaluation is permitted",
"production_root": str(root), "source_motion_split_disjoint": True,
"raw_class_split_counts": dict(raw_counts), "raw_background_languages": dict(raw_lang),
"raw_hours": dict(raw_hours), "excluded_sessions": len(excluded),
"exclusion_reason_counts": dict(Counter(f"{r['split']}/{why}" for r in excluded for why in r["reasons"])),
"recipe_manifest_sha256": manifest_hash,
"source_contract_sha256": sha256_file(root / "manifests/source_contract.json")}
write_json(output / "acceptance.json", report)
write_json(output / "excluded_sessions.json", excluded)
for split, selected in rows.items():
selected.sort(key=lambda r: r["session_id"])
selection = {"format": SELECTION_FORMAT if split == "train" else "hfd_third_party_scale_validation_v1",
**policy, "split": split, "seed": SEED, "baseline_semantics_version": 24,
"production_root": str(root), "recipe_manifest_sha256": manifest_hash,
"source_contract_sha256": report["source_contract_sha256"],
"session_ids": [r["session_id"] for r in selected], "sessions": selected,
"counts": dict(Counter(r["session_type"] for r in selected)),
"background_language_counts": dict(Counter(r["background_language"] for r in selected)),
"background_high_count": sum(r["background_high"] for r in selected),
"hours": sum(r["duration_seconds"] for r in selected) / 3600,
"probe_session_ids": fixed_panel(selected, 1),
"evaluation_session_ids": fixed_panel(selected, 10),
"video_session_ids": review_ids(selected),
"exclusion_policy": "include all target commands regardless of stop keywords; retain background integrity checks and natural class proportions"}
write_json(output / ("selection.json" if split == "train" else "validation.json"), selection)
def verify(pair):
path, expected = pair
if sha256_file(path) != expected:
raise ValueError(f"Frozen payload changed: {path}")
try:
with ThreadPoolExecutor(max_workers=16) as pool:
for index, _ in enumerate(pool.map(verify, checks), 1):
if index % 2000 == 0:
print(f"[scale payload hashes] {index}/{len(checks)}", flush=True)
contract = read(root / "manifests/source_contract.json")
for name in ("target_metadata", "motion_metadata", "source_assignment", "source_production_marker"):
if sha256_file(Path(contract[f"{name}_path"])) != contract[f"{name}_sha256"]:
raise ValueError(f"Source contract changed: {name}")
report.update(passed=True, checked_payloads=len(checks),
manifest_sha256=sha256_file(output / "selection.json"),
validation_sha256=sha256_file(output / "validation.json"),
loader_sha256=sha256_file(PROJECT / "src/psi/data/seed/third_party_frozen_dataset.py"),
timeline_builder_sha256=sha256_file(PROJECT / "src/psi/data/seed/full_duplex_audio_command_dataset.py"),
verification="all selected WAV/NPZ hashes; existing complete audit; runtime full-target replay per training item")
except Exception as error:
report["error"] = repr(error)
raise
finally:
write_json(output / "acceptance.json", report)
print(json.dumps(report, ensure_ascii=False, indent=2), flush=True)
def validate(manifest):
from psi.data.seed.third_party_frozen_dataset import sha256_file, ALL_TARGET_INSTRUCTIONS
manifest = Path(manifest).resolve()
if not manifest.is_relative_to(PROJECT / "data"):
raise ValueError("Scale manifest must be local")
selection = read(manifest); report = read(manifest.parent / "acceptance.json")
validation = read(manifest.parent / "validation.json")
if (report.get("passed") is not True or report.get("format") != "hfd_third_party_scale_readiness_v1"
or selection.get("scaleup") is not True or selection.get("split") != "train"
or any(item.get("target_instruction_policy") != ALL_TARGET_INSTRUCTIONS for item in (selection, validation, report))
or validation.get("split") != "validation" or report["train_count"] != len(selection["session_ids"])
or report["validation_count"] != len(validation["session_ids"])
or set(selection["session_ids"]) & set(validation["session_ids"])
or any(report.get(key) != 0 for key in ("visual_inputs", "other_task_inputs", "heldout_inputs"))):
raise ValueError("Invalid full-scale train/validation readiness")
root = Path(selection["production_root"]).resolve()
hashes = {manifest: report["manifest_sha256"], manifest.parent / "validation.json": report["validation_sha256"],
root / "manifests/session_recipes.jsonl": report["recipe_manifest_sha256"],
root / "manifests/source_contract.json": report["source_contract_sha256"],
PROJECT / "src/psi/data/seed/third_party_frozen_dataset.py": report["loader_sha256"],
PROJECT / "src/psi/data/seed/full_duplex_audio_command_dataset.py": report["timeline_builder_sha256"]}
for path, expected in hashes.items():
if sha256_file(path) != expected:
raise ValueError(f"Scale preparation identity changed: {path}")
return selection, validation, report
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--production-root", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/production")
parser.add_argument("--output-root", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/training_scale_all_commands")
args = parser.parse_args()
os.environ["CUDA_VISIBLE_DEVICES"] = ""
prepare(args.production_root, args.output_root)
|