code-HFD_lzk / scripts /data /prepare_third_party_scale.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
8562e9b verified
Raw History Blame Contribute Delete
9.97 kB
#!/usr/bin/env python3
"""Freeze all eligible production train/validation IDs without changing data."""
from __future__ import annotations
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import json
import math
import os
from pathlib import Path
import sys
PROJECT = Path(__file__).resolve().parents[2]
sys.path[:0] = [str(PROJECT), str(PROJECT / "src")]
from scripts.third_party_strategy_plan import write_json
from scripts.third_party_scale_plan import SEED, PROFILES, fixed_panel, review_ids, safe_relative_path
def read(path):
return json.loads(Path(path).read_text())
def prepare(root, output):
from psi.data.seed.third_party_frozen_dataset import (
selection_exclusion_reasons, ALL_TARGET_INSTRUCTIONS, read_jsonl, sha256_file, SELECTION_FORMAT)
policy = {"scaleup": True, "target_instruction_policy": ALL_TARGET_INSTRUCTIONS}
root, output = root.resolve(), output.resolve()
if not output.is_relative_to(PROJECT / "data") or output == root or output.exists():
raise ValueError("Use a fresh preparation directory under HFD_lzk/data")
audit = read(root / "reports/strict_audit.json")
manifest = root / "manifests/session_recipes.jsonl"
manifest_hash = sha256_file(manifest)
if audit.get("passed") is not True or audit.get("recipe_sha256") != manifest_hash:
raise ValueError("Production audit/hash mismatch")
rows = {"train": [], "validation": []}
sources = {"train": set(), "validation": set()}
raw_counts, raw_lang, raw_hours = Counter(), Counter(), Counter()
excluded, checks, seen = [], [], set()
for recipe in read_jsonl(manifest):
sid, split = recipe["session_id"], recipe["split"]
if sid in seen or split not in rows:
raise ValueError("Duplicate/unknown production session")
seen.add(sid)
raw_counts[f"{split}/{recipe['session_type']}"] += 1
raw_lang[f"{split}/{recipe['background_assignment']['language']}"] += 1
raw_hours[split] += recipe["audio"]["duration_seconds"] / 3600
# Keep every target instruction, including stop commands. Technical
# background checks remain identical for both original data splits.
reasons = [r for r in selection_exclusion_reasons(recipe, policy) if r != "heldout_split"]
if reasons:
excluded.append({"session_id": sid, "split": split, "reasons": reasons})
continue
row = {"session_id": sid, "session_type": recipe["session_type"], "split": split,
"background_language": recipe["background_assignment"]["language"],
"background_high": recipe["mix_metadata"]["measured_background_sir_db"] < 0,
"dense_steps": int(recipe["timeline"]["action_steps"]),
"semantic_steps": int(recipe["timeline"]["semantic_window_count"]),
"duration_seconds": recipe["audio"]["duration_seconds"]}
rows[split].append(row)
sources[split].update(turn["motion_filename"] for turn in recipe["turns"])
for key, hash_key in (("audio", "sha256"), ("timeline", "file_sha256")):
checks.append((safe_relative_path(root, recipe[key]["path"]), recipe[key][hash_key]))
if sources["train"] & sources["validation"]:
raise ValueError("Train/validation target motion primitives overlap")
if len(seen) != audit["checked_sessions"]:
raise ValueError("Production count changed")
output.mkdir(parents=True)
report = {"format": "hfd_third_party_scale_readiness_v1", "passed": False,
"target_instruction_policy": ALL_TARGET_INSTRUCTIONS,
"train_count": len(rows["train"]), "validation_count": len(rows["validation"]),
"visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0,
"heldout_inputs_scope": "training loader; separate heldout evaluation is permitted",
"production_root": str(root), "source_motion_split_disjoint": True,
"raw_class_split_counts": dict(raw_counts), "raw_background_languages": dict(raw_lang),
"raw_hours": dict(raw_hours), "excluded_sessions": len(excluded),
"exclusion_reason_counts": dict(Counter(f"{r['split']}/{why}" for r in excluded for why in r["reasons"])),
"recipe_manifest_sha256": manifest_hash,
"source_contract_sha256": sha256_file(root / "manifests/source_contract.json")}
write_json(output / "acceptance.json", report)
write_json(output / "excluded_sessions.json", excluded)
for split, selected in rows.items():
selected.sort(key=lambda r: r["session_id"])
selection = {"format": SELECTION_FORMAT if split == "train" else "hfd_third_party_scale_validation_v1",
**policy, "split": split, "seed": SEED, "baseline_semantics_version": 24,
"production_root": str(root), "recipe_manifest_sha256": manifest_hash,
"source_contract_sha256": report["source_contract_sha256"],
"session_ids": [r["session_id"] for r in selected], "sessions": selected,
"counts": dict(Counter(r["session_type"] for r in selected)),
"background_language_counts": dict(Counter(r["background_language"] for r in selected)),
"background_high_count": sum(r["background_high"] for r in selected),
"hours": sum(r["duration_seconds"] for r in selected) / 3600,
"probe_session_ids": fixed_panel(selected, 1),
"evaluation_session_ids": fixed_panel(selected, 10),
"video_session_ids": review_ids(selected),
"exclusion_policy": "include all target commands regardless of stop keywords; retain background integrity checks and natural class proportions"}
write_json(output / ("selection.json" if split == "train" else "validation.json"), selection)
def verify(pair):
path, expected = pair
if sha256_file(path) != expected:
raise ValueError(f"Frozen payload changed: {path}")
try:
with ThreadPoolExecutor(max_workers=16) as pool:
for index, _ in enumerate(pool.map(verify, checks), 1):
if index % 2000 == 0:
print(f"[scale payload hashes] {index}/{len(checks)}", flush=True)
contract = read(root / "manifests/source_contract.json")
for name in ("target_metadata", "motion_metadata", "source_assignment", "source_production_marker"):
if sha256_file(Path(contract[f"{name}_path"])) != contract[f"{name}_sha256"]:
raise ValueError(f"Source contract changed: {name}")
report.update(passed=True, checked_payloads=len(checks),
manifest_sha256=sha256_file(output / "selection.json"),
validation_sha256=sha256_file(output / "validation.json"),
loader_sha256=sha256_file(PROJECT / "src/psi/data/seed/third_party_frozen_dataset.py"),
timeline_builder_sha256=sha256_file(PROJECT / "src/psi/data/seed/full_duplex_audio_command_dataset.py"),
verification="all selected WAV/NPZ hashes; existing complete audit; runtime full-target replay per training item")
except Exception as error:
report["error"] = repr(error)
raise
finally:
write_json(output / "acceptance.json", report)
print(json.dumps(report, ensure_ascii=False, indent=2), flush=True)
def validate(manifest):
from psi.data.seed.third_party_frozen_dataset import sha256_file, ALL_TARGET_INSTRUCTIONS
manifest = Path(manifest).resolve()
if not manifest.is_relative_to(PROJECT / "data"):
raise ValueError("Scale manifest must be local")
selection = read(manifest); report = read(manifest.parent / "acceptance.json")
validation = read(manifest.parent / "validation.json")
if (report.get("passed") is not True or report.get("format") != "hfd_third_party_scale_readiness_v1"
or selection.get("scaleup") is not True or selection.get("split") != "train"
or any(item.get("target_instruction_policy") != ALL_TARGET_INSTRUCTIONS for item in (selection, validation, report))
or validation.get("split") != "validation" or report["train_count"] != len(selection["session_ids"])
or report["validation_count"] != len(validation["session_ids"])
or set(selection["session_ids"]) & set(validation["session_ids"])
or any(report.get(key) != 0 for key in ("visual_inputs", "other_task_inputs", "heldout_inputs"))):
raise ValueError("Invalid full-scale train/validation readiness")
root = Path(selection["production_root"]).resolve()
hashes = {manifest: report["manifest_sha256"], manifest.parent / "validation.json": report["validation_sha256"],
root / "manifests/session_recipes.jsonl": report["recipe_manifest_sha256"],
root / "manifests/source_contract.json": report["source_contract_sha256"],
PROJECT / "src/psi/data/seed/third_party_frozen_dataset.py": report["loader_sha256"],
PROJECT / "src/psi/data/seed/full_duplex_audio_command_dataset.py": report["timeline_builder_sha256"]}
for path, expected in hashes.items():
if sha256_file(path) != expected:
raise ValueError(f"Scale preparation identity changed: {path}")
return selection, validation, report
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--production-root", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/production")
parser.add_argument("--output-root", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/training_scale_all_commands")
args = parser.parse_args()
os.environ["CUDA_VISIBLE_DEVICES"] = ""
prepare(args.production_root, args.output_root)