Download scripts/data/prepare_third_party_scale.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 9.97 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/prepare_third_party_scale.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/data/prepare_third_party_scale.py
-
curl -L -o prepare_third_party_scale.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/prepare_third_party_scale.py
9.97 kB
| #!/usr/bin/env python3 | |
| """Freeze all eligible production train/validation IDs without changing data.""" | |
| from __future__ import annotations | |
| import argparse | |
| from collections import Counter | |
| from concurrent.futures import ThreadPoolExecutor | |
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| import sys | |
| PROJECT = Path(__file__).resolve().parents[2] | |
| sys.path[:0] = [str(PROJECT), str(PROJECT / "src")] | |
| from scripts.third_party_strategy_plan import write_json | |
| from scripts.third_party_scale_plan import SEED, PROFILES, fixed_panel, review_ids, safe_relative_path | |
| def read(path): | |
| return json.loads(Path(path).read_text()) | |
| def prepare(root, output): | |
| from psi.data.seed.third_party_frozen_dataset import ( | |
| selection_exclusion_reasons, ALL_TARGET_INSTRUCTIONS, read_jsonl, sha256_file, SELECTION_FORMAT) | |
| policy = {"scaleup": True, "target_instruction_policy": ALL_TARGET_INSTRUCTIONS} | |
| root, output = root.resolve(), output.resolve() | |
| if not output.is_relative_to(PROJECT / "data") or output == root or output.exists(): | |
| raise ValueError("Use a fresh preparation directory under HFD_lzk/data") | |
| audit = read(root / "reports/strict_audit.json") | |
| manifest = root / "manifests/session_recipes.jsonl" | |
| manifest_hash = sha256_file(manifest) | |
| if audit.get("passed") is not True or audit.get("recipe_sha256") != manifest_hash: | |
| raise ValueError("Production audit/hash mismatch") | |
| rows = {"train": [], "validation": []} | |
| sources = {"train": set(), "validation": set()} | |
| raw_counts, raw_lang, raw_hours = Counter(), Counter(), Counter() | |
| excluded, checks, seen = [], [], set() | |
| for recipe in read_jsonl(manifest): | |
| sid, split = recipe["session_id"], recipe["split"] | |
| if sid in seen or split not in rows: | |
| raise ValueError("Duplicate/unknown production session") | |
| seen.add(sid) | |
| raw_counts[f"{split}/{recipe['session_type']}"] += 1 | |
| raw_lang[f"{split}/{recipe['background_assignment']['language']}"] += 1 | |
| raw_hours[split] += recipe["audio"]["duration_seconds"] / 3600 | |
| # Keep every target instruction, including stop commands. Technical | |
| # background checks remain identical for both original data splits. | |
| reasons = [r for r in selection_exclusion_reasons(recipe, policy) if r != "heldout_split"] | |
| if reasons: | |
| excluded.append({"session_id": sid, "split": split, "reasons": reasons}) | |
| continue | |
| row = {"session_id": sid, "session_type": recipe["session_type"], "split": split, | |
| "background_language": recipe["background_assignment"]["language"], | |
| "background_high": recipe["mix_metadata"]["measured_background_sir_db"] < 0, | |
| "dense_steps": int(recipe["timeline"]["action_steps"]), | |
| "semantic_steps": int(recipe["timeline"]["semantic_window_count"]), | |
| "duration_seconds": recipe["audio"]["duration_seconds"]} | |
| rows[split].append(row) | |
| sources[split].update(turn["motion_filename"] for turn in recipe["turns"]) | |
| for key, hash_key in (("audio", "sha256"), ("timeline", "file_sha256")): | |
| checks.append((safe_relative_path(root, recipe[key]["path"]), recipe[key][hash_key])) | |
| if sources["train"] & sources["validation"]: | |
| raise ValueError("Train/validation target motion primitives overlap") | |
| if len(seen) != audit["checked_sessions"]: | |
| raise ValueError("Production count changed") | |
| output.mkdir(parents=True) | |
| report = {"format": "hfd_third_party_scale_readiness_v1", "passed": False, | |
| "target_instruction_policy": ALL_TARGET_INSTRUCTIONS, | |
| "train_count": len(rows["train"]), "validation_count": len(rows["validation"]), | |
| "visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0, | |
| "heldout_inputs_scope": "training loader; separate heldout evaluation is permitted", | |
| "production_root": str(root), "source_motion_split_disjoint": True, | |
| "raw_class_split_counts": dict(raw_counts), "raw_background_languages": dict(raw_lang), | |
| "raw_hours": dict(raw_hours), "excluded_sessions": len(excluded), | |
| "exclusion_reason_counts": dict(Counter(f"{r['split']}/{why}" for r in excluded for why in r["reasons"])), | |
| "recipe_manifest_sha256": manifest_hash, | |
| "source_contract_sha256": sha256_file(root / "manifests/source_contract.json")} | |
| write_json(output / "acceptance.json", report) | |
| write_json(output / "excluded_sessions.json", excluded) | |
| for split, selected in rows.items(): | |
| selected.sort(key=lambda r: r["session_id"]) | |
| selection = {"format": SELECTION_FORMAT if split == "train" else "hfd_third_party_scale_validation_v1", | |
| **policy, "split": split, "seed": SEED, "baseline_semantics_version": 24, | |
| "production_root": str(root), "recipe_manifest_sha256": manifest_hash, | |
| "source_contract_sha256": report["source_contract_sha256"], | |
| "session_ids": [r["session_id"] for r in selected], "sessions": selected, | |
| "counts": dict(Counter(r["session_type"] for r in selected)), | |
| "background_language_counts": dict(Counter(r["background_language"] for r in selected)), | |
| "background_high_count": sum(r["background_high"] for r in selected), | |
| "hours": sum(r["duration_seconds"] for r in selected) / 3600, | |
| "probe_session_ids": fixed_panel(selected, 1), | |
| "evaluation_session_ids": fixed_panel(selected, 10), | |
| "video_session_ids": review_ids(selected), | |
| "exclusion_policy": "include all target commands regardless of stop keywords; retain background integrity checks and natural class proportions"} | |
| write_json(output / ("selection.json" if split == "train" else "validation.json"), selection) | |
| def verify(pair): | |
| path, expected = pair | |
| if sha256_file(path) != expected: | |
| raise ValueError(f"Frozen payload changed: {path}") | |
| try: | |
| with ThreadPoolExecutor(max_workers=16) as pool: | |
| for index, _ in enumerate(pool.map(verify, checks), 1): | |
| if index % 2000 == 0: | |
| print(f"[scale payload hashes] {index}/{len(checks)}", flush=True) | |
| contract = read(root / "manifests/source_contract.json") | |
| for name in ("target_metadata", "motion_metadata", "source_assignment", "source_production_marker"): | |
| if sha256_file(Path(contract[f"{name}_path"])) != contract[f"{name}_sha256"]: | |
| raise ValueError(f"Source contract changed: {name}") | |
| report.update(passed=True, checked_payloads=len(checks), | |
| manifest_sha256=sha256_file(output / "selection.json"), | |
| validation_sha256=sha256_file(output / "validation.json"), | |
| loader_sha256=sha256_file(PROJECT / "src/psi/data/seed/third_party_frozen_dataset.py"), | |
| timeline_builder_sha256=sha256_file(PROJECT / "src/psi/data/seed/full_duplex_audio_command_dataset.py"), | |
| verification="all selected WAV/NPZ hashes; existing complete audit; runtime full-target replay per training item") | |
| except Exception as error: | |
| report["error"] = repr(error) | |
| raise | |
| finally: | |
| write_json(output / "acceptance.json", report) | |
| print(json.dumps(report, ensure_ascii=False, indent=2), flush=True) | |
| def validate(manifest): | |
| from psi.data.seed.third_party_frozen_dataset import sha256_file, ALL_TARGET_INSTRUCTIONS | |
| manifest = Path(manifest).resolve() | |
| if not manifest.is_relative_to(PROJECT / "data"): | |
| raise ValueError("Scale manifest must be local") | |
| selection = read(manifest); report = read(manifest.parent / "acceptance.json") | |
| validation = read(manifest.parent / "validation.json") | |
| if (report.get("passed") is not True or report.get("format") != "hfd_third_party_scale_readiness_v1" | |
| or selection.get("scaleup") is not True or selection.get("split") != "train" | |
| or any(item.get("target_instruction_policy") != ALL_TARGET_INSTRUCTIONS for item in (selection, validation, report)) | |
| or validation.get("split") != "validation" or report["train_count"] != len(selection["session_ids"]) | |
| or report["validation_count"] != len(validation["session_ids"]) | |
| or set(selection["session_ids"]) & set(validation["session_ids"]) | |
| or any(report.get(key) != 0 for key in ("visual_inputs", "other_task_inputs", "heldout_inputs"))): | |
| raise ValueError("Invalid full-scale train/validation readiness") | |
| root = Path(selection["production_root"]).resolve() | |
| hashes = {manifest: report["manifest_sha256"], manifest.parent / "validation.json": report["validation_sha256"], | |
| root / "manifests/session_recipes.jsonl": report["recipe_manifest_sha256"], | |
| root / "manifests/source_contract.json": report["source_contract_sha256"], | |
| PROJECT / "src/psi/data/seed/third_party_frozen_dataset.py": report["loader_sha256"], | |
| PROJECT / "src/psi/data/seed/full_duplex_audio_command_dataset.py": report["timeline_builder_sha256"]} | |
| for path, expected in hashes.items(): | |
| if sha256_file(path) != expected: | |
| raise ValueError(f"Scale preparation identity changed: {path}") | |
| return selection, validation, report | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--production-root", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/production") | |
| parser.add_argument("--output-root", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/training_scale_all_commands") | |
| args = parser.parse_args() | |
| os.environ["CUDA_VISIBLE_DEVICES"] = "" | |
| prepare(args.production_root, args.output_root) | |