File size: 9,969 Bytes
8562e9b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
#!/usr/bin/env python3
"""Freeze all eligible production train/validation IDs without changing data."""
from __future__ import annotations

import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import json
import math
import os
from pathlib import Path
import sys

PROJECT = Path(__file__).resolve().parents[2]
sys.path[:0] = [str(PROJECT), str(PROJECT / "src")]
from scripts.third_party_strategy_plan import write_json
from scripts.third_party_scale_plan import SEED, PROFILES, fixed_panel, review_ids, safe_relative_path


def read(path):
    return json.loads(Path(path).read_text())


def prepare(root, output):
    from psi.data.seed.third_party_frozen_dataset import (
        selection_exclusion_reasons, ALL_TARGET_INSTRUCTIONS, read_jsonl, sha256_file, SELECTION_FORMAT)
    policy = {"scaleup": True, "target_instruction_policy": ALL_TARGET_INSTRUCTIONS}
    root, output = root.resolve(), output.resolve()
    if not output.is_relative_to(PROJECT / "data") or output == root or output.exists():
        raise ValueError("Use a fresh preparation directory under HFD_lzk/data")
    audit = read(root / "reports/strict_audit.json")
    manifest = root / "manifests/session_recipes.jsonl"
    manifest_hash = sha256_file(manifest)
    if audit.get("passed") is not True or audit.get("recipe_sha256") != manifest_hash:
        raise ValueError("Production audit/hash mismatch")
    rows = {"train": [], "validation": []}
    sources = {"train": set(), "validation": set()}
    raw_counts, raw_lang, raw_hours = Counter(), Counter(), Counter()
    excluded, checks, seen = [], [], set()
    for recipe in read_jsonl(manifest):
        sid, split = recipe["session_id"], recipe["split"]
        if sid in seen or split not in rows:
            raise ValueError("Duplicate/unknown production session")
        seen.add(sid)
        raw_counts[f"{split}/{recipe['session_type']}"] += 1
        raw_lang[f"{split}/{recipe['background_assignment']['language']}"] += 1
        raw_hours[split] += recipe["audio"]["duration_seconds"] / 3600
        # Keep every target instruction, including stop commands. Technical
        # background checks remain identical for both original data splits.
        reasons = [r for r in selection_exclusion_reasons(recipe, policy) if r != "heldout_split"]
        if reasons:
            excluded.append({"session_id": sid, "split": split, "reasons": reasons})
            continue
        row = {"session_id": sid, "session_type": recipe["session_type"], "split": split,
               "background_language": recipe["background_assignment"]["language"],
               "background_high": recipe["mix_metadata"]["measured_background_sir_db"] < 0,
               "dense_steps": int(recipe["timeline"]["action_steps"]),
               "semantic_steps": int(recipe["timeline"]["semantic_window_count"]),
               "duration_seconds": recipe["audio"]["duration_seconds"]}
        rows[split].append(row)
        sources[split].update(turn["motion_filename"] for turn in recipe["turns"])
        for key, hash_key in (("audio", "sha256"), ("timeline", "file_sha256")):
            checks.append((safe_relative_path(root, recipe[key]["path"]), recipe[key][hash_key]))
    if sources["train"] & sources["validation"]:
        raise ValueError("Train/validation target motion primitives overlap")
    if len(seen) != audit["checked_sessions"]:
        raise ValueError("Production count changed")
    output.mkdir(parents=True)
    report = {"format": "hfd_third_party_scale_readiness_v1", "passed": False,
              "target_instruction_policy": ALL_TARGET_INSTRUCTIONS,
              "train_count": len(rows["train"]), "validation_count": len(rows["validation"]),
              "visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0,
              "heldout_inputs_scope": "training loader; separate heldout evaluation is permitted",
              "production_root": str(root), "source_motion_split_disjoint": True,
              "raw_class_split_counts": dict(raw_counts), "raw_background_languages": dict(raw_lang),
              "raw_hours": dict(raw_hours), "excluded_sessions": len(excluded),
              "exclusion_reason_counts": dict(Counter(f"{r['split']}/{why}" for r in excluded for why in r["reasons"])),
              "recipe_manifest_sha256": manifest_hash,
              "source_contract_sha256": sha256_file(root / "manifests/source_contract.json")}
    write_json(output / "acceptance.json", report)
    write_json(output / "excluded_sessions.json", excluded)
    for split, selected in rows.items():
        selected.sort(key=lambda r: r["session_id"])
        selection = {"format": SELECTION_FORMAT if split == "train" else "hfd_third_party_scale_validation_v1",
                     **policy, "split": split, "seed": SEED, "baseline_semantics_version": 24,
                     "production_root": str(root), "recipe_manifest_sha256": manifest_hash,
                     "source_contract_sha256": report["source_contract_sha256"],
                     "session_ids": [r["session_id"] for r in selected], "sessions": selected,
                     "counts": dict(Counter(r["session_type"] for r in selected)),
                     "background_language_counts": dict(Counter(r["background_language"] for r in selected)),
                     "background_high_count": sum(r["background_high"] for r in selected),
                     "hours": sum(r["duration_seconds"] for r in selected) / 3600,
                     "probe_session_ids": fixed_panel(selected, 1),
                     "evaluation_session_ids": fixed_panel(selected, 10),
                     "video_session_ids": review_ids(selected),
                     "exclusion_policy": "include all target commands regardless of stop keywords; retain background integrity checks and natural class proportions"}
        write_json(output / ("selection.json" if split == "train" else "validation.json"), selection)
    def verify(pair):
        path, expected = pair
        if sha256_file(path) != expected:
            raise ValueError(f"Frozen payload changed: {path}")
    try:
        with ThreadPoolExecutor(max_workers=16) as pool:
            for index, _ in enumerate(pool.map(verify, checks), 1):
                if index % 2000 == 0:
                    print(f"[scale payload hashes] {index}/{len(checks)}", flush=True)
        contract = read(root / "manifests/source_contract.json")
        for name in ("target_metadata", "motion_metadata", "source_assignment", "source_production_marker"):
            if sha256_file(Path(contract[f"{name}_path"])) != contract[f"{name}_sha256"]:
                raise ValueError(f"Source contract changed: {name}")
        report.update(passed=True, checked_payloads=len(checks),
                      manifest_sha256=sha256_file(output / "selection.json"),
                      validation_sha256=sha256_file(output / "validation.json"),
                      loader_sha256=sha256_file(PROJECT / "src/psi/data/seed/third_party_frozen_dataset.py"),
                      timeline_builder_sha256=sha256_file(PROJECT / "src/psi/data/seed/full_duplex_audio_command_dataset.py"),
                      verification="all selected WAV/NPZ hashes; existing complete audit; runtime full-target replay per training item")
    except Exception as error:
        report["error"] = repr(error)
        raise
    finally:
        write_json(output / "acceptance.json", report)
    print(json.dumps(report, ensure_ascii=False, indent=2), flush=True)


def validate(manifest):
    from psi.data.seed.third_party_frozen_dataset import sha256_file, ALL_TARGET_INSTRUCTIONS
    manifest = Path(manifest).resolve()
    if not manifest.is_relative_to(PROJECT / "data"):
        raise ValueError("Scale manifest must be local")
    selection = read(manifest); report = read(manifest.parent / "acceptance.json")
    validation = read(manifest.parent / "validation.json")
    if (report.get("passed") is not True or report.get("format") != "hfd_third_party_scale_readiness_v1"
            or selection.get("scaleup") is not True or selection.get("split") != "train"
            or any(item.get("target_instruction_policy") != ALL_TARGET_INSTRUCTIONS for item in (selection, validation, report))
            or validation.get("split") != "validation" or report["train_count"] != len(selection["session_ids"])
            or report["validation_count"] != len(validation["session_ids"])
            or set(selection["session_ids"]) & set(validation["session_ids"])
            or any(report.get(key) != 0 for key in ("visual_inputs", "other_task_inputs", "heldout_inputs"))):
        raise ValueError("Invalid full-scale train/validation readiness")
    root = Path(selection["production_root"]).resolve()
    hashes = {manifest: report["manifest_sha256"], manifest.parent / "validation.json": report["validation_sha256"],
              root / "manifests/session_recipes.jsonl": report["recipe_manifest_sha256"],
              root / "manifests/source_contract.json": report["source_contract_sha256"],
              PROJECT / "src/psi/data/seed/third_party_frozen_dataset.py": report["loader_sha256"],
              PROJECT / "src/psi/data/seed/full_duplex_audio_command_dataset.py": report["timeline_builder_sha256"]}
    for path, expected in hashes.items():
        if sha256_file(path) != expected:
            raise ValueError(f"Scale preparation identity changed: {path}")
    return selection, validation, report


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--production-root", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/production")
    parser.add_argument("--output-root", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/training_scale_all_commands")
    args = parser.parse_args()
    os.environ["CUDA_VISIBLE_DEVICES"] = ""
    prepare(args.production_root, args.output_root)