File size: 10,688 Bytes
8562e9b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
#!/usr/bin/env python3
"""Freeze the train-only 6 x 50 pilot and CPU-accept every complete example."""
from __future__ import annotations

import argparse
import json
import os
import sys
from collections import Counter
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parents[2]
sys.path[:0] = [str(PROJECT_ROOT), str(PROJECT_ROOT / "src")]

from psi.data.seed.third_party_frozen_dataset import (  # noqa: E402
    PROFILES, SELECTION_FORMAT, ThirdPartyFrozenDataset, read_jsonl,
    select_balanced_recipes, sha256_file,
)
from scripts.data.hfd_third_party_common import write_json_atomic  # noqa: E402


def validate_acceptance(selection_path: str | Path, production_root: str | Path,
                        acceptance_path: str | Path | None = None, *,
                        deep_payload: bool = True) -> dict:
    """Read-only launch gate; no model, CUDA allocation, or keeper changes."""
    selection_path, production_root = Path(selection_path).resolve(), Path(production_root).resolve()
    acceptance_path = Path(acceptance_path).resolve() if acceptance_path else selection_path.parent / "acceptance.json"
    selection = json.loads(selection_path.read_text())
    report = json.loads(acceptance_path.read_text())
    if selection.get("format") != SELECTION_FORMAT or selection.get("split") != "train":
        raise RuntimeError("Launch requires the frozen train-only selection")
    expected_counts = {profile: 50 for profile in PROFILES}
    ids = selection.get("session_ids", [])
    if (len(ids) != 300 or len(set(ids)) != 300 or selection.get("counts") != expected_counts
            or selection.get("background_language_counts") != {"zh": 150, "en": 150}):
        raise RuntimeError("Frozen 300-session quota changed")
    probes = selection.get("probe_session_ids", [])
    if len(probes) != 12 or len(set(probes)) != 12 or not set(probes).issubset(ids):
        raise RuntimeError("Invalid fixed training probe IDs")
    if (report.get("passed") is not True or report.get("checked_sessions") != 300
            or report.get("train_count") != 300
            or any(report.get(key) != 0 for key in ("visual_inputs", "other_task_inputs", "heldout_inputs"))):
        raise RuntimeError("Full CPU acceptance has not passed")
    if (Path(selection["production_root"]).resolve() != production_root
            or Path(report["production_root"]).resolve() != production_root
            or Path(report["selection_path"]).resolve() != selection_path):
        raise RuntimeError("CPU acceptance refers to another selection/root")
    checks = {
        selection_path: report["manifest_sha256"],
        production_root / "manifests/session_recipes.jsonl": selection["recipe_manifest_sha256"],
        production_root / "manifests/source_contract.json": selection["source_contract_sha256"],
        PROJECT_ROOT / "src/psi/data/seed/third_party_frozen_dataset.py": report["loader_sha256"],
        PROJECT_ROOT / "src/psi/data/seed/full_duplex_audio_command_dataset.py": report["timeline_builder_sha256"],
        Path(report["runtime_lengths_path"]): report["runtime_lengths_sha256"],
    }
    for path, expected in checks.items():
        if sha256_file(path) != expected:
            raise RuntimeError(f"Accepted input/code hash changed; rerun CPU acceptance: {path}")
    contract = json.loads((production_root / "manifests/source_contract.json").read_text())
    for name in ("target_metadata", "motion_metadata", "source_assignment", "source_production_marker"):
        if sha256_file(Path(contract[f"{name}_path"])) != contract[f"{name}_sha256"]:
            raise RuntimeError(f"Original source contract changed: {name}")
    if deep_payload:
        wanted, seen = set(ids), set()
        for recipe in read_jsonl(production_root / "manifests/session_recipes.jsonl"):
            session_id = recipe["session_id"]
            if session_id not in wanted:
                continue
            if session_id in seen or recipe.get("split") != "train":
                raise RuntimeError("Duplicate or heldout accepted session")
            seen.add(session_id)
            for name, hash_key in (("audio", "sha256"), ("timeline", "file_sha256")):
                path = production_root / recipe[name]["path"]
                if sha256_file(path) != recipe[name][hash_key]:
                    raise RuntimeError(f"Frozen {name} payload changed: {session_id}")
        if seen != wanted:
            raise RuntimeError("Accepted sessions missing from production")
    return report


def prepare(production_root: Path, output_root: Path, *, seed: int = 26100401) -> dict:
    production_root, output_root = production_root.resolve(), output_root.resolve()
    output_root.relative_to(PROJECT_ROOT)
    if output_root == PROJECT_ROOT or production_root == output_root:
        raise ValueError("Preparation output must be a dedicated LZK work directory")
    output_root.mkdir(parents=True, exist_ok=True)
    selection_path = output_root / "selection.json"
    acceptance_path = output_root / "acceptance.json"
    manifest = production_root / "manifests/session_recipes.jsonl"
    selected, probe_ids, exclusions = select_balanced_recipes(read_jsonl(manifest), seed=seed)
    selection = {
        "format": SELECTION_FORMAT, "production_root": str(production_root),
        "split": "train", "seed": seed, "baseline_semantics_version": 24,
        "recipe_manifest_sha256": sha256_file(manifest),
        "source_contract_sha256": sha256_file(production_root / "manifests/source_contract.json"),
        "session_ids": [row["session_id"] for row in selected],
        "probe_session_ids": probe_ids, "sessions": selected,
        "counts": dict(Counter(row["session_type"] for row in selected)),
        "background_language_counts": dict(Counter(row["background_language"] for row in selected)),
        "background_high_count": sum(row["background_high"] for row in selected),
        "exclusion_policy": "train_only_calibrated_background_no_stop_ambiguity_clone_active_ge160_v1",
        "evaluation_policy": "fixed_train_probe_not_heldout_generalization",
        "exclusion_reason_counts": dict(Counter(reason for row in exclusions for reason in row["reasons"])),
    }
    if selection_path.exists():
        if json.loads(selection_path.read_text()) != selection:
            raise RuntimeError("Refusing to overwrite a different frozen training selection")
    else:
        write_json_atomic(selection_path, selection)
    write_json_atomic(output_root / "excluded_sessions.json", exclusions)
    manifest_hash = sha256_file(selection_path)
    status = {
        "format": "hfd_third_party_training_acceptance_v1", "passed": False,
        "selection_path": str(selection_path), "manifest_sha256": manifest_hash,
        "recipe_manifest_sha256": selection["recipe_manifest_sha256"],
        "production_root": str(production_root), "train_count": 300,
        "visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0,
        "checked_sessions": 0, "device": "cpu",
        "baseline_semantics_version": 24,
        "loader_sha256": sha256_file(PROJECT_ROOT / "src/psi/data/seed/third_party_frozen_dataset.py"),
        "timeline_builder_sha256": sha256_file(PROJECT_ROOT / "src/psi/data/seed/full_duplex_audio_command_dataset.py"),
    }
    write_json_atomic(acceptance_path, status)
    try:
        dataset = ThirdPartyFrozenDataset(production_root=production_root, selection_path=selection_path)
        lengths = []
        for index, recipe in enumerate(dataset.recipes):
            item = dataset[index]
            if len(item["steps"]) != dataset.estimated_stream_steps(index):
                raise RuntimeError("Exact runtime length disagrees with frozen recipe")
            if item["control_command_events"] or item["backchannel_events"] or item["disfluency_events"]:
                raise RuntimeError("Unexpected other-task event in third-party pilot")
            if any(any(key in step for key in ("images", "image", "video", "pixel_values"))
                   for step in item["steps"]):
                raise RuntimeError("Unexpected visual input in audio-only pilot")
            lengths.append({"session_id": recipe["session_id"], "dense_steps": len(item["steps"]),
                            "semantic_steps": int(item["semantic_steps"]),
                            "audio_sha256": recipe["audio"]["sha256"],
                            "timeline_content_sha256": recipe["timeline"]["content_sha256"]})
            status["checked_sessions"] = index + 1
            if (index + 1) % 10 == 0 or index == 0:
                write_json_atomic(acceptance_path, status)
                print(f"[third-party CPU acceptance] {index + 1}/{len(dataset)}", flush=True)
            del item
        if len(dataset) != 300 or selection["counts"] != {profile: 50 for profile in PROFILES}:
            raise RuntimeError("Pilot quota mismatch")
        write_json_atomic(output_root / "runtime_lengths.json", lengths)
        status.update(passed=True, checked_sessions=len(dataset),
                      dense_steps=sum(row["dense_steps"] for row in lengths),
                      semantic_steps=sum(row["semantic_steps"] for row in lengths),
                      runtime_lengths_path=str(output_root / "runtime_lengths.json"),
                      runtime_lengths_sha256=sha256_file(output_root / "runtime_lengths.json"),
                      profile_counts=selection["counts"], probe_session_ids=probe_ids,
                      verification="all compact arrays + sparse targets + target source hashes + exact mixed WAV windows")
    except Exception as error:
        status["error"] = f"{type(error).__name__}: {error}"
        write_json_atomic(acceptance_path, status)
        raise
    write_json_atomic(acceptance_path, status)
    print(json.dumps(status, ensure_ascii=False, indent=2), flush=True)
    return status


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--production-root", type=Path,
                        default=PROJECT_ROOT / "data/hfd_third_party_multi_v5/production")
    parser.add_argument("--output-root", type=Path,
                        default=PROJECT_ROOT / "data/hfd_third_party_multi_v5/training_pilot_300")
    parser.add_argument("--seed", type=int, default=26100401)
    args = parser.parse_args()
    # This script never initializes a CUDA model, including when inherited from
    # a GPU shell. Source/GT reconstruction uses numpy/scipy and CPU torch.
    os.environ["CUDA_VISIBLE_DEVICES"] = ""
    prepare(args.production_root, args.output_root, seed=args.seed)


if __name__ == "__main__":
    main()