code-HFD / src /psi /data /seed /third_party_dataset.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
13.3 kB
"""Audited third-party microphones, current HFD motion construction.
No imports from the donor repository. Its frozen target *timing* is checked,
not its old poses: listen-hold and root40 remain owned by the current pipeline.
Original sessions are retained once; approved third-party sessions are appended
once and shuffled/refilled by the normal sampler (no extra task loss weight).
"""
from __future__ import annotations
import copy
import hashlib
import json
import random
import wave
from collections import Counter
from functools import lru_cache
from pathlib import Path
import numpy as np
from torch.utils.data import Dataset
PROFILES = {"single_short", "multi_single", "multi_babble", "mixed_short",
"continuous", "same_voice_aside"}
def file_hash(path):
h = hashlib.sha256()
with Path(path).open("rb") as f:
for chunk in iter(lambda: f.read(1024 * 1024), b""):
h.update(chunk)
return h.hexdigest()
def contained(root, relative):
path = (root / relative).resolve()
if not path.is_relative_to(root):
raise ValueError(f"Third-party path escapes production root: {relative}")
return path
@lru_cache(maxsize=1024)
def verify_primitive(directory, expected_hash, stamps):
"""Hash each loaded primitive once per file revision, not every window."""
digest = hashlib.sha256()
for filename in ("joint_pos.csv", "body_pos.csv", "body_quat.csv"):
digest.update(filename.encode() + b"\0")
digest.update(file_hash(Path(directory) / filename).encode() + b"\0")
if digest.hexdigest() != expected_hash:
raise ValueError(f"Third-party primitive motion changed: {directory}")
def read_recipes(root, selection_path, split):
"""Compact metadata only; never put audio or augmented poses in the index."""
from psi.data.seed.startup_index_cache import cached_index
root = Path(root).resolve()
selection = json.loads(Path(selection_path).read_text())
expected = "train" if split == "train" else "validation"
expected_format = ('hfd_third_party_training_selection_v1' if split == 'train'
else 'hfd_third_party_scale_validation_v1')
if (selection.get('format') != expected_format or selection.get('scaleup') is not True
or selection.get('baseline_semantics_version') != 24):
raise ValueError('Expected approved full-scale third-party selection')
if selection.get("split") != expected:
raise ValueError("Third-party selection split mismatch")
if Path(selection["production_root"]).resolve() != root:
raise ValueError("Third-party selection production root mismatch")
manifest = root / "manifests/session_recipes.jsonl"
if file_hash(manifest) != selection["recipe_manifest_sha256"]:
raise ValueError("Third-party recipe hash mismatch")
if file_hash(root / "manifests/source_contract.json") != selection["source_contract_sha256"]:
raise ValueError("Third-party source contract hash mismatch")
ids = selection["session_ids"]
if not ids or len(set(ids)) != len(ids):
raise ValueError("Empty/duplicate third-party selection")
def build():
wanted = set(ids)
found = {}
with manifest.open() as f:
for line in f:
row = json.loads(line)
sid = row["session_id"]
if sid not in wanted:
continue
if sid in found or row["split"] != expected:
raise ValueError(f"Duplicate or wrong-split third-party session: {sid}")
if row["session_type"] not in PROFILES or row["interaction_mode"] != "turn-taking":
raise ValueError(f"Unsupported third-party recipe: {sid}")
mix = row["mix_metadata"]
if mix.get("status") != "calibrated" or not mix.get("background_applied"):
raise ValueError(f"Unaudited background: {sid}")
if not np.isfinite(mix.get('measured_background_sir_db', np.nan)):
raise ValueError(f"Unmeasurable third-party background: {sid}")
for event in row['interference_events']:
if event.get('type') == 'reference_clone' or event.get('voice_relation') == 'reference_clone':
if event.get('boundary_unmeasurable') or event.get('active_sample_count',0) < 160:
raise ValueError(f"Rejected same-voice boundary: {sid}")
if not 2 <= len(row["turns"]) <= 4:
raise ValueError(f"Unexpected donor turn count: {sid}")
if len({t["voice_name"] for t in row["turns"]}) != 1:
raise ValueError(f"Speaker changes within third-party session: {sid}")
for section in ("audio", "timeline"):
if not contained(root, row[section]["path"]).is_file():
raise FileNotFoundError(row[section]["path"])
keys = ("session_id", "session_type", "split", "timeline_seed", "turns",
"audio", "timeline", "base_background", "supervision_targets",
"composition_variant_id", "composition_salt", "session_voice_name",
"target_speech_events")
compact = {k: row[k] for k in keys}
compact["background_language"] = row["background_assignment"]["language"]
compact["interference_intervals"] = [
{"start_seconds": e["start_sample"] / 16000.,
"end_seconds": e["end_sample"] / 16000.}
for e in row["interference_events"]]
found[sid] = compact
if set(found) != wanted:
raise ValueError("Selected third-party recipes missing")
return tuple(found[sid] for sid in ids)
return cached_index(f"third-party-{split}", selection, build, code_paths=(Path(__file__),))
def replace_microphone(item, recipe, root):
"""Fail closed on timing changes; no old-pose checks and no re-mixing."""
sid = recipe["session_id"]
audio = recipe["audio"]
if tuple(audio[k] for k in ("sample_rate", "channels", "sample_width_bytes",
"window_samples", "stride_samples")) != (16000, 1, 2, 9600, 6400):
raise ValueError("Invalid third-party waveform geometry")
timeline = contained(root, recipe["timeline"]["path"])
if file_hash(timeline) != recipe["timeline"]["file_sha256"]:
raise ValueError(f"Third-party timeline corrupted: {sid}")
steps = item["steps"]
with np.load(timeline, allow_pickle=False) as stored:
for key in stored.files:
if key == "states": # Intentionally regenerated by latest listen-hold construction.
continue
actual = np.asarray([s[key] for s in steps], dtype=stored[key].dtype)
if actual.shape != stored[key].shape or not np.array_equal(actual, stored[key]):
raise ValueError(f"Third-party timing/supervision drift: {sid}/{key}")
if len(steps) != recipe["timeline"]["action_steps"]:
raise ValueError(f"Third-party action timeline length drift: {sid}")
for expected in recipe["supervision_targets"]:
step = steps[expected["step_index"]]
for key, value in expected.items():
if isinstance(value, float):
equal = np.isclose(step[key], value, rtol=0, atol=1e-6)
else:
equal = step[key] == value
if not equal:
raise ValueError(f"Third-party sparse target drift: {sid}/{key}")
actual_speech = item["microphone_speech_intervals"]
if len(actual_speech) != len(recipe["target_speech_events"]):
raise ValueError("Third-party target speech event count changed")
for actual, expected in zip(actual_speech, recipe["target_speech_events"]):
if any(abs(round(actual[k] * 16000) - expected[e]) > 1 for k, e in
(("start_seconds", "start_sample"), ("end_seconds", "end_sample"))):
raise ValueError(f"Third-party target speech shifted: {sid}")
path = contained(root, audio["path"])
if file_hash(path) != audio["sha256"]:
raise ValueError(f"Third-party audio corrupted: {sid}")
with wave.open(str(path), "rb") as f:
if (f.getframerate(), f.getnchannels(), f.getsampwidth()) != (16000, 1, 2):
raise ValueError("Third-party WAV header mismatch")
waveform = np.frombuffer(f.readframes(f.getnframes()), "<i2").astype(np.float32) / 32768.
count = int(audio["window_count"])
if len(waveform) != audio["sample_count"] or len(waveform) != (count - 1) * 6400 + 9600:
raise ValueError("Third-party WAV length mismatch")
semantic = [s for s in steps if s["update_semantic_cache"]]
if [s["semantic_window_index"] for s in semantic] != list(range(count)):
raise ValueError("Third-party semantic windows are not consecutive")
for step in steps:
step["source_id"] = f"third-party::{sid}"
step["source_audio_key"] = sid
if step["update_semantic_cache"]:
start = int(step["semantic_window_index"]) * 6400
step["audio_waveform"] = waveform[start:start + 9600].copy()
step["audio_sampling_rate"] = 16000
step.pop("vlm_hidden_states", None)
item.update(source_id=f"third-party::{sid}", source_audio_key=sid,
interaction_task="third-party", third_party_profile=recipe["session_type"],
third_party_background_language=recipe["background_language"],
third_party_interference_intervals=recipe["interference_intervals"],
frozen_audio_path=str(path))
# Keep the underlying motion-mode (turn-taking) for established loss routing.
# Task identity is separate from control labels and physical motion mode.
return item
class ThirdPartyAugmentedDataset(Dataset):
def __init__(self, raw_dataset, production_root, selection_path, split):
self.raw_dataset = raw_dataset
self.production_root = Path(production_root).resolve()
self.recipes = read_recipes(self.production_root, selection_path, split)
by_row = {int(row["row_id"]): row for row in raw_dataset.records}
sessions = []
for index, recipe in enumerate(self.recipes):
turns = []
for turn in recipe["turns"]:
row = by_row.get(int(turn["row_id"]))
if row is None or row["filename"] != turn["motion_filename"] or row["audio_dir"] != turn["audio_dir"]:
raise ValueError(f"Third-party primitive missing from CURRENT {split} split: {turn['row_id']}")
if Path(row['motion_dir']).resolve() != Path(turn['motion_dir']).resolve():
raise ValueError(f"Third-party primitive path changed: {turn['row_id']}")
row = dict(row)
row.update(_session_voice_name=turn["voice_name"],
_session_voice_variant=recipe["composition_variant_id"],
_session_composition_variant=recipe["composition_variant_id"],
_session_composition_salt=recipe["composition_salt"],
_session_interaction_mode="turn-taking", _session_pool_index=index,
_third_party=True)
turns.append(row)
sessions.append(tuple(turns))
self.replay = copy.copy(raw_dataset)
self.replay.sessions = tuple(sessions)
self.replay.session_min_turns = 1
self.replay.session_max_turns = 4
self.sessions = (*raw_dataset.sessions, *sessions)
self.third_party_summary = {"original_sessions": len(raw_dataset), "added_sessions": len(sessions),
"profiles": dict(Counter(r["session_type"] for r in self.recipes))}
print(f"[THIRD PARTY] {split}: {self.third_party_summary}", flush=True)
def __getattr__(self, name):
if name == "raw_dataset":
raise AttributeError(name)
return getattr(self.raw_dataset, name)
def __len__(self):
return len(self.sessions)
def estimated_stream_steps(self, index):
if index < len(self.raw_dataset):
return self.raw_dataset.estimated_stream_steps(index)
return int(self.recipes[index - len(self.raw_dataset)]["timeline"]["action_steps"])
def __getitem__(self, index):
if index < len(self.raw_dataset):
return self.raw_dataset[index]
index -= len(self.raw_dataset)
recipe = self.recipes[index]
for turn in recipe['turns']:
directory = Path(turn['motion_dir'])
stamps = tuple((p.stat().st_size, p.stat().st_mtime_ns) for p in
(directory / name for name in ('joint_pos.csv','body_pos.csv','body_quat.csv')))
verify_primitive(str(directory), turn['motion_sha256'], stamps)
state = random.getstate()
try:
random.seed(recipe["timeline_seed"])
item = self.replay._build_source(index)
finally:
random.setstate(state)
return replace_microphone(item, recipe, self.production_root)