#!/usr/bin/env python3 """Non-destructive, resumable CPU-only visual dataset consolidation. Hardlinks fail closed across filesystems (never silently copy a TB). Read-only source manifests and immutable audit receipts remain usable by live workers. Only the two destination directories images/ and videos/ are created. """ from __future__ import annotations import argparse from collections import Counter, defaultdict, deque from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone import fcntl from functools import lru_cache import errno import hashlib import json import os from pathlib import Path import statistics import sys import time import unicodedata import wave sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src")) from psi.data.seed.visual_dataset import FORMAT, VOICES from psi.data.seed.causal_video_qa import valid_point def retry_io(action, context): for attempt in range(4): try: return action() except OSError as exc: if exc.errno not in (errno.EIO, errno.ESTALE, errno.ETIMEDOUT) or attempt == 3: raise OSError(exc.errno, f"{context}: {exc}") from exc print(json.dumps(dict(stage="transient_io_retry", context=str(context), attempt=attempt+1)), flush=True) time.sleep(.25 * (2 ** attempt)) def _digest(path): h = hashlib.sha256() with Path(path).open("rb") as handle: for block in iter(lambda: handle.read(4 << 20), b""): h.update(block) return h.hexdigest() def digest(path): return retry_io(lambda: _digest(path), path) def bounded_map(pool, fn, items, workers): """Bound in-flight metadata work and cancel queued work after a failure.""" source = iter(items) pending = deque() for _ in range(workers * 2): item = next(source, None) if item is not None: pending.append(pool.submit(fn, item)) try: while pending: yield pending.popleft().result() item = next(source, None) if item is not None: pending.append(pool.submit(fn, item)) finally: for future in pending: future.cancel() def atomic_json(path, data): path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_suffix(path.suffix + ".partial") with tmp.open("w") as handle: json.dump(data, handle, ensure_ascii=False, separators=(",", ":")) handle.flush() os.fsync(handle.fileno()) os.replace(tmp, path) def norm(text): return " ".join(unicodedata.normalize("NFKC", text).casefold().split()) def qa_identity(media_id, row): # Never collapse different answers or genuinely different questions. return hashlib.sha256(json.dumps([media_id, norm(row["question"]), norm(row["answer"])], ensure_ascii=False).encode()).hexdigest() @lru_cache(maxsize=65536) def resolved_parent(parent): return Path(parent).resolve(strict=True) @lru_cache(maxsize=65536) def ensure_directory(path): Path(path).mkdir(parents=True, exist_ok=True) def source_path(source): source = Path(source) source = resolved_parent(str(source.parent)) / source.name if source.is_symlink(): source = source.resolve(strict=True) return source def link_file(source, folder, relative): source = source_path(source) target = folder / relative ensure_directory(str(target.parent)) try: os.link(source, target) except FileExistsError: if not os.path.samefile(source, target): # CAS receipts can have equal bytes but separate inodes. if source.stat().st_size != target.stat().st_size or digest(source) != digest(target): raise ValueError(f"Conflicting existing file: {target}") return str(relative) @lru_cache(maxsize=131072) def receipt_link(source, folder): sha = digest(source) return link_file(source, folder, Path("receipts") / sha[:2] / (sha + ".json")) def media_check(item): path, expected, kind = item actual = digest(path) if actual != expected: raise ValueError(f"Media SHA mismatch: {path}") if kind == "images": from PIL import Image with Image.open(path) as image: image = image.convert("RGB") content = hashlib.sha256(str(image.size).encode() + image.tobytes()).hexdigest() else: content = actual return path, content def durations(values): edges = [(0, 10), (10, 20), (20, 30), (30, 60), (60, 120), (120, float("inf"))] return dict(count=len(values), min=min(values) if values else None, median=statistics.median(values) if values else None, mean=statistics.mean(values) if values else None, max=max(values) if values else None, bins={f"{lo}-{hi}s": sum(lo <= v < hi for v in values) for lo, hi in edges}) def load_groups(manifests): groups = [] inputs = [] for manifest in manifests: raw = Path(manifest).read_bytes() sha = hashlib.sha256(raw).hexdigest() obj = json.loads(raw) if obj.get("format") != "hfd_overlay_assets_v1": raise ValueError(f"Unknown manifest: {manifest}") release = Path(manifest).parent / "admission/acceptance.json" released = False if release.exists(): mark = json.loads(release.read_text()) released = mark.get("passed") is True and mark.get("manifest_sha256") == sha local = defaultdict(list) for row in obj["assets"]: local[row["asset_id"]].append(row) for rows in local.values(): groups.append(dict(rows=rows, source_manifest=str(Path(manifest).resolve()), source_sha256=sha, released=released)) inputs.append(dict(path=str(Path(manifest).resolve()), sha256=sha, groups=len(local), rows=len(obj["assets"]), provenance=obj.get("provenance", {}))) print(json.dumps(dict(stage="read_manifest", groups=len(local), rows=len(obj["assets"]))), flush=True) return groups, inputs def audit_snapshot(root): result = {} if root is None: return result # No partial files, no moving target manifest; each review is immutable. for shard in sorted(Path(root).glob("shard*")): contract_path = shard / "contract.json" contract = json.loads(contract_path.read_text()) if contract.get("version") != "causal_prefix_qa_v2": raise ValueError("Incompatible causal review policy") sha = digest(contract_path) for path in sorted((shard / "reviews").glob("*.json")): r = json.loads(path.read_text()) if r["contract_sha256"] != sha or r["asset_id"] in result: raise ValueError(f"Invalid/duplicate audit: {path}") result[r["asset_id"]] = (path, r, contract["source_sha256"]) return result def admission(group, audits): row = group["rows"][0] if not group["released"]: return "pending", {} if not row.get("video_path"): return "accepted", {} entry = audits.get(row["asset_id"]) if entry: path, review, source_sha = entry if review["video_sha256"] != row["video_sha256"] or source_sha != group["source_sha256"]: raise ValueError(f"Audit/source identity mismatch: {row['asset_id']}") points = review.get("causal_qa_points", []) if review.get("accepted") is True: if not points or not all(valid_point(p, row["duration_seconds"]) for p in points): raise ValueError("Invalid causal QA times") return "accepted", dict(causal_qa_points=points, live_qa_receipt=str(path), earliest_answer_video_seconds=min(p["question_end_seconds"] for p in points)) return "rejected", dict(live_qa_receipt=str(path)) if group["released"] and row.get("live_qa_receipt") and ( row.get("live_qa_windows") or row.get("causal_qa_points")): return "accepted", {k: row[k] for k in ("live_qa_receipt", "live_qa_windows", "causal_qa_points", "earliest_answer_video_seconds") if k in row} return "pending", {} def prepare_group(group, identity, audits): rows = group["rows"] by_voice = {r["voice"]: r for r in rows} if len(rows) != 8 or set(by_voice) != set(VOICES): raise ValueError(f"Not exactly eight distinct voices: {rows[0]['asset_id']}") row = by_voice["female_01_en"] kind = "videos" if row.get("video_path") else "images" media_key = "video" if kind == "videos" else "image" for r in rows: if (r.get(media_key + "_sha256"), r["split"], r.get("visual_features_path")) != ( row[media_key + "_sha256"], row["split"], row.get("visual_features_path")): raise ValueError(f"Voice variants disagree: {row['asset_id']}") for lang in ("en", "zh"): texts = {(norm(r["question"]), norm(r["answer"])) for r in rows if r["voice"].endswith(lang)} if len(texts) != 1: raise ValueError(f"Voices disagree on QA text: {row['asset_id']}") if not row["question"].strip() or not row["answer"].strip(): raise ValueError("Empty QA") if kind == "videos" and not (0 < float(row["duration_seconds"]) < 86400): raise ValueError("Invalid video duration") status, fields = admission(group, audits) return dict(group=group, row=row, by_voice=by_voice, kind=kind, media_key=media_key, media_id=identity, qa_id=qa_identity(identity, row), status=status, fields=fields) def build_record(items, destination): first = items[0] row, kind, media_id = first["row"], first["kind"], first["media_id"] folder = destination / kind media = row[first["media_key"] + "_path"] # Features are tied to the selected exact media bytes, never to a near duplicate. feature = row["visual_features_path"] if Path(feature).stat().st_size == 0: raise ValueError(f"Empty feature: {feature}") record = dict(id=media_id, kind=kind, split=row["split"], media_path=link_file(media, folder, Path("media") / media_id[:2] / (media_id + Path(media).suffix.lower())), media_sha256=row[first["media_key"] + "_sha256"], features_path=link_file(feature, folder, Path("features") / media_id[:2] / (media_id + ".pt")), duration_seconds=float(row["duration_seconds"]) if kind == "videos" else None, questions=[]) if row.get("parent_video_id"): record["parent_video_id"] = row["parent_video_id"] audio_bytes = 0 for item in items: qa = dict(id=item["qa_id"], status=item["status"], voices={}, **item["fields"]) qa["provenance"] = dict(asset_id=item["row"]["asset_id"], manifest_sha256=item["group"]["source_sha256"]) for key in ("visual_receipt", "live_qa_receipt"): source = qa.get(key) or item["row"].get(key) if source: qa[key] = receipt_link(source, folder) for voice in VOICES: r = item["by_voice"][voice] source = r["question_audio_path"] actual = digest(source) if actual != r["question_audio_sha256"]: raise ValueError(f"Audio SHA mismatch: {source}") with wave.open(source, "rb") as wav: if (wav.getframerate(), wav.getnchannels(), wav.getsampwidth()) != (16000, 1, 2) or wav.getnframes() == 0: raise ValueError(f"Invalid question audio: {source}") seconds = wav.getnframes() / wav.getframerate() audio_bytes += Path(source).stat().st_size values = dict(question=r["question"], answer=r["answer"], audio_path=link_file(source, folder, Path("seed_audio") / qa["id"] / (voice + ".wav")), audio_sha256=actual, audio_seconds=seconds) for key in ("asr_receipt", "question_translation_receipt", "answer_translation_receipt"): if r.get(key): values[key] = receipt_link(r[key], folder) qa["voices"][voice] = values record["questions"].append(qa) return record, audio_bytes def consolidate(manifests, output, *, audit_root=None, workers=8): output = Path(output).resolve() for kind in ("images", "videos"): (output / kind).mkdir(parents=True, exist_ok=True) with (output / "images/build.lock").open("a") as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) return _build(manifests, output, audit_root, workers) def _build(manifests, output, audit_root, workers): started = time.time() generation = datetime.now(timezone.utc).isoformat() groups, inputs = load_groups(manifests) audits = audit_snapshot(audit_root) print(json.dumps(dict(stage="audit_snapshot", receipts=len(audits))), flush=True) media_inputs = {} for g in groups: r = g["rows"][0] key = "video" if r.get("video_path") else "image" path = str(source_path(r[key + "_path"])) media_inputs[path] = (path, r[key + "_sha256"], key + "s") identities = {} cache_path = output / "images/media_checks.json" cache = json.loads(cache_path.read_text()) if cache_path.exists() else {} signatures = {} pending_media = [] for path, item in media_inputs.items(): stat = Path(path).stat() signature = [item[1], stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns] signatures[path] = signature if cache.get(path, {}).get("signature") == signature: identities[path] = cache[path]["identity"] else: pending_media.append(item) last = time.time() with ThreadPoolExecutor(max_workers=workers) as pool: for path, identity in bounded_map(pool, lambda x: retry_io(lambda: media_check(x), x[0]), pending_media, workers): identities[path] = identity if time.time() - last > 30: print(json.dumps(dict(stage="verify_media", done=len(identities), total=len(media_inputs))), flush=True) last = time.time() print(json.dumps(dict(stage="verify_media_complete", total=len(identities))), flush=True) atomic_json(cache_path, {p: dict(signature=signatures[p], identity=i) for p, i in identities.items()}) unique = {} exclusions = [] duplicate_qa = 0 split_ids = defaultdict(set) for group in groups: row = group["rows"][0] path = str(source_path(row.get("video_path") or row["image_path"])) item = prepare_group(group, identities[path], audits) split_ids[item["media_id"]].add(row["split"]) if row.get("parent_video_id"): split_ids["parent:" + row["parent_video_id"]].add(row["split"]) key = item["qa_id"] if key in unique: duplicate_qa += 1 old = unique[key] # Preserve the strongest audit, deterministic tie-break; do not mix voices. priority = {"accepted": 2, "pending": 1, "rejected": 0} if priority[item["status"]] > priority[old["status"]]: unique[key] = item else: unique[key] = item grouped = defaultdict(list) for item in unique.values(): r = item["row"] conflicts = len(split_ids[item["media_id"]]) > 1 or ( r.get("parent_video_id") and len(split_ids["parent:" + r["parent_video_id"]]) > 1) if conflicts or item["status"] == "rejected": exclusions.append(dict(kind=item["kind"], id=item["qa_id"], asset_id=r["asset_id"], reason="split_conflict" if conflicts else "review_rejected")) continue grouped[item["media_id"]].append(item) # Protect reruns: selected media/feature provider remains deterministic. jobs = [sorted(v, key=lambda x: x["qa_id"]) for _, v in sorted(grouped.items())] counts = {k: Counter() for k in ("images", "videos")} length = {k: [] for k in ("all", "accepted")} handles = {k: (output / k / "manifest.jsonl.partial").open("w") for k in counts} last = time.time() try: with ThreadPoolExecutor(max_workers=workers) as pool: work = lambda x: retry_io(lambda: build_record(x, output), x[0]["qa_id"]) for i, (record, audio_bytes) in enumerate(bounded_map(pool, work, jobs, workers), 1): kind = record["kind"] handles[kind].write(json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n") c = counts[kind] c["media"] += 1 c["questions"] += len(record["questions"]) c["question_audio_bytes"] += audio_bytes c["voice_rows"] += 8 * len(record["questions"]) statuses = {q["status"] for q in record["questions"]} for q in record["questions"]: c[q["status"] + "_questions"] += 1 if "accepted" in statuses: c["accepted_media"] += 1 if kind == "videos": length["all"].append(record["duration_seconds"]) if "accepted" in statuses: length["accepted"].append(record["duration_seconds"]) if time.time() - last > 30: progress = dict(stage="verify_and_link", done=i, total=len(jobs), elapsed=time.time()-started) atomic_json(output / "images/build_progress.json", progress) print(json.dumps(progress), flush=True) last = time.time() for handle in handles.values(): handle.flush() os.fsync(handle.fileno()) finally: for handle in handles.values(): handle.close() result = {} for kind in counts: folder = output / kind os.replace(folder / "manifest.jsonl.partial", folder / "manifest.jsonl") atomic_json(folder / "inputs.json", inputs) atomic_json(folder / "excluded.json", [e for e in exclusions if e["kind"] == kind]) summary = dict(format=FORMAT, generation=generation, counts=dict(counts[kind]), expected_voices=list(VOICES), all_questions_have_eight_voices=True, manifest_sha256=digest(folder / "manifest.jsonl"), source_groups=len(groups), duplicate_qa_removed=duplicate_qa, audit_receipts_snapshot=len(audits), dedup="image decoded RGB identity; video exact file SHA256; QA normalized English question+answer per visual; train/val parent conflicts excluded", not_claimed="No semantic/perceptual video near-duplicate or human quality guarantee", storage="hardlinks; no source deletion; immutable files must not be edited in place") if kind == "videos": summary["duration_seconds"] = {k: durations(v) for k, v in length.items()} atomic_json(folder / "summary.json", summary) result[kind] = summary atomic_json(output / "images/build_progress.json", dict(stage="complete", elapsed=time.time()-started)) return result def refresh_reviews(output, audit_root): """Incrementally admit existing pending records, without touching media/voices. Metadata and copied immutable receipts only. A failed refresh leaves the old publication intact or causes the reader to fail closed on generation mismatch. """ output = Path(output).resolve() with (output / "images/build.lock").open("a") as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) summaries = {k: json.loads((output/k/"summary.json").read_text()) for k in ("images", "videos")} if summaries["images"]["generation"] != summaries["videos"]["generation"]: raise ValueError("Incomplete publication; rebuild before refreshing") audits = audit_snapshot(audit_root) changed = 0 counts = Counter() length = {"all": [], "accepted": []} folder = output / "videos" tmp = folder / "manifest.jsonl.partial" with (folder / "manifest.jsonl").open() as source, tmp.open("w") as dest: for line in source: row = json.loads(line) for qa in row["questions"]: if qa["status"] != "pending": continue entry = audits.get(qa["provenance"]["asset_id"]) if entry is None: continue path, review, source_sha = entry if review["video_sha256"] != row["media_sha256"] or source_sha != qa["provenance"]["manifest_sha256"]: raise ValueError("Review source changed") points = review.get("causal_qa_points", []) if review["accepted"] and (not points or not all(valid_point(p, row["duration_seconds"]) for p in points)): raise ValueError("Invalid new causal points") qa["status"] = "accepted" if review["accepted"] else "rejected" qa["live_qa_receipt"] = receipt_link(path, folder) if review["accepted"]: qa["causal_qa_points"] = points qa["earliest_answer_video_seconds"] = min(p["question_end_seconds"] for p in points) changed += 1 counts["media"] += 1 counts["questions"] += len(row["questions"]) counts["voice_rows"] += 8 * len(row["questions"]) for qa in row["questions"]: counts[qa["status"] + "_questions"] += 1 length["all"].append(row["duration_seconds"]) if any(q["status"] == "accepted" for q in row["questions"]): counts["accepted_media"] += 1 length["accepted"].append(row["duration_seconds"]) dest.write(json.dumps(row, ensure_ascii=False, separators=(",", ":")) + "\n") dest.flush() os.fsync(dest.fileno()) if not changed: tmp.unlink() return dict(changed=0) # First invalidate generations, so no reader can see a mixed release. generation = datetime.now(timezone.utc).isoformat() summaries["images"]["generation"] = generation atomic_json(output / "images/summary.json", summaries["images"]) os.replace(tmp, folder / "manifest.jsonl") video = summaries["videos"] video["generation"] = generation video["counts"].update(counts) video["counts"]["pending_questions"] = counts["pending_questions"] video["duration_seconds"] = {k: durations(v) for k, v in length.items()} video["manifest_sha256"] = digest(folder / "manifest.jsonl") video["audit_receipts_snapshot"] = len(audits) atomic_json(folder / "summary.json", video) return dict(changed=changed, counts=video["counts"]) def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument("--manifests", type=Path, nargs="+") p.add_argument("--output", type=Path, required=True) p.add_argument("--audit-root", type=Path) p.add_argument("--workers", type=int, default=8) p.add_argument("--refresh-reviews", action="store_true", help="Only update review status of packaged videos") p.add_argument("--watch", action="store_true", help="Refresh every 30 min until all packaged videos are reviewed") a = p.parse_args() if a.workers < 1: p.error("workers must be positive") if a.refresh_reviews: if not a.audit_root: p.error("--refresh-reviews requires --audit-root") while True: print(json.dumps(refresh_reviews(a.output, a.audit_root), ensure_ascii=False), flush=True) summary = json.loads((a.output / "videos/summary.json").read_text()) if not a.watch or summary["counts"].get("pending_questions", 0) == 0: break time.sleep(1800) else: if not a.manifests: p.error("--manifests is required for building") print(json.dumps(consolidate(a.manifests, a.output, audit_root=a.audit_root, workers=a.workers), ensure_ascii=False)) if __name__ == "__main__": main()