code-HFD / scripts /data /consolidate_visual_dataset.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
24.6 kB
#!/usr/bin/env python3
"""Non-destructive, resumable CPU-only visual dataset consolidation.
Hardlinks fail closed across filesystems (never silently copy a TB). Read-only
source manifests and immutable audit receipts remain usable by live workers.
Only the two destination directories images/ and videos/ are created.
"""
from __future__ import annotations
import argparse
from collections import Counter, defaultdict, deque
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import fcntl
from functools import lru_cache
import errno
import hashlib
import json
import os
from pathlib import Path
import statistics
import sys
import time
import unicodedata
import wave
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src"))
from psi.data.seed.visual_dataset import FORMAT, VOICES
from psi.data.seed.causal_video_qa import valid_point
def retry_io(action, context):
for attempt in range(4):
try:
return action()
except OSError as exc:
if exc.errno not in (errno.EIO, errno.ESTALE, errno.ETIMEDOUT) or attempt == 3:
raise OSError(exc.errno, f"{context}: {exc}") from exc
print(json.dumps(dict(stage="transient_io_retry", context=str(context), attempt=attempt+1)), flush=True)
time.sleep(.25 * (2 ** attempt))
def _digest(path):
h = hashlib.sha256()
with Path(path).open("rb") as handle:
for block in iter(lambda: handle.read(4 << 20), b""):
h.update(block)
return h.hexdigest()
def digest(path):
return retry_io(lambda: _digest(path), path)
def bounded_map(pool, fn, items, workers):
"""Bound in-flight metadata work and cancel queued work after a failure."""
source = iter(items)
pending = deque()
for _ in range(workers * 2):
item = next(source, None)
if item is not None:
pending.append(pool.submit(fn, item))
try:
while pending:
yield pending.popleft().result()
item = next(source, None)
if item is not None:
pending.append(pool.submit(fn, item))
finally:
for future in pending:
future.cancel()
def atomic_json(path, data):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(path.suffix + ".partial")
with tmp.open("w") as handle:
json.dump(data, handle, ensure_ascii=False, separators=(",", ":"))
handle.flush()
os.fsync(handle.fileno())
os.replace(tmp, path)
def norm(text):
return " ".join(unicodedata.normalize("NFKC", text).casefold().split())
def qa_identity(media_id, row):
# Never collapse different answers or genuinely different questions.
return hashlib.sha256(json.dumps([media_id, norm(row["question"]), norm(row["answer"])],
ensure_ascii=False).encode()).hexdigest()
@lru_cache(maxsize=65536)
def resolved_parent(parent):
return Path(parent).resolve(strict=True)
@lru_cache(maxsize=65536)
def ensure_directory(path):
Path(path).mkdir(parents=True, exist_ok=True)
def source_path(source):
source = Path(source)
source = resolved_parent(str(source.parent)) / source.name
if source.is_symlink():
source = source.resolve(strict=True)
return source
def link_file(source, folder, relative):
source = source_path(source)
target = folder / relative
ensure_directory(str(target.parent))
try:
os.link(source, target)
except FileExistsError:
if not os.path.samefile(source, target):
# CAS receipts can have equal bytes but separate inodes.
if source.stat().st_size != target.stat().st_size or digest(source) != digest(target):
raise ValueError(f"Conflicting existing file: {target}")
return str(relative)
@lru_cache(maxsize=131072)
def receipt_link(source, folder):
sha = digest(source)
return link_file(source, folder, Path("receipts") / sha[:2] / (sha + ".json"))
def media_check(item):
path, expected, kind = item
actual = digest(path)
if actual != expected:
raise ValueError(f"Media SHA mismatch: {path}")
if kind == "images":
from PIL import Image
with Image.open(path) as image:
image = image.convert("RGB")
content = hashlib.sha256(str(image.size).encode() + image.tobytes()).hexdigest()
else:
content = actual
return path, content
def durations(values):
edges = [(0, 10), (10, 20), (20, 30), (30, 60), (60, 120), (120, float("inf"))]
return dict(count=len(values), min=min(values) if values else None,
median=statistics.median(values) if values else None,
mean=statistics.mean(values) if values else None,
max=max(values) if values else None,
bins={f"{lo}-{hi}s": sum(lo <= v < hi for v in values) for lo, hi in edges})
def load_groups(manifests):
groups = []
inputs = []
for manifest in manifests:
raw = Path(manifest).read_bytes()
sha = hashlib.sha256(raw).hexdigest()
obj = json.loads(raw)
if obj.get("format") != "hfd_overlay_assets_v1":
raise ValueError(f"Unknown manifest: {manifest}")
release = Path(manifest).parent / "admission/acceptance.json"
released = False
if release.exists():
mark = json.loads(release.read_text())
released = mark.get("passed") is True and mark.get("manifest_sha256") == sha
local = defaultdict(list)
for row in obj["assets"]:
local[row["asset_id"]].append(row)
for rows in local.values():
groups.append(dict(rows=rows, source_manifest=str(Path(manifest).resolve()),
source_sha256=sha, released=released))
inputs.append(dict(path=str(Path(manifest).resolve()), sha256=sha,
groups=len(local), rows=len(obj["assets"]),
provenance=obj.get("provenance", {})))
print(json.dumps(dict(stage="read_manifest", groups=len(local), rows=len(obj["assets"]))), flush=True)
return groups, inputs
def audit_snapshot(root):
result = {}
if root is None:
return result
# No partial files, no moving target manifest; each review is immutable.
for shard in sorted(Path(root).glob("shard*")):
contract_path = shard / "contract.json"
contract = json.loads(contract_path.read_text())
if contract.get("version") != "causal_prefix_qa_v2":
raise ValueError("Incompatible causal review policy")
sha = digest(contract_path)
for path in sorted((shard / "reviews").glob("*.json")):
r = json.loads(path.read_text())
if r["contract_sha256"] != sha or r["asset_id"] in result:
raise ValueError(f"Invalid/duplicate audit: {path}")
result[r["asset_id"]] = (path, r, contract["source_sha256"])
return result
def admission(group, audits):
row = group["rows"][0]
if not group["released"]:
return "pending", {}
if not row.get("video_path"):
return "accepted", {}
entry = audits.get(row["asset_id"])
if entry:
path, review, source_sha = entry
if review["video_sha256"] != row["video_sha256"] or source_sha != group["source_sha256"]:
raise ValueError(f"Audit/source identity mismatch: {row['asset_id']}")
points = review.get("causal_qa_points", [])
if review.get("accepted") is True:
if not points or not all(valid_point(p, row["duration_seconds"]) for p in points):
raise ValueError("Invalid causal QA times")
return "accepted", dict(causal_qa_points=points, live_qa_receipt=str(path),
earliest_answer_video_seconds=min(p["question_end_seconds"] for p in points))
return "rejected", dict(live_qa_receipt=str(path))
if group["released"] and row.get("live_qa_receipt") and (
row.get("live_qa_windows") or row.get("causal_qa_points")):
return "accepted", {k: row[k] for k in ("live_qa_receipt", "live_qa_windows",
"causal_qa_points", "earliest_answer_video_seconds") if k in row}
return "pending", {}
def prepare_group(group, identity, audits):
rows = group["rows"]
by_voice = {r["voice"]: r for r in rows}
if len(rows) != 8 or set(by_voice) != set(VOICES):
raise ValueError(f"Not exactly eight distinct voices: {rows[0]['asset_id']}")
row = by_voice["female_01_en"]
kind = "videos" if row.get("video_path") else "images"
media_key = "video" if kind == "videos" else "image"
for r in rows:
if (r.get(media_key + "_sha256"), r["split"], r.get("visual_features_path")) != (
row[media_key + "_sha256"], row["split"], row.get("visual_features_path")):
raise ValueError(f"Voice variants disagree: {row['asset_id']}")
for lang in ("en", "zh"):
texts = {(norm(r["question"]), norm(r["answer"])) for r in rows if r["voice"].endswith(lang)}
if len(texts) != 1:
raise ValueError(f"Voices disagree on QA text: {row['asset_id']}")
if not row["question"].strip() or not row["answer"].strip():
raise ValueError("Empty QA")
if kind == "videos" and not (0 < float(row["duration_seconds"]) < 86400):
raise ValueError("Invalid video duration")
status, fields = admission(group, audits)
return dict(group=group, row=row, by_voice=by_voice, kind=kind, media_key=media_key,
media_id=identity, qa_id=qa_identity(identity, row), status=status, fields=fields)
def build_record(items, destination):
first = items[0]
row, kind, media_id = first["row"], first["kind"], first["media_id"]
folder = destination / kind
media = row[first["media_key"] + "_path"]
# Features are tied to the selected exact media bytes, never to a near duplicate.
feature = row["visual_features_path"]
if Path(feature).stat().st_size == 0:
raise ValueError(f"Empty feature: {feature}")
record = dict(id=media_id, kind=kind, split=row["split"],
media_path=link_file(media, folder, Path("media") / media_id[:2] / (media_id + Path(media).suffix.lower())),
media_sha256=row[first["media_key"] + "_sha256"],
features_path=link_file(feature, folder, Path("features") / media_id[:2] / (media_id + ".pt")),
duration_seconds=float(row["duration_seconds"]) if kind == "videos" else None,
questions=[])
if row.get("parent_video_id"):
record["parent_video_id"] = row["parent_video_id"]
audio_bytes = 0
for item in items:
qa = dict(id=item["qa_id"], status=item["status"], voices={}, **item["fields"])
qa["provenance"] = dict(asset_id=item["row"]["asset_id"],
manifest_sha256=item["group"]["source_sha256"])
for key in ("visual_receipt", "live_qa_receipt"):
source = qa.get(key) or item["row"].get(key)
if source:
qa[key] = receipt_link(source, folder)
for voice in VOICES:
r = item["by_voice"][voice]
source = r["question_audio_path"]
actual = digest(source)
if actual != r["question_audio_sha256"]:
raise ValueError(f"Audio SHA mismatch: {source}")
with wave.open(source, "rb") as wav:
if (wav.getframerate(), wav.getnchannels(), wav.getsampwidth()) != (16000, 1, 2) or wav.getnframes() == 0:
raise ValueError(f"Invalid question audio: {source}")
seconds = wav.getnframes() / wav.getframerate()
audio_bytes += Path(source).stat().st_size
values = dict(question=r["question"], answer=r["answer"],
audio_path=link_file(source, folder, Path("seed_audio") / qa["id"] / (voice + ".wav")),
audio_sha256=actual, audio_seconds=seconds)
for key in ("asr_receipt", "question_translation_receipt", "answer_translation_receipt"):
if r.get(key):
values[key] = receipt_link(r[key], folder)
qa["voices"][voice] = values
record["questions"].append(qa)
return record, audio_bytes
def consolidate(manifests, output, *, audit_root=None, workers=8):
output = Path(output).resolve()
for kind in ("images", "videos"):
(output / kind).mkdir(parents=True, exist_ok=True)
with (output / "images/build.lock").open("a") as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
return _build(manifests, output, audit_root, workers)
def _build(manifests, output, audit_root, workers):
started = time.time()
generation = datetime.now(timezone.utc).isoformat()
groups, inputs = load_groups(manifests)
audits = audit_snapshot(audit_root)
print(json.dumps(dict(stage="audit_snapshot", receipts=len(audits))), flush=True)
media_inputs = {}
for g in groups:
r = g["rows"][0]
key = "video" if r.get("video_path") else "image"
path = str(source_path(r[key + "_path"]))
media_inputs[path] = (path, r[key + "_sha256"], key + "s")
identities = {}
cache_path = output / "images/media_checks.json"
cache = json.loads(cache_path.read_text()) if cache_path.exists() else {}
signatures = {}
pending_media = []
for path, item in media_inputs.items():
stat = Path(path).stat()
signature = [item[1], stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns]
signatures[path] = signature
if cache.get(path, {}).get("signature") == signature:
identities[path] = cache[path]["identity"]
else:
pending_media.append(item)
last = time.time()
with ThreadPoolExecutor(max_workers=workers) as pool:
for path, identity in bounded_map(pool, lambda x: retry_io(lambda: media_check(x), x[0]), pending_media, workers):
identities[path] = identity
if time.time() - last > 30:
print(json.dumps(dict(stage="verify_media", done=len(identities), total=len(media_inputs))), flush=True)
last = time.time()
print(json.dumps(dict(stage="verify_media_complete", total=len(identities))), flush=True)
atomic_json(cache_path, {p: dict(signature=signatures[p], identity=i) for p, i in identities.items()})
unique = {}
exclusions = []
duplicate_qa = 0
split_ids = defaultdict(set)
for group in groups:
row = group["rows"][0]
path = str(source_path(row.get("video_path") or row["image_path"]))
item = prepare_group(group, identities[path], audits)
split_ids[item["media_id"]].add(row["split"])
if row.get("parent_video_id"):
split_ids["parent:" + row["parent_video_id"]].add(row["split"])
key = item["qa_id"]
if key in unique:
duplicate_qa += 1
old = unique[key]
# Preserve the strongest audit, deterministic tie-break; do not mix voices.
priority = {"accepted": 2, "pending": 1, "rejected": 0}
if priority[item["status"]] > priority[old["status"]]:
unique[key] = item
else:
unique[key] = item
grouped = defaultdict(list)
for item in unique.values():
r = item["row"]
conflicts = len(split_ids[item["media_id"]]) > 1 or (
r.get("parent_video_id") and len(split_ids["parent:" + r["parent_video_id"]]) > 1)
if conflicts or item["status"] == "rejected":
exclusions.append(dict(kind=item["kind"], id=item["qa_id"], asset_id=r["asset_id"],
reason="split_conflict" if conflicts else "review_rejected"))
continue
grouped[item["media_id"]].append(item)
# Protect reruns: selected media/feature provider remains deterministic.
jobs = [sorted(v, key=lambda x: x["qa_id"]) for _, v in sorted(grouped.items())]
counts = {k: Counter() for k in ("images", "videos")}
length = {k: [] for k in ("all", "accepted")}
handles = {k: (output / k / "manifest.jsonl.partial").open("w") for k in counts}
last = time.time()
try:
with ThreadPoolExecutor(max_workers=workers) as pool:
work = lambda x: retry_io(lambda: build_record(x, output), x[0]["qa_id"])
for i, (record, audio_bytes) in enumerate(bounded_map(pool, work, jobs, workers), 1):
kind = record["kind"]
handles[kind].write(json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n")
c = counts[kind]
c["media"] += 1
c["questions"] += len(record["questions"])
c["question_audio_bytes"] += audio_bytes
c["voice_rows"] += 8 * len(record["questions"])
statuses = {q["status"] for q in record["questions"]}
for q in record["questions"]:
c[q["status"] + "_questions"] += 1
if "accepted" in statuses:
c["accepted_media"] += 1
if kind == "videos":
length["all"].append(record["duration_seconds"])
if "accepted" in statuses:
length["accepted"].append(record["duration_seconds"])
if time.time() - last > 30:
progress = dict(stage="verify_and_link", done=i, total=len(jobs), elapsed=time.time()-started)
atomic_json(output / "images/build_progress.json", progress)
print(json.dumps(progress), flush=True)
last = time.time()
for handle in handles.values():
handle.flush()
os.fsync(handle.fileno())
finally:
for handle in handles.values():
handle.close()
result = {}
for kind in counts:
folder = output / kind
os.replace(folder / "manifest.jsonl.partial", folder / "manifest.jsonl")
atomic_json(folder / "inputs.json", inputs)
atomic_json(folder / "excluded.json", [e for e in exclusions if e["kind"] == kind])
summary = dict(format=FORMAT, generation=generation, counts=dict(counts[kind]),
expected_voices=list(VOICES), all_questions_have_eight_voices=True,
manifest_sha256=digest(folder / "manifest.jsonl"),
source_groups=len(groups), duplicate_qa_removed=duplicate_qa,
audit_receipts_snapshot=len(audits),
dedup="image decoded RGB identity; video exact file SHA256; QA normalized English question+answer per visual; train/val parent conflicts excluded",
not_claimed="No semantic/perceptual video near-duplicate or human quality guarantee",
storage="hardlinks; no source deletion; immutable files must not be edited in place")
if kind == "videos":
summary["duration_seconds"] = {k: durations(v) for k, v in length.items()}
atomic_json(folder / "summary.json", summary)
result[kind] = summary
atomic_json(output / "images/build_progress.json", dict(stage="complete", elapsed=time.time()-started))
return result
def refresh_reviews(output, audit_root):
"""Incrementally admit existing pending records, without touching media/voices.
Metadata and copied immutable receipts only. A failed refresh leaves the old
publication intact or causes the reader to fail closed on generation mismatch.
"""
output = Path(output).resolve()
with (output / "images/build.lock").open("a") as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
summaries = {k: json.loads((output/k/"summary.json").read_text()) for k in ("images", "videos")}
if summaries["images"]["generation"] != summaries["videos"]["generation"]:
raise ValueError("Incomplete publication; rebuild before refreshing")
audits = audit_snapshot(audit_root)
changed = 0
counts = Counter()
length = {"all": [], "accepted": []}
folder = output / "videos"
tmp = folder / "manifest.jsonl.partial"
with (folder / "manifest.jsonl").open() as source, tmp.open("w") as dest:
for line in source:
row = json.loads(line)
for qa in row["questions"]:
if qa["status"] != "pending":
continue
entry = audits.get(qa["provenance"]["asset_id"])
if entry is None:
continue
path, review, source_sha = entry
if review["video_sha256"] != row["media_sha256"] or source_sha != qa["provenance"]["manifest_sha256"]:
raise ValueError("Review source changed")
points = review.get("causal_qa_points", [])
if review["accepted"] and (not points or not all(valid_point(p, row["duration_seconds"]) for p in points)):
raise ValueError("Invalid new causal points")
qa["status"] = "accepted" if review["accepted"] else "rejected"
qa["live_qa_receipt"] = receipt_link(path, folder)
if review["accepted"]:
qa["causal_qa_points"] = points
qa["earliest_answer_video_seconds"] = min(p["question_end_seconds"] for p in points)
changed += 1
counts["media"] += 1
counts["questions"] += len(row["questions"])
counts["voice_rows"] += 8 * len(row["questions"])
for qa in row["questions"]:
counts[qa["status"] + "_questions"] += 1
length["all"].append(row["duration_seconds"])
if any(q["status"] == "accepted" for q in row["questions"]):
counts["accepted_media"] += 1
length["accepted"].append(row["duration_seconds"])
dest.write(json.dumps(row, ensure_ascii=False, separators=(",", ":")) + "\n")
dest.flush()
os.fsync(dest.fileno())
if not changed:
tmp.unlink()
return dict(changed=0)
# First invalidate generations, so no reader can see a mixed release.
generation = datetime.now(timezone.utc).isoformat()
summaries["images"]["generation"] = generation
atomic_json(output / "images/summary.json", summaries["images"])
os.replace(tmp, folder / "manifest.jsonl")
video = summaries["videos"]
video["generation"] = generation
video["counts"].update(counts)
video["counts"]["pending_questions"] = counts["pending_questions"]
video["duration_seconds"] = {k: durations(v) for k, v in length.items()}
video["manifest_sha256"] = digest(folder / "manifest.jsonl")
video["audit_receipts_snapshot"] = len(audits)
atomic_json(folder / "summary.json", video)
return dict(changed=changed, counts=video["counts"])
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--manifests", type=Path, nargs="+")
p.add_argument("--output", type=Path, required=True)
p.add_argument("--audit-root", type=Path)
p.add_argument("--workers", type=int, default=8)
p.add_argument("--refresh-reviews", action="store_true", help="Only update review status of packaged videos")
p.add_argument("--watch", action="store_true", help="Refresh every 30 min until all packaged videos are reviewed")
a = p.parse_args()
if a.workers < 1:
p.error("workers must be positive")
if a.refresh_reviews:
if not a.audit_root:
p.error("--refresh-reviews requires --audit-root")
while True:
print(json.dumps(refresh_reviews(a.output, a.audit_root), ensure_ascii=False), flush=True)
summary = json.loads((a.output / "videos/summary.json").read_text())
if not a.watch or summary["counts"].get("pending_questions", 0) == 0:
break
time.sleep(1800)
else:
if not a.manifests:
p.error("--manifests is required for building")
print(json.dumps(consolidate(a.manifests, a.output, audit_root=a.audit_root, workers=a.workers), ensure_ascii=False))
if __name__ == "__main__":
main()