code-HFD / src /psi /data /seed /visual_dataset.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
8.4 kB
"""Portable image/video QA records. No training/model dependencies or GPU use."""
from __future__ import annotations
import json
import fcntl
import hashlib
import stat
from functools import lru_cache
from pathlib import Path
FORMAT = "hfd_visual_dataset_v1"
VOICES = tuple(f"{sex}_{n:02d}_{lang}" for lang in ("en", "zh")
for sex in ("female", "male") for n in (1, 2))
@lru_cache(maxsize=131072)
def _parent(path):
return Path(path).resolve()
@lru_cache(maxsize=262144)
def resolve_file(folder, relative):
folder = _parent(str(folder))
relative = Path(relative)
if relative.is_absolute() or '..' in relative.parts or not relative.name:
raise ValueError(f"Dataset path escapes package: {relative}")
parent = folder
for part in relative.parts[:-1]:
parent = _child_directory(str(parent), part)
path = parent / relative.name
if path.is_symlink():
path = path.resolve()
if not path.is_relative_to(folder):
raise ValueError(f"Dataset path escapes package: {relative}")
return str(path)
@lru_cache(maxsize=131072)
def _child_directory(parent, name):
# parent is already resolved. Path.resolve() for every new audio directory
# repeats lstat on the whole CPFS ancestry; inspect only its new component.
path = Path(parent) / name
try:
mode = path.lstat().st_mode
except FileNotFoundError:
return path # Same missing-file behavior as resolve(strict=False).
return path.resolve() if stat.S_ISLNK(mode) else path
def _manifest_file(folder, relative):
"""Validate metadata without issuing a million CPFS stats at startup."""
relative = Path(relative)
if relative.is_absolute() or '..' in relative.parts or not relative.name:
raise ValueError(f'Dataset path escapes package: {relative}')
return str(_parent(str(folder)) / relative)
def resolve_overlay_row(row):
"""Resolve every selected dependency BEFORE reading any selected file."""
folder = row.get('_visual_bundle_root')
if folder is None:
return row
result = dict(row)
result.pop('_visual_bundle_root')
for key in ('image_path', 'video_path', 'visual_features_path', 'question_audio_path',
'visual_receipt', 'live_qa_receipt', 'asr_receipt',
'question_translation_receipt', 'answer_translation_receipt'):
if result.get(key):
relative = Path(result[key]).relative_to(folder)
result[key] = resolve_file(folder, str(relative))
return result
def _digest(path):
value = hashlib.sha256()
with Path(path).open('rb') as handle:
for block in iter(lambda: handle.read(8 * 1024 * 1024), b''):
value.update(block)
return value.hexdigest()
def _verify_publication(root, summaries, base_model):
# Called under the same publication lock as record iteration. No old
# source directories or media rehashing are needed at training startup.
for kind, summary in zip(('images', 'videos'), summaries):
if _digest(root / kind / 'manifest.jsonl') != summary['manifest_sha256']:
raise ValueError('Unified visual manifest changed after publication')
if base_model is not None and summaries[1]['counts'].get('accepted_questions', 0):
inputs = json.loads((root / 'videos/inputs.json').read_text())
contracts = [r.get('provenance', {}).get('contract') for r in inputs]
contracts = [c for c in contracts if c is not None]
if not contracts:
raise ValueError('Unified video package has no encoder contract')
expected = {'format': 'hfd_dynamic_camera_v1', 'camera_fps': 8,
'side': 224, 'audio_window': .6, 'semantic_stride': .4,
'dtype': 'bfloat16', 'vision_packet_batch': 1}
expected.update(model_config_sha256=_digest(Path(base_model) / 'config.json'),
weight_index_sha256=_digest(Path(base_model) / 'model.safetensors.index.json'))
if any(any(c.get(k) != v for k, v in expected.items()) for c in contracts):
raise ValueError('Unified video frontend does not match active Qwen/geometry')
def records(root, kind=None, *, accepted_only=True, split=None, verify=False, base_model=None):
"""One record per unique visual; questions contain all eight voice variants.
By default pending/rejected QA cannot enter a training reader. This is an
asset-level admission, not evidence that a complete training run passed.
"""
root = Path(root)
with (root / "images/build.lock").open("r") as lock:
fcntl.flock(lock, fcntl.LOCK_SH)
summaries = [json.loads((root / k / "summary.json").read_text())
for k in ("images", "videos")]
if any(s["format"] != FORMAT for s in summaries):
raise ValueError("Unsupported dataset")
if summaries[0]["generation"] != summaries[1]["generation"]:
raise RuntimeError("Dataset publication in progress; retry after build completes")
if verify:
_verify_publication(root, summaries, base_model)
kinds = ("images", "videos") if kind is None else (kind,)
for k in kinds:
if k not in ("images", "videos"):
raise ValueError(k)
with (root / k / "manifest.jsonl").open() as handle:
for line in handle:
row = json.loads(line)
if split is not None and row["split"] != split:
continue
row["questions"] = [q for q in row["questions"]
if not accepted_only or q["status"] == "accepted"]
if row["questions"]:
yield row
def overlay_rows(root, kind=None, *, split=None, sample_per_group=False, verify=False,
base_model=None, lazy_paths=False):
"""Adapt both modalities to the existing motion-overlay row schema.
All dependency paths are resolved inside this bundle; original source
directories are not necessary. Does not modify production configuration.
"""
root = Path(root)
resolve = _manifest_file if lazy_paths else resolve_file
seen_groups = set()
for record in records(root, kind, split=split, verify=verify, base_model=base_model):
group = (record["kind"], record["split"])
if sample_per_group:
if group in seen_groups:
continue
seen_groups.add(group)
record["questions"] = record["questions"][:1]
folder = root / record["kind"]
media_key = "video" if record["kind"] == "videos" else "image"
base = dict(asset_id=record["id"], split=record["split"],
data_domain="unified_" + record["kind"],
visual_features_path=resolve(folder, record["features_path"]),
**{media_key + "_path": resolve(folder, record["media_path"]),
media_key + "_sha256": record["media_sha256"]})
if lazy_paths:
base['_visual_bundle_root'] = str(_parent(str(folder)))
if media_key == "video":
base.update(duration_seconds=record["duration_seconds"],
parent_video_id=record.get("parent_video_id", record["id"]))
for qa in record["questions"]:
if set(qa["voices"]) != set(VOICES):
raise ValueError("Incomplete eight-voice QA")
fields = {key: qa[key] for key in ("causal_qa_points", "live_qa_windows",
"earliest_answer_video_seconds") if key in qa}
for key in ("visual_receipt", "live_qa_receipt"):
if qa.get(key):
fields[key] = resolve(folder, qa[key])
for voice, values in qa["voices"].items():
row = dict(base, **fields, asset_id=qa["id"], voice=voice,
question=values["question"], answer=values["answer"],
question_audio_path=resolve(folder, values["audio_path"]),
question_audio_sha256=values["audio_sha256"])
for key in ("asr_receipt", "question_translation_receipt", "answer_translation_receipt"):
if values.get(key):
row[key] = resolve(folder, values[key])
yield row