"""Portable image/video QA records. No training/model dependencies or GPU use.""" from __future__ import annotations import json import fcntl import hashlib import stat from functools import lru_cache from pathlib import Path FORMAT = "hfd_visual_dataset_v1" VOICES = tuple(f"{sex}_{n:02d}_{lang}" for lang in ("en", "zh") for sex in ("female", "male") for n in (1, 2)) @lru_cache(maxsize=131072) def _parent(path): return Path(path).resolve() @lru_cache(maxsize=262144) def resolve_file(folder, relative): folder = _parent(str(folder)) relative = Path(relative) if relative.is_absolute() or '..' in relative.parts or not relative.name: raise ValueError(f"Dataset path escapes package: {relative}") parent = folder for part in relative.parts[:-1]: parent = _child_directory(str(parent), part) path = parent / relative.name if path.is_symlink(): path = path.resolve() if not path.is_relative_to(folder): raise ValueError(f"Dataset path escapes package: {relative}") return str(path) @lru_cache(maxsize=131072) def _child_directory(parent, name): # parent is already resolved. Path.resolve() for every new audio directory # repeats lstat on the whole CPFS ancestry; inspect only its new component. path = Path(parent) / name try: mode = path.lstat().st_mode except FileNotFoundError: return path # Same missing-file behavior as resolve(strict=False). return path.resolve() if stat.S_ISLNK(mode) else path def _manifest_file(folder, relative): """Validate metadata without issuing a million CPFS stats at startup.""" relative = Path(relative) if relative.is_absolute() or '..' in relative.parts or not relative.name: raise ValueError(f'Dataset path escapes package: {relative}') return str(_parent(str(folder)) / relative) def resolve_overlay_row(row): """Resolve every selected dependency BEFORE reading any selected file.""" folder = row.get('_visual_bundle_root') if folder is None: return row result = dict(row) result.pop('_visual_bundle_root') for key in ('image_path', 'video_path', 'visual_features_path', 'question_audio_path', 'visual_receipt', 'live_qa_receipt', 'asr_receipt', 'question_translation_receipt', 'answer_translation_receipt'): if result.get(key): relative = Path(result[key]).relative_to(folder) result[key] = resolve_file(folder, str(relative)) return result def _digest(path): value = hashlib.sha256() with Path(path).open('rb') as handle: for block in iter(lambda: handle.read(8 * 1024 * 1024), b''): value.update(block) return value.hexdigest() def _verify_publication(root, summaries, base_model): # Called under the same publication lock as record iteration. No old # source directories or media rehashing are needed at training startup. for kind, summary in zip(('images', 'videos'), summaries): if _digest(root / kind / 'manifest.jsonl') != summary['manifest_sha256']: raise ValueError('Unified visual manifest changed after publication') if base_model is not None and summaries[1]['counts'].get('accepted_questions', 0): inputs = json.loads((root / 'videos/inputs.json').read_text()) contracts = [r.get('provenance', {}).get('contract') for r in inputs] contracts = [c for c in contracts if c is not None] if not contracts: raise ValueError('Unified video package has no encoder contract') expected = {'format': 'hfd_dynamic_camera_v1', 'camera_fps': 8, 'side': 224, 'audio_window': .6, 'semantic_stride': .4, 'dtype': 'bfloat16', 'vision_packet_batch': 1} expected.update(model_config_sha256=_digest(Path(base_model) / 'config.json'), weight_index_sha256=_digest(Path(base_model) / 'model.safetensors.index.json')) if any(any(c.get(k) != v for k, v in expected.items()) for c in contracts): raise ValueError('Unified video frontend does not match active Qwen/geometry') def records(root, kind=None, *, accepted_only=True, split=None, verify=False, base_model=None): """One record per unique visual; questions contain all eight voice variants. By default pending/rejected QA cannot enter a training reader. This is an asset-level admission, not evidence that a complete training run passed. """ root = Path(root) with (root / "images/build.lock").open("r") as lock: fcntl.flock(lock, fcntl.LOCK_SH) summaries = [json.loads((root / k / "summary.json").read_text()) for k in ("images", "videos")] if any(s["format"] != FORMAT for s in summaries): raise ValueError("Unsupported dataset") if summaries[0]["generation"] != summaries[1]["generation"]: raise RuntimeError("Dataset publication in progress; retry after build completes") if verify: _verify_publication(root, summaries, base_model) kinds = ("images", "videos") if kind is None else (kind,) for k in kinds: if k not in ("images", "videos"): raise ValueError(k) with (root / k / "manifest.jsonl").open() as handle: for line in handle: row = json.loads(line) if split is not None and row["split"] != split: continue row["questions"] = [q for q in row["questions"] if not accepted_only or q["status"] == "accepted"] if row["questions"]: yield row def overlay_rows(root, kind=None, *, split=None, sample_per_group=False, verify=False, base_model=None, lazy_paths=False): """Adapt both modalities to the existing motion-overlay row schema. All dependency paths are resolved inside this bundle; original source directories are not necessary. Does not modify production configuration. """ root = Path(root) resolve = _manifest_file if lazy_paths else resolve_file seen_groups = set() for record in records(root, kind, split=split, verify=verify, base_model=base_model): group = (record["kind"], record["split"]) if sample_per_group: if group in seen_groups: continue seen_groups.add(group) record["questions"] = record["questions"][:1] folder = root / record["kind"] media_key = "video" if record["kind"] == "videos" else "image" base = dict(asset_id=record["id"], split=record["split"], data_domain="unified_" + record["kind"], visual_features_path=resolve(folder, record["features_path"]), **{media_key + "_path": resolve(folder, record["media_path"]), media_key + "_sha256": record["media_sha256"]}) if lazy_paths: base['_visual_bundle_root'] = str(_parent(str(folder))) if media_key == "video": base.update(duration_seconds=record["duration_seconds"], parent_video_id=record.get("parent_video_id", record["id"])) for qa in record["questions"]: if set(qa["voices"]) != set(VOICES): raise ValueError("Incomplete eight-voice QA") fields = {key: qa[key] for key in ("causal_qa_points", "live_qa_windows", "earliest_answer_video_seconds") if key in qa} for key in ("visual_receipt", "live_qa_receipt"): if qa.get(key): fields[key] = resolve(folder, qa[key]) for voice, values in qa["voices"].items(): row = dict(base, **fields, asset_id=qa["id"], voice=voice, question=values["question"], answer=values["answer"], question_audio_path=resolve(folder, values["audio_path"]), question_audio_sha256=values["audio_sha256"]) for key in ("asr_receipt", "question_translation_receipt", "answer_translation_receipt"): if values.get(key): row[key] = resolve(folder, values[key]) yield row