Download src/psi/data/seed/visual_dataset.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 8.4 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/data/seed/visual_dataset.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/data/seed/visual_dataset.py
-
curl -L -o visual_dataset.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/data/seed/visual_dataset.py
8.4 kB
| """Portable image/video QA records. No training/model dependencies or GPU use.""" | |
| from __future__ import annotations | |
| import json | |
| import fcntl | |
| import hashlib | |
| import stat | |
| from functools import lru_cache | |
| from pathlib import Path | |
| FORMAT = "hfd_visual_dataset_v1" | |
| VOICES = tuple(f"{sex}_{n:02d}_{lang}" for lang in ("en", "zh") | |
| for sex in ("female", "male") for n in (1, 2)) | |
| def _parent(path): | |
| return Path(path).resolve() | |
| def resolve_file(folder, relative): | |
| folder = _parent(str(folder)) | |
| relative = Path(relative) | |
| if relative.is_absolute() or '..' in relative.parts or not relative.name: | |
| raise ValueError(f"Dataset path escapes package: {relative}") | |
| parent = folder | |
| for part in relative.parts[:-1]: | |
| parent = _child_directory(str(parent), part) | |
| path = parent / relative.name | |
| if path.is_symlink(): | |
| path = path.resolve() | |
| if not path.is_relative_to(folder): | |
| raise ValueError(f"Dataset path escapes package: {relative}") | |
| return str(path) | |
| def _child_directory(parent, name): | |
| # parent is already resolved. Path.resolve() for every new audio directory | |
| # repeats lstat on the whole CPFS ancestry; inspect only its new component. | |
| path = Path(parent) / name | |
| try: | |
| mode = path.lstat().st_mode | |
| except FileNotFoundError: | |
| return path # Same missing-file behavior as resolve(strict=False). | |
| return path.resolve() if stat.S_ISLNK(mode) else path | |
| def _manifest_file(folder, relative): | |
| """Validate metadata without issuing a million CPFS stats at startup.""" | |
| relative = Path(relative) | |
| if relative.is_absolute() or '..' in relative.parts or not relative.name: | |
| raise ValueError(f'Dataset path escapes package: {relative}') | |
| return str(_parent(str(folder)) / relative) | |
| def resolve_overlay_row(row): | |
| """Resolve every selected dependency BEFORE reading any selected file.""" | |
| folder = row.get('_visual_bundle_root') | |
| if folder is None: | |
| return row | |
| result = dict(row) | |
| result.pop('_visual_bundle_root') | |
| for key in ('image_path', 'video_path', 'visual_features_path', 'question_audio_path', | |
| 'visual_receipt', 'live_qa_receipt', 'asr_receipt', | |
| 'question_translation_receipt', 'answer_translation_receipt'): | |
| if result.get(key): | |
| relative = Path(result[key]).relative_to(folder) | |
| result[key] = resolve_file(folder, str(relative)) | |
| return result | |
| def _digest(path): | |
| value = hashlib.sha256() | |
| with Path(path).open('rb') as handle: | |
| for block in iter(lambda: handle.read(8 * 1024 * 1024), b''): | |
| value.update(block) | |
| return value.hexdigest() | |
| def _verify_publication(root, summaries, base_model): | |
| # Called under the same publication lock as record iteration. No old | |
| # source directories or media rehashing are needed at training startup. | |
| for kind, summary in zip(('images', 'videos'), summaries): | |
| if _digest(root / kind / 'manifest.jsonl') != summary['manifest_sha256']: | |
| raise ValueError('Unified visual manifest changed after publication') | |
| if base_model is not None and summaries[1]['counts'].get('accepted_questions', 0): | |
| inputs = json.loads((root / 'videos/inputs.json').read_text()) | |
| contracts = [r.get('provenance', {}).get('contract') for r in inputs] | |
| contracts = [c for c in contracts if c is not None] | |
| if not contracts: | |
| raise ValueError('Unified video package has no encoder contract') | |
| expected = {'format': 'hfd_dynamic_camera_v1', 'camera_fps': 8, | |
| 'side': 224, 'audio_window': .6, 'semantic_stride': .4, | |
| 'dtype': 'bfloat16', 'vision_packet_batch': 1} | |
| expected.update(model_config_sha256=_digest(Path(base_model) / 'config.json'), | |
| weight_index_sha256=_digest(Path(base_model) / 'model.safetensors.index.json')) | |
| if any(any(c.get(k) != v for k, v in expected.items()) for c in contracts): | |
| raise ValueError('Unified video frontend does not match active Qwen/geometry') | |
| def records(root, kind=None, *, accepted_only=True, split=None, verify=False, base_model=None): | |
| """One record per unique visual; questions contain all eight voice variants. | |
| By default pending/rejected QA cannot enter a training reader. This is an | |
| asset-level admission, not evidence that a complete training run passed. | |
| """ | |
| root = Path(root) | |
| with (root / "images/build.lock").open("r") as lock: | |
| fcntl.flock(lock, fcntl.LOCK_SH) | |
| summaries = [json.loads((root / k / "summary.json").read_text()) | |
| for k in ("images", "videos")] | |
| if any(s["format"] != FORMAT for s in summaries): | |
| raise ValueError("Unsupported dataset") | |
| if summaries[0]["generation"] != summaries[1]["generation"]: | |
| raise RuntimeError("Dataset publication in progress; retry after build completes") | |
| if verify: | |
| _verify_publication(root, summaries, base_model) | |
| kinds = ("images", "videos") if kind is None else (kind,) | |
| for k in kinds: | |
| if k not in ("images", "videos"): | |
| raise ValueError(k) | |
| with (root / k / "manifest.jsonl").open() as handle: | |
| for line in handle: | |
| row = json.loads(line) | |
| if split is not None and row["split"] != split: | |
| continue | |
| row["questions"] = [q for q in row["questions"] | |
| if not accepted_only or q["status"] == "accepted"] | |
| if row["questions"]: | |
| yield row | |
| def overlay_rows(root, kind=None, *, split=None, sample_per_group=False, verify=False, | |
| base_model=None, lazy_paths=False): | |
| """Adapt both modalities to the existing motion-overlay row schema. | |
| All dependency paths are resolved inside this bundle; original source | |
| directories are not necessary. Does not modify production configuration. | |
| """ | |
| root = Path(root) | |
| resolve = _manifest_file if lazy_paths else resolve_file | |
| seen_groups = set() | |
| for record in records(root, kind, split=split, verify=verify, base_model=base_model): | |
| group = (record["kind"], record["split"]) | |
| if sample_per_group: | |
| if group in seen_groups: | |
| continue | |
| seen_groups.add(group) | |
| record["questions"] = record["questions"][:1] | |
| folder = root / record["kind"] | |
| media_key = "video" if record["kind"] == "videos" else "image" | |
| base = dict(asset_id=record["id"], split=record["split"], | |
| data_domain="unified_" + record["kind"], | |
| visual_features_path=resolve(folder, record["features_path"]), | |
| **{media_key + "_path": resolve(folder, record["media_path"]), | |
| media_key + "_sha256": record["media_sha256"]}) | |
| if lazy_paths: | |
| base['_visual_bundle_root'] = str(_parent(str(folder))) | |
| if media_key == "video": | |
| base.update(duration_seconds=record["duration_seconds"], | |
| parent_video_id=record.get("parent_video_id", record["id"])) | |
| for qa in record["questions"]: | |
| if set(qa["voices"]) != set(VOICES): | |
| raise ValueError("Incomplete eight-voice QA") | |
| fields = {key: qa[key] for key in ("causal_qa_points", "live_qa_windows", | |
| "earliest_answer_video_seconds") if key in qa} | |
| for key in ("visual_receipt", "live_qa_receipt"): | |
| if qa.get(key): | |
| fields[key] = resolve(folder, qa[key]) | |
| for voice, values in qa["voices"].items(): | |
| row = dict(base, **fields, asset_id=qa["id"], voice=voice, | |
| question=values["question"], answer=values["answer"], | |
| question_audio_path=resolve(folder, values["audio_path"]), | |
| question_audio_sha256=values["audio_sha256"]) | |
| for key in ("asr_receipt", "question_translation_receipt", "answer_translation_receipt"): | |
| if values.get(key): | |
| row[key] = resolve(folder, values[key]) | |
| yield row | |