Download benchmark/external_visual.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 7.95 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/external_visual.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/benchmark/external_visual.py
-
curl -L -o external_visual.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/external_visual.py
7.95 kB
| """Benchmark-only acquisition and provenance. No training dataset is modified. | |
| `fetch --media` resumes official OVO source archives (not redundant chunked | |
| archives). `inventory` is a structural filter, NOT a visual quality approval. | |
| """ | |
| import argparse | |
| from collections import Counter | |
| from concurrent.futures import ThreadPoolExecutor | |
| import io | |
| import json | |
| from pathlib import Path, PurePosixPath | |
| import shutil | |
| import tarfile | |
| import time | |
| import requests | |
| from common import sha256, write_json | |
| DEFAULT = Path('/mnt/workspace/hfd_bench_external_261013') | |
| OVO = 'https://raw.githubusercontent.com/JoeLeelyf/OVO-Bench/main/data/ovo_bench_new.json' | |
| TASKS = {'EPM', 'HLD', 'ASI', 'STU', 'OJR', 'ATR', 'ACR', 'OCR'} | |
| DEFERRED_ROOTS = ('Ego4D/', 'OpenEQA/') # Do not infer signed access from a mirror. | |
| def fetch(root, media=False): | |
| from huggingface_hub import hf_hub_download | |
| root.mkdir(parents=True, exist_ok=True) | |
| path = root/'annotations/ovo.json' | |
| if not path.exists(): | |
| response = requests.get(OVO, timeout=60); response.raise_for_status() | |
| write_json(path, response.json()) | |
| write_json(path.with_suffix('.provenance.json'), | |
| dict(url=OVO, sha256=sha256(path), retrieved_at=time.time())) | |
| jobs = [('JoeLeelyf/OVO-Bench', 'README.md'), | |
| ('stdKonjac/LiveSports-3K', 'README.md'), | |
| ('stdKonjac/LiveSports-3K', 'LiveSports-3K-CC.json'), | |
| ('stdKonjac/LiveSports-3K', 'LiveSports-3K-QA.json'), | |
| ('Hothan/Omni-DuplexEval', 'README.md')] | |
| if media: | |
| jobs += [('JoeLeelyf/OVO-Bench', 'src_videos.tar.part'+s) | |
| for s in ('aa', 'ab', 'ac', 'ad', 'ae')] | |
| revisions = {} | |
| for repo, _ in jobs: | |
| if repo not in revisions: | |
| response = requests.get('https://huggingface.co/api/datasets/'+repo, timeout=60) | |
| response.raise_for_status(); revisions[repo] = response.json()['sha'] | |
| frozen = root/'revisions.json' | |
| if frozen.exists(): | |
| revisions = json.loads(frozen.read_text()) | |
| else: | |
| write_json(frozen, revisions) | |
| def one(job): | |
| repo, name = job | |
| out = hf_hub_download(repo, name, repo_type='dataset', revision=revisions[repo], | |
| local_dir=root/'downloads'/repo.split('/')[-1]) | |
| print('DOWNLOADED', repo, name, Path(out).stat().st_size, flush=True) | |
| return out | |
| with ThreadPoolExecutor(max_workers=4) as pool: | |
| list(pool.map(one, jobs)) | |
| def exclusion(row): | |
| if row['task'] not in TASKS: | |
| return 'anticipation_or_proactive_task_not_prefix_qa' | |
| if row['video'].startswith(DEFERRED_ROOTS): | |
| return 'source_access_not_confirmed' | |
| if row['video'].startswith('YouTube_Games/'): | |
| return 'game_visual_not_primary_robot_scenario' | |
| if not 5 <= float(row['realtime']) <= 120: | |
| return 'query_outside_current_session_length_budget' | |
| if not row.get('question', '').strip() or not row.get('answer', '').strip(): | |
| return 'empty_text' | |
| if not 0 <= int(row['gt']) < len(row['options']): | |
| return 'invalid_option_index' | |
| # Do not require answer == option: official full-sentence references often | |
| # paraphrase the correct option. Preserve both, never overwrite either. | |
| return None | |
| def inventory(root): | |
| rows = json.loads((root/'annotations/ovo.json').read_text()) | |
| accepted, rejected = [], [] | |
| for row in rows: | |
| reason = exclusion(row) | |
| if reason: | |
| rejected.append(dict(id=row['id'], reason=reason)); continue | |
| accepted.append(dict(id=f"ovo_{row['id']:04d}", source='OVO-Bench', | |
| original=row, video_relative=row['video'], question_time=float(row['realtime']), | |
| question=row['question'].strip(), answer=row['answer'].strip(), | |
| options=row['options'], correct_option=int(row['gt']), | |
| visual_review_status='pending', | |
| protocol='HFD spoken open QA adaptation; original options retained for later evaluation', | |
| # No time relocation, video speed change, loop or history crop. | |
| video_start_seconds=0.0)) | |
| write_json(root/'candidates.json', accepted) | |
| write_json(root/'structural_rejections.json', rejected) | |
| summary = dict(official_records=len(rows), structurally_eligible=len(accepted), | |
| unique_videos=len({r['video_relative'] for r in accepted}), | |
| tasks=dict(Counter(r['original']['task'] for r in accepted)), | |
| rejections=dict(Counter(r['reason'] for r in rejected)), | |
| visually_approved=0, training_ready=False, | |
| other_sources={ | |
| 'Omni-DuplexEval': 'Deferred: continuous descriptions / proactive events need a different response protocol, not fabricated one-shot QA.', | |
| 'LiveSports-3K-CC': 'Not used as factual QA: ASR commentary includes nonvisual biography/context.', | |
| 'LiveSports-3K-QA': 'Potential alternative, not CC. Audio-dependent questions require filtering; not silently substituted.'}) | |
| write_json(root/'inventory.json', summary) | |
| print(json.dumps(summary, ensure_ascii=False, indent=2), flush=True) | |
| class JoinedReader(io.RawIOBase): | |
| """Read split tar in order without writing a second 46 GB joined archive.""" | |
| def __init__(self, paths): | |
| self.paths = iter(paths); self.current = None | |
| def readable(self): | |
| return True | |
| def readinto(self, buffer): | |
| if self.current is None: | |
| path = next(self.paths, None) | |
| if path is None: | |
| return 0 | |
| self.current = path.open('rb') | |
| count = self.current.readinto(buffer) | |
| if count: | |
| return count | |
| self.current.close(); self.current = None | |
| return self.readinto(buffer) | |
| def close(self): | |
| if self.current is not None: | |
| self.current.close() | |
| super().close() | |
| def extract(root): | |
| wanted = {r['video_relative'] for r in json.loads((root/'candidates.json').read_text())} | |
| parts = [root/'downloads/OVO-Bench'/('src_videos.tar.part'+s) for s in ('aa','ab','ac','ad','ae')] | |
| found = set(); sizes = {} | |
| with JoinedReader(parts) as joined, io.BufferedReader(joined, 8*1024*1024) as stream: | |
| with tarfile.open(fileobj=stream, mode='r|') as archive: | |
| for member in archive: | |
| name = PurePosixPath(member.name) | |
| if name.is_absolute() or '..' in name.parts or not member.isfile(): | |
| continue | |
| matches = [str(PurePosixPath(*name.parts[i:])) for i in range(len(name.parts))] | |
| key = next((s for s in matches if s in wanted), None) | |
| if key is None: | |
| continue | |
| dest = root/'media/OVO-Bench'/key | |
| if dest.is_file() and dest.stat().st_size == member.size: | |
| found.add(key); sizes[key] = member.size; continue | |
| dest.parent.mkdir(parents=True, exist_ok=True) | |
| with archive.extractfile(member) as source, dest.with_suffix('.partial').open('wb') as out: | |
| shutil.copyfileobj(source, out, 8*1024*1024) | |
| dest.with_suffix('.partial').replace(dest) | |
| found.add(key); sizes[key] = member.size | |
| print('EXTRACTED',len(found),'/',len(wanted),key,flush=True) | |
| write_json(root/'extraction.json', dict(found=sorted(found), missing=sorted(wanted-found), bytes=sum(sizes.values()))) | |
| if wanted-found: | |
| raise RuntimeError(f'Missing {len(wanted-found)} requested videos; inspect extraction.json') | |
| if __name__ == '__main__': | |
| p = argparse.ArgumentParser(description=__doc__) | |
| p.add_argument('command', choices=['fetch','inventory','extract']) | |
| p.add_argument('--root', type=Path, default=DEFAULT) | |
| p.add_argument('--media', action='store_true') | |
| a = p.parse_args() | |
| if a.command == 'fetch': fetch(a.root, a.media) | |
| elif a.command == 'inventory': inventory(a.root) | |
| else: extract(a.root) | |