File size: 7,953 Bytes
fb0074a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
"""Benchmark-only acquisition and provenance. No training dataset is modified.

`fetch --media` resumes official OVO source archives (not redundant chunked
archives). `inventory` is a structural filter, NOT a visual quality approval.
"""
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import io
import json
from pathlib import Path, PurePosixPath
import shutil
import tarfile
import time

import requests
from common import sha256, write_json

DEFAULT = Path('/mnt/workspace/hfd_bench_external_261013')
OVO = 'https://raw.githubusercontent.com/JoeLeelyf/OVO-Bench/main/data/ovo_bench_new.json'
TASKS = {'EPM', 'HLD', 'ASI', 'STU', 'OJR', 'ATR', 'ACR', 'OCR'}
DEFERRED_ROOTS = ('Ego4D/', 'OpenEQA/')  # Do not infer signed access from a mirror.


def fetch(root, media=False):
    from huggingface_hub import hf_hub_download
    root.mkdir(parents=True, exist_ok=True)
    path = root/'annotations/ovo.json'
    if not path.exists():
        response = requests.get(OVO, timeout=60); response.raise_for_status()
        write_json(path, response.json())
        write_json(path.with_suffix('.provenance.json'),
                   dict(url=OVO, sha256=sha256(path), retrieved_at=time.time()))
    jobs = [('JoeLeelyf/OVO-Bench', 'README.md'),
            ('stdKonjac/LiveSports-3K', 'README.md'),
            ('stdKonjac/LiveSports-3K', 'LiveSports-3K-CC.json'),
            ('stdKonjac/LiveSports-3K', 'LiveSports-3K-QA.json'),
            ('Hothan/Omni-DuplexEval', 'README.md')]
    if media:
        jobs += [('JoeLeelyf/OVO-Bench', 'src_videos.tar.part'+s)
                 for s in ('aa', 'ab', 'ac', 'ad', 'ae')]
    revisions = {}
    for repo, _ in jobs:
        if repo not in revisions:
            response = requests.get('https://huggingface.co/api/datasets/'+repo, timeout=60)
            response.raise_for_status(); revisions[repo] = response.json()['sha']
    frozen = root/'revisions.json'
    if frozen.exists():
        revisions = json.loads(frozen.read_text())
    else:
        write_json(frozen, revisions)
    def one(job):
        repo, name = job
        out = hf_hub_download(repo, name, repo_type='dataset', revision=revisions[repo],
                              local_dir=root/'downloads'/repo.split('/')[-1])
        print('DOWNLOADED', repo, name, Path(out).stat().st_size, flush=True)
        return out
    with ThreadPoolExecutor(max_workers=4) as pool:
        list(pool.map(one, jobs))


def exclusion(row):
    if row['task'] not in TASKS:
        return 'anticipation_or_proactive_task_not_prefix_qa'
    if row['video'].startswith(DEFERRED_ROOTS):
        return 'source_access_not_confirmed'
    if row['video'].startswith('YouTube_Games/'):
        return 'game_visual_not_primary_robot_scenario'
    if not 5 <= float(row['realtime']) <= 120:
        return 'query_outside_current_session_length_budget'
    if not row.get('question', '').strip() or not row.get('answer', '').strip():
        return 'empty_text'
    if not 0 <= int(row['gt']) < len(row['options']):
        return 'invalid_option_index'
    # Do not require answer == option: official full-sentence references often
    # paraphrase the correct option. Preserve both, never overwrite either.
    return None


def inventory(root):
    rows = json.loads((root/'annotations/ovo.json').read_text())
    accepted, rejected = [], []
    for row in rows:
        reason = exclusion(row)
        if reason:
            rejected.append(dict(id=row['id'], reason=reason)); continue
        accepted.append(dict(id=f"ovo_{row['id']:04d}", source='OVO-Bench',
            original=row, video_relative=row['video'], question_time=float(row['realtime']),
            question=row['question'].strip(), answer=row['answer'].strip(),
            options=row['options'], correct_option=int(row['gt']),
            visual_review_status='pending',
            protocol='HFD spoken open QA adaptation; original options retained for later evaluation',
            # No time relocation, video speed change, loop or history crop.
            video_start_seconds=0.0))
    write_json(root/'candidates.json', accepted)
    write_json(root/'structural_rejections.json', rejected)
    summary = dict(official_records=len(rows), structurally_eligible=len(accepted),
        unique_videos=len({r['video_relative'] for r in accepted}),
        tasks=dict(Counter(r['original']['task'] for r in accepted)),
        rejections=dict(Counter(r['reason'] for r in rejected)),
        visually_approved=0, training_ready=False,
        other_sources={
            'Omni-DuplexEval': 'Deferred: continuous descriptions / proactive events need a different response protocol, not fabricated one-shot QA.',
            'LiveSports-3K-CC': 'Not used as factual QA: ASR commentary includes nonvisual biography/context.',
            'LiveSports-3K-QA': 'Potential alternative, not CC. Audio-dependent questions require filtering; not silently substituted.'})
    write_json(root/'inventory.json', summary)
    print(json.dumps(summary, ensure_ascii=False, indent=2), flush=True)


class JoinedReader(io.RawIOBase):
    """Read split tar in order without writing a second 46 GB joined archive."""
    def __init__(self, paths):
        self.paths = iter(paths); self.current = None

    def readable(self):
        return True

    def readinto(self, buffer):
        if self.current is None:
            path = next(self.paths, None)
            if path is None:
                return 0
            self.current = path.open('rb')
        count = self.current.readinto(buffer)
        if count:
            return count
        self.current.close(); self.current = None
        return self.readinto(buffer)

    def close(self):
        if self.current is not None:
            self.current.close()
        super().close()


def extract(root):
    wanted = {r['video_relative'] for r in json.loads((root/'candidates.json').read_text())}
    parts = [root/'downloads/OVO-Bench'/('src_videos.tar.part'+s) for s in ('aa','ab','ac','ad','ae')]
    found = set(); sizes = {}
    with JoinedReader(parts) as joined, io.BufferedReader(joined, 8*1024*1024) as stream:
        with tarfile.open(fileobj=stream, mode='r|') as archive:
            for member in archive:
                name = PurePosixPath(member.name)
                if name.is_absolute() or '..' in name.parts or not member.isfile():
                    continue
                matches = [str(PurePosixPath(*name.parts[i:])) for i in range(len(name.parts))]
                key = next((s for s in matches if s in wanted), None)
                if key is None:
                    continue
                dest = root/'media/OVO-Bench'/key
                if dest.is_file() and dest.stat().st_size == member.size:
                    found.add(key); sizes[key] = member.size; continue
                dest.parent.mkdir(parents=True, exist_ok=True)
                with archive.extractfile(member) as source, dest.with_suffix('.partial').open('wb') as out:
                    shutil.copyfileobj(source, out, 8*1024*1024)
                dest.with_suffix('.partial').replace(dest)
                found.add(key); sizes[key] = member.size
                print('EXTRACTED',len(found),'/',len(wanted),key,flush=True)
    write_json(root/'extraction.json', dict(found=sorted(found), missing=sorted(wanted-found), bytes=sum(sizes.values())))
    if wanted-found:
        raise RuntimeError(f'Missing {len(wanted-found)} requested videos; inspect extraction.json')


if __name__ == '__main__':
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument('command', choices=['fetch','inventory','extract'])
    p.add_argument('--root', type=Path, default=DEFAULT)
    p.add_argument('--media', action='store_true')
    a = p.parse_args()
    if a.command == 'fetch': fetch(a.root, a.media)
    elif a.command == 'inventory': inventory(a.root)
    else: extract(a.root)