StreamingBench / prepare_benchmark_instant.py
GuanHuankang's picture
Add files using upload-large-folder tool
4599420 verified
Raw History Blame Contribute Delete
3.05 kB
from pathlib import Path
import glob
import json
import csv
import tqdm
import hashlib
from convert_utils.base_convertor import BaseConvertor
from time_utils.time_utils import format_time, time_to_seconds, seconds_to_time
class PrepareBenchmarkTemplate(BaseConvertor):
VIDEO_ROOT = "/home/dyvm6xra/dyvm6xrauser36/dyvm6xrauser36/huankguan2/Project/OmniEval/benchmarks/streamingbench/StreamingBench/videos"
def read_data(self, data_file: str) -> list:
with open(data_file, mode="r", encoding="utf-8") as f:
reader = csv.DictReader(f)
data = list(reader)
return data
def get_video_uuid(self, item) -> str:
video_uuid = hashlib.sha256(
self.get_video_path(item).encode("utf-8")
).hexdigest()[:8]
return video_uuid
def get_video_path(self, item) -> str:
task_name, sample_name, sample_id, _ = item["question_id"].split("_")
video_path = (
f"{self.VIDEO_ROOT}/{task_name}/{sample_name}_{sample_id}/video.mp4"
)
return video_path
def get_stream_addr(self, item) -> str:
return f"/home/dyvm6xra/dyvm6xrauser36/dyvm6xrauser36/huankguan2/Project/OmniEval/benchmarks/streamingbench/stream_chunks/{self.get_video_uuid(item)}"
def build_sqa(self, item) -> list:
"""event_id: int, timestamp: str"""
question_t = format_time(item["time_stamp"])
response_t = question_t
event_ts = [question_t, response_t]
def auto_fill_label(options):
options = eval(options)
for i in range(len(options)):
p = str(chr(ord('A') + i)) + ". "
if not options[i].startswith(p):
options[i] = p + options[i]
return str(options)
sqa = []
for i, ts in enumerate(self.deduplicate_and_sorting(event_ts)):
event = {
"event_id": i + 1,
"timestamp": ts,
"question_id": item["question_id"],
}
if ts == question_t:
event = event | {
"task_type": item["task_type"],
"question": item["question"],
"options": auto_fill_label(item["options"]),
}
if ts == response_t:
event = event | {"response": item["answer"]}
sqa.append(event)
return sqa
inputs = [
(
"contextual_understanding.jsonl",
["StreamingBench/annotations/Contextual_Understanding.csv"],
),
(
"omni_source_understanding.jsonl",
["StreamingBench/annotations/Omni_Source_Understanding.csv"],
),
(
"real_time_visual_understanding.jsonl",
["StreamingBench/annotations/Real_Time_Visual_Understanding.csv"],
),
]
for outfile, samples in inputs:
converter = PrepareBenchmarkTemplate(outfile=outfile, start_id=1)
converter.clear()
for x in tqdm.tqdm(samples):
converter.convert(x)
print("done:", outfile)