StreamingBench / prepare_benchmark_sqa.py
GuanHuankang's picture
Add files using upload-large-folder tool
4599420 verified
Raw History Blame Contribute Delete
3.39 kB
from pathlib import Path
import glob
import json
import csv
import tqdm
import hashlib
from convert_utils.base_convertor import BaseConvertor
from time_utils.time_utils import format_time, time_to_seconds, seconds_to_time
class PrepareBenchmarkTemplate(BaseConvertor):
VIDEO_ROOT = "/home/dyvm6xra/dyvm6xrauser36/dyvm6xrauser36/huankguan2/Project/OmniEval/benchmarks/streamingbench/StreamingBench/videos"
def read_data(self, data_file: str) -> list:
with open(data_file, mode="r", encoding="utf-8") as f:
reader = csv.DictReader(f)
data = list(reader)
sample_list = {}
for item in data:
task_name, sample_name, sample_id, _ = item["question_id"].split("_")
video_path = f"{self.VIDEO_ROOT}/{task_name}/{sample_name}_{sample_id}/video.mp4"
video_uuid = hashlib.sha256(video_path.encode('utf-8')).hexdigest()[:8]
seq = sample_list.get(video_uuid, {'video_path': video_path, 'video_uuid': video_uuid, 'qa': []})
seq['qa'].append(item)
sample_list[video_uuid] = seq
return list(sample_list.values())
def get_video_uuid(self, item) -> str:
return item["video_uuid"]
def get_video_path(self, item) -> str:
return item["video_path"]
def get_stream_addr(self, item) -> str:
return f"/home/dyvm6xra/dyvm6xrauser36/dyvm6xrauser36/huankguan2/Project/OmniEval/benchmarks/streamingbench/stream_chunks/{self.get_video_uuid(item)}"
def build_sqa(self, item) -> list:
sqa = []
for subitem in item['qa']:
sqa.extend(self.build_qa(subitem))
sqa.sort(key = lambda qa: time_to_seconds(qa["timestamp"]))
for i in range(len(sqa)):
sqa[i]['event_id'] = i + 1
return sqa
def build_qa(self, item):
"""event_id: int, timestamp: str"""
question_t = format_time(item["time_stamp"])
response_t = format_time(item.get("ground_truth_time_stamp", question_t))
event_ts = [question_t, response_t]
def auto_fill_label(options):
options = eval(options)
for i in range(len(options)):
p = str(chr(ord('A') + i)) + ". "
if not options[i].startswith(p):
options[i] = p + options[i]
return str(options)
qa = []
for i, ts in enumerate(self.deduplicate_and_sorting(event_ts)):
event = {
"event_id": i + 1,
"timestamp": ts,
"question_id": item["question_id"],
}
if ts == question_t:
event = event | {
"task_type": item["task_type"],
"question": item["question"],
"options": auto_fill_label(item["options"]),
}
if ts == response_t:
event = event | {"response": item["answer"]}
qa.append(event)
return qa
inputs = [
(
"sequential_question_answering.jsonl",
["StreamingBench/annotations/Sequential_Question_Answering.csv"],
),
]
for outfile, samples in inputs:
converter = PrepareBenchmarkTemplate(outfile=outfile, start_id=1)
converter.clear()
for x in tqdm.tqdm(samples):
converter.convert(x)
print("done:", outfile)