StreamingBench / prepare_benchmark_future.py
GuanHuankang's picture
Upload prepare_benchmark_future.py with huggingface_hub
57d82c4 verified
Raw History Blame Contribute Delete
3.34 kB
from pathlib import Path
import glob
import json
import csv
import tqdm
import hashlib
from convert_utils.base_convertor import BaseConvertor
from time_utils.time_utils import format_time, time_to_seconds, seconds_to_time
prompt = '''You are an advanced video question-answering AI assistant. You have been provided with some frames from the video and a multiple-choice question related to the video. Your task is to carefully analyze the video and provide the best answer to question, choosing from the four options provided. Respond with only the letter (A, B, C, or D) of the correct option.
Question: {}
Is it the right time to output {}?
You can only answer yes or no. The answer is:
'''
class PrepareBenchmarkTemplate(BaseConvertor):
VIDEO_ROOT = "/home/dyvm6xra/dyvm6xrauser36/dyvm6xrauser36/huankguan2/Project/OmniEval/benchmarks/streamingbench/StreamingBench/videos"
def read_data(self, data_file: str) -> list:
with open(data_file, mode="r", encoding="utf-8") as f:
reader = csv.DictReader(f)
data = list(reader)
return data
def get_video_uuid(self, item) -> str:
video_uuid = hashlib.sha256(
self.get_video_path(item).encode("utf-8")
).hexdigest()[:8]
return video_uuid
def get_video_path(self, item) -> str:
task_name, sample_name, sample_id, _ = item["question_id"].split("_")
video_path = (
f"{self.VIDEO_ROOT}/{task_name}/{sample_name}_{sample_id}/video.mp4"
)
return video_path
def get_stream_addr(self, item) -> str:
return f"/home/dyvm6xra/dyvm6xrauser36/dyvm6xrauser36/huankguan2/Project/OmniEval/benchmarks/streamingbench/stream_chunks/{self.get_video_uuid(item)}"
def build_sqa(self, item) -> list:
"""event_id: int, timestamp: str"""
question_t = format_time(item["time_stamp"])
response_t = format_time(item.get("ground_truth_time_stamp", question_t))
# question_id = item["question_id"]
event_dict = {
"question_id": item["question_id"],
"task_type": item["task_type"],
"question": prompt.format(item["question"], item["ground_truth_output"]),
"response": None
}
qt = time_to_seconds(question_t)
rt = time_to_seconds(response_t)
round_t = [ max(qt, rt+i) for i in range(-4, 5)]
event_ts = [question_t] + [seconds_to_time(t) for t in round_t]
sqa = []
for i, ts in enumerate(self.deduplicate_and_sorting(event_ts)):
event = {
"event_id": i + 1,
"timestamp": ts,
}
if ts >= response_t:
event_dict["response"] = "yes"
else:
event_dict["response"] = "no"
sqa.append(event | event_dict)
return sqa
inputs = [
(
"proactive_output_50.jsonl",
["StreamingBench/annotations/Proactive_Output_50.csv"],
),
(
"proactive_output.jsonl",
["StreamingBench/annotations/Proactive_Output.csv"],
),
]
for outfile, samples in inputs:
converter = PrepareBenchmarkTemplate(outfile=outfile, start_id=1)
converter.clear()
for x in tqdm.tqdm(samples):
converter.convert(x)
print("done:", outfile)