import json import random import pandas as pd from tqdm import tqdm import os import re DEFINED_COMPREHENSION_PROMPTS = [ "Give me a summary of the {}.", "What is the {} about?", "Tell me about the {}.", "Give me a brief summary of the {}.", "What is the main idea of the {}?", "Give me a short summary of the {}.", "Generate a caption about {}.", "Summarize the {}.", "What is the {}?", "What can you see from the {}?", "What is the main point of the {}?", "Give me a short description of the {}.", "What is the {} trying to say?", "Give me a brief description of the {}.", "What is the {} saying?", "What is the {} trying to convey?", "What is the {} trying to express?", "Can you outline the essence of the {}?", "Describe the core message of the {}.", "Provide an overview of the {}.", "What's the central theme of the {}?", "Could you encapsulate the essence of the {}?", "I'm curious about the essence of the {}.", "Could you distill the {} into a few sentences?", "Tell me the key points of the {}.", "What are the key elements of the {}?", "I'd like to know more about the {}.", "What's the primary focus of the {}?", "What's the {} really about?", "Give me an essence of the {}.", "What's the {} conveying?", "What's the central idea behind the {}?" ] DEFINED_GENERATION_PROMPTS = { "Generate a {} about {}.", "Create a {} about {}.", "Show me a {} about {}.", } IMAGE_KEYWORDS = ['picture', 'image', 'scene', 'photograph', 'snapshot', 'illustration', 'vision', 'photo', 'drawing', 'painting', 'view', 'vista'] VIDEO_KEYWORDS = ['video', 'scene', 'film', 'clip', 'visual content', 'movie', 'motion picture', 'footage', 'cinematic work', 'video clip', 'video recording', 'video content'] AUDIO_KEYWORDS = ['sound', 'audio', 'recording', 'melody', 'voice', 'music', 'rhythm', 'tune', 'song', 'soundtrack', 'audio clip', 'audio recording', 'audio content'] INSTRUCTION_PROMPT = ['{} me a {} about ', '{} me a {} of ', '{} {} from ', '{} a {} where ', 'I\'d love to {} a {} of ', 'I\'d like to {} a {} of ', 'I\'d like you to {} a {} of ', 'please {} a {} of ', 'Can you {} me a {} of ', 'Could you {} me a {} of ', 'I want you to {} a {} of ', 'I need you to {} a {} of ', 'I would like you to {} a {} of '] PRODUCE_KEYWORDS = ['generate', 'show', 'synthesize', 'produce', 'create', 'yield', 'form', 'manufacture', 'fabricate', 'compose', 'originate', 'make', 'render', 'illustrate', 'demonstrate', 'exhibit', 'display', 'depict', 'present', 'visualize', 'design', 'craft', 'develop', 'construct', 'build', 'forge', 'shape', 'mold', 'fashion', 'conjure'] RESPONSE_TEMPLATE = ['Sure, this is the {} you want.', 'Here is a {} for your reference', 'for reference, is this ok?', 'Ok.', 'No problem.', 'Sure.', 'how about this?', 'i would like to help.', 'this is what you want.', 'this is what you look for.', 'Certainly!', 'Sure thing!', 'of course', 'Absolutely!', 'Definitely', 'Certainly!', 'Sure thing!', 'Of course!'] def load_data(file_path): with open(file_path, 'r') as f: data = json.load(f) return data def preprocess_generation_data(data, modality='image', save_path=None): """ Preprocess the data for generation task, add a conversation between human and GPT """ new_data = [] for idx, d in tqdm(enumerate(data), total=len(data)): caption = d['caption'] if modality == 'image': image_name = d['image_name'] instruction = random.choice(INSTRUCTION_PROMPT).format(random.choice(PRODUCE_KEYWORDS), random.choice(IMAGE_KEYWORDS)) + caption new_data.append({ "output_image": image_name, "image_captions": caption, "image_caption_embeddings": f'{image_name}.npy', "conversations": [ { "from": "human", "value": instruction }, { "from": "gpt", "value": random.choice(RESPONSE_TEMPLATE).format(modality) + " " + "" } ] }) elif modality == 'video': video_name = d['video_name'] instruction = random.choice(INSTRUCTION_PROMPT).format(random.choice(PRODUCE_KEYWORDS), random.choice(VIDEO_KEYWORDS)) + caption new_data.append({ "output_video": video_name, "video_captions": caption, "video_caption_embeddings": f'{video_name}.npy', "conversations": [ { "from": "human", "value": instruction }, { "from": "gpt", "value": random.choice(RESPONSE_TEMPLATE).format(modality) + " " + "