File size: 3,600 Bytes
0ab2566 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | from typing import Dict, Optional, List
from dataclasses import dataclass, field
import transformers
from supported_models import MODEL_HF_PATH, MODEL_FAMILIES
@dataclass
class ModelArguments:
model_id: str = field(default="llava-1.5-7b")
model_local_path: Optional[str] = field(default=None)
def __post_init__(self):
assert self.model_id in MODEL_HF_PATH, f"Unknown model_id: {self.model_id}"
self.model_hf_path: str = MODEL_HF_PATH[self.model_id]
assert self.model_id in MODEL_FAMILIES, f"Unknown model_id: {self.model_id}"
self.model_family_id: str = MODEL_FAMILIES[self.model_id]
if not self.model_local_path:
self.model_local_path = self.model_hf_path
@dataclass
class DataArguments:
data_path: str = field(
default=None, metadata={"help": "Path to the training data json file."}
)
eval_data_path: Optional[str] = field(
default=None, metadata={"help": "Path to the evaluation data json file."}
)
image_folder: Optional[str] = field(default=None)
video_folder: Optional[str] = field(default=None)
num_frames: Optional[int] = field(default=8)
user_key: Optional[str] = field(default="human")
assistant_key: Optional[str] = field(default="gpt")
image_data_path: str = field(
default=None, metadata={"help": "Path to the image data json file."}
)
text_data_path: str = field(
default=None, metadata={"help": "Path to the text data json file."}
)
query_data_path: str = field(
default=None, metadata={"help": "Path to the query data json file."}
)
cand_pool_path: str = field(
default=None, metadata={"help": "Path to the cand pool data json file."}
)
instructions_path: str = field(
default=None, metadata={"help": "Path to the instructions data json file."}
)
rerank_data_path: str = field(
default=None, metadata={"help": "Path to the rerank data json file."}
)
image_path_prefix: str = field(
default=None, metadata={"help": "Path to the image files."}
)
train_data_path: str = field(
default=None, metadata={"help": "Path to the rerank data json file."}
)
eval_data_path: str = field(
default=None, metadata={"help": "Path to the rerank data json file."}
)
num_train_docs: int = field(default=10)
full_crop_mode: bool = False
class_mode: int = field(default=0)
use_crop: bool = False
dataset_type: str = "TongkuanPointWiseDataset"
format_prompt: str = "format_prompt/short_cot_format.jinja"
max_full_tokens: int = field(default=20)
max_crop_tokens: int = field(default=60)
max_visual_tokens: int = field(default=100)
@dataclass
class TrainingArguments(transformers.TrainingArguments):
model_max_length: int = field(
default=1024,
metadata={
"help": "Maximum sequence length. Sequences will be right padded (and possibly truncated)."
},
)
use_flash_attn: bool = False
train_vision_encoder: bool = False
train_vision_projector: bool = False
vision_projector_lr: float = None
tp_size: int = field(default=1)
def __post_init__(self):
super().__post_init__()
self.remove_unused_columns = False
@dataclass
class LoraArguments:
use_lora: bool = True
use_vision_lora: bool = True
q_lora: bool = False
lora_r: int = 64
lora_alpha: int = 16
lora_dropout: float = 0.05
lora_weight_path: str = ""
lora_bias: str = "none"
vision_lora_r: int = 16
vision_lora_alpha: int = 16
|