renpas22 commited on
Commit
e20135f
·
1 Parent(s): 61dbc34

Add VLM support to trainer with auto-detection

Browse files
Files changed (1) hide show
  1. src/reasoning/step_level_cot.py +75 -10
src/reasoning/step_level_cot.py CHANGED
@@ -114,19 +114,84 @@ class StepLevelCoTTrainer:
114
  logger.info("StepLevelCoTTrainer initialized")
115
 
116
  def _load_model(self) -> None:
117
- """Load vision-language model."""
118
  logger.info(f"Loading model: {self.config.pretrained_model_name_or_path}")
119
 
120
- # Load Qwen-Image model
121
- self.pipeline = QwenImagePipeline.from_pretrained(
122
- self.config.pretrained_model_name_or_path,
123
- torch_dtype=torch.float16 if self.config.mixed_precision == "fp16" else torch.float32,
124
- )
125
 
126
- self.model = self.pipeline.transformer
127
- self.vae = self.pipeline.vae
128
- self.text_encoder = self.pipeline.text_encoder
129
- self.tokenizer = self.pipeline.tokenizer
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
130
 
131
  # Add special tokens for chain-of-thought reasoning
132
  self._add_special_tokens()
 
114
  logger.info("StepLevelCoTTrainer initialized")
115
 
116
  def _load_model(self) -> None:
117
+ """Load vision-language model (supports both VLM and diffusion models)."""
118
  logger.info(f"Loading model: {self.config.pretrained_model_name_or_path}")
119
 
120
+ model_name = self.config.pretrained_model_name_or_path.lower()
121
+
122
+ # Detect model type
123
+ is_vlm = any(x in model_name for x in ["qwen2", "qwen-vl", "internvl", "llava", "cogvlm"])
124
+ is_diffusion = any(x in model_name for x in ["flux", "qwen-image", "stable-diffusion", "sd"])
125
 
126
+ if is_vlm:
127
+ # Load as Vision-Language Model
128
+ from transformers import AutoProcessor, AutoModelForVision2Seq, BitsAndBytesConfig
129
+
130
+ logger.info("Loading as Vision-Language Model...")
131
+
132
+ # Prepare quantization config if enabled
133
+ quantization_config = None
134
+ if self.config.get("quantize", False):
135
+ quantization_config = BitsAndBytesConfig(
136
+ load_in_8bit=True,
137
+ llm_int8_threshold=6.0,
138
+ llm_int8_has_fp16_weight=False,
139
+ )
140
+
141
+ # Load processor and model
142
+ self.tokenizer = AutoProcessor.from_pretrained(
143
+ self.config.pretrained_model_name_or_path,
144
+ trust_remote_code=True
145
+ )
146
+
147
+ torch_dtype = torch.float16 if self.config.mixed_precision == "fp16" else torch.bfloat16
148
+
149
+ self.model = AutoModelForVision2Seq.from_pretrained(
150
+ self.config.pretrained_model_name_or_path,
151
+ torch_dtype=torch_dtype,
152
+ quantization_config=quantization_config,
153
+ trust_remote_code=True,
154
+ device_map="auto"
155
+ )
156
+
157
+ # For VLMs, model itself is the main component
158
+ self.pipeline = None
159
+ self.vae = None
160
+ self.text_encoder = None
161
+
162
+ elif is_diffusion:
163
+ # Load as Diffusion Model
164
+ logger.info("Loading as Diffusion Model...")
165
+
166
+ self.pipeline = QwenImagePipeline.from_pretrained(
167
+ self.config.pretrained_model_name_or_path,
168
+ torch_dtype=torch.float16 if self.config.mixed_precision == "fp16" else torch.float32,
169
+ )
170
+
171
+ self.model = self.pipeline.transformer
172
+ self.vae = self.pipeline.vae
173
+ self.text_encoder = self.pipeline.text_encoder
174
+ self.tokenizer = self.pipeline.tokenizer
175
+ else:
176
+ # Try loading as VLM by default
177
+ logger.warning(f"Unknown model type, attempting to load as VLM...")
178
+ from transformers import AutoProcessor, AutoModelForVision2Seq
179
+
180
+ self.tokenizer = AutoProcessor.from_pretrained(
181
+ self.config.pretrained_model_name_or_path,
182
+ trust_remote_code=True
183
+ )
184
+
185
+ self.model = AutoModelForVision2Seq.from_pretrained(
186
+ self.config.pretrained_model_name_or_path,
187
+ torch_dtype=torch.bfloat16,
188
+ trust_remote_code=True,
189
+ device_map="auto"
190
+ )
191
+
192
+ self.pipeline = None
193
+ self.vae = None
194
+ self.text_encoder = None
195
 
196
  # Add special tokens for chain-of-thought reasoning
197
  self._add_special_tokens()