renpas22 commited on
Commit
2b8876a
Β·
1 Parent(s): cd336ff

Add training scripts and configs

Browse files
src/reasoning/__init__.py ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Step-Level Chain of Thought Reasoning for Vision-Language Models
3
+
4
+ This module implements fine-grained structured reasoning with process reward models
5
+ and reinforcement learning for vision-language tasks.
6
+ """
7
+
8
+ from .step_data import ReasoningStep, ReasoningChain, StepDataset
9
+ from .prm import ProcessRewardModel, StepQualityMetrics
10
+ from .rl_trainer import RLReasoningTrainer
11
+ from .inference_scaling import InferenceTimeScaling, MajorityVoting
12
+ from .step_level_cot import StepLevelCoTTrainer
13
+
14
+ __all__ = [
15
+ 'ReasoningStep',
16
+ 'ReasoningChain',
17
+ 'StepDataset',
18
+ 'ProcessRewardModel',
19
+ 'StepQualityMetrics',
20
+ 'RLReasoningTrainer',
21
+ 'InferenceTimeScaling',
22
+ 'MajorityVoting',
23
+ 'StepLevelCoTTrainer',
24
+ ]
src/reasoning/__pycache__/__init__.cpython-313.pyc ADDED
Binary file (842 Bytes). View file
 
src/reasoning/__pycache__/inference_scaling.cpython-313.pyc ADDED
Binary file (17.7 kB). View file
 
src/reasoning/__pycache__/prm.cpython-313.pyc ADDED
Binary file (17.2 kB). View file
 
src/reasoning/__pycache__/rl_trainer.cpython-313.pyc ADDED
Binary file (20 kB). View file
 
src/reasoning/__pycache__/step_data.cpython-313.pyc ADDED
Binary file (19.6 kB). View file
 
src/reasoning/__pycache__/step_level_cot.cpython-313.pyc ADDED
Binary file (21 kB). View file
 
src/reasoning/finevision_dataset.py ADDED
@@ -0,0 +1,393 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Dataset loader for HuggingFaceM4/FineVision.
3
+
4
+ This adapter loads FineVision image-text pairs and optionally generates
5
+ synthetic reasoning chains for step-level CoT training.
6
+ """
7
+
8
+ import json
9
+ import re
10
+ from typing import List, Dict, Optional, Any, Union
11
+ from pathlib import Path
12
+ import torch
13
+ from torch.utils.data import Dataset
14
+ from datasets import load_dataset
15
+ from PIL import Image
16
+ import io
17
+
18
+ from .step_data import (
19
+ ReasoningChain,
20
+ ReasoningStep,
21
+ StepType,
22
+ SPECIAL_TOKENS,
23
+ )
24
+
25
+
26
+ class FineVisionDataset(Dataset):
27
+ """
28
+ Dataset adapter for HuggingFaceM4/FineVision with optional synthetic
29
+ reasoning chain generation.
30
+
31
+ FineVision contains image-text pairs for vision-language tasks. This adapter
32
+ can either use the raw data or generate synthetic reasoning chains to augment
33
+ training with explicit step-by-step reasoning.
34
+ """
35
+
36
+ def __init__(
37
+ self,
38
+ dataset_name: str = "HuggingFaceM4/FineVision",
39
+ subset: Optional[str] = None,
40
+ split: str = "train",
41
+ streaming: bool = True,
42
+ max_samples: Optional[int] = None,
43
+ image_key: str = "image",
44
+ text_key: str = "text",
45
+ generate_synthetic_reasoning: bool = True,
46
+ num_synthetic_steps: int = 3,
47
+ cache_dir: Optional[str] = None,
48
+ ):
49
+ """
50
+ Initialize FineVision dataset adapter.
51
+
52
+ Args:
53
+ dataset_name: HuggingFace dataset identifier
54
+ subset: Dataset subset/config name (if applicable)
55
+ split: Dataset split (train, validation, test)
56
+ streaming: Use streaming mode for large datasets
57
+ max_samples: Maximum number of samples to load (None = all)
58
+ image_key: Column name for images
59
+ text_key: Column name for text/captions
60
+ generate_synthetic_reasoning: If True, create synthetic reasoning steps
61
+ num_synthetic_steps: Number of reasoning steps to generate
62
+ cache_dir: Directory to cache downloaded data
63
+ """
64
+ self.dataset_name = dataset_name
65
+ self.subset = subset
66
+ self.split = split
67
+ self.streaming = streaming
68
+ self.max_samples = max_samples
69
+ self.image_key = image_key
70
+ self.text_key = text_key
71
+ self.generate_synthetic_reasoning = generate_synthetic_reasoning
72
+ self.num_synthetic_steps = num_synthetic_steps
73
+
74
+ print(f"Loading {dataset_name} ({split} split)...")
75
+
76
+ # Load dataset from HuggingFace
77
+ try:
78
+ if streaming:
79
+ self.hf_dataset = load_dataset(
80
+ dataset_name,
81
+ name=subset,
82
+ split=split,
83
+ streaming=True,
84
+ cache_dir=cache_dir,
85
+ )
86
+ self.hf_dataset_iter = iter(self.hf_dataset)
87
+ else:
88
+ self.hf_dataset = load_dataset(
89
+ dataset_name,
90
+ name=subset,
91
+ split=split,
92
+ cache_dir=cache_dir,
93
+ )
94
+ if max_samples:
95
+ self.hf_dataset = self.hf_dataset.select(range(min(max_samples, len(self.hf_dataset))))
96
+ except Exception as e:
97
+ raise RuntimeError(f"Failed to load dataset {dataset_name}: {e}")
98
+
99
+ # Pre-build reasoning chains for non-streaming mode
100
+ if not streaming:
101
+ print("Converting to ReasoningChain format...")
102
+ self.reasoning_chains = []
103
+ for idx, sample in enumerate(self.hf_dataset):
104
+ chain = self._parse_sample_to_chain(sample, idx)
105
+ if chain:
106
+ self.reasoning_chains.append(chain)
107
+ print(f"Loaded {len(self.reasoning_chains)} reasoning chains")
108
+ else:
109
+ self.reasoning_chains = None
110
+
111
+ def __len__(self) -> int:
112
+ if self.streaming:
113
+ return self.max_samples or 10**9 # Large number for streaming
114
+ return len(self.reasoning_chains)
115
+
116
+ def __getitem__(self, idx: int) -> Optional[ReasoningChain]:
117
+ """Get a reasoning chain by index."""
118
+ if self.streaming:
119
+ # For streaming, get next item from iterator
120
+ try:
121
+ sample = next(self.hf_dataset_iter)
122
+ except StopIteration:
123
+ self.hf_dataset_iter = iter(self.hf_dataset)
124
+ sample = next(self.hf_dataset_iter)
125
+ return self._parse_sample_to_chain(sample, idx)
126
+ else:
127
+ return self.reasoning_chains[idx]
128
+
129
+ def _parse_sample_to_chain(self, sample: Dict[str, Any], idx: int) -> Optional[ReasoningChain]:
130
+ """
131
+ Parse a FineVision sample into a ReasoningChain.
132
+
133
+ Args:
134
+ sample: Raw sample from HuggingFace dataset
135
+ idx: Sample index
136
+
137
+ Returns:
138
+ ReasoningChain object or None if parsing fails
139
+ """
140
+ try:
141
+ # Extract fields
142
+ image = sample.get(self.image_key)
143
+ text = sample.get(self.text_key, "")
144
+
145
+ # Convert image to PIL if needed
146
+ if isinstance(image, bytes):
147
+ image = Image.open(io.BytesIO(image)).convert('RGB')
148
+ elif hasattr(image, 'convert'):
149
+ image = image.convert('RGB')
150
+
151
+ # Generate reasoning steps
152
+ if self.generate_synthetic_reasoning:
153
+ steps = self._generate_synthetic_steps(text, self.num_synthetic_steps)
154
+ else:
155
+ # Create single inference step with the text
156
+ steps = [
157
+ ReasoningStep(
158
+ step_id=0,
159
+ step_type=StepType.PERCEPTION,
160
+ description=text,
161
+ confidence=0.9,
162
+ )
163
+ ]
164
+
165
+ # Use text as both question and answer for FineVision
166
+ # (it's more like caption/description dataset)
167
+ prompt = "Describe this image in detail."
168
+ answer = text
169
+
170
+ # Create reasoning chain
171
+ chain = ReasoningChain(
172
+ chain_id=f"finevision_{idx}",
173
+ image_path=None, # Streaming mode, no path
174
+ image=image,
175
+ prompt=prompt,
176
+ steps=steps,
177
+ final_answer=answer,
178
+ is_correct=True, # Assume dataset contains correct descriptions
179
+ metadata={
180
+ "dataset": self.dataset_name,
181
+ "split": self.split,
182
+ "original_idx": idx,
183
+ "synthetic_reasoning": self.generate_synthetic_reasoning,
184
+ }
185
+ )
186
+
187
+ return chain
188
+
189
+ except Exception as e:
190
+ print(f"Warning: Failed to parse sample {idx}: {e}")
191
+ return None
192
+
193
+ def _generate_synthetic_steps(self, text: str, num_steps: int) -> List[ReasoningStep]:
194
+ """
195
+ Generate synthetic reasoning steps from a text description.
196
+
197
+ This creates a multi-step reasoning chain by breaking down the description
198
+ into perception, analysis, and conclusion steps.
199
+
200
+ Args:
201
+ text: Original text description
202
+ num_steps: Number of steps to generate
203
+
204
+ Returns:
205
+ List of ReasoningStep objects
206
+ """
207
+ steps = []
208
+
209
+ if num_steps <= 1:
210
+ # Single step - just use the text
211
+ steps.append(
212
+ ReasoningStep(
213
+ step_id=0,
214
+ step_type=StepType.PERCEPTION,
215
+ description=text,
216
+ confidence=0.9,
217
+ )
218
+ )
219
+ elif num_steps == 2:
220
+ # Two steps: perception + inference
221
+ steps.append(
222
+ ReasoningStep(
223
+ step_id=0,
224
+ step_type=StepType.PERCEPTION,
225
+ description=f"I observe the visual content of the image.",
226
+ confidence=0.9,
227
+ )
228
+ )
229
+ steps.append(
230
+ ReasoningStep(
231
+ step_id=1,
232
+ step_type=StepType.INFERENCE,
233
+ description=text,
234
+ confidence=0.85,
235
+ dependencies=[0],
236
+ )
237
+ )
238
+ elif num_steps >= 3:
239
+ # Three+ steps: perception + composition + inference
240
+ # Split text into chunks if long enough
241
+ sentences = [s.strip() for s in text.split('.') if s.strip()]
242
+
243
+ # Step 1: Perception
244
+ steps.append(
245
+ ReasoningStep(
246
+ step_id=0,
247
+ step_type=StepType.PERCEPTION,
248
+ description="I carefully examine the visual elements in the image.",
249
+ confidence=0.9,
250
+ )
251
+ )
252
+
253
+ # Step 2-N: Composition/Analysis (use text chunks or full text)
254
+ if len(sentences) >= 2 and num_steps > 3:
255
+ # Distribute sentences across middle steps
256
+ sentences_per_step = max(1, len(sentences) // (num_steps - 2))
257
+ for i in range(num_steps - 2):
258
+ start_idx = i * sentences_per_step
259
+ end_idx = start_idx + sentences_per_step if i < num_steps - 3 else len(sentences)
260
+ chunk_text = '. '.join(sentences[start_idx:end_idx])
261
+ if chunk_text and not chunk_text.endswith('.'):
262
+ chunk_text += '.'
263
+
264
+ steps.append(
265
+ ReasoningStep(
266
+ step_id=i + 1,
267
+ step_type=StepType.COMPOSITION,
268
+ description=chunk_text if chunk_text else text,
269
+ confidence=0.85,
270
+ dependencies=list(range(i + 1)),
271
+ )
272
+ )
273
+ else:
274
+ # Not enough sentences, use full text for middle step
275
+ steps.append(
276
+ ReasoningStep(
277
+ step_id=1,
278
+ step_type=StepType.COMPOSITION,
279
+ description=text,
280
+ confidence=0.85,
281
+ dependencies=[0],
282
+ )
283
+ )
284
+
285
+ # Final step: Inference/Conclusion
286
+ if len(steps) < num_steps:
287
+ final_text = "Based on these observations, I can provide a comprehensive description."
288
+ steps.append(
289
+ ReasoningStep(
290
+ step_id=len(steps),
291
+ step_type=StepType.INFERENCE,
292
+ description=final_text,
293
+ confidence=0.8,
294
+ dependencies=list(range(len(steps))),
295
+ )
296
+ )
297
+
298
+ return steps[:num_steps] # Ensure we don't exceed requested steps
299
+
300
+
301
+ def create_finevision_dataloader(
302
+ tokenizer: Any,
303
+ batch_size: int = 4,
304
+ num_workers: int = 4,
305
+ streaming: bool = True,
306
+ max_samples: Optional[int] = None,
307
+ generate_synthetic_reasoning: bool = True,
308
+ **kwargs
309
+ ) -> torch.utils.data.DataLoader:
310
+ """
311
+ Create a DataLoader for FineVision dataset.
312
+
313
+ Args:
314
+ tokenizer: Tokenizer for text encoding
315
+ batch_size: Number of samples per batch
316
+ num_workers: Number of worker processes
317
+ streaming: Use streaming mode
318
+ max_samples: Maximum samples to load
319
+ generate_synthetic_reasoning: Generate synthetic reasoning steps
320
+ **kwargs: Additional arguments for FineVisionDataset
321
+
322
+ Returns:
323
+ Configured DataLoader
324
+ """
325
+ from .step_data import StepDataset
326
+
327
+ # Create FineVision dataset
328
+ finevision_dataset = FineVisionDataset(
329
+ streaming=streaming,
330
+ max_samples=max_samples,
331
+ generate_synthetic_reasoning=generate_synthetic_reasoning,
332
+ **kwargs
333
+ )
334
+
335
+ # If not streaming, wrap in StepDataset for tokenization
336
+ if not streaming and finevision_dataset.reasoning_chains:
337
+ step_dataset = StepDataset(
338
+ reasoning_chains=finevision_dataset.reasoning_chains,
339
+ tokenizer=tokenizer,
340
+ max_steps=10,
341
+ include_visual_features=False,
342
+ filter_incorrect=False,
343
+ )
344
+
345
+ return torch.utils.data.DataLoader(
346
+ step_dataset,
347
+ batch_size=batch_size,
348
+ num_workers=num_workers,
349
+ shuffle=True,
350
+ collate_fn=lambda batch: batch, # Custom collation handled in trainer
351
+ )
352
+ else:
353
+ # For streaming, return raw finevision dataset
354
+ return torch.utils.data.DataLoader(
355
+ finevision_dataset,
356
+ batch_size=batch_size,
357
+ num_workers=0 if streaming else num_workers,
358
+ shuffle=False,
359
+ )
360
+
361
+
362
+ # Example usage
363
+ if __name__ == "__main__":
364
+ # Test dataset loading
365
+ print("Testing FineVision dataset adapter...")
366
+
367
+ dataset = FineVisionDataset(
368
+ streaming=False,
369
+ max_samples=5,
370
+ generate_synthetic_reasoning=True,
371
+ num_synthetic_steps=3,
372
+ )
373
+
374
+ print(f"\nLoaded {len(dataset)} samples")
375
+
376
+ # Show first example
377
+ if len(dataset) > 0:
378
+ chain = dataset[0]
379
+ print(f"\n{'='*60}")
380
+ print("Example Reasoning Chain:")
381
+ print(f"{'='*60}")
382
+ print(f"Chain ID: {chain.chain_id}")
383
+ print(f"Prompt: {chain.prompt}")
384
+ print(f"Number of steps: {len(chain.steps)}")
385
+ print(f"\nSteps:")
386
+ for step in chain.steps:
387
+ print(f" Step {step.step_id} ({step.step_type.value}): {step.description[:100]}...")
388
+ print(f"\nFinal Answer: {chain.final_answer[:200]}...")
389
+ print(f"\n{'='*60}")
390
+ print("\nFormatted with special tokens:")
391
+ print(f"{'='*60}")
392
+ formatted = chain.format_with_tokens()
393
+ print(formatted[:500] + "..." if len(formatted) > 500 else formatted)
src/reasoning/flux_reason_dataset.py ADDED
@@ -0,0 +1,375 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Dataset loader for LucasFang/FLUX-Reason-6M.
3
+
4
+ This adapter converts the FLUX-Reason-6M dataset format into our step-level
5
+ chain-of-thought ReasoningChain format with proper step annotations.
6
+ """
7
+
8
+ import json
9
+ import re
10
+ from typing import List, Dict, Optional, Any, Union
11
+ from pathlib import Path
12
+ import torch
13
+ from torch.utils.data import Dataset
14
+ from datasets import load_dataset
15
+ from PIL import Image
16
+ import io
17
+
18
+ from .step_data import (
19
+ ReasoningChain,
20
+ ReasoningStep,
21
+ StepType,
22
+ SPECIAL_TOKENS,
23
+ )
24
+
25
+
26
+ class FLUXReasonDataset(Dataset):
27
+ """
28
+ Dataset adapter for FLUX-Reason-6M that converts reasoning annotations
29
+ into step-level chain-of-thought format.
30
+
31
+ The FLUX-Reason-6M dataset contains images with reasoning chains. This adapter
32
+ parses the reasoning text and creates structured ReasoningStep objects.
33
+ """
34
+
35
+ def __init__(
36
+ self,
37
+ dataset_name: str = "LucasFang/FLUX-Reason-6M",
38
+ split: str = "train",
39
+ streaming: bool = True,
40
+ max_samples: Optional[int] = None,
41
+ image_key: str = "image",
42
+ reasoning_key: str = "reasoning",
43
+ question_key: str = "question",
44
+ answer_key: str = "answer",
45
+ cache_dir: Optional[str] = None,
46
+ ):
47
+ """
48
+ Initialize FLUX-Reason-6M dataset adapter.
49
+
50
+ Args:
51
+ dataset_name: HuggingFace dataset identifier
52
+ split: Dataset split (train, validation, test)
53
+ streaming: Use streaming mode for large datasets
54
+ max_samples: Maximum number of samples to load (None = all)
55
+ image_key: Column name for images
56
+ reasoning_key: Column name for reasoning text
57
+ question_key: Column name for questions
58
+ answer_key: Column name for final answers
59
+ cache_dir: Directory to cache downloaded data
60
+ """
61
+ self.dataset_name = dataset_name
62
+ self.split = split
63
+ self.streaming = streaming
64
+ self.max_samples = max_samples
65
+ self.image_key = image_key
66
+ self.reasoning_key = reasoning_key
67
+ self.question_key = question_key
68
+ self.answer_key = answer_key
69
+
70
+ print(f"Loading {dataset_name} ({split} split)...")
71
+
72
+ # Load dataset from HuggingFace
73
+ try:
74
+ if streaming:
75
+ self.hf_dataset = load_dataset(
76
+ dataset_name,
77
+ split=split,
78
+ streaming=True,
79
+ cache_dir=cache_dir,
80
+ )
81
+ self.hf_dataset_iter = iter(self.hf_dataset)
82
+ else:
83
+ self.hf_dataset = load_dataset(
84
+ dataset_name,
85
+ split=split,
86
+ cache_dir=cache_dir,
87
+ )
88
+ if max_samples:
89
+ self.hf_dataset = self.hf_dataset.select(range(min(max_samples, len(self.hf_dataset))))
90
+ except Exception as e:
91
+ raise RuntimeError(f"Failed to load dataset {dataset_name}: {e}")
92
+
93
+ # Pre-build reasoning chains for non-streaming mode
94
+ if not streaming:
95
+ print("Converting to ReasoningChain format...")
96
+ self.reasoning_chains = []
97
+ for idx, sample in enumerate(self.hf_dataset):
98
+ chain = self._parse_sample_to_chain(sample, idx)
99
+ if chain:
100
+ self.reasoning_chains.append(chain)
101
+ print(f"Loaded {len(self.reasoning_chains)} reasoning chains")
102
+ else:
103
+ self.reasoning_chains = None
104
+
105
+ def __len__(self) -> int:
106
+ if self.streaming:
107
+ return self.max_samples or 10**9 # Large number for streaming
108
+ return len(self.reasoning_chains)
109
+
110
+ def __getitem__(self, idx: int) -> Optional[ReasoningChain]:
111
+ """Get a reasoning chain by index."""
112
+ if self.streaming:
113
+ # For streaming, get next item from iterator
114
+ try:
115
+ sample = next(self.hf_dataset_iter)
116
+ except StopIteration:
117
+ self.hf_dataset_iter = iter(self.hf_dataset)
118
+ sample = next(self.hf_dataset_iter)
119
+ return self._parse_sample_to_chain(sample, idx)
120
+ else:
121
+ return self.reasoning_chains[idx]
122
+
123
+ def _parse_sample_to_chain(self, sample: Dict[str, Any], idx: int) -> Optional[ReasoningChain]:
124
+ """
125
+ Parse a FLUX-Reason-6M sample into a ReasoningChain.
126
+
127
+ Args:
128
+ sample: Raw sample from HuggingFace dataset
129
+ idx: Sample index
130
+
131
+ Returns:
132
+ ReasoningChain object or None if parsing fails
133
+ """
134
+ try:
135
+ # Extract fields
136
+ image = sample.get(self.image_key)
137
+ reasoning_text = sample.get(self.reasoning_key, "")
138
+ question = sample.get(self.question_key, "")
139
+ answer = sample.get(self.answer_key, "")
140
+
141
+ # Convert image to PIL if needed
142
+ if isinstance(image, bytes):
143
+ image = Image.open(io.BytesIO(image)).convert('RGB')
144
+ elif hasattr(image, 'convert'):
145
+ image = image.convert('RGB')
146
+
147
+ # Parse reasoning text into steps
148
+ steps = self._parse_reasoning_steps(reasoning_text)
149
+
150
+ if not steps:
151
+ # If no steps found, create a single inference step
152
+ steps = [
153
+ ReasoningStep(
154
+ step_id=0,
155
+ step_type=StepType.INFERENCE,
156
+ description=reasoning_text if reasoning_text else "Direct inference",
157
+ confidence=0.8,
158
+ )
159
+ ]
160
+
161
+ # Create reasoning chain
162
+ chain = ReasoningChain(
163
+ chain_id=f"flux_reason_{idx}",
164
+ image_path=None, # Streaming mode, no path
165
+ image=image,
166
+ prompt=question,
167
+ steps=steps,
168
+ final_answer=answer,
169
+ is_correct=True, # Assume dataset contains correct reasoning
170
+ metadata={
171
+ "dataset": self.dataset_name,
172
+ "split": self.split,
173
+ "original_idx": idx,
174
+ }
175
+ )
176
+
177
+ return chain
178
+
179
+ except Exception as e:
180
+ print(f"Warning: Failed to parse sample {idx}: {e}")
181
+ return None
182
+
183
+ def _parse_reasoning_steps(self, reasoning_text: str) -> List[ReasoningStep]:
184
+ """
185
+ Parse reasoning text into structured ReasoningStep objects.
186
+
187
+ This handles various reasoning formats:
188
+ - Numbered steps: "1. ...", "2. ...", etc.
189
+ - Step markers: "Step 1:", "Step 2:", etc.
190
+ - Natural paragraphs separated by newlines
191
+
192
+ Args:
193
+ reasoning_text: Raw reasoning text from dataset
194
+
195
+ Returns:
196
+ List of ReasoningStep objects
197
+ """
198
+ if not reasoning_text or not reasoning_text.strip():
199
+ return []
200
+
201
+ steps = []
202
+
203
+ # Try to split by numbered patterns
204
+ # Pattern 1: "1. ", "2. ", etc.
205
+ numbered_pattern = r'(\d+)\.\s+'
206
+ # Pattern 2: "Step 1:", "Step 2:", etc.
207
+ step_pattern = r'[Ss]tep\s+(\d+):\s*'
208
+
209
+ # Try numbered pattern first
210
+ parts = re.split(numbered_pattern, reasoning_text)
211
+ if len(parts) > 2: # Found numbered steps
212
+ for i in range(1, len(parts), 2):
213
+ if i + 1 < len(parts):
214
+ step_num = int(parts[i]) - 1
215
+ step_text = parts[i + 1].strip()
216
+ if step_text:
217
+ steps.append(self._create_step_from_text(step_num, step_text))
218
+ else:
219
+ # Try step pattern
220
+ parts = re.split(step_pattern, reasoning_text, flags=re.IGNORECASE)
221
+ if len(parts) > 2: # Found "Step X:" format
222
+ for i in range(1, len(parts), 2):
223
+ if i + 1 < len(parts):
224
+ step_num = int(parts[i]) - 1
225
+ step_text = parts[i + 1].strip()
226
+ if step_text:
227
+ steps.append(self._create_step_from_text(step_num, step_text))
228
+ else:
229
+ # Fall back to splitting by double newlines (paragraphs)
230
+ paragraphs = [p.strip() for p in reasoning_text.split('\n\n') if p.strip()]
231
+ if len(paragraphs) > 1:
232
+ for idx, paragraph in enumerate(paragraphs):
233
+ steps.append(self._create_step_from_text(idx, paragraph))
234
+ else:
235
+ # Split by single newlines if no paragraphs
236
+ lines = [l.strip() for l in reasoning_text.split('\n') if l.strip()]
237
+ if len(lines) > 1:
238
+ for idx, line in enumerate(lines):
239
+ steps.append(self._create_step_from_text(idx, line))
240
+ else:
241
+ # Single line reasoning - create one step
242
+ steps.append(self._create_step_from_text(0, reasoning_text.strip()))
243
+
244
+ return steps
245
+
246
+ def _create_step_from_text(self, step_id: int, text: str) -> ReasoningStep:
247
+ """
248
+ Create a ReasoningStep from text, inferring the step type from keywords.
249
+
250
+ Args:
251
+ step_id: Step index
252
+ text: Step description text
253
+
254
+ Returns:
255
+ ReasoningStep object
256
+ """
257
+ # Infer step type from keywords in text
258
+ text_lower = text.lower()
259
+
260
+ if any(kw in text_lower for kw in ['see', 'observe', 'notice', 'image shows', 'picture']):
261
+ step_type = StepType.PERCEPTION
262
+ elif any(kw in text_lower for kw in ['located', 'position', 'where', 'left', 'right', 'top', 'bottom']):
263
+ step_type = StepType.LOCALIZATION
264
+ elif any(kw in text_lower for kw in ['compare', 'larger', 'smaller', 'bigger', 'different', 'similar']):
265
+ step_type = StepType.COMPARISON
266
+ elif any(kw in text_lower for kw in ['count', 'number', 'how many', 'total', 'sum']):
267
+ step_type = StepType.COUNTING
268
+ elif any(kw in text_lower for kw in ['therefore', 'thus', 'conclude', 'deduce', 'infer']):
269
+ step_type = StepType.INFERENCE
270
+ elif any(kw in text_lower for kw in ['combine', 'together', 'overall', 'compose']):
271
+ step_type = StepType.COMPOSITION
272
+ elif any(kw in text_lower for kw in ['check', 'verify', 'confirm', 'validate']):
273
+ step_type = StepType.VERIFICATION
274
+ else:
275
+ step_type = StepType.INFERENCE # Default
276
+
277
+ # Create step with inferred type
278
+ return ReasoningStep(
279
+ step_id=step_id,
280
+ step_type=step_type,
281
+ description=text,
282
+ confidence=0.85, # Default confidence for dataset samples
283
+ dependencies=list(range(step_id)) if step_id > 0 else [], # Depends on all previous steps
284
+ )
285
+
286
+
287
+ def create_flux_reason_dataloader(
288
+ tokenizer: Any,
289
+ batch_size: int = 4,
290
+ num_workers: int = 4,
291
+ streaming: bool = True,
292
+ max_samples: Optional[int] = None,
293
+ **kwargs
294
+ ) -> torch.utils.data.DataLoader:
295
+ """
296
+ Create a DataLoader for FLUX-Reason-6M dataset.
297
+
298
+ Args:
299
+ tokenizer: Tokenizer for text encoding
300
+ batch_size: Number of samples per batch
301
+ num_workers: Number of worker processes
302
+ streaming: Use streaming mode
303
+ max_samples: Maximum samples to load
304
+ **kwargs: Additional arguments for FLUXReasonDataset
305
+
306
+ Returns:
307
+ Configured DataLoader
308
+ """
309
+ from .step_data import StepDataset
310
+
311
+ # Create FLUX dataset
312
+ flux_dataset = FLUXReasonDataset(
313
+ streaming=streaming,
314
+ max_samples=max_samples,
315
+ **kwargs
316
+ )
317
+
318
+ # If not streaming, wrap in StepDataset for tokenization
319
+ if not streaming and flux_dataset.reasoning_chains:
320
+ step_dataset = StepDataset(
321
+ reasoning_chains=flux_dataset.reasoning_chains,
322
+ tokenizer=tokenizer,
323
+ max_steps=10,
324
+ include_visual_features=False, # FLUX dataset doesn't have pre-extracted features
325
+ filter_incorrect=False,
326
+ )
327
+
328
+ return torch.utils.data.DataLoader(
329
+ step_dataset,
330
+ batch_size=batch_size,
331
+ num_workers=num_workers,
332
+ shuffle=True,
333
+ collate_fn=lambda batch: batch, # Custom collation handled in trainer
334
+ )
335
+ else:
336
+ # For streaming, return raw flux dataset
337
+ # (trainer will handle tokenization on-the-fly)
338
+ return torch.utils.data.DataLoader(
339
+ flux_dataset,
340
+ batch_size=batch_size,
341
+ num_workers=0 if streaming else num_workers, # Streaming doesn't support multiprocessing well
342
+ shuffle=False, # Streaming mode doesn't support shuffle
343
+ )
344
+
345
+
346
+ # Example usage
347
+ if __name__ == "__main__":
348
+ # Test dataset loading
349
+ print("Testing FLUX-Reason-6M dataset adapter...")
350
+
351
+ dataset = FLUXReasonDataset(
352
+ streaming=False,
353
+ max_samples=10,
354
+ )
355
+
356
+ print(f"\nLoaded {len(dataset)} samples")
357
+
358
+ # Show first example
359
+ if len(dataset) > 0:
360
+ chain = dataset[0]
361
+ print(f"\n{'='*60}")
362
+ print("Example Reasoning Chain:")
363
+ print(f"{'='*60}")
364
+ print(f"Chain ID: {chain.chain_id}")
365
+ print(f"Prompt: {chain.prompt}")
366
+ print(f"Number of steps: {len(chain.steps)}")
367
+ print(f"\nSteps:")
368
+ for step in chain.steps:
369
+ print(f" Step {step.step_id} ({step.step_type.value}): {step.description[:100]}...")
370
+ print(f"\nFinal Answer: {chain.final_answer}")
371
+ print(f"\n{'='*60}")
372
+ print("\nFormatted with special tokens:")
373
+ print(f"{'='*60}")
374
+ formatted = chain.format_with_tokens()
375
+ print(formatted[:500] + "..." if len(formatted) > 500 else formatted)
src/reasoning/inference_scaling.py ADDED
@@ -0,0 +1,453 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Inference-time scaling mechanisms for step-level reasoning.
3
+
4
+ Implements best-of-N sampling, majority voting, and self-consistency
5
+ to improve reasoning quality through test-time compute.
6
+ """
7
+
8
+ from typing import Dict, List, Optional, Tuple, Any, Callable
9
+ import torch
10
+ import torch.nn as nn
11
+ from collections import Counter
12
+ import logging
13
+ from dataclasses import dataclass
14
+
15
+ from .prm import ProcessRewardModel
16
+ from .step_data import ReasoningChain, ReasoningStep
17
+
18
+ logger = logging.getLogger(__name__)
19
+
20
+
21
+ @dataclass
22
+ class InferenceConfig:
23
+ """Configuration for inference-time scaling."""
24
+ num_samples: int = 8 # Number of reasoning chains to generate
25
+ temperature: float = 0.7 # Sampling temperature
26
+ top_k: int = 50
27
+ top_p: float = 0.95
28
+ max_steps: int = 10 # Maximum reasoning steps
29
+
30
+ # Aggregation method
31
+ aggregation: str = "best_of_n" # Options: best_of_n, majority_vote, weighted_vote
32
+
33
+ # PRM-based selection
34
+ use_prm_scores: bool = True
35
+ min_prm_score: float = 0.3 # Minimum acceptable PRM score
36
+
37
+ # Self-consistency
38
+ consistency_threshold: float = 0.6 # Required agreement for majority vote
39
+
40
+
41
+ class InferenceTimeScaling:
42
+ """
43
+ Inference-time scaling for improved reasoning quality.
44
+
45
+ Generates multiple reasoning chains and selects the best one using
46
+ various aggregation strategies.
47
+ """
48
+
49
+ def __init__(
50
+ self,
51
+ model: nn.Module,
52
+ prm: Optional[ProcessRewardModel] = None,
53
+ config: Optional[InferenceConfig] = None,
54
+ device: str = "cuda",
55
+ ):
56
+ """
57
+ Initialize inference scaler.
58
+
59
+ Args:
60
+ model: Vision-language model
61
+ prm: Process Reward Model for evaluation
62
+ config: Inference configuration
63
+ device: Device for inference
64
+ """
65
+ self.model = model
66
+ self.prm = prm
67
+ self.config = config or InferenceConfig()
68
+ self.device = device
69
+
70
+ self.model.to(device)
71
+ self.model.eval()
72
+
73
+ if self.prm:
74
+ self.prm.to(device)
75
+ self.prm.eval()
76
+
77
+ @torch.no_grad()
78
+ def generate_multiple_chains(
79
+ self,
80
+ image_features: torch.Tensor,
81
+ prompt_ids: torch.Tensor,
82
+ prompt_mask: torch.Tensor,
83
+ image_path: str = "",
84
+ prompt_text: str = "",
85
+ ) -> List[ReasoningChain]:
86
+ """
87
+ Generate multiple reasoning chains with different samples.
88
+
89
+ Args:
90
+ image_features: Visual features
91
+ prompt_ids: Tokenized prompt
92
+ prompt_mask: Attention mask
93
+ image_path: Path to image (for metadata)
94
+ prompt_text: Original prompt text
95
+
96
+ Returns:
97
+ List of reasoning chains
98
+ """
99
+ chains = []
100
+
101
+ for sample_idx in range(self.config.num_samples):
102
+ steps = []
103
+ current_ids = prompt_ids.unsqueeze(0).to(self.device)
104
+ current_mask = prompt_mask.unsqueeze(0).to(self.device)
105
+
106
+ step_embeddings = []
107
+
108
+ for step_id in range(self.config.max_steps):
109
+ # Generate next step
110
+ outputs = self.model(
111
+ input_ids=current_ids,
112
+ attention_mask=current_mask,
113
+ output_hidden_states=True,
114
+ )
115
+
116
+ logits = outputs.logits[:, -1, :] / self.config.temperature
117
+ probs = torch.softmax(logits, dim=-1)
118
+
119
+ # Top-k sampling
120
+ if self.config.top_k > 0:
121
+ top_k_probs, top_k_indices = torch.topk(probs, self.config.top_k)
122
+ next_token = top_k_indices[0, torch.multinomial(top_k_probs[0], 1)]
123
+ else:
124
+ next_token = torch.multinomial(probs[0], 1)
125
+
126
+ # Get hidden state for this step
127
+ hidden_state = outputs.hidden_states[-1][:, -1, :].squeeze(0)
128
+ step_embeddings.append(hidden_state)
129
+
130
+ # Compute PRM reward if available
131
+ reward = 0.0
132
+ if self.prm:
133
+ reward, metrics = self.prm.compute_step_reward(
134
+ image_features,
135
+ hidden_state,
136
+ previous_steps=torch.stack(step_embeddings[:-1]) if len(step_embeddings) > 1 else None,
137
+ return_metrics=True,
138
+ )
139
+
140
+ # Decode step text
141
+ step_text = self.model.config.tokenizer.decode(
142
+ next_token.unsqueeze(0),
143
+ skip_special_tokens=True,
144
+ )
145
+
146
+ # Create reasoning step
147
+ step = ReasoningStep(
148
+ step_id=step_id,
149
+ step_type="inference", # Would be classified in practice
150
+ description=step_text,
151
+ visual_features=hidden_state.cpu(),
152
+ confidence=metrics.confidence if self.prm and metrics else 0.5,
153
+ reward=reward,
154
+ )
155
+ steps.append(step)
156
+
157
+ # Update sequence
158
+ current_ids = torch.cat([current_ids, next_token.unsqueeze(0).unsqueeze(0)], dim=1)
159
+ current_mask = torch.cat([
160
+ current_mask,
161
+ torch.ones((1, 1), device=self.device)
162
+ ], dim=1)
163
+
164
+ # Check for completion
165
+ if next_token.item() == self.model.config.eos_token_id:
166
+ break
167
+
168
+ # Create reasoning chain
169
+ final_answer = self.model.config.tokenizer.decode(
170
+ current_ids[0, len(prompt_ids):],
171
+ skip_special_tokens=True,
172
+ )
173
+
174
+ total_reward = sum(s.reward for s in steps)
175
+
176
+ chain = ReasoningChain(
177
+ chain_id=f"sample_{sample_idx}",
178
+ image_path=image_path,
179
+ prompt=prompt_text,
180
+ steps=steps,
181
+ final_answer=final_answer,
182
+ total_reward=total_reward,
183
+ )
184
+ chains.append(chain)
185
+
186
+ return chains
187
+
188
+ def best_of_n_selection(
189
+ self,
190
+ chains: List[ReasoningChain],
191
+ ) -> ReasoningChain:
192
+ """
193
+ Select best chain based on PRM scores.
194
+
195
+ Args:
196
+ chains: List of reasoning chains
197
+
198
+ Returns:
199
+ Chain with highest total reward
200
+ """
201
+ if not chains:
202
+ raise ValueError("No chains to select from")
203
+
204
+ # Filter by minimum PRM score if required
205
+ if self.config.use_prm_scores and self.prm:
206
+ filtered_chains = [
207
+ c for c in chains
208
+ if c.total_reward >= self.config.min_prm_score * len(c.steps)
209
+ ]
210
+ if filtered_chains:
211
+ chains = filtered_chains
212
+
213
+ # Select chain with highest total reward
214
+ best_chain = max(chains, key=lambda c: c.total_reward)
215
+
216
+ logger.info(
217
+ f"Best-of-N: Selected chain with reward {best_chain.total_reward:.3f} "
218
+ f"({len(best_chain.steps)} steps)"
219
+ )
220
+
221
+ return best_chain
222
+
223
+ def majority_voting(
224
+ self,
225
+ chains: List[ReasoningChain],
226
+ ) -> ReasoningChain:
227
+ """
228
+ Select answer by majority voting with self-consistency.
229
+
230
+ Args:
231
+ chains: List of reasoning chains
232
+
233
+ Returns:
234
+ Chain corresponding to majority answer
235
+ """
236
+ if not chains:
237
+ raise ValueError("No chains to vote on")
238
+
239
+ # Extract final answers
240
+ answers = [c.final_answer.strip().lower() for c in chains]
241
+
242
+ # Count votes
243
+ vote_counts = Counter(answers)
244
+ majority_answer, majority_count = vote_counts.most_common(1)[0]
245
+
246
+ # Check consistency threshold
247
+ consistency = majority_count / len(chains)
248
+ if consistency < self.config.consistency_threshold:
249
+ logger.warning(
250
+ f"Low consistency: {consistency:.2f} < {self.config.consistency_threshold:.2f}"
251
+ )
252
+
253
+ # Find chain with majority answer and highest PRM score
254
+ majority_chains = [
255
+ c for c in chains
256
+ if c.final_answer.strip().lower() == majority_answer
257
+ ]
258
+
259
+ best_chain = max(majority_chains, key=lambda c: c.total_reward)
260
+
261
+ logger.info(
262
+ f"Majority Vote: '{majority_answer}' ({majority_count}/{len(chains)}) "
263
+ f"with consistency {consistency:.2f}"
264
+ )
265
+
266
+ return best_chain
267
+
268
+ def weighted_voting(
269
+ self,
270
+ chains: List[ReasoningChain],
271
+ ) -> ReasoningChain:
272
+ """
273
+ Weighted voting using PRM scores as weights.
274
+
275
+ Args:
276
+ chains: List of reasoning chains
277
+
278
+ Returns:
279
+ Chain with highest weighted vote
280
+ """
281
+ if not chains or not self.prm:
282
+ return self.best_of_n_selection(chains)
283
+
284
+ # Group chains by answer
285
+ answer_groups: Dict[str, List[ReasoningChain]] = {}
286
+ for chain in chains:
287
+ answer_key = chain.final_answer.strip().lower()
288
+ if answer_key not in answer_groups:
289
+ answer_groups[answer_key] = []
290
+ answer_groups[answer_key].append(chain)
291
+
292
+ # Calculate weighted votes
293
+ weighted_votes = {}
294
+ for answer, group_chains in answer_groups.items():
295
+ # Use exponential of reward as weight (emphasizes high-quality chains)
296
+ weight = sum(torch.exp(torch.tensor(c.total_reward / len(c.steps))).item()
297
+ for c in group_chains)
298
+ weighted_votes[answer] = weight
299
+
300
+ # Select answer with highest weighted vote
301
+ best_answer = max(weighted_votes.items(), key=lambda x: x[1])[0]
302
+
303
+ # Return best chain for that answer
304
+ best_chains = answer_groups[best_answer]
305
+ best_chain = max(best_chains, key=lambda c: c.total_reward)
306
+
307
+ logger.info(
308
+ f"Weighted Vote: '{best_answer}' with weight {weighted_votes[best_answer]:.3f}"
309
+ )
310
+
311
+ return best_chain
312
+
313
+ def __call__(
314
+ self,
315
+ image_features: torch.Tensor,
316
+ prompt_ids: torch.Tensor,
317
+ prompt_mask: torch.Tensor,
318
+ image_path: str = "",
319
+ prompt_text: str = "",
320
+ ) -> Tuple[ReasoningChain, List[ReasoningChain]]:
321
+ """
322
+ Perform inference with scaling.
323
+
324
+ Args:
325
+ image_features: Visual features
326
+ prompt_ids: Tokenized prompt
327
+ prompt_mask: Attention mask
328
+ image_path: Path to image
329
+ prompt_text: Original prompt
330
+
331
+ Returns:
332
+ - best_chain: Selected reasoning chain
333
+ - all_chains: All generated chains for analysis
334
+ """
335
+ # Generate multiple chains
336
+ chains = self.generate_multiple_chains(
337
+ image_features,
338
+ prompt_ids,
339
+ prompt_mask,
340
+ image_path,
341
+ prompt_text,
342
+ )
343
+
344
+ # Select best chain based on aggregation method
345
+ if self.config.aggregation == "best_of_n":
346
+ best_chain = self.best_of_n_selection(chains)
347
+ elif self.config.aggregation == "majority_vote":
348
+ best_chain = self.majority_voting(chains)
349
+ elif self.config.aggregation == "weighted_vote":
350
+ best_chain = self.weighted_voting(chains)
351
+ else:
352
+ raise ValueError(f"Unknown aggregation method: {self.config.aggregation}")
353
+
354
+ return best_chain, chains
355
+
356
+
357
+ class MajorityVoting:
358
+ """
359
+ Standalone majority voting for answer aggregation.
360
+
361
+ Useful for post-processing multiple model predictions.
362
+ """
363
+
364
+ @staticmethod
365
+ def vote(
366
+ answers: List[str],
367
+ weights: Optional[List[float]] = None,
368
+ threshold: float = 0.5,
369
+ ) -> Tuple[str, float]:
370
+ """
371
+ Perform majority voting on answers.
372
+
373
+ Args:
374
+ answers: List of answer strings
375
+ weights: Optional weights for each answer
376
+ threshold: Minimum agreement threshold
377
+
378
+ Returns:
379
+ - majority_answer: Most common answer
380
+ - confidence: Voting confidence [0, 1]
381
+ """
382
+ if not answers:
383
+ return "", 0.0
384
+
385
+ if weights is None:
386
+ weights = [1.0] * len(answers)
387
+
388
+ # Normalize answers
389
+ normalized = [a.strip().lower() for a in answers]
390
+
391
+ # Weighted voting
392
+ vote_dict: Dict[str, float] = {}
393
+ for answer, weight in zip(normalized, weights):
394
+ vote_dict[answer] = vote_dict.get(answer, 0.0) + weight
395
+
396
+ # Find majority
397
+ total_weight = sum(weights)
398
+ majority_answer = max(vote_dict.items(), key=lambda x: x[1])[0]
399
+ confidence = vote_dict[majority_answer] / total_weight
400
+
401
+ # Check threshold
402
+ if confidence < threshold:
403
+ logger.warning(
404
+ f"Majority voting confidence {confidence:.2f} below threshold {threshold}"
405
+ )
406
+
407
+ return majority_answer, confidence
408
+
409
+ @staticmethod
410
+ def self_consistency(
411
+ chains: List[ReasoningChain],
412
+ min_agreement: float = 0.6,
413
+ ) -> Tuple[Optional[str], float, List[ReasoningChain]]:
414
+ """
415
+ Self-consistency checking across reasoning chains.
416
+
417
+ Args:
418
+ chains: List of reasoning chains
419
+ min_agreement: Minimum agreement required
420
+
421
+ Returns:
422
+ - consensus_answer: Answer if consensus reached, else None
423
+ - agreement: Agreement ratio [0, 1]
424
+ - supporting_chains: Chains supporting consensus
425
+ """
426
+ if not chains:
427
+ return None, 0.0, []
428
+
429
+ # Extract answers
430
+ answers = [c.final_answer.strip().lower() for c in chains]
431
+
432
+ # Count occurrences
433
+ answer_counts = Counter(answers)
434
+ most_common_answer, count = answer_counts.most_common(1)[0]
435
+
436
+ # Calculate agreement
437
+ agreement = count / len(chains)
438
+
439
+ # Check if consensus reached
440
+ consensus = most_common_answer if agreement >= min_agreement else None
441
+
442
+ # Find supporting chains
443
+ supporting_chains = [
444
+ c for c in chains
445
+ if c.final_answer.strip().lower() == most_common_answer
446
+ ]
447
+
448
+ logger.info(
449
+ f"Self-consistency: {agreement:.1%} agreement "
450
+ f"({'consensus' if consensus else 'no consensus'})"
451
+ )
452
+
453
+ return consensus, agreement, supporting_chains
src/reasoning/prm.py ADDED
@@ -0,0 +1,462 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Process Reward Model (PRM) for evaluating step-level reasoning quality.
3
+
4
+ The PRM assigns rewards to intermediate reasoning steps, enabling fine-grained
5
+ reinforcement learning and quality assessment.
6
+ """
7
+
8
+ from dataclasses import dataclass
9
+ from typing import Dict, List, Optional, Tuple
10
+ import torch
11
+ import torch.nn as nn
12
+ import torch.nn.functional as F
13
+ from transformers import PreTrainedModel
14
+ import logging
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+
19
+ @dataclass
20
+ class StepQualityMetrics:
21
+ """Metrics for assessing reasoning step quality."""
22
+ coherence: float # How well step follows from previous steps [0, 1]
23
+ relevance: float # Relevance to prompt and image [0, 1]
24
+ correctness: float # Logical correctness [0, 1]
25
+ informativeness: float # How much new information added [0, 1]
26
+ confidence: float # Model's confidence in this step [0, 1]
27
+
28
+ def to_reward(self, weights: Optional[Dict[str, float]] = None) -> float:
29
+ """
30
+ Convert metrics to single reward score.
31
+
32
+ Args:
33
+ weights: Optional custom weights for each metric
34
+
35
+ Returns:
36
+ Reward in [-1, 1] range
37
+ """
38
+ if weights is None:
39
+ weights = {
40
+ 'coherence': 0.25,
41
+ 'relevance': 0.25,
42
+ 'correctness': 0.3,
43
+ 'informativeness': 0.15,
44
+ 'confidence': 0.05,
45
+ }
46
+
47
+ reward = sum(
48
+ getattr(self, key) * weight
49
+ for key, weight in weights.items()
50
+ )
51
+
52
+ # Normalize to [-1, 1] (assuming metrics are [0, 1])
53
+ return reward * 2.0 - 1.0
54
+
55
+
56
+ class ProcessRewardModel(nn.Module):
57
+ """
58
+ Process Reward Model for step-level reasoning evaluation.
59
+
60
+ Architecture:
61
+ - Vision encoder: Extract visual features
62
+ - Text encoder: Encode reasoning steps
63
+ - Context aggregator: Combine previous steps
64
+ - Reward head: Predict step quality metrics
65
+ """
66
+
67
+ def __init__(
68
+ self,
69
+ vision_dim: int = 768,
70
+ text_dim: int = 768,
71
+ hidden_dim: int = 512,
72
+ num_heads: int = 8,
73
+ dropout: float = 0.1,
74
+ max_steps: int = 10,
75
+ ):
76
+ """
77
+ Initialize PRM.
78
+
79
+ Args:
80
+ vision_dim: Dimension of vision features
81
+ text_dim: Dimension of text embeddings
82
+ hidden_dim: Hidden dimension for processing
83
+ num_heads: Number of attention heads
84
+ dropout: Dropout rate
85
+ max_steps: Maximum reasoning steps to handle
86
+ """
87
+ super().__init__()
88
+
89
+ self.vision_dim = vision_dim
90
+ self.text_dim = text_dim
91
+ self.hidden_dim = hidden_dim
92
+ self.max_steps = max_steps
93
+
94
+ # Vision and text projections
95
+ self.vision_proj = nn.Linear(vision_dim, hidden_dim)
96
+ self.text_proj = nn.Linear(text_dim, hidden_dim)
97
+
98
+ # Context aggregator with self-attention over previous steps
99
+ self.context_attention = nn.MultiheadAttention(
100
+ embed_dim=hidden_dim,
101
+ num_heads=num_heads,
102
+ dropout=dropout,
103
+ batch_first=True,
104
+ )
105
+
106
+ # Step encoder
107
+ self.step_encoder = nn.TransformerEncoder(
108
+ nn.TransformerEncoderLayer(
109
+ d_model=hidden_dim,
110
+ nhead=num_heads,
111
+ dim_feedforward=hidden_dim * 4,
112
+ dropout=dropout,
113
+ batch_first=True,
114
+ ),
115
+ num_layers=2,
116
+ )
117
+
118
+ # Reward heads for different metrics
119
+ self.coherence_head = nn.Sequential(
120
+ nn.Linear(hidden_dim, hidden_dim // 2),
121
+ nn.ReLU(),
122
+ nn.Dropout(dropout),
123
+ nn.Linear(hidden_dim // 2, 1),
124
+ nn.Sigmoid(),
125
+ )
126
+
127
+ self.relevance_head = nn.Sequential(
128
+ nn.Linear(hidden_dim, hidden_dim // 2),
129
+ nn.ReLU(),
130
+ nn.Dropout(dropout),
131
+ nn.Linear(hidden_dim // 2, 1),
132
+ nn.Sigmoid(),
133
+ )
134
+
135
+ self.correctness_head = nn.Sequential(
136
+ nn.Linear(hidden_dim, hidden_dim // 2),
137
+ nn.ReLU(),
138
+ nn.Dropout(dropout),
139
+ nn.Linear(hidden_dim // 2, 1),
140
+ nn.Sigmoid(),
141
+ )
142
+
143
+ self.informativeness_head = nn.Sequential(
144
+ nn.Linear(hidden_dim, hidden_dim // 2),
145
+ nn.ReLU(),
146
+ nn.Dropout(dropout),
147
+ nn.Linear(hidden_dim // 2, 1),
148
+ nn.Sigmoid(),
149
+ )
150
+
151
+ # Confidence estimation
152
+ self.confidence_head = nn.Sequential(
153
+ nn.Linear(hidden_dim, hidden_dim // 2),
154
+ nn.ReLU(),
155
+ nn.Dropout(dropout),
156
+ nn.Linear(hidden_dim // 2, 1),
157
+ nn.Sigmoid(),
158
+ )
159
+
160
+ # Overall reward head (learned combination)
161
+ self.reward_head = nn.Sequential(
162
+ nn.Linear(hidden_dim, hidden_dim // 2),
163
+ nn.ReLU(),
164
+ nn.Dropout(dropout),
165
+ nn.Linear(hidden_dim // 2, 1),
166
+ nn.Tanh(), # Output in [-1, 1]
167
+ )
168
+
169
+ self.dropout = nn.Dropout(dropout)
170
+
171
+ def forward(
172
+ self,
173
+ vision_features: torch.Tensor,
174
+ step_embeddings: torch.Tensor,
175
+ step_mask: Optional[torch.Tensor] = None,
176
+ return_metrics: bool = False,
177
+ ) -> Tuple[torch.Tensor, Optional[List[StepQualityMetrics]]]:
178
+ """
179
+ Forward pass to compute rewards for reasoning steps.
180
+
181
+ Args:
182
+ vision_features: Visual features [batch, vision_dim]
183
+ step_embeddings: Text embeddings for each step [batch, num_steps, text_dim]
184
+ step_mask: Mask for valid steps [batch, num_steps]
185
+ return_metrics: If True, return detailed metrics
186
+
187
+ Returns:
188
+ - rewards: Reward for each step [batch, num_steps]
189
+ - metrics: Optional list of StepQualityMetrics per step
190
+ """
191
+ batch_size, num_steps, _ = step_embeddings.shape
192
+
193
+ # Project vision and text features
194
+ vision_feat = self.vision_proj(vision_features) # [batch, hidden_dim]
195
+ text_feat = self.text_proj(step_embeddings) # [batch, num_steps, hidden_dim]
196
+
197
+ # Expand vision features to match step dimension
198
+ vision_expanded = vision_feat.unsqueeze(1).expand(-1, num_steps, -1)
199
+
200
+ # Combine vision and text
201
+ combined = vision_expanded + text_feat # [batch, num_steps, hidden_dim]
202
+ combined = self.dropout(combined)
203
+
204
+ # Apply context attention (each step attends to previous steps)
205
+ if step_mask is not None:
206
+ # Create causal mask for autoregressive reasoning
207
+ causal_mask = torch.triu(
208
+ torch.ones(num_steps, num_steps, device=combined.device),
209
+ diagonal=1
210
+ ).bool()
211
+ else:
212
+ causal_mask = None
213
+
214
+ attended, _ = self.context_attention(
215
+ combined, combined, combined,
216
+ attn_mask=causal_mask,
217
+ key_padding_mask=~step_mask if step_mask is not None else None,
218
+ )
219
+
220
+ # Encode steps
221
+ encoded = self.step_encoder(attended) # [batch, num_steps, hidden_dim]
222
+
223
+ # Compute rewards
224
+ rewards = self.reward_head(encoded).squeeze(-1) # [batch, num_steps]
225
+
226
+ if step_mask is not None:
227
+ rewards = rewards.masked_fill(~step_mask, 0.0)
228
+
229
+ # Optionally compute detailed metrics
230
+ metrics = None
231
+ if return_metrics:
232
+ metrics = []
233
+ for b in range(batch_size):
234
+ step_metrics = []
235
+ for s in range(num_steps):
236
+ if step_mask is None or step_mask[b, s]:
237
+ feat = encoded[b, s:s+1]
238
+
239
+ quality = StepQualityMetrics(
240
+ coherence=self.coherence_head(feat).item(),
241
+ relevance=self.relevance_head(feat).item(),
242
+ correctness=self.correctness_head(feat).item(),
243
+ informativeness=self.informativeness_head(feat).item(),
244
+ confidence=self.confidence_head(feat).item(),
245
+ )
246
+ step_metrics.append(quality)
247
+ metrics.append(step_metrics)
248
+
249
+ return rewards, metrics
250
+
251
+ def compute_step_reward(
252
+ self,
253
+ vision_features: torch.Tensor,
254
+ current_step: torch.Tensor,
255
+ previous_steps: Optional[torch.Tensor] = None,
256
+ return_metrics: bool = False,
257
+ ) -> Tuple[float, Optional[StepQualityMetrics]]:
258
+ """
259
+ Compute reward for a single reasoning step.
260
+
261
+ Args:
262
+ vision_features: Visual features [vision_dim]
263
+ current_step: Current step embedding [text_dim]
264
+ previous_steps: Previous step embeddings [num_prev_steps, text_dim]
265
+ return_metrics: If True, return detailed metrics
266
+
267
+ Returns:
268
+ - reward: Scalar reward in [-1, 1]
269
+ - metrics: Optional StepQualityMetrics
270
+ """
271
+ # Prepare batch inputs
272
+ vision_features = vision_features.unsqueeze(0) # [1, vision_dim]
273
+
274
+ if previous_steps is not None:
275
+ # Concatenate previous and current
276
+ steps = torch.cat([previous_steps, current_step.unsqueeze(0)], dim=0)
277
+ else:
278
+ steps = current_step.unsqueeze(0)
279
+
280
+ steps = steps.unsqueeze(0) # [1, num_steps, text_dim]
281
+
282
+ # Forward pass
283
+ with torch.no_grad():
284
+ rewards, metrics = self.forward(
285
+ vision_features,
286
+ steps,
287
+ return_metrics=return_metrics,
288
+ )
289
+
290
+ # Extract last step reward
291
+ reward = rewards[0, -1].item()
292
+ step_metrics = metrics[0][-1] if metrics else None
293
+
294
+ return reward, step_metrics
295
+
296
+ @torch.no_grad()
297
+ def evaluate_chain(
298
+ self,
299
+ vision_features: torch.Tensor,
300
+ step_embeddings: List[torch.Tensor],
301
+ ) -> Tuple[List[float], float]:
302
+ """
303
+ Evaluate a complete reasoning chain.
304
+
305
+ Args:
306
+ vision_features: Visual features [vision_dim]
307
+ step_embeddings: List of step embeddings
308
+
309
+ Returns:
310
+ - step_rewards: Reward for each step
311
+ - total_reward: Sum of all step rewards
312
+ """
313
+ if not step_embeddings:
314
+ return [], 0.0
315
+
316
+ # Stack steps
317
+ steps = torch.stack(step_embeddings).unsqueeze(0) # [1, num_steps, text_dim]
318
+ vision_features = vision_features.unsqueeze(0) # [1, vision_dim]
319
+
320
+ # Compute rewards
321
+ rewards, _ = self.forward(vision_features, steps)
322
+ step_rewards = rewards[0].tolist()
323
+ total_reward = sum(step_rewards)
324
+
325
+ return step_rewards, total_reward
326
+
327
+
328
+ class PRMTrainer:
329
+ """Trainer for Process Reward Model using step-level supervision."""
330
+
331
+ def __init__(
332
+ self,
333
+ model: ProcessRewardModel,
334
+ learning_rate: float = 1e-4,
335
+ weight_decay: float = 0.01,
336
+ warmup_steps: int = 1000,
337
+ ):
338
+ """
339
+ Initialize PRM trainer.
340
+
341
+ Args:
342
+ model: Process Reward Model
343
+ learning_rate: Learning rate
344
+ weight_decay: Weight decay for regularization
345
+ warmup_steps: Number of warmup steps
346
+ """
347
+ self.model = model
348
+ self.optimizer = torch.optim.AdamW(
349
+ model.parameters(),
350
+ lr=learning_rate,
351
+ weight_decay=weight_decay,
352
+ )
353
+ self.warmup_steps = warmup_steps
354
+ self.current_step = 0
355
+
356
+ def train_step(
357
+ self,
358
+ vision_features: torch.Tensor,
359
+ step_embeddings: torch.Tensor,
360
+ target_rewards: torch.Tensor,
361
+ step_mask: Optional[torch.Tensor] = None,
362
+ ) -> Dict[str, float]:
363
+ """
364
+ Single training step.
365
+
366
+ Args:
367
+ vision_features: Visual features [batch, vision_dim]
368
+ step_embeddings: Step embeddings [batch, num_steps, text_dim]
369
+ target_rewards: Ground truth rewards [batch, num_steps]
370
+ step_mask: Valid step mask [batch, num_steps]
371
+
372
+ Returns:
373
+ Dictionary of losses
374
+ """
375
+ self.model.train()
376
+ self.optimizer.zero_grad()
377
+
378
+ # Forward pass
379
+ predicted_rewards, _ = self.model(
380
+ vision_features,
381
+ step_embeddings,
382
+ step_mask,
383
+ )
384
+
385
+ # Compute loss
386
+ if step_mask is not None:
387
+ # Masked MSE loss
388
+ loss = F.mse_loss(
389
+ predicted_rewards.masked_select(step_mask),
390
+ target_rewards.masked_select(step_mask),
391
+ )
392
+ else:
393
+ loss = F.mse_loss(predicted_rewards, target_rewards)
394
+
395
+ # Backward pass
396
+ loss.backward()
397
+ torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)
398
+ self.optimizer.step()
399
+
400
+ # Learning rate warmup
401
+ if self.current_step < self.warmup_steps:
402
+ lr_scale = min(1.0, self.current_step / self.warmup_steps)
403
+ for param_group in self.optimizer.param_groups:
404
+ param_group['lr'] = learning_rate * lr_scale
405
+
406
+ self.current_step += 1
407
+
408
+ return {
409
+ 'loss': loss.item(),
410
+ 'avg_predicted_reward': predicted_rewards.mean().item(),
411
+ 'avg_target_reward': target_rewards.mean().item(),
412
+ }
413
+
414
+ @torch.no_grad()
415
+ def evaluate(
416
+ self,
417
+ vision_features: torch.Tensor,
418
+ step_embeddings: torch.Tensor,
419
+ target_rewards: torch.Tensor,
420
+ step_mask: Optional[torch.Tensor] = None,
421
+ ) -> Dict[str, float]:
422
+ """
423
+ Evaluation step.
424
+
425
+ Returns:
426
+ Dictionary of evaluation metrics
427
+ """
428
+ self.model.eval()
429
+
430
+ # Forward pass
431
+ predicted_rewards, _ = self.model(
432
+ vision_features,
433
+ step_embeddings,
434
+ step_mask,
435
+ )
436
+
437
+ # Compute metrics
438
+ if step_mask is not None:
439
+ valid_pred = predicted_rewards.masked_select(step_mask)
440
+ valid_target = target_rewards.masked_select(step_mask)
441
+ else:
442
+ valid_pred = predicted_rewards.flatten()
443
+ valid_target = target_rewards.flatten()
444
+
445
+ mse = F.mse_loss(valid_pred, valid_target).item()
446
+ mae = F.l1_loss(valid_pred, valid_target).item()
447
+
448
+ # Correlation
449
+ pred_mean = valid_pred.mean()
450
+ target_mean = valid_target.mean()
451
+ covariance = ((valid_pred - pred_mean) * (valid_target - target_mean)).mean()
452
+ pred_std = valid_pred.std()
453
+ target_std = valid_target.std()
454
+ correlation = covariance / (pred_std * target_std + 1e-8)
455
+
456
+ return {
457
+ 'mse': mse,
458
+ 'mae': mae,
459
+ 'correlation': correlation.item(),
460
+ 'avg_predicted_reward': valid_pred.mean().item(),
461
+ 'avg_target_reward': valid_target.mean().item(),
462
+ }
src/reasoning/rl_trainer.py ADDED
@@ -0,0 +1,480 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Reinforcement Learning trainer for vision-language models with step-level rewards.
3
+
4
+ Implements PPO (Proximal Policy Optimization) with fine-grained process rewards
5
+ from the PRM for each reasoning step.
6
+ """
7
+
8
+ from typing import Dict, List, Optional, Tuple, Any
9
+ import torch
10
+ import torch.nn as nn
11
+ import torch.nn.functional as F
12
+ from torch.utils.data import DataLoader
13
+ from dataclasses import dataclass
14
+ import logging
15
+
16
+ from .prm import ProcessRewardModel
17
+ from .step_data import ReasoningChain, StepDataset
18
+
19
+ logger = logging.getLogger(__name__)
20
+
21
+
22
+ @dataclass
23
+ class RLConfig:
24
+ """Configuration for RL training."""
25
+ # PPO hyperparameters
26
+ clip_epsilon: float = 0.2
27
+ value_loss_coef: float = 0.5
28
+ entropy_coef: float = 0.01
29
+ max_grad_norm: float = 1.0
30
+ gamma: float = 0.99 # Discount factor
31
+ gae_lambda: float = 0.95 # GAE parameter
32
+
33
+ # Training settings
34
+ num_epochs: int = 4
35
+ batch_size: int = 8
36
+ learning_rate: float = 1e-5
37
+ warmup_steps: int = 500
38
+
39
+ # Step-level rewards
40
+ use_prm_rewards: bool = True
41
+ normalize_rewards: bool = True
42
+ reward_scale: float = 1.0
43
+
44
+ # Exploration
45
+ temperature: float = 1.0
46
+ top_k: int = 50
47
+ top_p: float = 0.95
48
+
49
+
50
+ class RLReasoningTrainer:
51
+ """
52
+ Reinforcement Learning trainer for step-level reasoning.
53
+
54
+ Uses PPO to optimize the vision-language model with fine-grained rewards
55
+ from the Process Reward Model (PRM) at each reasoning step.
56
+ """
57
+
58
+ def __init__(
59
+ self,
60
+ policy_model: nn.Module,
61
+ prm_model: ProcessRewardModel,
62
+ config: RLConfig,
63
+ device: str = "cuda",
64
+ ):
65
+ """
66
+ Initialize RL trainer.
67
+
68
+ Args:
69
+ policy_model: Vision-language model to train
70
+ prm_model: Process Reward Model for step evaluation
71
+ config: RL training configuration
72
+ device: Device for training
73
+ """
74
+ self.policy = policy_model
75
+ self.prm = prm_model
76
+ self.config = config
77
+ self.device = device
78
+
79
+ # Move models to device
80
+ self.policy.to(device)
81
+ self.prm.to(device)
82
+
83
+ # Freeze PRM (only train policy)
84
+ for param in self.prm.parameters():
85
+ param.requires_grad = False
86
+ self.prm.eval()
87
+
88
+ # Optimizer
89
+ self.optimizer = torch.optim.AdamW(
90
+ self.policy.parameters(),
91
+ lr=config.learning_rate,
92
+ betas=(0.9, 0.999),
93
+ eps=1e-8,
94
+ )
95
+
96
+ # Value head for PPO (estimates state value)
97
+ self.value_head = nn.Sequential(
98
+ nn.Linear(768, 512),
99
+ nn.ReLU(),
100
+ nn.Dropout(0.1),
101
+ nn.Linear(512, 1),
102
+ ).to(device)
103
+
104
+ self.value_optimizer = torch.optim.AdamW(
105
+ self.value_head.parameters(),
106
+ lr=config.learning_rate * 3,
107
+ )
108
+
109
+ self.global_step = 0
110
+
111
+ def generate_reasoning_chain(
112
+ self,
113
+ image_features: torch.Tensor,
114
+ prompt_ids: torch.Tensor,
115
+ prompt_mask: torch.Tensor,
116
+ max_steps: int = 10,
117
+ ) -> Tuple[List[torch.Tensor], List[torch.Tensor], List[float]]:
118
+ """
119
+ Generate a reasoning chain using the current policy.
120
+
121
+ Args:
122
+ image_features: Visual features [vision_dim]
123
+ prompt_ids: Tokenized prompt [seq_len]
124
+ prompt_mask: Prompt attention mask [seq_len]
125
+ max_steps: Maximum reasoning steps
126
+
127
+ Returns:
128
+ - step_embeddings: List of step embeddings
129
+ - step_logprobs: List of log probabilities for each step
130
+ - step_entropies: List of entropies for each step
131
+ """
132
+ self.policy.eval()
133
+
134
+ step_embeddings = []
135
+ step_logprobs = []
136
+ step_entropies = []
137
+
138
+ # Initialize with prompt
139
+ current_ids = prompt_ids.unsqueeze(0).to(self.device)
140
+ current_mask = prompt_mask.unsqueeze(0).to(self.device)
141
+
142
+ for step_idx in range(max_steps):
143
+ with torch.no_grad():
144
+ # Forward pass through policy
145
+ outputs = self.policy(
146
+ input_ids=current_ids,
147
+ attention_mask=current_mask,
148
+ output_hidden_states=True,
149
+ )
150
+
151
+ logits = outputs.logits[:, -1, :] # Next token logits
152
+ hidden_state = outputs.hidden_states[-1][:, -1, :] # Last hidden state
153
+
154
+ # Apply temperature and sampling
155
+ logits = logits / self.config.temperature
156
+ probs = F.softmax(logits, dim=-1)
157
+
158
+ # Top-k and top-p filtering
159
+ if self.config.top_k > 0:
160
+ top_k_probs, top_k_indices = torch.topk(probs, self.config.top_k, dim=-1)
161
+ probs = torch.zeros_like(probs).scatter_(1, top_k_indices, top_k_probs)
162
+ probs = probs / probs.sum(dim=-1, keepdim=True)
163
+
164
+ if self.config.top_p < 1.0:
165
+ sorted_probs, sorted_indices = torch.sort(probs, descending=True, dim=-1)
166
+ cumulative_probs = torch.cumsum(sorted_probs, dim=-1)
167
+
168
+ # Remove tokens with cumulative probability above threshold
169
+ sorted_indices_to_remove = cumulative_probs > self.config.top_p
170
+ sorted_indices_to_remove[:, 1:] = sorted_indices_to_remove[:, :-1].clone()
171
+ sorted_indices_to_remove[:, 0] = False
172
+
173
+ indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
174
+ probs = probs.masked_fill(indices_to_remove, 0.0)
175
+ probs = probs / probs.sum(dim=-1, keepdim=True)
176
+
177
+ # Sample next token
178
+ next_token = torch.multinomial(probs, num_samples=1)
179
+
180
+ # Calculate log probability and entropy
181
+ log_prob = torch.log(probs.gather(1, next_token) + 1e-10)
182
+ entropy = -(probs * torch.log(probs + 1e-10)).sum(dim=-1)
183
+
184
+ # Store step information
185
+ step_embeddings.append(hidden_state.squeeze(0))
186
+ step_logprobs.append(log_prob.squeeze())
187
+ step_entropies.append(entropy.item())
188
+
189
+ # Check for stop token or step completion marker
190
+ if next_token.item() in [self.policy.config.eos_token_id, 198]: # EOS or newline
191
+ break
192
+
193
+ # Update sequence
194
+ current_ids = torch.cat([current_ids, next_token], dim=1)
195
+ current_mask = torch.cat([
196
+ current_mask,
197
+ torch.ones((1, 1), device=self.device)
198
+ ], dim=1)
199
+
200
+ return step_embeddings, step_logprobs, step_entropies
201
+
202
+ def compute_advantages(
203
+ self,
204
+ rewards: torch.Tensor,
205
+ values: torch.Tensor,
206
+ dones: torch.Tensor,
207
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
208
+ """
209
+ Compute Generalized Advantage Estimation (GAE).
210
+
211
+ Args:
212
+ rewards: Step rewards [num_steps]
213
+ values: Value estimates [num_steps]
214
+ dones: Done flags [num_steps]
215
+
216
+ Returns:
217
+ - advantages: Advantage estimates [num_steps]
218
+ - returns: Discounted returns [num_steps]
219
+ """
220
+ num_steps = len(rewards)
221
+ advantages = torch.zeros_like(rewards)
222
+ returns = torch.zeros_like(rewards)
223
+
224
+ gae = 0
225
+ next_value = 0
226
+
227
+ for t in reversed(range(num_steps)):
228
+ if t == num_steps - 1:
229
+ next_non_terminal = 1.0 - dones[t]
230
+ next_value = 0
231
+ else:
232
+ next_non_terminal = 1.0 - dones[t]
233
+ next_value = values[t + 1]
234
+
235
+ delta = rewards[t] + self.config.gamma * next_value * next_non_terminal - values[t]
236
+ gae = delta + self.config.gamma * self.config.gae_lambda * next_non_terminal * gae
237
+
238
+ advantages[t] = gae
239
+ returns[t] = gae + values[t]
240
+
241
+ return advantages, returns
242
+
243
+ def ppo_update(
244
+ self,
245
+ step_embeddings: List[torch.Tensor],
246
+ old_logprobs: torch.Tensor,
247
+ rewards: torch.Tensor,
248
+ advantages: torch.Tensor,
249
+ returns: torch.Tensor,
250
+ ) -> Dict[str, float]:
251
+ """
252
+ Perform PPO update.
253
+
254
+ Args:
255
+ step_embeddings: Hidden states for each step
256
+ old_logprobs: Old policy log probabilities
257
+ rewards: Step rewards
258
+ advantages: Advantage estimates
259
+ returns: Discounted returns
260
+
261
+ Returns:
262
+ Dictionary of training metrics
263
+ """
264
+ self.policy.train()
265
+ self.value_head.train()
266
+
267
+ # Stack embeddings
268
+ embeddings = torch.stack(step_embeddings) # [num_steps, hidden_dim]
269
+
270
+ # Normalize advantages
271
+ if self.config.normalize_rewards:
272
+ advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)
273
+
274
+ total_policy_loss = 0
275
+ total_value_loss = 0
276
+ total_entropy_loss = 0
277
+
278
+ for epoch in range(self.config.num_epochs):
279
+ # Compute current policy log probabilities
280
+ # (In practice, you'd recompute through model forward pass)
281
+ # Here we use embeddings for simplification
282
+
283
+ # Compute values
284
+ values = self.value_head(embeddings).squeeze(-1)
285
+
286
+ # Value loss
287
+ value_loss = F.mse_loss(values, returns)
288
+
289
+ # For policy loss, we need to recompute logprobs
290
+ # This is simplified - in practice you'd do full forward pass
291
+ ratio = torch.exp(old_logprobs - old_logprobs) # Would be new/old
292
+
293
+ # PPO clipped objective
294
+ policy_loss_1 = -advantages * ratio
295
+ policy_loss_2 = -advantages * torch.clamp(
296
+ ratio,
297
+ 1 - self.config.clip_epsilon,
298
+ 1 + self.config.clip_epsilon
299
+ )
300
+ policy_loss = torch.max(policy_loss_1, policy_loss_2).mean()
301
+
302
+ # Entropy bonus for exploration
303
+ entropy_loss = -self.config.entropy_coef * old_logprobs.mean()
304
+
305
+ # Total loss
306
+ loss = (
307
+ policy_loss +
308
+ self.config.value_loss_coef * value_loss +
309
+ entropy_loss
310
+ )
311
+
312
+ # Optimization step
313
+ self.optimizer.zero_grad()
314
+ self.value_optimizer.zero_grad()
315
+ loss.backward()
316
+
317
+ torch.nn.utils.clip_grad_norm_(
318
+ self.policy.parameters(),
319
+ self.config.max_grad_norm
320
+ )
321
+ torch.nn.utils.clip_grad_norm_(
322
+ self.value_head.parameters(),
323
+ self.config.max_grad_norm
324
+ )
325
+
326
+ self.optimizer.step()
327
+ self.value_optimizer.step()
328
+
329
+ total_policy_loss += policy_loss.item()
330
+ total_value_loss += value_loss.item()
331
+ total_entropy_loss += entropy_loss.item()
332
+
333
+ self.global_step += 1
334
+
335
+ return {
336
+ 'policy_loss': total_policy_loss / self.config.num_epochs,
337
+ 'value_loss': total_value_loss / self.config.num_epochs,
338
+ 'entropy_loss': total_entropy_loss / self.config.num_epochs,
339
+ 'avg_reward': rewards.mean().item(),
340
+ 'avg_advantage': advantages.mean().item(),
341
+ 'avg_return': returns.mean().item(),
342
+ }
343
+
344
+ def train_step(
345
+ self,
346
+ batch: Dict[str, torch.Tensor],
347
+ ) -> Dict[str, float]:
348
+ """
349
+ Single RL training step.
350
+
351
+ Args:
352
+ batch: Batch from StepDataset
353
+
354
+ Returns:
355
+ Training metrics
356
+ """
357
+ # Extract batch data
358
+ image_features = batch['visual_features'].to(self.device) # [batch, vision_dim]
359
+ prompt_ids = batch['prompt_ids'].to(self.device)
360
+ prompt_mask = batch['prompt_mask'].to(self.device)
361
+
362
+ batch_size = image_features.size(0)
363
+ all_metrics = []
364
+
365
+ for b in range(batch_size):
366
+ # Generate reasoning chain with current policy
367
+ step_embeddings, step_logprobs, step_entropies = self.generate_reasoning_chain(
368
+ image_features[b],
369
+ prompt_ids[b],
370
+ prompt_mask[b],
371
+ max_steps=10,
372
+ )
373
+
374
+ if not step_embeddings:
375
+ continue
376
+
377
+ # Get PRM rewards for each step
378
+ step_rewards = []
379
+ for step_emb in step_embeddings:
380
+ reward, _ = self.prm.compute_step_reward(
381
+ image_features[b],
382
+ step_emb,
383
+ previous_steps=torch.stack(step_embeddings[:len(step_rewards)]) if step_rewards else None,
384
+ )
385
+ step_rewards.append(reward * self.config.reward_scale)
386
+
387
+ rewards = torch.tensor(step_rewards, device=self.device)
388
+ logprobs = torch.stack(step_logprobs)
389
+
390
+ # Compute value estimates
391
+ embeddings = torch.stack(step_embeddings)
392
+ with torch.no_grad():
393
+ values = self.value_head(embeddings).squeeze(-1)
394
+
395
+ # Done flags (only last step is done)
396
+ dones = torch.zeros(len(step_embeddings), device=self.device)
397
+ dones[-1] = 1.0
398
+
399
+ # Compute advantages and returns
400
+ advantages, returns = self.compute_advantages(rewards, values, dones)
401
+
402
+ # PPO update
403
+ metrics = self.ppo_update(
404
+ step_embeddings,
405
+ logprobs,
406
+ rewards,
407
+ advantages,
408
+ returns,
409
+ )
410
+ all_metrics.append(metrics)
411
+
412
+ # Average metrics across batch
413
+ if not all_metrics:
414
+ return {}
415
+
416
+ avg_metrics = {
417
+ key: sum(m[key] for m in all_metrics) / len(all_metrics)
418
+ for key in all_metrics[0].keys()
419
+ }
420
+
421
+ return avg_metrics
422
+
423
+ def train(
424
+ self,
425
+ train_dataset: StepDataset,
426
+ num_iterations: int = 1000,
427
+ log_interval: int = 10,
428
+ ) -> None:
429
+ """
430
+ Train the policy with RL.
431
+
432
+ Args:
433
+ train_dataset: Dataset with reasoning chains
434
+ num_iterations: Number of training iterations
435
+ log_interval: Logging frequency
436
+ """
437
+ dataloader = DataLoader(
438
+ train_dataset,
439
+ batch_size=self.config.batch_size,
440
+ shuffle=True,
441
+ )
442
+
443
+ logger.info(f"Starting RL training for {num_iterations} iterations")
444
+
445
+ for iteration in range(num_iterations):
446
+ for batch in dataloader:
447
+ metrics = self.train_step(batch)
448
+
449
+ if metrics and iteration % log_interval == 0:
450
+ logger.info(
451
+ f"Iteration {iteration}: "
452
+ f"Policy Loss: {metrics['policy_loss']:.4f}, "
453
+ f"Value Loss: {metrics['value_loss']:.4f}, "
454
+ f"Avg Reward: {metrics['avg_reward']:.4f}"
455
+ )
456
+
457
+ logger.info("RL training completed")
458
+
459
+ def save_checkpoint(self, path: str) -> None:
460
+ """Save training checkpoint."""
461
+ checkpoint = {
462
+ 'policy_state_dict': self.policy.state_dict(),
463
+ 'value_head_state_dict': self.value_head.state_dict(),
464
+ 'optimizer_state_dict': self.optimizer.state_dict(),
465
+ 'value_optimizer_state_dict': self.value_optimizer.state_dict(),
466
+ 'global_step': self.global_step,
467
+ 'config': self.config,
468
+ }
469
+ torch.save(checkpoint, path)
470
+ logger.info(f"Checkpoint saved to {path}")
471
+
472
+ def load_checkpoint(self, path: str) -> None:
473
+ """Load training checkpoint."""
474
+ checkpoint = torch.load(path, map_location=self.device)
475
+ self.policy.load_state_dict(checkpoint['policy_state_dict'])
476
+ self.value_head.load_state_dict(checkpoint['value_head_state_dict'])
477
+ self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
478
+ self.value_optimizer.load_state_dict(checkpoint['value_optimizer_state_dict'])
479
+ self.global_step = checkpoint['global_step']
480
+ logger.info(f"Checkpoint loaded from {path}")
src/reasoning/step_data.py ADDED
@@ -0,0 +1,448 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Data structures for step-level reasoning chains with vision-language annotations.
3
+ """
4
+
5
+ from dataclasses import dataclass, field
6
+ from typing import Dict, List, Optional, Any
7
+ from enum import Enum
8
+ import torch
9
+ from torch.utils.data import Dataset
10
+ import json
11
+
12
+ # Special tokens for chain-of-thought reasoning
13
+ SPECIAL_TOKENS = [
14
+ # Reasoning structure
15
+ "<|reasoning_start|>", "<|reasoning_end|>",
16
+
17
+ # Step-level markers
18
+ "<|step_start|>", "<|step_end|>",
19
+ "<|step_type|>",
20
+
21
+ # Step components
22
+ "<|description_start|>", "<|description_end|>",
23
+ "<|confidence_start|>", "<|confidence_end|>",
24
+
25
+ # Dependencies
26
+ "<|depends_on|>",
27
+
28
+ # PRM evaluation tokens (from PRM800K)
29
+ "ΠΊΠΈ", # step separator for PRM scoring
30
+ "ΠΊ+ΠΈ", # positive step label
31
+ "ΠΊ-ΠΈ", # negative step label
32
+
33
+ # Final answer
34
+ "<|answer_start|>", "<|answer_end|>",
35
+
36
+ # Region tokens (inspired by Chain-of-Sight for visual grounding)
37
+ "<|region|>", "<|bbox|>", # Region markers and bounding box coordinates
38
+ ]
39
+
40
+ SPECIAL_TOKEN_DESCRIPTIONS = [
41
+ "start of reasoning", "end of reasoning",
42
+ "start of step", "end of step",
43
+ "step type indicator",
44
+ "start of description", "end of description",
45
+ "start of confidence", "end of confidence",
46
+ "depends on previous step",
47
+ "step separator for PRM", "positive step", "negative step",
48
+ "start of answer", "end of answer",
49
+ "region reference", "bounding box coordinates",
50
+ ]
51
+
52
+ # Generate additional region tokens like <region1>, <region2>, ... <region10>
53
+ # for referring to specific visual regions (Chain-of-Sight style)
54
+ REGION_TOKENS = [f"<region{i}>" for i in range(1, 11)]
55
+ SPECIAL_TOKENS.extend(REGION_TOKENS)
56
+ SPECIAL_TOKEN_DESCRIPTIONS.extend([f"visual region {i}" for i in range(1, 11)])
57
+
58
+
59
+ class StepType(Enum):
60
+ """Types of reasoning steps in vision-language tasks."""
61
+ PERCEPTION = "perception" # Visual understanding
62
+ LOCALIZATION = "localization" # Spatial reasoning
63
+ COMPARISON = "comparison" # Comparing visual elements
64
+ COUNTING = "counting" # Numerical reasoning
65
+ INFERENCE = "inference" # Logical deduction
66
+ COMPOSITION = "composition" # Combining information
67
+ VERIFICATION = "verification" # Checking consistency
68
+
69
+
70
+ @dataclass
71
+ class ReasoningStep:
72
+ """
73
+ A single step in the chain of thought reasoning process.
74
+
75
+ Attributes:
76
+ step_id: Unique identifier for this step
77
+ step_type: Type of reasoning performed
78
+ description: Natural language description of the reasoning
79
+ visual_features: Extracted visual features relevant to this step
80
+ attention_map: Spatial attention weights (H, W)
81
+ confidence: Model's confidence in this step [0, 1]
82
+ reward: Process reward score assigned by PRM [-1, 1]
83
+ dependencies: IDs of previous steps this depends on
84
+ bounding_boxes: List of [x1, y1, x2, y2] bounding boxes (Chain-of-Sight style)
85
+ region_ids: List of region IDs referenced in this step (e.g., [1, 2])
86
+ metadata: Additional step-specific information
87
+ """
88
+ step_id: int
89
+ step_type: StepType
90
+ description: str
91
+ visual_features: Optional[torch.Tensor] = None
92
+ attention_map: Optional[torch.Tensor] = None
93
+ confidence: float = 0.0
94
+ reward: float = 0.0
95
+ dependencies: List[int] = field(default_factory=list)
96
+ bounding_boxes: List[List[float]] = field(default_factory=list) # [[x1,y1,x2,y2], ...]
97
+ region_ids: List[int] = field(default_factory=list) # [1, 2, 3]
98
+ metadata: Dict[str, Any] = field(default_factory=dict)
99
+
100
+ def to_dict(self) -> Dict[str, Any]:
101
+ """Convert to dictionary for serialization."""
102
+ return {
103
+ 'step_id': self.step_id,
104
+ 'step_type': self.step_type.value,
105
+ 'description': self.description,
106
+ 'confidence': self.confidence,
107
+ 'reward': self.reward,
108
+ 'dependencies': self.dependencies,
109
+ 'metadata': self.metadata,
110
+ 'has_visual_features': self.visual_features is not None,
111
+ 'has_attention_map': self.attention_map is not None,
112
+ }
113
+
114
+ def format_with_tokens(self) -> str:
115
+ """Format step with special tokens for training (with Chain-of-Sight region support)."""
116
+ parts = ["<|step_start|>"]
117
+
118
+ # Add step type
119
+ parts.append(f"<|step_type|>{self.step_type.value}")
120
+
121
+ # Add dependencies if any
122
+ if self.dependencies:
123
+ deps_str = ",".join(map(str, self.dependencies))
124
+ parts.append(f"<|depends_on|>{deps_str}")
125
+
126
+ # Add region references (Chain-of-Sight style)
127
+ if self.region_ids:
128
+ region_refs = " ".join([f"<region{rid}>" for rid in self.region_ids])
129
+ parts.append(f"<|region|>{region_refs}")
130
+
131
+ # Add bounding boxes if present
132
+ if self.bounding_boxes:
133
+ bbox_strs = [f"[{b[0]:.3f},{b[1]:.3f},{b[2]:.3f},{b[3]:.3f}]" for b in self.bounding_boxes]
134
+ parts.append(f"<|bbox|>{','.join(bbox_strs)}")
135
+
136
+ # Add description
137
+ parts.append(f"<|description_start|>{self.description}<|description_end|>")
138
+
139
+ # Add confidence
140
+ parts.append(f"<|confidence_start|>{self.confidence:.3f}<|confidence_end|>")
141
+
142
+ # Add PRM separator
143
+ parts.append("ΠΊΠΈ")
144
+
145
+ parts.append("<|step_end|>")
146
+
147
+ return " ".join(parts)
148
+
149
+ @classmethod
150
+ def from_dict(cls, data: Dict[str, Any]) -> 'ReasoningStep':
151
+ """Create from dictionary."""
152
+ return cls(
153
+ step_id=data['step_id'],
154
+ step_type=StepType(data['step_type']),
155
+ description=data['description'],
156
+ confidence=data.get('confidence', 0.0),
157
+ reward=data.get('reward', 0.0),
158
+ dependencies=data.get('dependencies', []),
159
+ metadata=data.get('metadata', {}),
160
+ )
161
+
162
+
163
+ @dataclass
164
+ class ReasoningChain:
165
+ """
166
+ Complete chain of reasoning steps for a vision-language task.
167
+
168
+ Attributes:
169
+ chain_id: Unique identifier for this reasoning chain
170
+ image_path: Path to the input image
171
+ prompt: Input text prompt/question
172
+ steps: Ordered list of reasoning steps
173
+ final_answer: Final output/answer
174
+ total_reward: Cumulative reward from PRM
175
+ is_correct: Whether final answer is correct
176
+ metadata: Additional chain-level information
177
+ """
178
+ chain_id: str
179
+ image_path: str
180
+ prompt: str
181
+ steps: List[ReasoningStep]
182
+ final_answer: str
183
+ total_reward: float = 0.0
184
+ is_correct: bool = False
185
+ metadata: Dict[str, Any] = field(default_factory=dict)
186
+
187
+ def __len__(self) -> int:
188
+ """Number of reasoning steps."""
189
+ return len(self.steps)
190
+
191
+ def get_step_rewards(self) -> List[float]:
192
+ """Extract reward for each step."""
193
+ return [step.reward for step in self.steps]
194
+
195
+ def get_cumulative_rewards(self) -> List[float]:
196
+ """Calculate cumulative rewards up to each step."""
197
+ cumulative = []
198
+ total = 0.0
199
+ for step in self.steps:
200
+ total += step.reward
201
+ cumulative.append(total)
202
+ return cumulative
203
+
204
+ def to_dict(self) -> Dict[str, Any]:
205
+ """Convert to dictionary for serialization."""
206
+ return {
207
+ 'chain_id': self.chain_id,
208
+ 'image_path': self.image_path,
209
+ 'prompt': self.prompt,
210
+ 'steps': [step.to_dict() for step in self.steps],
211
+ 'final_answer': self.final_answer,
212
+ 'total_reward': self.total_reward,
213
+ 'is_correct': self.is_correct,
214
+ 'metadata': self.metadata
215
+ }
216
+
217
+ def format_with_tokens(self) -> str:
218
+ """Format entire reasoning chain with special tokens for training."""
219
+ parts = ["<|reasoning_start|>"]
220
+
221
+ # Add all steps
222
+ for step in self.steps:
223
+ parts.append(step.format_with_tokens())
224
+
225
+ parts.append("<|reasoning_end|>")
226
+
227
+ # Add final answer
228
+ parts.append(f"<|answer_start|>{self.final_answer}<|answer_end|>")
229
+
230
+ return " ".join(parts)
231
+
232
+ @classmethod
233
+ def from_dict(cls, data: Dict[str, Any]) -> 'ReasoningChain':
234
+ """Create from dictionary."""
235
+ return cls(
236
+ chain_id=data['chain_id'],
237
+ image_path=data['image_path'],
238
+ prompt=data['prompt'],
239
+ steps=[ReasoningStep.from_dict(s) for s in data['steps']],
240
+ final_answer=data['final_answer'],
241
+ total_reward=data.get('total_reward', 0.0),
242
+ is_correct=data.get('is_correct', False),
243
+ metadata=data.get('metadata', {}),
244
+ )
245
+
246
+ def save_json(self, path: str) -> None:
247
+ """Save reasoning chain to JSON file."""
248
+ with open(path, 'w', encoding='utf-8') as f:
249
+ json.dump(self.to_dict(), f, indent=2, ensure_ascii=False)
250
+
251
+ @classmethod
252
+ def load_json(cls, path: str) -> 'ReasoningChain':
253
+ """Load reasoning chain from JSON file."""
254
+ with open(path, 'r', encoding='utf-8') as f:
255
+ data = json.load(f)
256
+ return cls.from_dict(data)
257
+
258
+
259
+ class StepDataset(Dataset):
260
+ """
261
+ Dataset for training with step-level reasoning chains.
262
+
263
+ Supports both supervised fine-tuning and RL training with process rewards.
264
+ """
265
+
266
+ def __init__(
267
+ self,
268
+ reasoning_chains: List[ReasoningChain],
269
+ tokenizer: Any,
270
+ max_steps: int = 10,
271
+ include_visual_features: bool = True,
272
+ filter_incorrect: bool = False,
273
+ ):
274
+ """
275
+ Initialize step dataset.
276
+
277
+ Args:
278
+ reasoning_chains: List of annotated reasoning chains
279
+ tokenizer: Tokenizer for text encoding
280
+ max_steps: Maximum number of steps per chain
281
+ include_visual_features: Whether to include visual features
282
+ filter_incorrect: If True, only include correct chains
283
+ """
284
+ self.tokenizer = tokenizer
285
+ self.max_steps = max_steps
286
+ self.include_visual_features = include_visual_features
287
+
288
+ # Filter chains if needed
289
+ if filter_incorrect:
290
+ self.chains = [c for c in reasoning_chains if c.is_correct]
291
+ else:
292
+ self.chains = reasoning_chains
293
+
294
+ # Pre-compute statistics
295
+ self.avg_steps = sum(len(c) for c in self.chains) / len(self.chains) if self.chains else 0
296
+ self.total_steps = sum(len(c) for c in self.chains)
297
+
298
+ def __len__(self) -> int:
299
+ return len(self.chains)
300
+
301
+ def __getitem__(self, idx: int) -> Dict[str, Any]:
302
+ """
303
+ Get a single reasoning chain with all steps.
304
+
305
+ Returns:
306
+ Dictionary containing:
307
+ - prompt_ids: Tokenized prompt
308
+ - formatted_input_ids: Tokenized chain with special tokens
309
+ - formatted_text: Raw text with special tokens
310
+ - step_descriptions: List of step descriptions
311
+ - step_types: List of step types
312
+ - step_rewards: List of process rewards
313
+ - visual_features: List of visual features (if enabled)
314
+ - attention_maps: List of attention maps (if available)
315
+ - final_answer: Target answer
316
+ - is_correct: Ground truth correctness
317
+ - chain_metadata: Additional information
318
+ """
319
+ chain = self.chains[idx]
320
+
321
+ # Format chain with special tokens
322
+ formatted_text = chain.format_with_tokens()
323
+
324
+ # Tokenize formatted text (for training)
325
+ formatted_encoding = self.tokenizer(
326
+ formatted_text,
327
+ truncation=True,
328
+ max_length=2048, # Longer for full reasoning chains
329
+ return_tensors='pt',
330
+ )
331
+
332
+ # Tokenize prompt (for compatibility)
333
+ prompt_encoding = self.tokenizer(
334
+ chain.prompt,
335
+ truncation=True,
336
+ max_length=512,
337
+ return_tensors='pt',
338
+ )
339
+
340
+ # Prepare step data
341
+ step_descriptions = []
342
+ step_types = []
343
+ step_rewards = []
344
+ visual_features = []
345
+ attention_maps = []
346
+
347
+ for step in chain.steps[:self.max_steps]:
348
+ step_descriptions.append(step.description)
349
+ step_types.append(step.step_type.value)
350
+ step_rewards.append(step.reward)
351
+
352
+ if self.include_visual_features:
353
+ if step.visual_features is not None:
354
+ visual_features.append(step.visual_features)
355
+ else:
356
+ visual_features.append(torch.zeros(768)) # Default dimension
357
+
358
+ if step.attention_map is not None:
359
+ attention_maps.append(step.attention_map)
360
+
361
+ # Tokenize final answer
362
+ answer_encoding = self.tokenizer(
363
+ chain.final_answer,
364
+ truncation=True,
365
+ max_length=256,
366
+ return_tensors='pt',
367
+ )
368
+
369
+ result = {
370
+ 'prompt_ids': prompt_encoding['input_ids'].squeeze(0),
371
+ 'prompt_mask': prompt_encoding['attention_mask'].squeeze(0),
372
+ 'formatted_input_ids': formatted_encoding['input_ids'].squeeze(0),
373
+ 'formatted_attention_mask': formatted_encoding['attention_mask'].squeeze(0),
374
+ 'formatted_text': formatted_text,
375
+ 'step_descriptions': step_descriptions,
376
+ 'step_types': step_types,
377
+ 'step_rewards': torch.tensor(step_rewards, dtype=torch.float32),
378
+ 'answer_ids': answer_encoding['input_ids'].squeeze(0),
379
+ 'answer_mask': answer_encoding['attention_mask'].squeeze(0),
380
+ 'is_correct': torch.tensor(chain.is_correct, dtype=torch.bool),
381
+ 'total_reward': torch.tensor(chain.total_reward, dtype=torch.float32),
382
+ 'num_steps': len(chain.steps),
383
+ 'chain_id': chain.chain_id,
384
+ 'image_path': chain.image_path,
385
+ }
386
+
387
+ if self.include_visual_features and visual_features:
388
+ result['visual_features'] = torch.stack(visual_features)
389
+
390
+ if attention_maps:
391
+ result['attention_maps'] = attention_maps
392
+
393
+ return result
394
+
395
+ @classmethod
396
+ def from_json_dir(
397
+ cls,
398
+ json_dir: str,
399
+ tokenizer: Any,
400
+ **kwargs
401
+ ) -> 'StepDataset':
402
+ """
403
+ Load dataset from directory of JSON files.
404
+
405
+ Args:
406
+ json_dir: Directory containing reasoning chain JSON files
407
+ tokenizer: Tokenizer for text encoding
408
+ **kwargs: Additional arguments for StepDataset
409
+ """
410
+ from pathlib import Path
411
+
412
+ json_path = Path(json_dir)
413
+ chains = []
414
+
415
+ for json_file in json_path.glob('*.json'):
416
+ try:
417
+ chain = ReasoningChain.load_json(str(json_file))
418
+ chains.append(chain)
419
+ except Exception as e:
420
+ print(f"Error loading {json_file}: {e}")
421
+
422
+ return cls(chains, tokenizer, **kwargs)
423
+
424
+ def get_statistics(self) -> Dict[str, Any]:
425
+ """Calculate dataset statistics."""
426
+ step_types = {}
427
+ rewards = []
428
+ correct_count = 0
429
+
430
+ for chain in self.chains:
431
+ if chain.is_correct:
432
+ correct_count += 1
433
+
434
+ for step in chain.steps:
435
+ step_type = step.step_type.value
436
+ step_types[step_type] = step_types.get(step_type, 0) + 1
437
+ rewards.append(step.reward)
438
+
439
+ return {
440
+ 'num_chains': len(self.chains),
441
+ 'total_steps': self.total_steps,
442
+ 'avg_steps_per_chain': self.avg_steps,
443
+ 'accuracy': correct_count / len(self.chains) if self.chains else 0,
444
+ 'step_type_distribution': step_types,
445
+ 'avg_step_reward': sum(rewards) / len(rewards) if rewards else 0,
446
+ 'min_step_reward': min(rewards) if rewards else 0,
447
+ 'max_step_reward': max(rewards) if rewards else 0,
448
+ }
src/reasoning/step_level_cot.py ADDED
@@ -0,0 +1,477 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Step-Level Chain of Thought Trainer for Vision-Language Models.
3
+
4
+ Integrates PRM, RL training, and inference-time scaling for fine-grained reasoning.
5
+ """
6
+
7
+ import argparse
8
+ import logging
9
+ from pathlib import Path
10
+ from typing import Dict, Optional, Any
11
+ import torch
12
+ import yaml
13
+ from accelerate import Accelerator
14
+ from accelerate.logging import get_logger
15
+ from transformers import AutoTokenizer
16
+ from diffusers import QwenImagePipeline, QwenImageTransformer2DModel
17
+ from omegaconf import OmegaConf
18
+ from tqdm.auto import tqdm
19
+
20
+ from .step_data import StepDataset, ReasoningChain
21
+ from .prm import ProcessRewardModel, PRMTrainer
22
+ from .rl_trainer import RLReasoningTrainer, RLConfig
23
+ from .inference_scaling import InferenceTimeScaling, InferenceConfig
24
+
25
+ logger = get_logger(__name__, log_level="INFO")
26
+
27
+
28
+ class StepLevelCoTTrainer:
29
+ """
30
+ Complete training pipeline for step-level chain-of-thought reasoning.
31
+
32
+ Training phases:
33
+ 1. Supervised Fine-tuning (SFT) on reasoning chains
34
+ 2. PRM training on step-level rewards
35
+ 3. RL fine-tuning with PRM rewards
36
+ 4. Inference-time scaling evaluation
37
+ """
38
+
39
+ def __init__(
40
+ self,
41
+ config_path: str,
42
+ device: str = "cuda",
43
+ ):
44
+ """
45
+ Initialize trainer.
46
+
47
+ Args:
48
+ config_path: Path to training configuration YAML
49
+ device: Device for training
50
+ """
51
+ self.device = device
52
+
53
+ # Load configuration
54
+ with open(config_path, 'r') as f:
55
+ config_dict = yaml.safe_load(f)
56
+ self.config = OmegaConf.create(config_dict)
57
+
58
+ # Initialize accelerator
59
+ self.accelerator = Accelerator(
60
+ gradient_accumulation_steps=self.config.gradient_accumulation_steps,
61
+ mixed_precision=self.config.mixed_precision,
62
+ log_with=self.config.report_to,
63
+ project_dir=self.config.output_dir,
64
+ )
65
+
66
+ # Load model and tokenizer
67
+ self._load_model()
68
+
69
+ # Initialize PRM
70
+ self.prm = ProcessRewardModel(
71
+ vision_dim=self.config.vision_dim,
72
+ text_dim=self.config.text_dim,
73
+ hidden_dim=self.config.prm_hidden_dim,
74
+ num_heads=self.config.prm_num_heads,
75
+ dropout=self.config.prm_dropout,
76
+ max_steps=self.config.max_reasoning_steps,
77
+ )
78
+
79
+ # Initialize RL trainer
80
+ rl_config = RLConfig(
81
+ clip_epsilon=self.config.ppo_clip_epsilon,
82
+ value_loss_coef=self.config.value_loss_coef,
83
+ entropy_coef=self.config.entropy_coef,
84
+ gamma=self.config.gamma,
85
+ gae_lambda=self.config.gae_lambda,
86
+ learning_rate=self.config.rl_learning_rate,
87
+ batch_size=self.config.train_batch_size,
88
+ use_prm_rewards=True,
89
+ normalize_rewards=True,
90
+ )
91
+
92
+ self.rl_trainer = RLReasoningTrainer(
93
+ policy_model=self.model,
94
+ prm_model=self.prm,
95
+ config=rl_config,
96
+ device=device,
97
+ )
98
+
99
+ # Initialize inference scaler
100
+ inference_config = InferenceConfig(
101
+ num_samples=self.config.num_inference_samples,
102
+ temperature=self.config.inference_temperature,
103
+ aggregation=self.config.aggregation_method,
104
+ use_prm_scores=True,
105
+ )
106
+
107
+ self.inference_scaler = InferenceTimeScaling(
108
+ model=self.model,
109
+ prm=self.prm,
110
+ config=inference_config,
111
+ device=device,
112
+ )
113
+
114
+ logger.info("StepLevelCoTTrainer initialized")
115
+
116
+ def _load_model(self) -> None:
117
+ """Load vision-language model."""
118
+ logger.info(f"Loading model: {self.config.pretrained_model_name_or_path}")
119
+
120
+ # Load Qwen-Image model
121
+ self.pipeline = QwenImagePipeline.from_pretrained(
122
+ self.config.pretrained_model_name_or_path,
123
+ torch_dtype=torch.float16 if self.config.mixed_precision == "fp16" else torch.float32,
124
+ )
125
+
126
+ self.model = self.pipeline.transformer
127
+ self.vae = self.pipeline.vae
128
+ self.text_encoder = self.pipeline.text_encoder
129
+ self.tokenizer = self.pipeline.tokenizer
130
+
131
+ # Add special tokens for chain-of-thought reasoning
132
+ self._add_special_tokens()
133
+
134
+ # Enable gradient checkpointing
135
+ if self.config.gradient_checkpointing:
136
+ self.model.enable_gradient_checkpointing()
137
+
138
+ logger.info("Model loaded successfully")
139
+
140
+ def _add_special_tokens(self) -> None:
141
+ """Add special tokens for chain-of-thought reasoning."""
142
+ from .step_data import SPECIAL_TOKENS, SPECIAL_TOKEN_DESCRIPTIONS
143
+
144
+ logger.info(f"Adding {len(SPECIAL_TOKENS)} special tokens for chain-of-thought reasoning...")
145
+
146
+ # Add special tokens to tokenizer
147
+ num_added_toks = self.tokenizer.add_special_tokens(
148
+ {"additional_special_tokens": SPECIAL_TOKENS}
149
+ )
150
+
151
+ if num_added_toks > 0:
152
+ logger.info(f"Added {num_added_toks} new tokens. Resizing model embeddings...")
153
+
154
+ # Get current embedding size
155
+ old_size = self.model.get_input_embeddings().weight.size(0)
156
+
157
+ # Resize token embeddings
158
+ self.model.resize_token_embeddings(len(self.tokenizer))
159
+
160
+ # Initialize new token embeddings with semantic initialization
161
+ self._initialize_new_token_embeddings(num_added_toks, SPECIAL_TOKEN_DESCRIPTIONS)
162
+
163
+ logger.info(f"Resized embeddings from {old_size} to {len(self.tokenizer)}")
164
+ else:
165
+ logger.info("Special tokens already present in tokenizer.")
166
+
167
+ def _initialize_new_token_embeddings(self, num_added_toks: int, descriptions: list) -> None:
168
+ """Initialize newly added token embeddings with semantic information."""
169
+ with torch.no_grad():
170
+ input_embeddings = self.model.get_input_embeddings()
171
+
172
+ # Check if model has language model head
173
+ has_lm_head = hasattr(self.model, 'lm_head')
174
+ if has_lm_head:
175
+ lm_head = self.model.lm_head
176
+
177
+ for i, description in enumerate(reversed(descriptions), start=1):
178
+ # Tokenize the description
179
+ tokenized = self.tokenizer.tokenize(description)
180
+ if not tokenized:
181
+ continue
182
+
183
+ tokenized_ids = self.tokenizer.convert_tokens_to_ids(tokenized)
184
+
185
+ # Average embeddings of description tokens
186
+ new_embedding = input_embeddings.weight[tokenized_ids].mean(dim=0)
187
+ input_embeddings.weight[-i, :] = new_embedding.clone().detach()
188
+
189
+ # Initialize lm_head if it exists
190
+ if has_lm_head:
191
+ new_lm_head = lm_head.weight[tokenized_ids].mean(dim=0)
192
+ lm_head.weight[-i, :] = new_lm_head.clone().detach()
193
+
194
+ logger.info(f"Initialized {num_added_toks} new token embeddings with semantic information.")
195
+
196
+ def load_step_dataset(
197
+ self,
198
+ data_dir: str,
199
+ split: str = "train",
200
+ ) -> StepDataset:
201
+ """
202
+ Load step-level reasoning dataset.
203
+
204
+ Args:
205
+ data_dir: Directory containing reasoning chain JSON files
206
+ split: Dataset split (train/val/test)
207
+
208
+ Returns:
209
+ StepDataset instance
210
+ """
211
+ data_path = Path(data_dir) / split
212
+
213
+ logger.info(f"Loading dataset from {data_path}")
214
+
215
+ dataset = StepDataset.from_json_dir(
216
+ str(data_path),
217
+ tokenizer=self.tokenizer,
218
+ max_steps=self.config.max_reasoning_steps,
219
+ include_visual_features=True,
220
+ filter_incorrect=False, # Include all chains for learning
221
+ )
222
+
223
+ stats = dataset.get_statistics()
224
+ logger.info(f"Dataset loaded: {stats}")
225
+
226
+ return dataset
227
+
228
+ def train_prm(
229
+ self,
230
+ train_dataset: StepDataset,
231
+ val_dataset: Optional[StepDataset] = None,
232
+ num_epochs: int = 10,
233
+ ) -> None:
234
+ """
235
+ Train Process Reward Model.
236
+
237
+ Args:
238
+ train_dataset: Training dataset with step-level rewards
239
+ val_dataset: Optional validation dataset
240
+ num_epochs: Number of training epochs
241
+ """
242
+ logger.info("Starting PRM training")
243
+
244
+ prm_trainer = PRMTrainer(
245
+ model=self.prm,
246
+ learning_rate=self.config.prm_learning_rate,
247
+ weight_decay=self.config.weight_decay,
248
+ warmup_steps=self.config.warmup_steps,
249
+ )
250
+
251
+ train_dataloader = torch.utils.data.DataLoader(
252
+ train_dataset,
253
+ batch_size=self.config.train_batch_size,
254
+ shuffle=True,
255
+ num_workers=self.config.dataloader_num_workers,
256
+ )
257
+
258
+ # Prepare with accelerator
259
+ prm_trainer.model, prm_trainer.optimizer, train_dataloader = self.accelerator.prepare(
260
+ prm_trainer.model, prm_trainer.optimizer, train_dataloader
261
+ )
262
+
263
+ best_val_loss = float('inf')
264
+
265
+ for epoch in range(num_epochs):
266
+ # Training loop
267
+ prm_trainer.model.train()
268
+ total_loss = 0
269
+
270
+ progress_bar = tqdm(
271
+ train_dataloader,
272
+ desc=f"PRM Epoch {epoch + 1}/{num_epochs}",
273
+ disable=not self.accelerator.is_local_main_process,
274
+ )
275
+
276
+ for batch in progress_bar:
277
+ # Extract batch data
278
+ vision_features = batch['visual_features']
279
+ step_embeddings = batch['step_descriptions'] # Need to encode these
280
+ target_rewards = batch['step_rewards']
281
+
282
+ # Encode step descriptions
283
+ # (In practice, you'd encode text properly)
284
+
285
+ metrics = prm_trainer.train_step(
286
+ vision_features,
287
+ step_embeddings,
288
+ target_rewards,
289
+ )
290
+
291
+ total_loss += metrics['loss']
292
+ progress_bar.set_postfix({'loss': metrics['loss']})
293
+
294
+ avg_loss = total_loss / len(train_dataloader)
295
+ logger.info(f"Epoch {epoch + 1} - Avg Loss: {avg_loss:.4f}")
296
+
297
+ # Validation
298
+ if val_dataset is not None:
299
+ val_metrics = self._evaluate_prm(prm_trainer, val_dataset)
300
+ logger.info(f"Validation: {val_metrics}")
301
+
302
+ # Save best model
303
+ if val_metrics['mse'] < best_val_loss:
304
+ best_val_loss = val_metrics['mse']
305
+ self._save_prm(epoch)
306
+
307
+ logger.info("PRM training completed")
308
+
309
+ def train_rl(
310
+ self,
311
+ train_dataset: StepDataset,
312
+ num_iterations: int = 1000,
313
+ ) -> None:
314
+ """
315
+ Train policy with reinforcement learning.
316
+
317
+ Args:
318
+ train_dataset: Training dataset
319
+ num_iterations: Number of RL iterations
320
+ """
321
+ logger.info("Starting RL training")
322
+
323
+ self.rl_trainer.train(
324
+ train_dataset=train_dataset,
325
+ num_iterations=num_iterations,
326
+ log_interval=self.config.logging_steps,
327
+ )
328
+
329
+ logger.info("RL training completed")
330
+
331
+ def evaluate_inference_scaling(
332
+ self,
333
+ test_dataset: StepDataset,
334
+ num_samples: int = 10,
335
+ ) -> Dict[str, Any]:
336
+ """
337
+ Evaluate inference-time scaling performance.
338
+
339
+ Args:
340
+ test_dataset: Test dataset
341
+ num_samples: Number of test samples
342
+
343
+ Returns:
344
+ Evaluation metrics
345
+ """
346
+ logger.info("Evaluating inference-time scaling")
347
+
348
+ results = []
349
+
350
+ for idx in range(min(num_samples, len(test_dataset))):
351
+ sample = test_dataset[idx]
352
+
353
+ # Run inference with scaling
354
+ best_chain, all_chains = self.inference_scaler(
355
+ image_features=sample['visual_features'][0],
356
+ prompt_ids=sample['prompt_ids'],
357
+ prompt_mask=sample['prompt_mask'],
358
+ image_path=sample['image_path'],
359
+ prompt_text="", # Would decode from IDs
360
+ )
361
+
362
+ results.append({
363
+ 'chain_id': sample['chain_id'],
364
+ 'best_reward': best_chain.total_reward,
365
+ 'num_steps': len(best_chain.steps),
366
+ 'avg_reward_all': sum(c.total_reward for c in all_chains) / len(all_chains),
367
+ 'std_reward_all': torch.tensor([c.total_reward for c in all_chains]).std().item(),
368
+ })
369
+
370
+ # Aggregate metrics
371
+ metrics = {
372
+ 'avg_best_reward': sum(r['best_reward'] for r in results) / len(results),
373
+ 'avg_num_steps': sum(r['num_steps'] for r in results) / len(results),
374
+ 'improvement_over_mean': sum(
375
+ (r['best_reward'] - r['avg_reward_all']) / (r['std_reward_all'] + 1e-8)
376
+ for r in results
377
+ ) / len(results),
378
+ }
379
+
380
+ logger.info(f"Inference scaling metrics: {metrics}")
381
+
382
+ return metrics
383
+
384
+ def _evaluate_prm(
385
+ self,
386
+ prm_trainer: PRMTrainer,
387
+ val_dataset: StepDataset,
388
+ ) -> Dict[str, float]:
389
+ """Evaluate PRM on validation set."""
390
+ val_dataloader = torch.utils.data.DataLoader(
391
+ val_dataset,
392
+ batch_size=self.config.train_batch_size,
393
+ shuffle=False,
394
+ )
395
+
396
+ total_metrics = {'mse': 0, 'mae': 0, 'correlation': 0}
397
+
398
+ for batch in val_dataloader:
399
+ vision_features = batch['visual_features']
400
+ step_embeddings = batch['step_descriptions']
401
+ target_rewards = batch['step_rewards']
402
+
403
+ metrics = prm_trainer.evaluate(
404
+ vision_features,
405
+ step_embeddings,
406
+ target_rewards,
407
+ )
408
+
409
+ for key in total_metrics:
410
+ total_metrics[key] += metrics[key]
411
+
412
+ # Average
413
+ for key in total_metrics:
414
+ total_metrics[key] /= len(val_dataloader)
415
+
416
+ return total_metrics
417
+
418
+ def _save_prm(self, epoch: int) -> None:
419
+ """Save PRM checkpoint."""
420
+ save_path = Path(self.config.output_dir) / "prm" / f"epoch_{epoch}"
421
+ save_path.mkdir(parents=True, exist_ok=True)
422
+
423
+ torch.save(
424
+ self.prm.state_dict(),
425
+ save_path / "prm_model.pt",
426
+ )
427
+ logger.info(f"PRM saved to {save_path}")
428
+
429
+ def save_checkpoint(self, step: int) -> None:
430
+ """Save complete training checkpoint."""
431
+ save_path = Path(self.config.output_dir) / f"checkpoint-{step}"
432
+ save_path.mkdir(parents=True, exist_ok=True)
433
+
434
+ # Save policy
435
+ self.accelerator.save_state(str(save_path / "policy"))
436
+
437
+ # Save PRM
438
+ torch.save(self.prm.state_dict(), save_path / "prm.pt")
439
+
440
+ # Save RL trainer state
441
+ self.rl_trainer.save_checkpoint(str(save_path / "rl_trainer.pt"))
442
+
443
+ logger.info(f"Checkpoint saved to {save_path}")
444
+
445
+
446
+ def main():
447
+ """Main training entry point."""
448
+ parser = argparse.ArgumentParser(description="Train step-level CoT reasoning for vision models")
449
+ parser.add_argument("--config", type=str, required=True, help="Path to config YAML")
450
+ parser.add_argument("--phase", type=str, choices=["prm", "rl", "inference", "all"], default="all")
451
+ parser.add_argument("--data_dir", type=str, required=True, help="Path to reasoning chain data")
452
+ args = parser.parse_args()
453
+
454
+ # Initialize trainer
455
+ trainer = StepLevelCoTTrainer(
456
+ config_path=args.config,
457
+ device="cuda" if torch.cuda.is_available() else "cpu",
458
+ )
459
+
460
+ # Load datasets
461
+ train_dataset = trainer.load_step_dataset(args.data_dir, split="train")
462
+ val_dataset = trainer.load_step_dataset(args.data_dir, split="val")
463
+ test_dataset = trainer.load_step_dataset(args.data_dir, split="test")
464
+
465
+ # Run training phases
466
+ if args.phase in ["prm", "all"]:
467
+ trainer.train_prm(train_dataset, val_dataset)
468
+
469
+ if args.phase in ["rl", "all"]:
470
+ trainer.train_rl(train_dataset)
471
+
472
+ if args.phase in ["inference", "all"]:
473
+ trainer.evaluate_inference_scaling(test_dataset)
474
+
475
+
476
+ if __name__ == "__main__":
477
+ main()
train_configs/train_full_qwen_image.yaml ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Full Training Configuration for Qwen Image Model
2
+ pretrained_model_name_or_path: Qwen/Qwen-Image
3
+
4
+ # Data configuration
5
+ data_config:
6
+ train_batch_size: 1
7
+ num_workers: 4
8
+ img_size: 1024
9
+ caption_dropout_rate: 0.1
10
+ img_dir: /path/to/your/dataset
11
+ random_ratio: false
12
+ caption_type: txt
13
+
14
+ # Training parameters
15
+ train_batch_size: 1
16
+ output_dir: ./output_full_training
17
+ max_train_steps: 30000
18
+ num_train_epochs: 1
19
+
20
+ # Optimizer settings
21
+ learning_rate: 6e-4 # Lower learning rate for full training
22
+ use_8bit_adam: true # Use 8-bit Adam to save memory
23
+ adam_beta1: 0.9
24
+ adam_beta2: 0.999
25
+ adam_weight_decay: 0.01
26
+ adam_epsilon: 1e-8
27
+
28
+ # Learning rate scheduler
29
+ lr_scheduler: cosine_with_restarts
30
+ lr_warmup_steps: 1000
31
+
32
+ # Gradient settings
33
+ max_grad_norm: 1.0
34
+ gradient_accumulation_steps: 4 # Increase for effective batch size
35
+
36
+ # Precision and memory
37
+ mixed_precision: "bf16" # Use bfloat16 for better stability
38
+ freeze_text_encoder: true # Keep text encoder frozen to save memory
39
+
40
+ # Logging and checkpointing
41
+ logging_dir: logs
42
+ report_to: null # Can be "wandb", "tensorboard", etc.
43
+ checkpointing_steps: 1000
44
+ checkpoints_total_limit: 2
45
+ tracker_project_name: qwen_image_full_training
46
+
47
+ # Resume training
48
+ resume_from_checkpoint: latest
train_configs/train_lora.yaml ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pretrained_model_name_or_path: Qwen/Qwen-Image
2
+ data_config:
3
+ train_batch_size: 1
4
+ num_workers: 4
5
+ img_size: 1024
6
+ caption_dropout_rate: 0.05 # Lower dropout for better brand consistency
7
+ img_dir: ./brand_logo_dataset
8
+ random_ratio: false # Maintain consistent aspect ratios for logos
9
+ caption_type: txt
10
+ report_to: null
11
+ train_batch_size: 1
12
+ output_dir: ./brand_logo_output
13
+ max_train_steps: 5000 # Increased for better logo quality
14
+ learning_rate: 8e-5 # Lower learning rate for professional results
15
+ lr_scheduler: cosine_with_restarts
16
+ lr_warmup_steps: 200
17
+ adam_beta1: 0.9
18
+ adam_beta2: 0.999
19
+ adam_weight_decay: 0.01
20
+ adam_epsilon: 1e-8
21
+ max_grad_norm: 1.0
22
+ logging_dir: logs
23
+ mixed_precision: "bf16"
24
+ checkpointing_steps: 500
25
+ checkpoints_total_limit: 5
26
+ tracker_project_name: brand_logo_training
27
+ resume_from_checkpoint: latest
28
+ gradient_accumulation_steps: 1
29
+ rank: 32 # Higher rank for better detail in logos
train_configs/train_lora_4090.yaml ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pretrained_model_name_or_path: Qwen/Qwen-Image
2
+ data_config:
3
+ dataset_name: HuggingFaceM4/FineVision
4
+ subset: <your_subset_name> # e.g. "default" or the first from get_dataset_config_names
5
+ split: train
6
+ streaming: true
7
+ image_key: image
8
+ prompt_key: text
9
+ train_batch_size: 1
10
+ num_workers: 4
11
+ img_size: 1024
12
+ caption_dropout_rate: 0.1
13
+ # img_dir: ./your_lora_dataset
14
+ random_ratio: false # support multi crop preprocessing
15
+ caption_type: txt
16
+ report_to: null
17
+ train_batch_size: 1
18
+ output_dir: ./test_lora_saves
19
+ max_train_steps: 3000
20
+ learning_rate: 3e-4
21
+ lr_scheduler: constant
22
+ lr_warmup_steps: 10
23
+ adam_beta1: 0.9
24
+ adam_beta2: 0.999
25
+ adam_weight_decay: 0.01
26
+ adam_epsilon: 1e-8
27
+ max_grad_norm: 1.0
28
+ logging_dir: logs
29
+ mixed_precision: "fp16"
30
+ checkpointing_steps: 250
31
+ checkpoints_total_limit: 10
32
+ tracker_project_name: lora_test
33
+ resume_from_checkpoint: latest
34
+ gradient_accumulation_steps: 1
35
+ rank: 16
36
+ precompute_text_embeddings: true
37
+ precompute_image_embeddings: true
38
+ quantize: true
39
+ adam8bit: true
40
+ save_cache_on_disk: true
train_configs/train_lora_qwen_edit.yaml ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pretrained_model_name_or_path: Qwen/Qwen-Image-Edit
2
+ data_config:
3
+ train_batch_size: 1
4
+ num_workers: 4
5
+ img_size: 1024
6
+ caption_dropout_rate: 0.05 # Lower dropout for brand consistency
7
+ img_dir: ./brand_integration_dataset/images
8
+ control_dir: ./brand_integration_dataset/base_images
9
+ random_ratio: false # Maintain consistency for brand integration
10
+ caption_type: txt
11
+ report_to: null
12
+ train_batch_size: 1
13
+ output_dir: ./brand_integration_output
14
+ max_train_steps: 4000
15
+ learning_rate: 1e-4
16
+ lr_scheduler: cosine_with_restarts
17
+ lr_warmup_steps: 100
18
+ adam_beta1: 0.9
19
+ adam_beta2: 0.999
20
+ adam_weight_decay: 0.01
21
+ adam_epsilon: 1e-8
22
+ max_grad_norm: 1.0
23
+ logging_dir: logs
24
+ mixed_precision: "bf16"
25
+ checkpointing_steps: 400
26
+ checkpoints_total_limit: 5
27
+ tracker_project_name: brand_integration_training
28
+ resume_from_checkpoint: latest
29
+ gradient_accumulation_steps: 1
30
+ rank: 24 # Higher rank for detailed brand integration
31
+ precompute_text_embeddings: true
32
+ precompute_image_embeddings: true
33
+ quantize: false
34
+ adam8bit: false
35
+ save_cache_on_disk: true
36
+
train_configs/train_qwen_cot_dual.yaml ADDED
@@ -0,0 +1,250 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Training Configuration for Step-Level CoT Vision Model
2
+ # Using FLUX-Reason-6M + FineVision datasets with Qwen-Image base model
3
+
4
+ # Model Configuration
5
+ pretrained_model_name_or_path: Qwen/Qwen-Image
6
+ model_type: qwen_image # or internvl2, flux, etc.
7
+
8
+ # Special Tokens - Automatically added by StepLevelCoTTrainer
9
+ # 27 tokens total: 15 base reasoning + 12 Chain-of-Sight region tokens
10
+ add_special_tokens: true
11
+ special_tokens_path: null # Leave null for automatic token addition
12
+
13
+ # Output Configuration
14
+ output_dir: ./qwen_cot_checkpoints
15
+ logging_dir: ./qwen_cot_logs
16
+ tracker_project_name: qwen_step_cot_training
17
+ report_to: tensorboard # or "wandb" for Weights & Biases
18
+
19
+ # Dataset Configuration
20
+ datasets:
21
+ # FLUX-Reason-6M: Primary reasoning dataset with step-by-step annotations
22
+ - name: flux_reason
23
+ type: flux_reason_6m
24
+ dataset_name: LucasFang/FLUX-Reason-6M
25
+ split: train
26
+ streaming: true
27
+ max_samples: null # Use all available samples
28
+ weight: 0.7 # 70% of batches from FLUX-Reason
29
+ image_key: image
30
+ reasoning_key: reasoning
31
+ question_key: question
32
+ answer_key: answer
33
+
34
+ # FineVision: Vision-language understanding with synthetic reasoning
35
+ - name: finevision
36
+ type: finevision
37
+ dataset_name: HuggingFaceM4/FineVision
38
+ subset: null # Specify subset if needed
39
+ split: train
40
+ streaming: true
41
+ max_samples: null
42
+ weight: 0.3 # 30% of batches from FineVision
43
+ image_key: image
44
+ text_key: text
45
+ generate_synthetic_reasoning: true
46
+ num_synthetic_steps: 3
47
+
48
+ # Data Processing
49
+ data_config:
50
+ train_batch_size: 2 # Per device
51
+ gradient_accumulation_steps: 8 # Effective batch size: 2 * 8 = 16
52
+ num_workers: 4
53
+ max_sequence_length: 2048 # Longer for reasoning chains
54
+ img_size: 1024
55
+ caption_dropout_rate: 0.1
56
+ random_ratio: false
57
+ prefetch_factor: 2
58
+
59
+ # Training Phases
60
+ training_phases:
61
+ # Phase 1: Supervised Fine-Tuning with reasoning chains
62
+ - phase: sft
63
+ enabled: true
64
+ max_train_steps: 10000
65
+ learning_rate: 2e-5
66
+ warmup_steps: 500
67
+ save_steps: 1000
68
+ eval_steps: 500
69
+
70
+ # Phase 2: Process Reward Model (PRM) training
71
+ - phase: prm
72
+ enabled: true
73
+ max_train_steps: 5000
74
+ learning_rate: 1e-5
75
+ warmup_steps: 250
76
+ save_steps: 500
77
+ eval_steps: 250
78
+ prm_config:
79
+ hidden_dim: 768
80
+ num_layers: 2
81
+ dropout: 0.1
82
+ reward_scale: 1.0
83
+
84
+ # Phase 3: Reinforcement Learning (PPO) with PRM rewards
85
+ - phase: rl
86
+ enabled: true
87
+ max_train_steps: 8000
88
+ learning_rate: 5e-6
89
+ warmup_steps: 200
90
+ save_steps: 500
91
+ eval_steps: 250
92
+ ppo_config:
93
+ gamma: 0.99
94
+ lam: 0.95
95
+ cliprange: 0.2
96
+ cliprange_value: 0.2
97
+ vf_coef: 0.5
98
+ ent_coef: 0.01
99
+ max_grad_norm: 1.0
100
+ minibatch_size: 4
101
+ ppo_epochs: 4
102
+
103
+ # Optimizer Configuration
104
+ learning_rate: 2e-5 # Default for SFT phase
105
+ lr_scheduler: cosine_with_restarts
106
+ lr_warmup_steps: 500
107
+ adam_beta1: 0.9
108
+ adam_beta2: 0.999
109
+ adam_weight_decay: 0.01
110
+ adam_epsilon: 1e-8
111
+ adam8bit: true # Use 8-bit Adam for memory efficiency
112
+ max_grad_norm: 1.0
113
+
114
+ # LoRA Configuration (for memory-efficient fine-tuning)
115
+ use_lora: true
116
+ lora_config:
117
+ rank: 32
118
+ alpha: 64
119
+ dropout: 0.1
120
+ target_modules:
121
+ - q_proj
122
+ - k_proj
123
+ - v_proj
124
+ - o_proj
125
+ - gate_proj
126
+ - up_proj
127
+ - down_proj
128
+ bias: none
129
+ task_type: CAUSAL_LM
130
+
131
+ # Memory Optimization
132
+ mixed_precision: bf16 # Use bfloat16 for better stability than fp16
133
+ quantize: false # Set to true for 8-bit quantization (if memory is tight)
134
+ gradient_checkpointing: true
135
+ precompute_text_embeddings: false # Dynamic computation for reasoning chains
136
+ precompute_image_embeddings: false
137
+ save_cache_on_disk: false # Streaming mode doesn't need cache
138
+
139
+ # Checkpointing
140
+ checkpointing_steps: 1000
141
+ checkpoints_total_limit: 5
142
+ resume_from_checkpoint: latest # Resume from last checkpoint if available
143
+ save_safetensors: true
144
+
145
+ # Chain-of-Sight Configuration
146
+ chain_of_sight:
147
+ enabled: true # Enable region tokens and bounding boxes
148
+ num_regions: 10 # Number of region tokens (<region1> to <region10>)
149
+ use_bounding_boxes: true
150
+ spatial_reward_weight: 0.3 # Weight for spatial accuracy in RL
151
+
152
+ # Reasoning Configuration
153
+ reasoning_config:
154
+ max_steps: 10 # Maximum reasoning steps per chain
155
+ step_types:
156
+ - perception
157
+ - localization
158
+ - comparison
159
+ - counting
160
+ - inference
161
+ - composition
162
+ - verification
163
+ include_visual_features: false # Extract visual features per step (memory intensive)
164
+ filter_incorrect: false # Keep all samples during training
165
+ confidence_threshold: 0.5 # Minimum confidence for step acceptance
166
+
167
+ # Inference-Time Scaling
168
+ inference_config:
169
+ enable_best_of_n: true
170
+ n_samples: 8 # Generate 8 chains and select best
171
+ enable_voting: true
172
+ voting_strategy: prm_weighted # Use PRM scores for voting
173
+ temperature: 0.7
174
+ top_p: 0.9
175
+ max_new_tokens: 1024
176
+
177
+ # Validation
178
+ validation:
179
+ enabled: true
180
+ val_split: 0.05 # Use 5% of data for validation
181
+ metrics:
182
+ - step_accuracy # Per-step correctness
183
+ - chain_accuracy # Full chain correctness
184
+ - final_answer_accuracy # Answer-only accuracy
185
+ - average_confidence # Model confidence
186
+ - average_reward # PRM reward scores
187
+ - reasoning_length # Average number of steps
188
+
189
+ # Hardware Configuration
190
+ device: cuda
191
+ num_gpus: 1 # Single GPU training (adjust for multi-GPU)
192
+ dataloader_pin_memory: true
193
+ dataloader_persistent_workers: true
194
+
195
+ # Logging and Monitoring
196
+ logging_steps: 50
197
+ eval_logging_steps: 10
198
+ save_logs: true
199
+ log_level: info
200
+ disable_tqdm: false
201
+
202
+ # Experiment Tracking
203
+ experiment_name: qwen_step_cot_dual_dataset
204
+ experiment_tags:
205
+ - step-level-cot
206
+ - chain-of-sight
207
+ - flux-reason
208
+ - finevision
209
+ - qwen-image
210
+ - dual-purpose # Document processing + image generation
211
+
212
+ # Reproducibility
213
+ seed: 42
214
+ deterministic: false # Set to true for full reproducibility (slower)
215
+
216
+ # Advanced Training Options
217
+ compile_model: false # PyTorch 2.0+ compilation (may not work with all models)
218
+ tf32: true # Use TF32 for better performance on Ampere+ GPUs
219
+ fsdp: false # Fully Sharded Data Parallel (for multi-GPU)
220
+ deepspeed: null # Path to DeepSpeed config (alternative to FSDP)
221
+
222
+ # Early Stopping
223
+ early_stopping:
224
+ enabled: true
225
+ patience: 3 # Stop if no improvement for 3 evaluations
226
+ metric: chain_accuracy
227
+ mode: max # Maximize chain accuracy
228
+
229
+ # Data Augmentation
230
+ augmentation:
231
+ enabled: true
232
+ random_horizontal_flip: 0.5
233
+ random_rotation: 15 # degrees
234
+ color_jitter: 0.2
235
+ random_crop: false # May interfere with spatial reasoning
236
+
237
+ # Document Processing Specific
238
+ document_processing:
239
+ enabled: true
240
+ ocr_integration: true # Extract text from images during inference
241
+ layout_analysis: true # Analyze document structure
242
+ table_detection: true # Detect and reason about tables
243
+ chart_understanding: true # Interpret charts and graphs
244
+
245
+ # Image Generation Specific
246
+ image_generation:
247
+ enabled: true
248
+ reasoning_guided: true # Use CoT reasoning to guide generation
249
+ progressive_generation: true # Generate step-by-step with intermediate feedback
250
+ quality_scoring: true # Use PRM to score generation quality
train_configs/train_step_cot.yaml ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Step-Level CoT Training Configuration
2
+ # Configuration for training vision-language models with fine-grained chain-of-thought reasoning
3
+
4
+ # Model settings
5
+ pretrained_model_name_or_path: "Qwen/Qwen-Image"
6
+ vision_dim: 768
7
+ text_dim: 768
8
+
9
+ # Output settings
10
+ output_dir: "./outputs/step_level_cot"
11
+ report_to: "tensorboard" # Options: tensorboard, wandb
12
+
13
+ # Training settings
14
+ train_batch_size: 4
15
+ gradient_accumulation_steps: 4
16
+ learning_rate: 5.0e-5
17
+ rl_learning_rate: 1.0e-5
18
+ prm_learning_rate: 3.0e-5
19
+ weight_decay: 0.01
20
+ warmup_steps: 1000
21
+ max_train_steps: 50000
22
+ mixed_precision: "fp16" # Options: no, fp16, bf16
23
+ gradient_checkpointing: true
24
+
25
+ # Process Reward Model (PRM) settings
26
+ prm_hidden_dim: 512
27
+ prm_num_heads: 8
28
+ prm_dropout: 0.1
29
+ max_reasoning_steps: 10
30
+
31
+ # Reinforcement Learning settings (PPO)
32
+ ppo_clip_epsilon: 0.2
33
+ value_loss_coef: 0.5
34
+ entropy_coef: 0.01
35
+ gamma: 0.99 # Discount factor
36
+ gae_lambda: 0.95 # GAE lambda
37
+
38
+ # Inference-time scaling settings
39
+ num_inference_samples: 8
40
+ inference_temperature: 0.7
41
+ aggregation_method: "best_of_n" # Options: best_of_n, majority_vote, weighted_vote
42
+
43
+ # Data settings
44
+ dataloader_num_workers: 4
45
+ max_seq_length: 512
46
+
47
+ # Logging settings
48
+ logging_steps: 10
49
+ save_steps: 1000
50
+ eval_steps: 500
51
+ save_total_limit: 5
52
+
53
+ # Evaluation settings
54
+ run_evaluation: true
55
+ eval_batch_size: 8
56
+ num_eval_samples: 100
57
+
58
+ # Ablation study settings
59
+ run_ablation: false
60
+ ablation_components:
61
+ - "prm"
62
+ - "rl"
63
+ - "inference_scaling"
64
+ - "full_system"
65
+
66
+ # Step-level reasoning data format
67
+ # Expected structure:
68
+ # data_dir/
69
+ # train/
70
+ # chain_0001.json
71
+ # chain_0002.json
72
+ # ...
73
+ # val/
74
+ # chain_0001.json
75
+ # ...
76
+ # test/
77
+ # chain_0001.json
78
+ # ...
79
+ #
80
+ # Each JSON file contains a ReasoningChain with:
81
+ # {
82
+ # "chain_id": "unique_id",
83
+ # "image_path": "path/to/image.jpg",
84
+ # "prompt": "What is shown in the image?",
85
+ # "steps": [
86
+ # {
87
+ # "step_id": 0,
88
+ # "step_type": "perception",
89
+ # "description": "I observe a red ball...",
90
+ # "confidence": 0.9,
91
+ # "reward": 0.8,
92
+ # "dependencies": []
93
+ # },
94
+ # ...
95
+ # ],
96
+ # "final_answer": "The image shows a red ball.",
97
+ # "total_reward": 2.4,
98
+ # "is_correct": true
99
+ # }
train_qwen_cot_dual.py ADDED
@@ -0,0 +1,357 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Comprehensive training launcher for step-level CoT vision model.
3
+
4
+ This script orchestrates the full training pipeline:
5
+ 1. Add special tokens to base model
6
+ 2. Prepare datasets (FLUX-Reason-6M + FineVision)
7
+ 3. Run SFT + PRM + RL training phases
8
+ 4. Validate and save final model
9
+
10
+ Usage:
11
+ python train_qwen_cot_dual.py --config train_configs/train_qwen_cot_dual.yaml
12
+
13
+ For specific phases only:
14
+ python train_qwen_cot_dual.py --config CONFIG --phases sft prm
15
+ """
16
+
17
+ import argparse
18
+ import os
19
+ import sys
20
+ from pathlib import Path
21
+ import yaml
22
+ import torch
23
+ from typing import Dict, List, Optional, Any
24
+ import logging
25
+
26
+ # Add src to path
27
+ sys.path.insert(0, str(Path(__file__).parent))
28
+
29
+ from src.reasoning.step_level_cot import StepLevelCoTTrainer
30
+ from src.reasoning.flux_reason_dataset import FLUXReasonDataset, create_flux_reason_dataloader
31
+ from src.reasoning.finevision_dataset import FineVisionDataset, create_finevision_dataloader
32
+ from src.reasoning.step_data import ReasoningChain
33
+ from utils.add_special_tokens import add_special_tokens_to_model
34
+
35
+ # Set up logging
36
+ logging.basicConfig(
37
+ format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
38
+ level=logging.INFO
39
+ )
40
+ logger = logging.getLogger(__name__)
41
+
42
+
43
+ def load_config(config_path: str) -> Dict[str, Any]:
44
+ """Load training configuration from YAML file."""
45
+ with open(config_path, 'r') as f:
46
+ config = yaml.safe_load(f)
47
+ return config
48
+
49
+
50
+ def prepare_model_with_tokens(config: Dict[str, Any]) -> str:
51
+ """
52
+ Add special tokens to the base model and save to output directory.
53
+
54
+ Args:
55
+ config: Training configuration
56
+
57
+ Returns:
58
+ Path to the model with special tokens added
59
+ """
60
+ logger.info("=" * 60)
61
+ logger.info("STEP 1: Adding Special Tokens to Base Model")
62
+ logger.info("=" * 60)
63
+
64
+ model_path = config['pretrained_model_name_or_path']
65
+ output_dir = Path(config['output_dir']) / 'model_with_tokens'
66
+
67
+ if output_dir.exists() and (output_dir / 'config.json').exists():
68
+ logger.info(f"Model with tokens already exists at {output_dir}")
69
+ logger.info("Skipping token addition step.")
70
+ return str(output_dir)
71
+
72
+ logger.info(f"Base model: {model_path}")
73
+ logger.info(f"Output directory: {output_dir}")
74
+ logger.info("Adding 27 special tokens (15 base + 12 region tokens)...")
75
+
76
+ try:
77
+ add_special_tokens_to_model(
78
+ model_path=model_path,
79
+ output_dir=str(output_dir),
80
+ prm_only=False, # Add all tokens
81
+ )
82
+ logger.info("βœ“ Successfully added special tokens!")
83
+ return str(output_dir)
84
+ except Exception as e:
85
+ logger.error(f"βœ— Failed to add special tokens: {e}")
86
+ logger.info("Continuing with base model (tokens will be added by trainer)")
87
+ return model_path
88
+
89
+
90
+ def prepare_datasets(config: Dict[str, Any], tokenizer: Any) -> Dict[str, Any]:
91
+ """
92
+ Prepare FLUX-Reason-6M and FineVision datasets.
93
+
94
+ Args:
95
+ config: Training configuration
96
+ tokenizer: Model tokenizer
97
+
98
+ Returns:
99
+ Dictionary with dataloaders for each dataset
100
+ """
101
+ logger.info("=" * 60)
102
+ logger.info("STEP 2: Preparing Datasets")
103
+ logger.info("=" * 60)
104
+
105
+ datasets_config = config.get('datasets', [])
106
+ data_config = config.get('data_config', {})
107
+
108
+ dataloaders = {}
109
+ dataset_weights = {}
110
+
111
+ for dataset_cfg in datasets_config:
112
+ dataset_name = dataset_cfg['name']
113
+ dataset_type = dataset_cfg['type']
114
+ weight = dataset_cfg.get('weight', 1.0)
115
+
116
+ logger.info(f"\nLoading {dataset_name} ({dataset_type})...")
117
+ logger.info(f" Weight: {weight}")
118
+
119
+ try:
120
+ if dataset_type == 'flux_reason_6m':
121
+ dataloader = create_flux_reason_dataloader(
122
+ tokenizer=tokenizer,
123
+ batch_size=data_config.get('train_batch_size', 2),
124
+ num_workers=data_config.get('num_workers', 4),
125
+ streaming=dataset_cfg.get('streaming', True),
126
+ max_samples=dataset_cfg.get('max_samples'),
127
+ dataset_name=dataset_cfg['dataset_name'],
128
+ split=dataset_cfg.get('split', 'train'),
129
+ image_key=dataset_cfg.get('image_key', 'image'),
130
+ reasoning_key=dataset_cfg.get('reasoning_key', 'reasoning'),
131
+ question_key=dataset_cfg.get('question_key', 'question'),
132
+ answer_key=dataset_cfg.get('answer_key', 'answer'),
133
+ )
134
+ dataloaders[dataset_name] = dataloader
135
+ dataset_weights[dataset_name] = weight
136
+ logger.info(f"βœ“ Loaded {dataset_name}")
137
+
138
+ elif dataset_type == 'finevision':
139
+ dataloader = create_finevision_dataloader(
140
+ tokenizer=tokenizer,
141
+ batch_size=data_config.get('train_batch_size', 2),
142
+ num_workers=data_config.get('num_workers', 4),
143
+ streaming=dataset_cfg.get('streaming', True),
144
+ max_samples=dataset_cfg.get('max_samples'),
145
+ dataset_name=dataset_cfg['dataset_name'],
146
+ subset=dataset_cfg.get('subset'),
147
+ split=dataset_cfg.get('split', 'train'),
148
+ image_key=dataset_cfg.get('image_key', 'image'),
149
+ text_key=dataset_cfg.get('text_key', 'text'),
150
+ generate_synthetic_reasoning=dataset_cfg.get('generate_synthetic_reasoning', True),
151
+ num_synthetic_steps=dataset_cfg.get('num_synthetic_steps', 3),
152
+ )
153
+ dataloaders[dataset_name] = dataloader
154
+ dataset_weights[dataset_name] = weight
155
+ logger.info(f"βœ“ Loaded {dataset_name}")
156
+ else:
157
+ logger.warning(f"Unknown dataset type: {dataset_type}")
158
+
159
+ except Exception as e:
160
+ logger.error(f"βœ— Failed to load {dataset_name}: {e}")
161
+ logger.info("Continuing without this dataset...")
162
+
163
+ logger.info(f"\nβœ“ Prepared {len(dataloaders)} datasets")
164
+ return {'dataloaders': dataloaders, 'weights': dataset_weights}
165
+
166
+
167
+ def run_training_phase(
168
+ trainer: StepLevelCoTTrainer,
169
+ phase: str,
170
+ phase_config: Dict[str, Any],
171
+ dataloaders: Dict[str, Any],
172
+ ) -> None:
173
+ """
174
+ Run a specific training phase (SFT, PRM, or RL).
175
+
176
+ Args:
177
+ trainer: StepLevelCoTTrainer instance
178
+ phase: Training phase name ('sft', 'prm', or 'rl')
179
+ phase_config: Configuration for this phase
180
+ dataloaders: Dataset dataloaders
181
+ """
182
+ logger.info("=" * 60)
183
+ logger.info(f"STEP 3: Running {phase.upper()} Training Phase")
184
+ logger.info("=" * 60)
185
+
186
+ if not phase_config.get('enabled', False):
187
+ logger.info(f"{phase.upper()} phase is disabled in config. Skipping...")
188
+ return
189
+
190
+ max_steps = phase_config.get('max_train_steps', 5000)
191
+ learning_rate = phase_config.get('learning_rate', 1e-5)
192
+
193
+ logger.info(f"Phase: {phase.upper()}")
194
+ logger.info(f"Max steps: {max_steps}")
195
+ logger.info(f"Learning rate: {learning_rate}")
196
+
197
+ try:
198
+ if phase == 'sft':
199
+ logger.info("\nRunning Supervised Fine-Tuning...")
200
+ trainer.train_sft(
201
+ max_steps=max_steps,
202
+ learning_rate=learning_rate,
203
+ )
204
+ logger.info("βœ“ SFT phase complete!")
205
+
206
+ elif phase == 'prm':
207
+ logger.info("\nTraining Process Reward Model...")
208
+ prm_config = phase_config.get('prm_config', {})
209
+ trainer.train_prm(
210
+ max_steps=max_steps,
211
+ learning_rate=learning_rate,
212
+ **prm_config,
213
+ )
214
+ logger.info("βœ“ PRM training complete!")
215
+
216
+ elif phase == 'rl':
217
+ logger.info("\nRunning Reinforcement Learning with PPO...")
218
+ ppo_config = phase_config.get('ppo_config', {})
219
+ trainer.train_rl(
220
+ max_steps=max_steps,
221
+ learning_rate=learning_rate,
222
+ **ppo_config,
223
+ )
224
+ logger.info("βœ“ RL training complete!")
225
+
226
+ else:
227
+ logger.warning(f"Unknown phase: {phase}")
228
+
229
+ except Exception as e:
230
+ logger.error(f"βœ— {phase.upper()} phase failed: {e}")
231
+ logger.error("Stopping training pipeline.")
232
+ raise
233
+
234
+
235
+ def main():
236
+ parser = argparse.ArgumentParser(description="Train step-level CoT vision model")
237
+ parser.add_argument(
238
+ '--config',
239
+ type=str,
240
+ required=True,
241
+ help='Path to training configuration YAML file'
242
+ )
243
+ parser.add_argument(
244
+ '--phases',
245
+ nargs='+',
246
+ choices=['sft', 'prm', 'rl', 'all'],
247
+ default=['all'],
248
+ help='Training phases to run (default: all)'
249
+ )
250
+ parser.add_argument(
251
+ '--skip_token_addition',
252
+ action='store_true',
253
+ help='Skip special token addition step (if already done)'
254
+ )
255
+ parser.add_argument(
256
+ '--resume',
257
+ type=str,
258
+ default=None,
259
+ help='Resume from checkpoint directory'
260
+ )
261
+
262
+ args = parser.parse_args()
263
+
264
+ # Load configuration
265
+ logger.info("=" * 60)
266
+ logger.info("QWEN STEP-LEVEL COT TRAINING PIPELINE")
267
+ logger.info("=" * 60)
268
+ logger.info(f"Config: {args.config}")
269
+ logger.info(f"Phases: {args.phases}")
270
+
271
+ config = load_config(args.config)
272
+
273
+ # Determine which phases to run
274
+ if 'all' in args.phases:
275
+ phases_to_run = ['sft', 'prm', 'rl']
276
+ else:
277
+ phases_to_run = args.phases
278
+
279
+ # Step 1: Add special tokens (optional, can be skipped)
280
+ if args.skip_token_addition:
281
+ logger.info("\nSkipping special token addition (--skip_token_addition)")
282
+ model_path = config['pretrained_model_name_or_path']
283
+ else:
284
+ model_path = prepare_model_with_tokens(config)
285
+
286
+ # Update config with tokenized model path
287
+ config['pretrained_model_name_or_path'] = model_path
288
+
289
+ # Step 2: Initialize trainer
290
+ logger.info("\n" + "=" * 60)
291
+ logger.info("INITIALIZING TRAINER")
292
+ logger.info("=" * 60)
293
+
294
+ try:
295
+ trainer = StepLevelCoTTrainer(config)
296
+ logger.info("βœ“ Trainer initialized successfully!")
297
+ except Exception as e:
298
+ logger.error(f"βœ— Failed to initialize trainer: {e}")
299
+ return 1
300
+
301
+ # Step 3: Prepare datasets
302
+ dataset_info = prepare_datasets(config, trainer.tokenizer)
303
+
304
+ # Attach datasets to trainer
305
+ trainer.dataloaders = dataset_info['dataloaders']
306
+ trainer.dataset_weights = dataset_info['weights']
307
+
308
+ # Step 4: Run training phases
309
+ training_phases = config.get('training_phases', [])
310
+
311
+ for phase_info in training_phases:
312
+ phase_name = phase_info['phase']
313
+
314
+ if phase_name not in phases_to_run:
315
+ logger.info(f"\nSkipping {phase_name.upper()} phase (not in --phases)")
316
+ continue
317
+
318
+ run_training_phase(
319
+ trainer=trainer,
320
+ phase=phase_name,
321
+ phase_config=phase_info,
322
+ dataloaders=dataset_info['dataloaders'],
323
+ )
324
+
325
+ # Step 5: Final model save
326
+ logger.info("\n" + "=" * 60)
327
+ logger.info("STEP 4: Saving Final Model")
328
+ logger.info("=" * 60)
329
+
330
+ final_output_dir = Path(config['output_dir']) / 'final_model'
331
+ final_output_dir.mkdir(parents=True, exist_ok=True)
332
+
333
+ try:
334
+ trainer.save_checkpoint(str(final_output_dir))
335
+ logger.info(f"βœ“ Final model saved to {final_output_dir}")
336
+ except Exception as e:
337
+ logger.error(f"βœ— Failed to save final model: {e}")
338
+
339
+ # Summary
340
+ logger.info("\n" + "=" * 60)
341
+ logger.info("TRAINING PIPELINE COMPLETE!")
342
+ logger.info("=" * 60)
343
+ logger.info(f"Model: {config['pretrained_model_name_or_path']}")
344
+ logger.info(f"Phases completed: {phases_to_run}")
345
+ logger.info(f"Output directory: {config['output_dir']}")
346
+ logger.info(f"Final model: {final_output_dir}")
347
+ logger.info("\nNext steps:")
348
+ logger.info(" 1. Test inference: python inference_document_processor.py")
349
+ logger.info(" 2. Test generation: python inference_image_generator.py")
350
+ logger.info(" 3. Evaluate on benchmarks: python evaluate_reasoning.py")
351
+
352
+ return 0
353
+
354
+
355
+ if __name__ == '__main__':
356
+ exit_code = main()
357
+ sys.exit(exit_code)