renpas22 commited on
Commit Β·
2b8876a
1
Parent(s): cd336ff
Add training scripts and configs
Browse files- src/reasoning/__init__.py +24 -0
- src/reasoning/__pycache__/__init__.cpython-313.pyc +0 -0
- src/reasoning/__pycache__/inference_scaling.cpython-313.pyc +0 -0
- src/reasoning/__pycache__/prm.cpython-313.pyc +0 -0
- src/reasoning/__pycache__/rl_trainer.cpython-313.pyc +0 -0
- src/reasoning/__pycache__/step_data.cpython-313.pyc +0 -0
- src/reasoning/__pycache__/step_level_cot.cpython-313.pyc +0 -0
- src/reasoning/finevision_dataset.py +393 -0
- src/reasoning/flux_reason_dataset.py +375 -0
- src/reasoning/inference_scaling.py +453 -0
- src/reasoning/prm.py +462 -0
- src/reasoning/rl_trainer.py +480 -0
- src/reasoning/step_data.py +448 -0
- src/reasoning/step_level_cot.py +477 -0
- train_configs/train_full_qwen_image.yaml +48 -0
- train_configs/train_lora.yaml +29 -0
- train_configs/train_lora_4090.yaml +40 -0
- train_configs/train_lora_qwen_edit.yaml +36 -0
- train_configs/train_qwen_cot_dual.yaml +250 -0
- train_configs/train_step_cot.yaml +99 -0
- train_qwen_cot_dual.py +357 -0
src/reasoning/__init__.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Step-Level Chain of Thought Reasoning for Vision-Language Models
|
| 3 |
+
|
| 4 |
+
This module implements fine-grained structured reasoning with process reward models
|
| 5 |
+
and reinforcement learning for vision-language tasks.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from .step_data import ReasoningStep, ReasoningChain, StepDataset
|
| 9 |
+
from .prm import ProcessRewardModel, StepQualityMetrics
|
| 10 |
+
from .rl_trainer import RLReasoningTrainer
|
| 11 |
+
from .inference_scaling import InferenceTimeScaling, MajorityVoting
|
| 12 |
+
from .step_level_cot import StepLevelCoTTrainer
|
| 13 |
+
|
| 14 |
+
__all__ = [
|
| 15 |
+
'ReasoningStep',
|
| 16 |
+
'ReasoningChain',
|
| 17 |
+
'StepDataset',
|
| 18 |
+
'ProcessRewardModel',
|
| 19 |
+
'StepQualityMetrics',
|
| 20 |
+
'RLReasoningTrainer',
|
| 21 |
+
'InferenceTimeScaling',
|
| 22 |
+
'MajorityVoting',
|
| 23 |
+
'StepLevelCoTTrainer',
|
| 24 |
+
]
|
src/reasoning/__pycache__/__init__.cpython-313.pyc
ADDED
|
Binary file (842 Bytes). View file
|
|
|
src/reasoning/__pycache__/inference_scaling.cpython-313.pyc
ADDED
|
Binary file (17.7 kB). View file
|
|
|
src/reasoning/__pycache__/prm.cpython-313.pyc
ADDED
|
Binary file (17.2 kB). View file
|
|
|
src/reasoning/__pycache__/rl_trainer.cpython-313.pyc
ADDED
|
Binary file (20 kB). View file
|
|
|
src/reasoning/__pycache__/step_data.cpython-313.pyc
ADDED
|
Binary file (19.6 kB). View file
|
|
|
src/reasoning/__pycache__/step_level_cot.cpython-313.pyc
ADDED
|
Binary file (21 kB). View file
|
|
|
src/reasoning/finevision_dataset.py
ADDED
|
@@ -0,0 +1,393 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Dataset loader for HuggingFaceM4/FineVision.
|
| 3 |
+
|
| 4 |
+
This adapter loads FineVision image-text pairs and optionally generates
|
| 5 |
+
synthetic reasoning chains for step-level CoT training.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import json
|
| 9 |
+
import re
|
| 10 |
+
from typing import List, Dict, Optional, Any, Union
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
import torch
|
| 13 |
+
from torch.utils.data import Dataset
|
| 14 |
+
from datasets import load_dataset
|
| 15 |
+
from PIL import Image
|
| 16 |
+
import io
|
| 17 |
+
|
| 18 |
+
from .step_data import (
|
| 19 |
+
ReasoningChain,
|
| 20 |
+
ReasoningStep,
|
| 21 |
+
StepType,
|
| 22 |
+
SPECIAL_TOKENS,
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class FineVisionDataset(Dataset):
|
| 27 |
+
"""
|
| 28 |
+
Dataset adapter for HuggingFaceM4/FineVision with optional synthetic
|
| 29 |
+
reasoning chain generation.
|
| 30 |
+
|
| 31 |
+
FineVision contains image-text pairs for vision-language tasks. This adapter
|
| 32 |
+
can either use the raw data or generate synthetic reasoning chains to augment
|
| 33 |
+
training with explicit step-by-step reasoning.
|
| 34 |
+
"""
|
| 35 |
+
|
| 36 |
+
def __init__(
|
| 37 |
+
self,
|
| 38 |
+
dataset_name: str = "HuggingFaceM4/FineVision",
|
| 39 |
+
subset: Optional[str] = None,
|
| 40 |
+
split: str = "train",
|
| 41 |
+
streaming: bool = True,
|
| 42 |
+
max_samples: Optional[int] = None,
|
| 43 |
+
image_key: str = "image",
|
| 44 |
+
text_key: str = "text",
|
| 45 |
+
generate_synthetic_reasoning: bool = True,
|
| 46 |
+
num_synthetic_steps: int = 3,
|
| 47 |
+
cache_dir: Optional[str] = None,
|
| 48 |
+
):
|
| 49 |
+
"""
|
| 50 |
+
Initialize FineVision dataset adapter.
|
| 51 |
+
|
| 52 |
+
Args:
|
| 53 |
+
dataset_name: HuggingFace dataset identifier
|
| 54 |
+
subset: Dataset subset/config name (if applicable)
|
| 55 |
+
split: Dataset split (train, validation, test)
|
| 56 |
+
streaming: Use streaming mode for large datasets
|
| 57 |
+
max_samples: Maximum number of samples to load (None = all)
|
| 58 |
+
image_key: Column name for images
|
| 59 |
+
text_key: Column name for text/captions
|
| 60 |
+
generate_synthetic_reasoning: If True, create synthetic reasoning steps
|
| 61 |
+
num_synthetic_steps: Number of reasoning steps to generate
|
| 62 |
+
cache_dir: Directory to cache downloaded data
|
| 63 |
+
"""
|
| 64 |
+
self.dataset_name = dataset_name
|
| 65 |
+
self.subset = subset
|
| 66 |
+
self.split = split
|
| 67 |
+
self.streaming = streaming
|
| 68 |
+
self.max_samples = max_samples
|
| 69 |
+
self.image_key = image_key
|
| 70 |
+
self.text_key = text_key
|
| 71 |
+
self.generate_synthetic_reasoning = generate_synthetic_reasoning
|
| 72 |
+
self.num_synthetic_steps = num_synthetic_steps
|
| 73 |
+
|
| 74 |
+
print(f"Loading {dataset_name} ({split} split)...")
|
| 75 |
+
|
| 76 |
+
# Load dataset from HuggingFace
|
| 77 |
+
try:
|
| 78 |
+
if streaming:
|
| 79 |
+
self.hf_dataset = load_dataset(
|
| 80 |
+
dataset_name,
|
| 81 |
+
name=subset,
|
| 82 |
+
split=split,
|
| 83 |
+
streaming=True,
|
| 84 |
+
cache_dir=cache_dir,
|
| 85 |
+
)
|
| 86 |
+
self.hf_dataset_iter = iter(self.hf_dataset)
|
| 87 |
+
else:
|
| 88 |
+
self.hf_dataset = load_dataset(
|
| 89 |
+
dataset_name,
|
| 90 |
+
name=subset,
|
| 91 |
+
split=split,
|
| 92 |
+
cache_dir=cache_dir,
|
| 93 |
+
)
|
| 94 |
+
if max_samples:
|
| 95 |
+
self.hf_dataset = self.hf_dataset.select(range(min(max_samples, len(self.hf_dataset))))
|
| 96 |
+
except Exception as e:
|
| 97 |
+
raise RuntimeError(f"Failed to load dataset {dataset_name}: {e}")
|
| 98 |
+
|
| 99 |
+
# Pre-build reasoning chains for non-streaming mode
|
| 100 |
+
if not streaming:
|
| 101 |
+
print("Converting to ReasoningChain format...")
|
| 102 |
+
self.reasoning_chains = []
|
| 103 |
+
for idx, sample in enumerate(self.hf_dataset):
|
| 104 |
+
chain = self._parse_sample_to_chain(sample, idx)
|
| 105 |
+
if chain:
|
| 106 |
+
self.reasoning_chains.append(chain)
|
| 107 |
+
print(f"Loaded {len(self.reasoning_chains)} reasoning chains")
|
| 108 |
+
else:
|
| 109 |
+
self.reasoning_chains = None
|
| 110 |
+
|
| 111 |
+
def __len__(self) -> int:
|
| 112 |
+
if self.streaming:
|
| 113 |
+
return self.max_samples or 10**9 # Large number for streaming
|
| 114 |
+
return len(self.reasoning_chains)
|
| 115 |
+
|
| 116 |
+
def __getitem__(self, idx: int) -> Optional[ReasoningChain]:
|
| 117 |
+
"""Get a reasoning chain by index."""
|
| 118 |
+
if self.streaming:
|
| 119 |
+
# For streaming, get next item from iterator
|
| 120 |
+
try:
|
| 121 |
+
sample = next(self.hf_dataset_iter)
|
| 122 |
+
except StopIteration:
|
| 123 |
+
self.hf_dataset_iter = iter(self.hf_dataset)
|
| 124 |
+
sample = next(self.hf_dataset_iter)
|
| 125 |
+
return self._parse_sample_to_chain(sample, idx)
|
| 126 |
+
else:
|
| 127 |
+
return self.reasoning_chains[idx]
|
| 128 |
+
|
| 129 |
+
def _parse_sample_to_chain(self, sample: Dict[str, Any], idx: int) -> Optional[ReasoningChain]:
|
| 130 |
+
"""
|
| 131 |
+
Parse a FineVision sample into a ReasoningChain.
|
| 132 |
+
|
| 133 |
+
Args:
|
| 134 |
+
sample: Raw sample from HuggingFace dataset
|
| 135 |
+
idx: Sample index
|
| 136 |
+
|
| 137 |
+
Returns:
|
| 138 |
+
ReasoningChain object or None if parsing fails
|
| 139 |
+
"""
|
| 140 |
+
try:
|
| 141 |
+
# Extract fields
|
| 142 |
+
image = sample.get(self.image_key)
|
| 143 |
+
text = sample.get(self.text_key, "")
|
| 144 |
+
|
| 145 |
+
# Convert image to PIL if needed
|
| 146 |
+
if isinstance(image, bytes):
|
| 147 |
+
image = Image.open(io.BytesIO(image)).convert('RGB')
|
| 148 |
+
elif hasattr(image, 'convert'):
|
| 149 |
+
image = image.convert('RGB')
|
| 150 |
+
|
| 151 |
+
# Generate reasoning steps
|
| 152 |
+
if self.generate_synthetic_reasoning:
|
| 153 |
+
steps = self._generate_synthetic_steps(text, self.num_synthetic_steps)
|
| 154 |
+
else:
|
| 155 |
+
# Create single inference step with the text
|
| 156 |
+
steps = [
|
| 157 |
+
ReasoningStep(
|
| 158 |
+
step_id=0,
|
| 159 |
+
step_type=StepType.PERCEPTION,
|
| 160 |
+
description=text,
|
| 161 |
+
confidence=0.9,
|
| 162 |
+
)
|
| 163 |
+
]
|
| 164 |
+
|
| 165 |
+
# Use text as both question and answer for FineVision
|
| 166 |
+
# (it's more like caption/description dataset)
|
| 167 |
+
prompt = "Describe this image in detail."
|
| 168 |
+
answer = text
|
| 169 |
+
|
| 170 |
+
# Create reasoning chain
|
| 171 |
+
chain = ReasoningChain(
|
| 172 |
+
chain_id=f"finevision_{idx}",
|
| 173 |
+
image_path=None, # Streaming mode, no path
|
| 174 |
+
image=image,
|
| 175 |
+
prompt=prompt,
|
| 176 |
+
steps=steps,
|
| 177 |
+
final_answer=answer,
|
| 178 |
+
is_correct=True, # Assume dataset contains correct descriptions
|
| 179 |
+
metadata={
|
| 180 |
+
"dataset": self.dataset_name,
|
| 181 |
+
"split": self.split,
|
| 182 |
+
"original_idx": idx,
|
| 183 |
+
"synthetic_reasoning": self.generate_synthetic_reasoning,
|
| 184 |
+
}
|
| 185 |
+
)
|
| 186 |
+
|
| 187 |
+
return chain
|
| 188 |
+
|
| 189 |
+
except Exception as e:
|
| 190 |
+
print(f"Warning: Failed to parse sample {idx}: {e}")
|
| 191 |
+
return None
|
| 192 |
+
|
| 193 |
+
def _generate_synthetic_steps(self, text: str, num_steps: int) -> List[ReasoningStep]:
|
| 194 |
+
"""
|
| 195 |
+
Generate synthetic reasoning steps from a text description.
|
| 196 |
+
|
| 197 |
+
This creates a multi-step reasoning chain by breaking down the description
|
| 198 |
+
into perception, analysis, and conclusion steps.
|
| 199 |
+
|
| 200 |
+
Args:
|
| 201 |
+
text: Original text description
|
| 202 |
+
num_steps: Number of steps to generate
|
| 203 |
+
|
| 204 |
+
Returns:
|
| 205 |
+
List of ReasoningStep objects
|
| 206 |
+
"""
|
| 207 |
+
steps = []
|
| 208 |
+
|
| 209 |
+
if num_steps <= 1:
|
| 210 |
+
# Single step - just use the text
|
| 211 |
+
steps.append(
|
| 212 |
+
ReasoningStep(
|
| 213 |
+
step_id=0,
|
| 214 |
+
step_type=StepType.PERCEPTION,
|
| 215 |
+
description=text,
|
| 216 |
+
confidence=0.9,
|
| 217 |
+
)
|
| 218 |
+
)
|
| 219 |
+
elif num_steps == 2:
|
| 220 |
+
# Two steps: perception + inference
|
| 221 |
+
steps.append(
|
| 222 |
+
ReasoningStep(
|
| 223 |
+
step_id=0,
|
| 224 |
+
step_type=StepType.PERCEPTION,
|
| 225 |
+
description=f"I observe the visual content of the image.",
|
| 226 |
+
confidence=0.9,
|
| 227 |
+
)
|
| 228 |
+
)
|
| 229 |
+
steps.append(
|
| 230 |
+
ReasoningStep(
|
| 231 |
+
step_id=1,
|
| 232 |
+
step_type=StepType.INFERENCE,
|
| 233 |
+
description=text,
|
| 234 |
+
confidence=0.85,
|
| 235 |
+
dependencies=[0],
|
| 236 |
+
)
|
| 237 |
+
)
|
| 238 |
+
elif num_steps >= 3:
|
| 239 |
+
# Three+ steps: perception + composition + inference
|
| 240 |
+
# Split text into chunks if long enough
|
| 241 |
+
sentences = [s.strip() for s in text.split('.') if s.strip()]
|
| 242 |
+
|
| 243 |
+
# Step 1: Perception
|
| 244 |
+
steps.append(
|
| 245 |
+
ReasoningStep(
|
| 246 |
+
step_id=0,
|
| 247 |
+
step_type=StepType.PERCEPTION,
|
| 248 |
+
description="I carefully examine the visual elements in the image.",
|
| 249 |
+
confidence=0.9,
|
| 250 |
+
)
|
| 251 |
+
)
|
| 252 |
+
|
| 253 |
+
# Step 2-N: Composition/Analysis (use text chunks or full text)
|
| 254 |
+
if len(sentences) >= 2 and num_steps > 3:
|
| 255 |
+
# Distribute sentences across middle steps
|
| 256 |
+
sentences_per_step = max(1, len(sentences) // (num_steps - 2))
|
| 257 |
+
for i in range(num_steps - 2):
|
| 258 |
+
start_idx = i * sentences_per_step
|
| 259 |
+
end_idx = start_idx + sentences_per_step if i < num_steps - 3 else len(sentences)
|
| 260 |
+
chunk_text = '. '.join(sentences[start_idx:end_idx])
|
| 261 |
+
if chunk_text and not chunk_text.endswith('.'):
|
| 262 |
+
chunk_text += '.'
|
| 263 |
+
|
| 264 |
+
steps.append(
|
| 265 |
+
ReasoningStep(
|
| 266 |
+
step_id=i + 1,
|
| 267 |
+
step_type=StepType.COMPOSITION,
|
| 268 |
+
description=chunk_text if chunk_text else text,
|
| 269 |
+
confidence=0.85,
|
| 270 |
+
dependencies=list(range(i + 1)),
|
| 271 |
+
)
|
| 272 |
+
)
|
| 273 |
+
else:
|
| 274 |
+
# Not enough sentences, use full text for middle step
|
| 275 |
+
steps.append(
|
| 276 |
+
ReasoningStep(
|
| 277 |
+
step_id=1,
|
| 278 |
+
step_type=StepType.COMPOSITION,
|
| 279 |
+
description=text,
|
| 280 |
+
confidence=0.85,
|
| 281 |
+
dependencies=[0],
|
| 282 |
+
)
|
| 283 |
+
)
|
| 284 |
+
|
| 285 |
+
# Final step: Inference/Conclusion
|
| 286 |
+
if len(steps) < num_steps:
|
| 287 |
+
final_text = "Based on these observations, I can provide a comprehensive description."
|
| 288 |
+
steps.append(
|
| 289 |
+
ReasoningStep(
|
| 290 |
+
step_id=len(steps),
|
| 291 |
+
step_type=StepType.INFERENCE,
|
| 292 |
+
description=final_text,
|
| 293 |
+
confidence=0.8,
|
| 294 |
+
dependencies=list(range(len(steps))),
|
| 295 |
+
)
|
| 296 |
+
)
|
| 297 |
+
|
| 298 |
+
return steps[:num_steps] # Ensure we don't exceed requested steps
|
| 299 |
+
|
| 300 |
+
|
| 301 |
+
def create_finevision_dataloader(
|
| 302 |
+
tokenizer: Any,
|
| 303 |
+
batch_size: int = 4,
|
| 304 |
+
num_workers: int = 4,
|
| 305 |
+
streaming: bool = True,
|
| 306 |
+
max_samples: Optional[int] = None,
|
| 307 |
+
generate_synthetic_reasoning: bool = True,
|
| 308 |
+
**kwargs
|
| 309 |
+
) -> torch.utils.data.DataLoader:
|
| 310 |
+
"""
|
| 311 |
+
Create a DataLoader for FineVision dataset.
|
| 312 |
+
|
| 313 |
+
Args:
|
| 314 |
+
tokenizer: Tokenizer for text encoding
|
| 315 |
+
batch_size: Number of samples per batch
|
| 316 |
+
num_workers: Number of worker processes
|
| 317 |
+
streaming: Use streaming mode
|
| 318 |
+
max_samples: Maximum samples to load
|
| 319 |
+
generate_synthetic_reasoning: Generate synthetic reasoning steps
|
| 320 |
+
**kwargs: Additional arguments for FineVisionDataset
|
| 321 |
+
|
| 322 |
+
Returns:
|
| 323 |
+
Configured DataLoader
|
| 324 |
+
"""
|
| 325 |
+
from .step_data import StepDataset
|
| 326 |
+
|
| 327 |
+
# Create FineVision dataset
|
| 328 |
+
finevision_dataset = FineVisionDataset(
|
| 329 |
+
streaming=streaming,
|
| 330 |
+
max_samples=max_samples,
|
| 331 |
+
generate_synthetic_reasoning=generate_synthetic_reasoning,
|
| 332 |
+
**kwargs
|
| 333 |
+
)
|
| 334 |
+
|
| 335 |
+
# If not streaming, wrap in StepDataset for tokenization
|
| 336 |
+
if not streaming and finevision_dataset.reasoning_chains:
|
| 337 |
+
step_dataset = StepDataset(
|
| 338 |
+
reasoning_chains=finevision_dataset.reasoning_chains,
|
| 339 |
+
tokenizer=tokenizer,
|
| 340 |
+
max_steps=10,
|
| 341 |
+
include_visual_features=False,
|
| 342 |
+
filter_incorrect=False,
|
| 343 |
+
)
|
| 344 |
+
|
| 345 |
+
return torch.utils.data.DataLoader(
|
| 346 |
+
step_dataset,
|
| 347 |
+
batch_size=batch_size,
|
| 348 |
+
num_workers=num_workers,
|
| 349 |
+
shuffle=True,
|
| 350 |
+
collate_fn=lambda batch: batch, # Custom collation handled in trainer
|
| 351 |
+
)
|
| 352 |
+
else:
|
| 353 |
+
# For streaming, return raw finevision dataset
|
| 354 |
+
return torch.utils.data.DataLoader(
|
| 355 |
+
finevision_dataset,
|
| 356 |
+
batch_size=batch_size,
|
| 357 |
+
num_workers=0 if streaming else num_workers,
|
| 358 |
+
shuffle=False,
|
| 359 |
+
)
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
# Example usage
|
| 363 |
+
if __name__ == "__main__":
|
| 364 |
+
# Test dataset loading
|
| 365 |
+
print("Testing FineVision dataset adapter...")
|
| 366 |
+
|
| 367 |
+
dataset = FineVisionDataset(
|
| 368 |
+
streaming=False,
|
| 369 |
+
max_samples=5,
|
| 370 |
+
generate_synthetic_reasoning=True,
|
| 371 |
+
num_synthetic_steps=3,
|
| 372 |
+
)
|
| 373 |
+
|
| 374 |
+
print(f"\nLoaded {len(dataset)} samples")
|
| 375 |
+
|
| 376 |
+
# Show first example
|
| 377 |
+
if len(dataset) > 0:
|
| 378 |
+
chain = dataset[0]
|
| 379 |
+
print(f"\n{'='*60}")
|
| 380 |
+
print("Example Reasoning Chain:")
|
| 381 |
+
print(f"{'='*60}")
|
| 382 |
+
print(f"Chain ID: {chain.chain_id}")
|
| 383 |
+
print(f"Prompt: {chain.prompt}")
|
| 384 |
+
print(f"Number of steps: {len(chain.steps)}")
|
| 385 |
+
print(f"\nSteps:")
|
| 386 |
+
for step in chain.steps:
|
| 387 |
+
print(f" Step {step.step_id} ({step.step_type.value}): {step.description[:100]}...")
|
| 388 |
+
print(f"\nFinal Answer: {chain.final_answer[:200]}...")
|
| 389 |
+
print(f"\n{'='*60}")
|
| 390 |
+
print("\nFormatted with special tokens:")
|
| 391 |
+
print(f"{'='*60}")
|
| 392 |
+
formatted = chain.format_with_tokens()
|
| 393 |
+
print(formatted[:500] + "..." if len(formatted) > 500 else formatted)
|
src/reasoning/flux_reason_dataset.py
ADDED
|
@@ -0,0 +1,375 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Dataset loader for LucasFang/FLUX-Reason-6M.
|
| 3 |
+
|
| 4 |
+
This adapter converts the FLUX-Reason-6M dataset format into our step-level
|
| 5 |
+
chain-of-thought ReasoningChain format with proper step annotations.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import json
|
| 9 |
+
import re
|
| 10 |
+
from typing import List, Dict, Optional, Any, Union
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
import torch
|
| 13 |
+
from torch.utils.data import Dataset
|
| 14 |
+
from datasets import load_dataset
|
| 15 |
+
from PIL import Image
|
| 16 |
+
import io
|
| 17 |
+
|
| 18 |
+
from .step_data import (
|
| 19 |
+
ReasoningChain,
|
| 20 |
+
ReasoningStep,
|
| 21 |
+
StepType,
|
| 22 |
+
SPECIAL_TOKENS,
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class FLUXReasonDataset(Dataset):
|
| 27 |
+
"""
|
| 28 |
+
Dataset adapter for FLUX-Reason-6M that converts reasoning annotations
|
| 29 |
+
into step-level chain-of-thought format.
|
| 30 |
+
|
| 31 |
+
The FLUX-Reason-6M dataset contains images with reasoning chains. This adapter
|
| 32 |
+
parses the reasoning text and creates structured ReasoningStep objects.
|
| 33 |
+
"""
|
| 34 |
+
|
| 35 |
+
def __init__(
|
| 36 |
+
self,
|
| 37 |
+
dataset_name: str = "LucasFang/FLUX-Reason-6M",
|
| 38 |
+
split: str = "train",
|
| 39 |
+
streaming: bool = True,
|
| 40 |
+
max_samples: Optional[int] = None,
|
| 41 |
+
image_key: str = "image",
|
| 42 |
+
reasoning_key: str = "reasoning",
|
| 43 |
+
question_key: str = "question",
|
| 44 |
+
answer_key: str = "answer",
|
| 45 |
+
cache_dir: Optional[str] = None,
|
| 46 |
+
):
|
| 47 |
+
"""
|
| 48 |
+
Initialize FLUX-Reason-6M dataset adapter.
|
| 49 |
+
|
| 50 |
+
Args:
|
| 51 |
+
dataset_name: HuggingFace dataset identifier
|
| 52 |
+
split: Dataset split (train, validation, test)
|
| 53 |
+
streaming: Use streaming mode for large datasets
|
| 54 |
+
max_samples: Maximum number of samples to load (None = all)
|
| 55 |
+
image_key: Column name for images
|
| 56 |
+
reasoning_key: Column name for reasoning text
|
| 57 |
+
question_key: Column name for questions
|
| 58 |
+
answer_key: Column name for final answers
|
| 59 |
+
cache_dir: Directory to cache downloaded data
|
| 60 |
+
"""
|
| 61 |
+
self.dataset_name = dataset_name
|
| 62 |
+
self.split = split
|
| 63 |
+
self.streaming = streaming
|
| 64 |
+
self.max_samples = max_samples
|
| 65 |
+
self.image_key = image_key
|
| 66 |
+
self.reasoning_key = reasoning_key
|
| 67 |
+
self.question_key = question_key
|
| 68 |
+
self.answer_key = answer_key
|
| 69 |
+
|
| 70 |
+
print(f"Loading {dataset_name} ({split} split)...")
|
| 71 |
+
|
| 72 |
+
# Load dataset from HuggingFace
|
| 73 |
+
try:
|
| 74 |
+
if streaming:
|
| 75 |
+
self.hf_dataset = load_dataset(
|
| 76 |
+
dataset_name,
|
| 77 |
+
split=split,
|
| 78 |
+
streaming=True,
|
| 79 |
+
cache_dir=cache_dir,
|
| 80 |
+
)
|
| 81 |
+
self.hf_dataset_iter = iter(self.hf_dataset)
|
| 82 |
+
else:
|
| 83 |
+
self.hf_dataset = load_dataset(
|
| 84 |
+
dataset_name,
|
| 85 |
+
split=split,
|
| 86 |
+
cache_dir=cache_dir,
|
| 87 |
+
)
|
| 88 |
+
if max_samples:
|
| 89 |
+
self.hf_dataset = self.hf_dataset.select(range(min(max_samples, len(self.hf_dataset))))
|
| 90 |
+
except Exception as e:
|
| 91 |
+
raise RuntimeError(f"Failed to load dataset {dataset_name}: {e}")
|
| 92 |
+
|
| 93 |
+
# Pre-build reasoning chains for non-streaming mode
|
| 94 |
+
if not streaming:
|
| 95 |
+
print("Converting to ReasoningChain format...")
|
| 96 |
+
self.reasoning_chains = []
|
| 97 |
+
for idx, sample in enumerate(self.hf_dataset):
|
| 98 |
+
chain = self._parse_sample_to_chain(sample, idx)
|
| 99 |
+
if chain:
|
| 100 |
+
self.reasoning_chains.append(chain)
|
| 101 |
+
print(f"Loaded {len(self.reasoning_chains)} reasoning chains")
|
| 102 |
+
else:
|
| 103 |
+
self.reasoning_chains = None
|
| 104 |
+
|
| 105 |
+
def __len__(self) -> int:
|
| 106 |
+
if self.streaming:
|
| 107 |
+
return self.max_samples or 10**9 # Large number for streaming
|
| 108 |
+
return len(self.reasoning_chains)
|
| 109 |
+
|
| 110 |
+
def __getitem__(self, idx: int) -> Optional[ReasoningChain]:
|
| 111 |
+
"""Get a reasoning chain by index."""
|
| 112 |
+
if self.streaming:
|
| 113 |
+
# For streaming, get next item from iterator
|
| 114 |
+
try:
|
| 115 |
+
sample = next(self.hf_dataset_iter)
|
| 116 |
+
except StopIteration:
|
| 117 |
+
self.hf_dataset_iter = iter(self.hf_dataset)
|
| 118 |
+
sample = next(self.hf_dataset_iter)
|
| 119 |
+
return self._parse_sample_to_chain(sample, idx)
|
| 120 |
+
else:
|
| 121 |
+
return self.reasoning_chains[idx]
|
| 122 |
+
|
| 123 |
+
def _parse_sample_to_chain(self, sample: Dict[str, Any], idx: int) -> Optional[ReasoningChain]:
|
| 124 |
+
"""
|
| 125 |
+
Parse a FLUX-Reason-6M sample into a ReasoningChain.
|
| 126 |
+
|
| 127 |
+
Args:
|
| 128 |
+
sample: Raw sample from HuggingFace dataset
|
| 129 |
+
idx: Sample index
|
| 130 |
+
|
| 131 |
+
Returns:
|
| 132 |
+
ReasoningChain object or None if parsing fails
|
| 133 |
+
"""
|
| 134 |
+
try:
|
| 135 |
+
# Extract fields
|
| 136 |
+
image = sample.get(self.image_key)
|
| 137 |
+
reasoning_text = sample.get(self.reasoning_key, "")
|
| 138 |
+
question = sample.get(self.question_key, "")
|
| 139 |
+
answer = sample.get(self.answer_key, "")
|
| 140 |
+
|
| 141 |
+
# Convert image to PIL if needed
|
| 142 |
+
if isinstance(image, bytes):
|
| 143 |
+
image = Image.open(io.BytesIO(image)).convert('RGB')
|
| 144 |
+
elif hasattr(image, 'convert'):
|
| 145 |
+
image = image.convert('RGB')
|
| 146 |
+
|
| 147 |
+
# Parse reasoning text into steps
|
| 148 |
+
steps = self._parse_reasoning_steps(reasoning_text)
|
| 149 |
+
|
| 150 |
+
if not steps:
|
| 151 |
+
# If no steps found, create a single inference step
|
| 152 |
+
steps = [
|
| 153 |
+
ReasoningStep(
|
| 154 |
+
step_id=0,
|
| 155 |
+
step_type=StepType.INFERENCE,
|
| 156 |
+
description=reasoning_text if reasoning_text else "Direct inference",
|
| 157 |
+
confidence=0.8,
|
| 158 |
+
)
|
| 159 |
+
]
|
| 160 |
+
|
| 161 |
+
# Create reasoning chain
|
| 162 |
+
chain = ReasoningChain(
|
| 163 |
+
chain_id=f"flux_reason_{idx}",
|
| 164 |
+
image_path=None, # Streaming mode, no path
|
| 165 |
+
image=image,
|
| 166 |
+
prompt=question,
|
| 167 |
+
steps=steps,
|
| 168 |
+
final_answer=answer,
|
| 169 |
+
is_correct=True, # Assume dataset contains correct reasoning
|
| 170 |
+
metadata={
|
| 171 |
+
"dataset": self.dataset_name,
|
| 172 |
+
"split": self.split,
|
| 173 |
+
"original_idx": idx,
|
| 174 |
+
}
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
return chain
|
| 178 |
+
|
| 179 |
+
except Exception as e:
|
| 180 |
+
print(f"Warning: Failed to parse sample {idx}: {e}")
|
| 181 |
+
return None
|
| 182 |
+
|
| 183 |
+
def _parse_reasoning_steps(self, reasoning_text: str) -> List[ReasoningStep]:
|
| 184 |
+
"""
|
| 185 |
+
Parse reasoning text into structured ReasoningStep objects.
|
| 186 |
+
|
| 187 |
+
This handles various reasoning formats:
|
| 188 |
+
- Numbered steps: "1. ...", "2. ...", etc.
|
| 189 |
+
- Step markers: "Step 1:", "Step 2:", etc.
|
| 190 |
+
- Natural paragraphs separated by newlines
|
| 191 |
+
|
| 192 |
+
Args:
|
| 193 |
+
reasoning_text: Raw reasoning text from dataset
|
| 194 |
+
|
| 195 |
+
Returns:
|
| 196 |
+
List of ReasoningStep objects
|
| 197 |
+
"""
|
| 198 |
+
if not reasoning_text or not reasoning_text.strip():
|
| 199 |
+
return []
|
| 200 |
+
|
| 201 |
+
steps = []
|
| 202 |
+
|
| 203 |
+
# Try to split by numbered patterns
|
| 204 |
+
# Pattern 1: "1. ", "2. ", etc.
|
| 205 |
+
numbered_pattern = r'(\d+)\.\s+'
|
| 206 |
+
# Pattern 2: "Step 1:", "Step 2:", etc.
|
| 207 |
+
step_pattern = r'[Ss]tep\s+(\d+):\s*'
|
| 208 |
+
|
| 209 |
+
# Try numbered pattern first
|
| 210 |
+
parts = re.split(numbered_pattern, reasoning_text)
|
| 211 |
+
if len(parts) > 2: # Found numbered steps
|
| 212 |
+
for i in range(1, len(parts), 2):
|
| 213 |
+
if i + 1 < len(parts):
|
| 214 |
+
step_num = int(parts[i]) - 1
|
| 215 |
+
step_text = parts[i + 1].strip()
|
| 216 |
+
if step_text:
|
| 217 |
+
steps.append(self._create_step_from_text(step_num, step_text))
|
| 218 |
+
else:
|
| 219 |
+
# Try step pattern
|
| 220 |
+
parts = re.split(step_pattern, reasoning_text, flags=re.IGNORECASE)
|
| 221 |
+
if len(parts) > 2: # Found "Step X:" format
|
| 222 |
+
for i in range(1, len(parts), 2):
|
| 223 |
+
if i + 1 < len(parts):
|
| 224 |
+
step_num = int(parts[i]) - 1
|
| 225 |
+
step_text = parts[i + 1].strip()
|
| 226 |
+
if step_text:
|
| 227 |
+
steps.append(self._create_step_from_text(step_num, step_text))
|
| 228 |
+
else:
|
| 229 |
+
# Fall back to splitting by double newlines (paragraphs)
|
| 230 |
+
paragraphs = [p.strip() for p in reasoning_text.split('\n\n') if p.strip()]
|
| 231 |
+
if len(paragraphs) > 1:
|
| 232 |
+
for idx, paragraph in enumerate(paragraphs):
|
| 233 |
+
steps.append(self._create_step_from_text(idx, paragraph))
|
| 234 |
+
else:
|
| 235 |
+
# Split by single newlines if no paragraphs
|
| 236 |
+
lines = [l.strip() for l in reasoning_text.split('\n') if l.strip()]
|
| 237 |
+
if len(lines) > 1:
|
| 238 |
+
for idx, line in enumerate(lines):
|
| 239 |
+
steps.append(self._create_step_from_text(idx, line))
|
| 240 |
+
else:
|
| 241 |
+
# Single line reasoning - create one step
|
| 242 |
+
steps.append(self._create_step_from_text(0, reasoning_text.strip()))
|
| 243 |
+
|
| 244 |
+
return steps
|
| 245 |
+
|
| 246 |
+
def _create_step_from_text(self, step_id: int, text: str) -> ReasoningStep:
|
| 247 |
+
"""
|
| 248 |
+
Create a ReasoningStep from text, inferring the step type from keywords.
|
| 249 |
+
|
| 250 |
+
Args:
|
| 251 |
+
step_id: Step index
|
| 252 |
+
text: Step description text
|
| 253 |
+
|
| 254 |
+
Returns:
|
| 255 |
+
ReasoningStep object
|
| 256 |
+
"""
|
| 257 |
+
# Infer step type from keywords in text
|
| 258 |
+
text_lower = text.lower()
|
| 259 |
+
|
| 260 |
+
if any(kw in text_lower for kw in ['see', 'observe', 'notice', 'image shows', 'picture']):
|
| 261 |
+
step_type = StepType.PERCEPTION
|
| 262 |
+
elif any(kw in text_lower for kw in ['located', 'position', 'where', 'left', 'right', 'top', 'bottom']):
|
| 263 |
+
step_type = StepType.LOCALIZATION
|
| 264 |
+
elif any(kw in text_lower for kw in ['compare', 'larger', 'smaller', 'bigger', 'different', 'similar']):
|
| 265 |
+
step_type = StepType.COMPARISON
|
| 266 |
+
elif any(kw in text_lower for kw in ['count', 'number', 'how many', 'total', 'sum']):
|
| 267 |
+
step_type = StepType.COUNTING
|
| 268 |
+
elif any(kw in text_lower for kw in ['therefore', 'thus', 'conclude', 'deduce', 'infer']):
|
| 269 |
+
step_type = StepType.INFERENCE
|
| 270 |
+
elif any(kw in text_lower for kw in ['combine', 'together', 'overall', 'compose']):
|
| 271 |
+
step_type = StepType.COMPOSITION
|
| 272 |
+
elif any(kw in text_lower for kw in ['check', 'verify', 'confirm', 'validate']):
|
| 273 |
+
step_type = StepType.VERIFICATION
|
| 274 |
+
else:
|
| 275 |
+
step_type = StepType.INFERENCE # Default
|
| 276 |
+
|
| 277 |
+
# Create step with inferred type
|
| 278 |
+
return ReasoningStep(
|
| 279 |
+
step_id=step_id,
|
| 280 |
+
step_type=step_type,
|
| 281 |
+
description=text,
|
| 282 |
+
confidence=0.85, # Default confidence for dataset samples
|
| 283 |
+
dependencies=list(range(step_id)) if step_id > 0 else [], # Depends on all previous steps
|
| 284 |
+
)
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def create_flux_reason_dataloader(
|
| 288 |
+
tokenizer: Any,
|
| 289 |
+
batch_size: int = 4,
|
| 290 |
+
num_workers: int = 4,
|
| 291 |
+
streaming: bool = True,
|
| 292 |
+
max_samples: Optional[int] = None,
|
| 293 |
+
**kwargs
|
| 294 |
+
) -> torch.utils.data.DataLoader:
|
| 295 |
+
"""
|
| 296 |
+
Create a DataLoader for FLUX-Reason-6M dataset.
|
| 297 |
+
|
| 298 |
+
Args:
|
| 299 |
+
tokenizer: Tokenizer for text encoding
|
| 300 |
+
batch_size: Number of samples per batch
|
| 301 |
+
num_workers: Number of worker processes
|
| 302 |
+
streaming: Use streaming mode
|
| 303 |
+
max_samples: Maximum samples to load
|
| 304 |
+
**kwargs: Additional arguments for FLUXReasonDataset
|
| 305 |
+
|
| 306 |
+
Returns:
|
| 307 |
+
Configured DataLoader
|
| 308 |
+
"""
|
| 309 |
+
from .step_data import StepDataset
|
| 310 |
+
|
| 311 |
+
# Create FLUX dataset
|
| 312 |
+
flux_dataset = FLUXReasonDataset(
|
| 313 |
+
streaming=streaming,
|
| 314 |
+
max_samples=max_samples,
|
| 315 |
+
**kwargs
|
| 316 |
+
)
|
| 317 |
+
|
| 318 |
+
# If not streaming, wrap in StepDataset for tokenization
|
| 319 |
+
if not streaming and flux_dataset.reasoning_chains:
|
| 320 |
+
step_dataset = StepDataset(
|
| 321 |
+
reasoning_chains=flux_dataset.reasoning_chains,
|
| 322 |
+
tokenizer=tokenizer,
|
| 323 |
+
max_steps=10,
|
| 324 |
+
include_visual_features=False, # FLUX dataset doesn't have pre-extracted features
|
| 325 |
+
filter_incorrect=False,
|
| 326 |
+
)
|
| 327 |
+
|
| 328 |
+
return torch.utils.data.DataLoader(
|
| 329 |
+
step_dataset,
|
| 330 |
+
batch_size=batch_size,
|
| 331 |
+
num_workers=num_workers,
|
| 332 |
+
shuffle=True,
|
| 333 |
+
collate_fn=lambda batch: batch, # Custom collation handled in trainer
|
| 334 |
+
)
|
| 335 |
+
else:
|
| 336 |
+
# For streaming, return raw flux dataset
|
| 337 |
+
# (trainer will handle tokenization on-the-fly)
|
| 338 |
+
return torch.utils.data.DataLoader(
|
| 339 |
+
flux_dataset,
|
| 340 |
+
batch_size=batch_size,
|
| 341 |
+
num_workers=0 if streaming else num_workers, # Streaming doesn't support multiprocessing well
|
| 342 |
+
shuffle=False, # Streaming mode doesn't support shuffle
|
| 343 |
+
)
|
| 344 |
+
|
| 345 |
+
|
| 346 |
+
# Example usage
|
| 347 |
+
if __name__ == "__main__":
|
| 348 |
+
# Test dataset loading
|
| 349 |
+
print("Testing FLUX-Reason-6M dataset adapter...")
|
| 350 |
+
|
| 351 |
+
dataset = FLUXReasonDataset(
|
| 352 |
+
streaming=False,
|
| 353 |
+
max_samples=10,
|
| 354 |
+
)
|
| 355 |
+
|
| 356 |
+
print(f"\nLoaded {len(dataset)} samples")
|
| 357 |
+
|
| 358 |
+
# Show first example
|
| 359 |
+
if len(dataset) > 0:
|
| 360 |
+
chain = dataset[0]
|
| 361 |
+
print(f"\n{'='*60}")
|
| 362 |
+
print("Example Reasoning Chain:")
|
| 363 |
+
print(f"{'='*60}")
|
| 364 |
+
print(f"Chain ID: {chain.chain_id}")
|
| 365 |
+
print(f"Prompt: {chain.prompt}")
|
| 366 |
+
print(f"Number of steps: {len(chain.steps)}")
|
| 367 |
+
print(f"\nSteps:")
|
| 368 |
+
for step in chain.steps:
|
| 369 |
+
print(f" Step {step.step_id} ({step.step_type.value}): {step.description[:100]}...")
|
| 370 |
+
print(f"\nFinal Answer: {chain.final_answer}")
|
| 371 |
+
print(f"\n{'='*60}")
|
| 372 |
+
print("\nFormatted with special tokens:")
|
| 373 |
+
print(f"{'='*60}")
|
| 374 |
+
formatted = chain.format_with_tokens()
|
| 375 |
+
print(formatted[:500] + "..." if len(formatted) > 500 else formatted)
|
src/reasoning/inference_scaling.py
ADDED
|
@@ -0,0 +1,453 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Inference-time scaling mechanisms for step-level reasoning.
|
| 3 |
+
|
| 4 |
+
Implements best-of-N sampling, majority voting, and self-consistency
|
| 5 |
+
to improve reasoning quality through test-time compute.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from typing import Dict, List, Optional, Tuple, Any, Callable
|
| 9 |
+
import torch
|
| 10 |
+
import torch.nn as nn
|
| 11 |
+
from collections import Counter
|
| 12 |
+
import logging
|
| 13 |
+
from dataclasses import dataclass
|
| 14 |
+
|
| 15 |
+
from .prm import ProcessRewardModel
|
| 16 |
+
from .step_data import ReasoningChain, ReasoningStep
|
| 17 |
+
|
| 18 |
+
logger = logging.getLogger(__name__)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@dataclass
|
| 22 |
+
class InferenceConfig:
|
| 23 |
+
"""Configuration for inference-time scaling."""
|
| 24 |
+
num_samples: int = 8 # Number of reasoning chains to generate
|
| 25 |
+
temperature: float = 0.7 # Sampling temperature
|
| 26 |
+
top_k: int = 50
|
| 27 |
+
top_p: float = 0.95
|
| 28 |
+
max_steps: int = 10 # Maximum reasoning steps
|
| 29 |
+
|
| 30 |
+
# Aggregation method
|
| 31 |
+
aggregation: str = "best_of_n" # Options: best_of_n, majority_vote, weighted_vote
|
| 32 |
+
|
| 33 |
+
# PRM-based selection
|
| 34 |
+
use_prm_scores: bool = True
|
| 35 |
+
min_prm_score: float = 0.3 # Minimum acceptable PRM score
|
| 36 |
+
|
| 37 |
+
# Self-consistency
|
| 38 |
+
consistency_threshold: float = 0.6 # Required agreement for majority vote
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class InferenceTimeScaling:
|
| 42 |
+
"""
|
| 43 |
+
Inference-time scaling for improved reasoning quality.
|
| 44 |
+
|
| 45 |
+
Generates multiple reasoning chains and selects the best one using
|
| 46 |
+
various aggregation strategies.
|
| 47 |
+
"""
|
| 48 |
+
|
| 49 |
+
def __init__(
|
| 50 |
+
self,
|
| 51 |
+
model: nn.Module,
|
| 52 |
+
prm: Optional[ProcessRewardModel] = None,
|
| 53 |
+
config: Optional[InferenceConfig] = None,
|
| 54 |
+
device: str = "cuda",
|
| 55 |
+
):
|
| 56 |
+
"""
|
| 57 |
+
Initialize inference scaler.
|
| 58 |
+
|
| 59 |
+
Args:
|
| 60 |
+
model: Vision-language model
|
| 61 |
+
prm: Process Reward Model for evaluation
|
| 62 |
+
config: Inference configuration
|
| 63 |
+
device: Device for inference
|
| 64 |
+
"""
|
| 65 |
+
self.model = model
|
| 66 |
+
self.prm = prm
|
| 67 |
+
self.config = config or InferenceConfig()
|
| 68 |
+
self.device = device
|
| 69 |
+
|
| 70 |
+
self.model.to(device)
|
| 71 |
+
self.model.eval()
|
| 72 |
+
|
| 73 |
+
if self.prm:
|
| 74 |
+
self.prm.to(device)
|
| 75 |
+
self.prm.eval()
|
| 76 |
+
|
| 77 |
+
@torch.no_grad()
|
| 78 |
+
def generate_multiple_chains(
|
| 79 |
+
self,
|
| 80 |
+
image_features: torch.Tensor,
|
| 81 |
+
prompt_ids: torch.Tensor,
|
| 82 |
+
prompt_mask: torch.Tensor,
|
| 83 |
+
image_path: str = "",
|
| 84 |
+
prompt_text: str = "",
|
| 85 |
+
) -> List[ReasoningChain]:
|
| 86 |
+
"""
|
| 87 |
+
Generate multiple reasoning chains with different samples.
|
| 88 |
+
|
| 89 |
+
Args:
|
| 90 |
+
image_features: Visual features
|
| 91 |
+
prompt_ids: Tokenized prompt
|
| 92 |
+
prompt_mask: Attention mask
|
| 93 |
+
image_path: Path to image (for metadata)
|
| 94 |
+
prompt_text: Original prompt text
|
| 95 |
+
|
| 96 |
+
Returns:
|
| 97 |
+
List of reasoning chains
|
| 98 |
+
"""
|
| 99 |
+
chains = []
|
| 100 |
+
|
| 101 |
+
for sample_idx in range(self.config.num_samples):
|
| 102 |
+
steps = []
|
| 103 |
+
current_ids = prompt_ids.unsqueeze(0).to(self.device)
|
| 104 |
+
current_mask = prompt_mask.unsqueeze(0).to(self.device)
|
| 105 |
+
|
| 106 |
+
step_embeddings = []
|
| 107 |
+
|
| 108 |
+
for step_id in range(self.config.max_steps):
|
| 109 |
+
# Generate next step
|
| 110 |
+
outputs = self.model(
|
| 111 |
+
input_ids=current_ids,
|
| 112 |
+
attention_mask=current_mask,
|
| 113 |
+
output_hidden_states=True,
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
logits = outputs.logits[:, -1, :] / self.config.temperature
|
| 117 |
+
probs = torch.softmax(logits, dim=-1)
|
| 118 |
+
|
| 119 |
+
# Top-k sampling
|
| 120 |
+
if self.config.top_k > 0:
|
| 121 |
+
top_k_probs, top_k_indices = torch.topk(probs, self.config.top_k)
|
| 122 |
+
next_token = top_k_indices[0, torch.multinomial(top_k_probs[0], 1)]
|
| 123 |
+
else:
|
| 124 |
+
next_token = torch.multinomial(probs[0], 1)
|
| 125 |
+
|
| 126 |
+
# Get hidden state for this step
|
| 127 |
+
hidden_state = outputs.hidden_states[-1][:, -1, :].squeeze(0)
|
| 128 |
+
step_embeddings.append(hidden_state)
|
| 129 |
+
|
| 130 |
+
# Compute PRM reward if available
|
| 131 |
+
reward = 0.0
|
| 132 |
+
if self.prm:
|
| 133 |
+
reward, metrics = self.prm.compute_step_reward(
|
| 134 |
+
image_features,
|
| 135 |
+
hidden_state,
|
| 136 |
+
previous_steps=torch.stack(step_embeddings[:-1]) if len(step_embeddings) > 1 else None,
|
| 137 |
+
return_metrics=True,
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
# Decode step text
|
| 141 |
+
step_text = self.model.config.tokenizer.decode(
|
| 142 |
+
next_token.unsqueeze(0),
|
| 143 |
+
skip_special_tokens=True,
|
| 144 |
+
)
|
| 145 |
+
|
| 146 |
+
# Create reasoning step
|
| 147 |
+
step = ReasoningStep(
|
| 148 |
+
step_id=step_id,
|
| 149 |
+
step_type="inference", # Would be classified in practice
|
| 150 |
+
description=step_text,
|
| 151 |
+
visual_features=hidden_state.cpu(),
|
| 152 |
+
confidence=metrics.confidence if self.prm and metrics else 0.5,
|
| 153 |
+
reward=reward,
|
| 154 |
+
)
|
| 155 |
+
steps.append(step)
|
| 156 |
+
|
| 157 |
+
# Update sequence
|
| 158 |
+
current_ids = torch.cat([current_ids, next_token.unsqueeze(0).unsqueeze(0)], dim=1)
|
| 159 |
+
current_mask = torch.cat([
|
| 160 |
+
current_mask,
|
| 161 |
+
torch.ones((1, 1), device=self.device)
|
| 162 |
+
], dim=1)
|
| 163 |
+
|
| 164 |
+
# Check for completion
|
| 165 |
+
if next_token.item() == self.model.config.eos_token_id:
|
| 166 |
+
break
|
| 167 |
+
|
| 168 |
+
# Create reasoning chain
|
| 169 |
+
final_answer = self.model.config.tokenizer.decode(
|
| 170 |
+
current_ids[0, len(prompt_ids):],
|
| 171 |
+
skip_special_tokens=True,
|
| 172 |
+
)
|
| 173 |
+
|
| 174 |
+
total_reward = sum(s.reward for s in steps)
|
| 175 |
+
|
| 176 |
+
chain = ReasoningChain(
|
| 177 |
+
chain_id=f"sample_{sample_idx}",
|
| 178 |
+
image_path=image_path,
|
| 179 |
+
prompt=prompt_text,
|
| 180 |
+
steps=steps,
|
| 181 |
+
final_answer=final_answer,
|
| 182 |
+
total_reward=total_reward,
|
| 183 |
+
)
|
| 184 |
+
chains.append(chain)
|
| 185 |
+
|
| 186 |
+
return chains
|
| 187 |
+
|
| 188 |
+
def best_of_n_selection(
|
| 189 |
+
self,
|
| 190 |
+
chains: List[ReasoningChain],
|
| 191 |
+
) -> ReasoningChain:
|
| 192 |
+
"""
|
| 193 |
+
Select best chain based on PRM scores.
|
| 194 |
+
|
| 195 |
+
Args:
|
| 196 |
+
chains: List of reasoning chains
|
| 197 |
+
|
| 198 |
+
Returns:
|
| 199 |
+
Chain with highest total reward
|
| 200 |
+
"""
|
| 201 |
+
if not chains:
|
| 202 |
+
raise ValueError("No chains to select from")
|
| 203 |
+
|
| 204 |
+
# Filter by minimum PRM score if required
|
| 205 |
+
if self.config.use_prm_scores and self.prm:
|
| 206 |
+
filtered_chains = [
|
| 207 |
+
c for c in chains
|
| 208 |
+
if c.total_reward >= self.config.min_prm_score * len(c.steps)
|
| 209 |
+
]
|
| 210 |
+
if filtered_chains:
|
| 211 |
+
chains = filtered_chains
|
| 212 |
+
|
| 213 |
+
# Select chain with highest total reward
|
| 214 |
+
best_chain = max(chains, key=lambda c: c.total_reward)
|
| 215 |
+
|
| 216 |
+
logger.info(
|
| 217 |
+
f"Best-of-N: Selected chain with reward {best_chain.total_reward:.3f} "
|
| 218 |
+
f"({len(best_chain.steps)} steps)"
|
| 219 |
+
)
|
| 220 |
+
|
| 221 |
+
return best_chain
|
| 222 |
+
|
| 223 |
+
def majority_voting(
|
| 224 |
+
self,
|
| 225 |
+
chains: List[ReasoningChain],
|
| 226 |
+
) -> ReasoningChain:
|
| 227 |
+
"""
|
| 228 |
+
Select answer by majority voting with self-consistency.
|
| 229 |
+
|
| 230 |
+
Args:
|
| 231 |
+
chains: List of reasoning chains
|
| 232 |
+
|
| 233 |
+
Returns:
|
| 234 |
+
Chain corresponding to majority answer
|
| 235 |
+
"""
|
| 236 |
+
if not chains:
|
| 237 |
+
raise ValueError("No chains to vote on")
|
| 238 |
+
|
| 239 |
+
# Extract final answers
|
| 240 |
+
answers = [c.final_answer.strip().lower() for c in chains]
|
| 241 |
+
|
| 242 |
+
# Count votes
|
| 243 |
+
vote_counts = Counter(answers)
|
| 244 |
+
majority_answer, majority_count = vote_counts.most_common(1)[0]
|
| 245 |
+
|
| 246 |
+
# Check consistency threshold
|
| 247 |
+
consistency = majority_count / len(chains)
|
| 248 |
+
if consistency < self.config.consistency_threshold:
|
| 249 |
+
logger.warning(
|
| 250 |
+
f"Low consistency: {consistency:.2f} < {self.config.consistency_threshold:.2f}"
|
| 251 |
+
)
|
| 252 |
+
|
| 253 |
+
# Find chain with majority answer and highest PRM score
|
| 254 |
+
majority_chains = [
|
| 255 |
+
c for c in chains
|
| 256 |
+
if c.final_answer.strip().lower() == majority_answer
|
| 257 |
+
]
|
| 258 |
+
|
| 259 |
+
best_chain = max(majority_chains, key=lambda c: c.total_reward)
|
| 260 |
+
|
| 261 |
+
logger.info(
|
| 262 |
+
f"Majority Vote: '{majority_answer}' ({majority_count}/{len(chains)}) "
|
| 263 |
+
f"with consistency {consistency:.2f}"
|
| 264 |
+
)
|
| 265 |
+
|
| 266 |
+
return best_chain
|
| 267 |
+
|
| 268 |
+
def weighted_voting(
|
| 269 |
+
self,
|
| 270 |
+
chains: List[ReasoningChain],
|
| 271 |
+
) -> ReasoningChain:
|
| 272 |
+
"""
|
| 273 |
+
Weighted voting using PRM scores as weights.
|
| 274 |
+
|
| 275 |
+
Args:
|
| 276 |
+
chains: List of reasoning chains
|
| 277 |
+
|
| 278 |
+
Returns:
|
| 279 |
+
Chain with highest weighted vote
|
| 280 |
+
"""
|
| 281 |
+
if not chains or not self.prm:
|
| 282 |
+
return self.best_of_n_selection(chains)
|
| 283 |
+
|
| 284 |
+
# Group chains by answer
|
| 285 |
+
answer_groups: Dict[str, List[ReasoningChain]] = {}
|
| 286 |
+
for chain in chains:
|
| 287 |
+
answer_key = chain.final_answer.strip().lower()
|
| 288 |
+
if answer_key not in answer_groups:
|
| 289 |
+
answer_groups[answer_key] = []
|
| 290 |
+
answer_groups[answer_key].append(chain)
|
| 291 |
+
|
| 292 |
+
# Calculate weighted votes
|
| 293 |
+
weighted_votes = {}
|
| 294 |
+
for answer, group_chains in answer_groups.items():
|
| 295 |
+
# Use exponential of reward as weight (emphasizes high-quality chains)
|
| 296 |
+
weight = sum(torch.exp(torch.tensor(c.total_reward / len(c.steps))).item()
|
| 297 |
+
for c in group_chains)
|
| 298 |
+
weighted_votes[answer] = weight
|
| 299 |
+
|
| 300 |
+
# Select answer with highest weighted vote
|
| 301 |
+
best_answer = max(weighted_votes.items(), key=lambda x: x[1])[0]
|
| 302 |
+
|
| 303 |
+
# Return best chain for that answer
|
| 304 |
+
best_chains = answer_groups[best_answer]
|
| 305 |
+
best_chain = max(best_chains, key=lambda c: c.total_reward)
|
| 306 |
+
|
| 307 |
+
logger.info(
|
| 308 |
+
f"Weighted Vote: '{best_answer}' with weight {weighted_votes[best_answer]:.3f}"
|
| 309 |
+
)
|
| 310 |
+
|
| 311 |
+
return best_chain
|
| 312 |
+
|
| 313 |
+
def __call__(
|
| 314 |
+
self,
|
| 315 |
+
image_features: torch.Tensor,
|
| 316 |
+
prompt_ids: torch.Tensor,
|
| 317 |
+
prompt_mask: torch.Tensor,
|
| 318 |
+
image_path: str = "",
|
| 319 |
+
prompt_text: str = "",
|
| 320 |
+
) -> Tuple[ReasoningChain, List[ReasoningChain]]:
|
| 321 |
+
"""
|
| 322 |
+
Perform inference with scaling.
|
| 323 |
+
|
| 324 |
+
Args:
|
| 325 |
+
image_features: Visual features
|
| 326 |
+
prompt_ids: Tokenized prompt
|
| 327 |
+
prompt_mask: Attention mask
|
| 328 |
+
image_path: Path to image
|
| 329 |
+
prompt_text: Original prompt
|
| 330 |
+
|
| 331 |
+
Returns:
|
| 332 |
+
- best_chain: Selected reasoning chain
|
| 333 |
+
- all_chains: All generated chains for analysis
|
| 334 |
+
"""
|
| 335 |
+
# Generate multiple chains
|
| 336 |
+
chains = self.generate_multiple_chains(
|
| 337 |
+
image_features,
|
| 338 |
+
prompt_ids,
|
| 339 |
+
prompt_mask,
|
| 340 |
+
image_path,
|
| 341 |
+
prompt_text,
|
| 342 |
+
)
|
| 343 |
+
|
| 344 |
+
# Select best chain based on aggregation method
|
| 345 |
+
if self.config.aggregation == "best_of_n":
|
| 346 |
+
best_chain = self.best_of_n_selection(chains)
|
| 347 |
+
elif self.config.aggregation == "majority_vote":
|
| 348 |
+
best_chain = self.majority_voting(chains)
|
| 349 |
+
elif self.config.aggregation == "weighted_vote":
|
| 350 |
+
best_chain = self.weighted_voting(chains)
|
| 351 |
+
else:
|
| 352 |
+
raise ValueError(f"Unknown aggregation method: {self.config.aggregation}")
|
| 353 |
+
|
| 354 |
+
return best_chain, chains
|
| 355 |
+
|
| 356 |
+
|
| 357 |
+
class MajorityVoting:
|
| 358 |
+
"""
|
| 359 |
+
Standalone majority voting for answer aggregation.
|
| 360 |
+
|
| 361 |
+
Useful for post-processing multiple model predictions.
|
| 362 |
+
"""
|
| 363 |
+
|
| 364 |
+
@staticmethod
|
| 365 |
+
def vote(
|
| 366 |
+
answers: List[str],
|
| 367 |
+
weights: Optional[List[float]] = None,
|
| 368 |
+
threshold: float = 0.5,
|
| 369 |
+
) -> Tuple[str, float]:
|
| 370 |
+
"""
|
| 371 |
+
Perform majority voting on answers.
|
| 372 |
+
|
| 373 |
+
Args:
|
| 374 |
+
answers: List of answer strings
|
| 375 |
+
weights: Optional weights for each answer
|
| 376 |
+
threshold: Minimum agreement threshold
|
| 377 |
+
|
| 378 |
+
Returns:
|
| 379 |
+
- majority_answer: Most common answer
|
| 380 |
+
- confidence: Voting confidence [0, 1]
|
| 381 |
+
"""
|
| 382 |
+
if not answers:
|
| 383 |
+
return "", 0.0
|
| 384 |
+
|
| 385 |
+
if weights is None:
|
| 386 |
+
weights = [1.0] * len(answers)
|
| 387 |
+
|
| 388 |
+
# Normalize answers
|
| 389 |
+
normalized = [a.strip().lower() for a in answers]
|
| 390 |
+
|
| 391 |
+
# Weighted voting
|
| 392 |
+
vote_dict: Dict[str, float] = {}
|
| 393 |
+
for answer, weight in zip(normalized, weights):
|
| 394 |
+
vote_dict[answer] = vote_dict.get(answer, 0.0) + weight
|
| 395 |
+
|
| 396 |
+
# Find majority
|
| 397 |
+
total_weight = sum(weights)
|
| 398 |
+
majority_answer = max(vote_dict.items(), key=lambda x: x[1])[0]
|
| 399 |
+
confidence = vote_dict[majority_answer] / total_weight
|
| 400 |
+
|
| 401 |
+
# Check threshold
|
| 402 |
+
if confidence < threshold:
|
| 403 |
+
logger.warning(
|
| 404 |
+
f"Majority voting confidence {confidence:.2f} below threshold {threshold}"
|
| 405 |
+
)
|
| 406 |
+
|
| 407 |
+
return majority_answer, confidence
|
| 408 |
+
|
| 409 |
+
@staticmethod
|
| 410 |
+
def self_consistency(
|
| 411 |
+
chains: List[ReasoningChain],
|
| 412 |
+
min_agreement: float = 0.6,
|
| 413 |
+
) -> Tuple[Optional[str], float, List[ReasoningChain]]:
|
| 414 |
+
"""
|
| 415 |
+
Self-consistency checking across reasoning chains.
|
| 416 |
+
|
| 417 |
+
Args:
|
| 418 |
+
chains: List of reasoning chains
|
| 419 |
+
min_agreement: Minimum agreement required
|
| 420 |
+
|
| 421 |
+
Returns:
|
| 422 |
+
- consensus_answer: Answer if consensus reached, else None
|
| 423 |
+
- agreement: Agreement ratio [0, 1]
|
| 424 |
+
- supporting_chains: Chains supporting consensus
|
| 425 |
+
"""
|
| 426 |
+
if not chains:
|
| 427 |
+
return None, 0.0, []
|
| 428 |
+
|
| 429 |
+
# Extract answers
|
| 430 |
+
answers = [c.final_answer.strip().lower() for c in chains]
|
| 431 |
+
|
| 432 |
+
# Count occurrences
|
| 433 |
+
answer_counts = Counter(answers)
|
| 434 |
+
most_common_answer, count = answer_counts.most_common(1)[0]
|
| 435 |
+
|
| 436 |
+
# Calculate agreement
|
| 437 |
+
agreement = count / len(chains)
|
| 438 |
+
|
| 439 |
+
# Check if consensus reached
|
| 440 |
+
consensus = most_common_answer if agreement >= min_agreement else None
|
| 441 |
+
|
| 442 |
+
# Find supporting chains
|
| 443 |
+
supporting_chains = [
|
| 444 |
+
c for c in chains
|
| 445 |
+
if c.final_answer.strip().lower() == most_common_answer
|
| 446 |
+
]
|
| 447 |
+
|
| 448 |
+
logger.info(
|
| 449 |
+
f"Self-consistency: {agreement:.1%} agreement "
|
| 450 |
+
f"({'consensus' if consensus else 'no consensus'})"
|
| 451 |
+
)
|
| 452 |
+
|
| 453 |
+
return consensus, agreement, supporting_chains
|
src/reasoning/prm.py
ADDED
|
@@ -0,0 +1,462 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Process Reward Model (PRM) for evaluating step-level reasoning quality.
|
| 3 |
+
|
| 4 |
+
The PRM assigns rewards to intermediate reasoning steps, enabling fine-grained
|
| 5 |
+
reinforcement learning and quality assessment.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from dataclasses import dataclass
|
| 9 |
+
from typing import Dict, List, Optional, Tuple
|
| 10 |
+
import torch
|
| 11 |
+
import torch.nn as nn
|
| 12 |
+
import torch.nn.functional as F
|
| 13 |
+
from transformers import PreTrainedModel
|
| 14 |
+
import logging
|
| 15 |
+
|
| 16 |
+
logger = logging.getLogger(__name__)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@dataclass
|
| 20 |
+
class StepQualityMetrics:
|
| 21 |
+
"""Metrics for assessing reasoning step quality."""
|
| 22 |
+
coherence: float # How well step follows from previous steps [0, 1]
|
| 23 |
+
relevance: float # Relevance to prompt and image [0, 1]
|
| 24 |
+
correctness: float # Logical correctness [0, 1]
|
| 25 |
+
informativeness: float # How much new information added [0, 1]
|
| 26 |
+
confidence: float # Model's confidence in this step [0, 1]
|
| 27 |
+
|
| 28 |
+
def to_reward(self, weights: Optional[Dict[str, float]] = None) -> float:
|
| 29 |
+
"""
|
| 30 |
+
Convert metrics to single reward score.
|
| 31 |
+
|
| 32 |
+
Args:
|
| 33 |
+
weights: Optional custom weights for each metric
|
| 34 |
+
|
| 35 |
+
Returns:
|
| 36 |
+
Reward in [-1, 1] range
|
| 37 |
+
"""
|
| 38 |
+
if weights is None:
|
| 39 |
+
weights = {
|
| 40 |
+
'coherence': 0.25,
|
| 41 |
+
'relevance': 0.25,
|
| 42 |
+
'correctness': 0.3,
|
| 43 |
+
'informativeness': 0.15,
|
| 44 |
+
'confidence': 0.05,
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
reward = sum(
|
| 48 |
+
getattr(self, key) * weight
|
| 49 |
+
for key, weight in weights.items()
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
# Normalize to [-1, 1] (assuming metrics are [0, 1])
|
| 53 |
+
return reward * 2.0 - 1.0
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
class ProcessRewardModel(nn.Module):
|
| 57 |
+
"""
|
| 58 |
+
Process Reward Model for step-level reasoning evaluation.
|
| 59 |
+
|
| 60 |
+
Architecture:
|
| 61 |
+
- Vision encoder: Extract visual features
|
| 62 |
+
- Text encoder: Encode reasoning steps
|
| 63 |
+
- Context aggregator: Combine previous steps
|
| 64 |
+
- Reward head: Predict step quality metrics
|
| 65 |
+
"""
|
| 66 |
+
|
| 67 |
+
def __init__(
|
| 68 |
+
self,
|
| 69 |
+
vision_dim: int = 768,
|
| 70 |
+
text_dim: int = 768,
|
| 71 |
+
hidden_dim: int = 512,
|
| 72 |
+
num_heads: int = 8,
|
| 73 |
+
dropout: float = 0.1,
|
| 74 |
+
max_steps: int = 10,
|
| 75 |
+
):
|
| 76 |
+
"""
|
| 77 |
+
Initialize PRM.
|
| 78 |
+
|
| 79 |
+
Args:
|
| 80 |
+
vision_dim: Dimension of vision features
|
| 81 |
+
text_dim: Dimension of text embeddings
|
| 82 |
+
hidden_dim: Hidden dimension for processing
|
| 83 |
+
num_heads: Number of attention heads
|
| 84 |
+
dropout: Dropout rate
|
| 85 |
+
max_steps: Maximum reasoning steps to handle
|
| 86 |
+
"""
|
| 87 |
+
super().__init__()
|
| 88 |
+
|
| 89 |
+
self.vision_dim = vision_dim
|
| 90 |
+
self.text_dim = text_dim
|
| 91 |
+
self.hidden_dim = hidden_dim
|
| 92 |
+
self.max_steps = max_steps
|
| 93 |
+
|
| 94 |
+
# Vision and text projections
|
| 95 |
+
self.vision_proj = nn.Linear(vision_dim, hidden_dim)
|
| 96 |
+
self.text_proj = nn.Linear(text_dim, hidden_dim)
|
| 97 |
+
|
| 98 |
+
# Context aggregator with self-attention over previous steps
|
| 99 |
+
self.context_attention = nn.MultiheadAttention(
|
| 100 |
+
embed_dim=hidden_dim,
|
| 101 |
+
num_heads=num_heads,
|
| 102 |
+
dropout=dropout,
|
| 103 |
+
batch_first=True,
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
# Step encoder
|
| 107 |
+
self.step_encoder = nn.TransformerEncoder(
|
| 108 |
+
nn.TransformerEncoderLayer(
|
| 109 |
+
d_model=hidden_dim,
|
| 110 |
+
nhead=num_heads,
|
| 111 |
+
dim_feedforward=hidden_dim * 4,
|
| 112 |
+
dropout=dropout,
|
| 113 |
+
batch_first=True,
|
| 114 |
+
),
|
| 115 |
+
num_layers=2,
|
| 116 |
+
)
|
| 117 |
+
|
| 118 |
+
# Reward heads for different metrics
|
| 119 |
+
self.coherence_head = nn.Sequential(
|
| 120 |
+
nn.Linear(hidden_dim, hidden_dim // 2),
|
| 121 |
+
nn.ReLU(),
|
| 122 |
+
nn.Dropout(dropout),
|
| 123 |
+
nn.Linear(hidden_dim // 2, 1),
|
| 124 |
+
nn.Sigmoid(),
|
| 125 |
+
)
|
| 126 |
+
|
| 127 |
+
self.relevance_head = nn.Sequential(
|
| 128 |
+
nn.Linear(hidden_dim, hidden_dim // 2),
|
| 129 |
+
nn.ReLU(),
|
| 130 |
+
nn.Dropout(dropout),
|
| 131 |
+
nn.Linear(hidden_dim // 2, 1),
|
| 132 |
+
nn.Sigmoid(),
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
self.correctness_head = nn.Sequential(
|
| 136 |
+
nn.Linear(hidden_dim, hidden_dim // 2),
|
| 137 |
+
nn.ReLU(),
|
| 138 |
+
nn.Dropout(dropout),
|
| 139 |
+
nn.Linear(hidden_dim // 2, 1),
|
| 140 |
+
nn.Sigmoid(),
|
| 141 |
+
)
|
| 142 |
+
|
| 143 |
+
self.informativeness_head = nn.Sequential(
|
| 144 |
+
nn.Linear(hidden_dim, hidden_dim // 2),
|
| 145 |
+
nn.ReLU(),
|
| 146 |
+
nn.Dropout(dropout),
|
| 147 |
+
nn.Linear(hidden_dim // 2, 1),
|
| 148 |
+
nn.Sigmoid(),
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
# Confidence estimation
|
| 152 |
+
self.confidence_head = nn.Sequential(
|
| 153 |
+
nn.Linear(hidden_dim, hidden_dim // 2),
|
| 154 |
+
nn.ReLU(),
|
| 155 |
+
nn.Dropout(dropout),
|
| 156 |
+
nn.Linear(hidden_dim // 2, 1),
|
| 157 |
+
nn.Sigmoid(),
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
# Overall reward head (learned combination)
|
| 161 |
+
self.reward_head = nn.Sequential(
|
| 162 |
+
nn.Linear(hidden_dim, hidden_dim // 2),
|
| 163 |
+
nn.ReLU(),
|
| 164 |
+
nn.Dropout(dropout),
|
| 165 |
+
nn.Linear(hidden_dim // 2, 1),
|
| 166 |
+
nn.Tanh(), # Output in [-1, 1]
|
| 167 |
+
)
|
| 168 |
+
|
| 169 |
+
self.dropout = nn.Dropout(dropout)
|
| 170 |
+
|
| 171 |
+
def forward(
|
| 172 |
+
self,
|
| 173 |
+
vision_features: torch.Tensor,
|
| 174 |
+
step_embeddings: torch.Tensor,
|
| 175 |
+
step_mask: Optional[torch.Tensor] = None,
|
| 176 |
+
return_metrics: bool = False,
|
| 177 |
+
) -> Tuple[torch.Tensor, Optional[List[StepQualityMetrics]]]:
|
| 178 |
+
"""
|
| 179 |
+
Forward pass to compute rewards for reasoning steps.
|
| 180 |
+
|
| 181 |
+
Args:
|
| 182 |
+
vision_features: Visual features [batch, vision_dim]
|
| 183 |
+
step_embeddings: Text embeddings for each step [batch, num_steps, text_dim]
|
| 184 |
+
step_mask: Mask for valid steps [batch, num_steps]
|
| 185 |
+
return_metrics: If True, return detailed metrics
|
| 186 |
+
|
| 187 |
+
Returns:
|
| 188 |
+
- rewards: Reward for each step [batch, num_steps]
|
| 189 |
+
- metrics: Optional list of StepQualityMetrics per step
|
| 190 |
+
"""
|
| 191 |
+
batch_size, num_steps, _ = step_embeddings.shape
|
| 192 |
+
|
| 193 |
+
# Project vision and text features
|
| 194 |
+
vision_feat = self.vision_proj(vision_features) # [batch, hidden_dim]
|
| 195 |
+
text_feat = self.text_proj(step_embeddings) # [batch, num_steps, hidden_dim]
|
| 196 |
+
|
| 197 |
+
# Expand vision features to match step dimension
|
| 198 |
+
vision_expanded = vision_feat.unsqueeze(1).expand(-1, num_steps, -1)
|
| 199 |
+
|
| 200 |
+
# Combine vision and text
|
| 201 |
+
combined = vision_expanded + text_feat # [batch, num_steps, hidden_dim]
|
| 202 |
+
combined = self.dropout(combined)
|
| 203 |
+
|
| 204 |
+
# Apply context attention (each step attends to previous steps)
|
| 205 |
+
if step_mask is not None:
|
| 206 |
+
# Create causal mask for autoregressive reasoning
|
| 207 |
+
causal_mask = torch.triu(
|
| 208 |
+
torch.ones(num_steps, num_steps, device=combined.device),
|
| 209 |
+
diagonal=1
|
| 210 |
+
).bool()
|
| 211 |
+
else:
|
| 212 |
+
causal_mask = None
|
| 213 |
+
|
| 214 |
+
attended, _ = self.context_attention(
|
| 215 |
+
combined, combined, combined,
|
| 216 |
+
attn_mask=causal_mask,
|
| 217 |
+
key_padding_mask=~step_mask if step_mask is not None else None,
|
| 218 |
+
)
|
| 219 |
+
|
| 220 |
+
# Encode steps
|
| 221 |
+
encoded = self.step_encoder(attended) # [batch, num_steps, hidden_dim]
|
| 222 |
+
|
| 223 |
+
# Compute rewards
|
| 224 |
+
rewards = self.reward_head(encoded).squeeze(-1) # [batch, num_steps]
|
| 225 |
+
|
| 226 |
+
if step_mask is not None:
|
| 227 |
+
rewards = rewards.masked_fill(~step_mask, 0.0)
|
| 228 |
+
|
| 229 |
+
# Optionally compute detailed metrics
|
| 230 |
+
metrics = None
|
| 231 |
+
if return_metrics:
|
| 232 |
+
metrics = []
|
| 233 |
+
for b in range(batch_size):
|
| 234 |
+
step_metrics = []
|
| 235 |
+
for s in range(num_steps):
|
| 236 |
+
if step_mask is None or step_mask[b, s]:
|
| 237 |
+
feat = encoded[b, s:s+1]
|
| 238 |
+
|
| 239 |
+
quality = StepQualityMetrics(
|
| 240 |
+
coherence=self.coherence_head(feat).item(),
|
| 241 |
+
relevance=self.relevance_head(feat).item(),
|
| 242 |
+
correctness=self.correctness_head(feat).item(),
|
| 243 |
+
informativeness=self.informativeness_head(feat).item(),
|
| 244 |
+
confidence=self.confidence_head(feat).item(),
|
| 245 |
+
)
|
| 246 |
+
step_metrics.append(quality)
|
| 247 |
+
metrics.append(step_metrics)
|
| 248 |
+
|
| 249 |
+
return rewards, metrics
|
| 250 |
+
|
| 251 |
+
def compute_step_reward(
|
| 252 |
+
self,
|
| 253 |
+
vision_features: torch.Tensor,
|
| 254 |
+
current_step: torch.Tensor,
|
| 255 |
+
previous_steps: Optional[torch.Tensor] = None,
|
| 256 |
+
return_metrics: bool = False,
|
| 257 |
+
) -> Tuple[float, Optional[StepQualityMetrics]]:
|
| 258 |
+
"""
|
| 259 |
+
Compute reward for a single reasoning step.
|
| 260 |
+
|
| 261 |
+
Args:
|
| 262 |
+
vision_features: Visual features [vision_dim]
|
| 263 |
+
current_step: Current step embedding [text_dim]
|
| 264 |
+
previous_steps: Previous step embeddings [num_prev_steps, text_dim]
|
| 265 |
+
return_metrics: If True, return detailed metrics
|
| 266 |
+
|
| 267 |
+
Returns:
|
| 268 |
+
- reward: Scalar reward in [-1, 1]
|
| 269 |
+
- metrics: Optional StepQualityMetrics
|
| 270 |
+
"""
|
| 271 |
+
# Prepare batch inputs
|
| 272 |
+
vision_features = vision_features.unsqueeze(0) # [1, vision_dim]
|
| 273 |
+
|
| 274 |
+
if previous_steps is not None:
|
| 275 |
+
# Concatenate previous and current
|
| 276 |
+
steps = torch.cat([previous_steps, current_step.unsqueeze(0)], dim=0)
|
| 277 |
+
else:
|
| 278 |
+
steps = current_step.unsqueeze(0)
|
| 279 |
+
|
| 280 |
+
steps = steps.unsqueeze(0) # [1, num_steps, text_dim]
|
| 281 |
+
|
| 282 |
+
# Forward pass
|
| 283 |
+
with torch.no_grad():
|
| 284 |
+
rewards, metrics = self.forward(
|
| 285 |
+
vision_features,
|
| 286 |
+
steps,
|
| 287 |
+
return_metrics=return_metrics,
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
# Extract last step reward
|
| 291 |
+
reward = rewards[0, -1].item()
|
| 292 |
+
step_metrics = metrics[0][-1] if metrics else None
|
| 293 |
+
|
| 294 |
+
return reward, step_metrics
|
| 295 |
+
|
| 296 |
+
@torch.no_grad()
|
| 297 |
+
def evaluate_chain(
|
| 298 |
+
self,
|
| 299 |
+
vision_features: torch.Tensor,
|
| 300 |
+
step_embeddings: List[torch.Tensor],
|
| 301 |
+
) -> Tuple[List[float], float]:
|
| 302 |
+
"""
|
| 303 |
+
Evaluate a complete reasoning chain.
|
| 304 |
+
|
| 305 |
+
Args:
|
| 306 |
+
vision_features: Visual features [vision_dim]
|
| 307 |
+
step_embeddings: List of step embeddings
|
| 308 |
+
|
| 309 |
+
Returns:
|
| 310 |
+
- step_rewards: Reward for each step
|
| 311 |
+
- total_reward: Sum of all step rewards
|
| 312 |
+
"""
|
| 313 |
+
if not step_embeddings:
|
| 314 |
+
return [], 0.0
|
| 315 |
+
|
| 316 |
+
# Stack steps
|
| 317 |
+
steps = torch.stack(step_embeddings).unsqueeze(0) # [1, num_steps, text_dim]
|
| 318 |
+
vision_features = vision_features.unsqueeze(0) # [1, vision_dim]
|
| 319 |
+
|
| 320 |
+
# Compute rewards
|
| 321 |
+
rewards, _ = self.forward(vision_features, steps)
|
| 322 |
+
step_rewards = rewards[0].tolist()
|
| 323 |
+
total_reward = sum(step_rewards)
|
| 324 |
+
|
| 325 |
+
return step_rewards, total_reward
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
class PRMTrainer:
|
| 329 |
+
"""Trainer for Process Reward Model using step-level supervision."""
|
| 330 |
+
|
| 331 |
+
def __init__(
|
| 332 |
+
self,
|
| 333 |
+
model: ProcessRewardModel,
|
| 334 |
+
learning_rate: float = 1e-4,
|
| 335 |
+
weight_decay: float = 0.01,
|
| 336 |
+
warmup_steps: int = 1000,
|
| 337 |
+
):
|
| 338 |
+
"""
|
| 339 |
+
Initialize PRM trainer.
|
| 340 |
+
|
| 341 |
+
Args:
|
| 342 |
+
model: Process Reward Model
|
| 343 |
+
learning_rate: Learning rate
|
| 344 |
+
weight_decay: Weight decay for regularization
|
| 345 |
+
warmup_steps: Number of warmup steps
|
| 346 |
+
"""
|
| 347 |
+
self.model = model
|
| 348 |
+
self.optimizer = torch.optim.AdamW(
|
| 349 |
+
model.parameters(),
|
| 350 |
+
lr=learning_rate,
|
| 351 |
+
weight_decay=weight_decay,
|
| 352 |
+
)
|
| 353 |
+
self.warmup_steps = warmup_steps
|
| 354 |
+
self.current_step = 0
|
| 355 |
+
|
| 356 |
+
def train_step(
|
| 357 |
+
self,
|
| 358 |
+
vision_features: torch.Tensor,
|
| 359 |
+
step_embeddings: torch.Tensor,
|
| 360 |
+
target_rewards: torch.Tensor,
|
| 361 |
+
step_mask: Optional[torch.Tensor] = None,
|
| 362 |
+
) -> Dict[str, float]:
|
| 363 |
+
"""
|
| 364 |
+
Single training step.
|
| 365 |
+
|
| 366 |
+
Args:
|
| 367 |
+
vision_features: Visual features [batch, vision_dim]
|
| 368 |
+
step_embeddings: Step embeddings [batch, num_steps, text_dim]
|
| 369 |
+
target_rewards: Ground truth rewards [batch, num_steps]
|
| 370 |
+
step_mask: Valid step mask [batch, num_steps]
|
| 371 |
+
|
| 372 |
+
Returns:
|
| 373 |
+
Dictionary of losses
|
| 374 |
+
"""
|
| 375 |
+
self.model.train()
|
| 376 |
+
self.optimizer.zero_grad()
|
| 377 |
+
|
| 378 |
+
# Forward pass
|
| 379 |
+
predicted_rewards, _ = self.model(
|
| 380 |
+
vision_features,
|
| 381 |
+
step_embeddings,
|
| 382 |
+
step_mask,
|
| 383 |
+
)
|
| 384 |
+
|
| 385 |
+
# Compute loss
|
| 386 |
+
if step_mask is not None:
|
| 387 |
+
# Masked MSE loss
|
| 388 |
+
loss = F.mse_loss(
|
| 389 |
+
predicted_rewards.masked_select(step_mask),
|
| 390 |
+
target_rewards.masked_select(step_mask),
|
| 391 |
+
)
|
| 392 |
+
else:
|
| 393 |
+
loss = F.mse_loss(predicted_rewards, target_rewards)
|
| 394 |
+
|
| 395 |
+
# Backward pass
|
| 396 |
+
loss.backward()
|
| 397 |
+
torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)
|
| 398 |
+
self.optimizer.step()
|
| 399 |
+
|
| 400 |
+
# Learning rate warmup
|
| 401 |
+
if self.current_step < self.warmup_steps:
|
| 402 |
+
lr_scale = min(1.0, self.current_step / self.warmup_steps)
|
| 403 |
+
for param_group in self.optimizer.param_groups:
|
| 404 |
+
param_group['lr'] = learning_rate * lr_scale
|
| 405 |
+
|
| 406 |
+
self.current_step += 1
|
| 407 |
+
|
| 408 |
+
return {
|
| 409 |
+
'loss': loss.item(),
|
| 410 |
+
'avg_predicted_reward': predicted_rewards.mean().item(),
|
| 411 |
+
'avg_target_reward': target_rewards.mean().item(),
|
| 412 |
+
}
|
| 413 |
+
|
| 414 |
+
@torch.no_grad()
|
| 415 |
+
def evaluate(
|
| 416 |
+
self,
|
| 417 |
+
vision_features: torch.Tensor,
|
| 418 |
+
step_embeddings: torch.Tensor,
|
| 419 |
+
target_rewards: torch.Tensor,
|
| 420 |
+
step_mask: Optional[torch.Tensor] = None,
|
| 421 |
+
) -> Dict[str, float]:
|
| 422 |
+
"""
|
| 423 |
+
Evaluation step.
|
| 424 |
+
|
| 425 |
+
Returns:
|
| 426 |
+
Dictionary of evaluation metrics
|
| 427 |
+
"""
|
| 428 |
+
self.model.eval()
|
| 429 |
+
|
| 430 |
+
# Forward pass
|
| 431 |
+
predicted_rewards, _ = self.model(
|
| 432 |
+
vision_features,
|
| 433 |
+
step_embeddings,
|
| 434 |
+
step_mask,
|
| 435 |
+
)
|
| 436 |
+
|
| 437 |
+
# Compute metrics
|
| 438 |
+
if step_mask is not None:
|
| 439 |
+
valid_pred = predicted_rewards.masked_select(step_mask)
|
| 440 |
+
valid_target = target_rewards.masked_select(step_mask)
|
| 441 |
+
else:
|
| 442 |
+
valid_pred = predicted_rewards.flatten()
|
| 443 |
+
valid_target = target_rewards.flatten()
|
| 444 |
+
|
| 445 |
+
mse = F.mse_loss(valid_pred, valid_target).item()
|
| 446 |
+
mae = F.l1_loss(valid_pred, valid_target).item()
|
| 447 |
+
|
| 448 |
+
# Correlation
|
| 449 |
+
pred_mean = valid_pred.mean()
|
| 450 |
+
target_mean = valid_target.mean()
|
| 451 |
+
covariance = ((valid_pred - pred_mean) * (valid_target - target_mean)).mean()
|
| 452 |
+
pred_std = valid_pred.std()
|
| 453 |
+
target_std = valid_target.std()
|
| 454 |
+
correlation = covariance / (pred_std * target_std + 1e-8)
|
| 455 |
+
|
| 456 |
+
return {
|
| 457 |
+
'mse': mse,
|
| 458 |
+
'mae': mae,
|
| 459 |
+
'correlation': correlation.item(),
|
| 460 |
+
'avg_predicted_reward': valid_pred.mean().item(),
|
| 461 |
+
'avg_target_reward': valid_target.mean().item(),
|
| 462 |
+
}
|
src/reasoning/rl_trainer.py
ADDED
|
@@ -0,0 +1,480 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Reinforcement Learning trainer for vision-language models with step-level rewards.
|
| 3 |
+
|
| 4 |
+
Implements PPO (Proximal Policy Optimization) with fine-grained process rewards
|
| 5 |
+
from the PRM for each reasoning step.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from typing import Dict, List, Optional, Tuple, Any
|
| 9 |
+
import torch
|
| 10 |
+
import torch.nn as nn
|
| 11 |
+
import torch.nn.functional as F
|
| 12 |
+
from torch.utils.data import DataLoader
|
| 13 |
+
from dataclasses import dataclass
|
| 14 |
+
import logging
|
| 15 |
+
|
| 16 |
+
from .prm import ProcessRewardModel
|
| 17 |
+
from .step_data import ReasoningChain, StepDataset
|
| 18 |
+
|
| 19 |
+
logger = logging.getLogger(__name__)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@dataclass
|
| 23 |
+
class RLConfig:
|
| 24 |
+
"""Configuration for RL training."""
|
| 25 |
+
# PPO hyperparameters
|
| 26 |
+
clip_epsilon: float = 0.2
|
| 27 |
+
value_loss_coef: float = 0.5
|
| 28 |
+
entropy_coef: float = 0.01
|
| 29 |
+
max_grad_norm: float = 1.0
|
| 30 |
+
gamma: float = 0.99 # Discount factor
|
| 31 |
+
gae_lambda: float = 0.95 # GAE parameter
|
| 32 |
+
|
| 33 |
+
# Training settings
|
| 34 |
+
num_epochs: int = 4
|
| 35 |
+
batch_size: int = 8
|
| 36 |
+
learning_rate: float = 1e-5
|
| 37 |
+
warmup_steps: int = 500
|
| 38 |
+
|
| 39 |
+
# Step-level rewards
|
| 40 |
+
use_prm_rewards: bool = True
|
| 41 |
+
normalize_rewards: bool = True
|
| 42 |
+
reward_scale: float = 1.0
|
| 43 |
+
|
| 44 |
+
# Exploration
|
| 45 |
+
temperature: float = 1.0
|
| 46 |
+
top_k: int = 50
|
| 47 |
+
top_p: float = 0.95
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
class RLReasoningTrainer:
|
| 51 |
+
"""
|
| 52 |
+
Reinforcement Learning trainer for step-level reasoning.
|
| 53 |
+
|
| 54 |
+
Uses PPO to optimize the vision-language model with fine-grained rewards
|
| 55 |
+
from the Process Reward Model (PRM) at each reasoning step.
|
| 56 |
+
"""
|
| 57 |
+
|
| 58 |
+
def __init__(
|
| 59 |
+
self,
|
| 60 |
+
policy_model: nn.Module,
|
| 61 |
+
prm_model: ProcessRewardModel,
|
| 62 |
+
config: RLConfig,
|
| 63 |
+
device: str = "cuda",
|
| 64 |
+
):
|
| 65 |
+
"""
|
| 66 |
+
Initialize RL trainer.
|
| 67 |
+
|
| 68 |
+
Args:
|
| 69 |
+
policy_model: Vision-language model to train
|
| 70 |
+
prm_model: Process Reward Model for step evaluation
|
| 71 |
+
config: RL training configuration
|
| 72 |
+
device: Device for training
|
| 73 |
+
"""
|
| 74 |
+
self.policy = policy_model
|
| 75 |
+
self.prm = prm_model
|
| 76 |
+
self.config = config
|
| 77 |
+
self.device = device
|
| 78 |
+
|
| 79 |
+
# Move models to device
|
| 80 |
+
self.policy.to(device)
|
| 81 |
+
self.prm.to(device)
|
| 82 |
+
|
| 83 |
+
# Freeze PRM (only train policy)
|
| 84 |
+
for param in self.prm.parameters():
|
| 85 |
+
param.requires_grad = False
|
| 86 |
+
self.prm.eval()
|
| 87 |
+
|
| 88 |
+
# Optimizer
|
| 89 |
+
self.optimizer = torch.optim.AdamW(
|
| 90 |
+
self.policy.parameters(),
|
| 91 |
+
lr=config.learning_rate,
|
| 92 |
+
betas=(0.9, 0.999),
|
| 93 |
+
eps=1e-8,
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
# Value head for PPO (estimates state value)
|
| 97 |
+
self.value_head = nn.Sequential(
|
| 98 |
+
nn.Linear(768, 512),
|
| 99 |
+
nn.ReLU(),
|
| 100 |
+
nn.Dropout(0.1),
|
| 101 |
+
nn.Linear(512, 1),
|
| 102 |
+
).to(device)
|
| 103 |
+
|
| 104 |
+
self.value_optimizer = torch.optim.AdamW(
|
| 105 |
+
self.value_head.parameters(),
|
| 106 |
+
lr=config.learning_rate * 3,
|
| 107 |
+
)
|
| 108 |
+
|
| 109 |
+
self.global_step = 0
|
| 110 |
+
|
| 111 |
+
def generate_reasoning_chain(
|
| 112 |
+
self,
|
| 113 |
+
image_features: torch.Tensor,
|
| 114 |
+
prompt_ids: torch.Tensor,
|
| 115 |
+
prompt_mask: torch.Tensor,
|
| 116 |
+
max_steps: int = 10,
|
| 117 |
+
) -> Tuple[List[torch.Tensor], List[torch.Tensor], List[float]]:
|
| 118 |
+
"""
|
| 119 |
+
Generate a reasoning chain using the current policy.
|
| 120 |
+
|
| 121 |
+
Args:
|
| 122 |
+
image_features: Visual features [vision_dim]
|
| 123 |
+
prompt_ids: Tokenized prompt [seq_len]
|
| 124 |
+
prompt_mask: Prompt attention mask [seq_len]
|
| 125 |
+
max_steps: Maximum reasoning steps
|
| 126 |
+
|
| 127 |
+
Returns:
|
| 128 |
+
- step_embeddings: List of step embeddings
|
| 129 |
+
- step_logprobs: List of log probabilities for each step
|
| 130 |
+
- step_entropies: List of entropies for each step
|
| 131 |
+
"""
|
| 132 |
+
self.policy.eval()
|
| 133 |
+
|
| 134 |
+
step_embeddings = []
|
| 135 |
+
step_logprobs = []
|
| 136 |
+
step_entropies = []
|
| 137 |
+
|
| 138 |
+
# Initialize with prompt
|
| 139 |
+
current_ids = prompt_ids.unsqueeze(0).to(self.device)
|
| 140 |
+
current_mask = prompt_mask.unsqueeze(0).to(self.device)
|
| 141 |
+
|
| 142 |
+
for step_idx in range(max_steps):
|
| 143 |
+
with torch.no_grad():
|
| 144 |
+
# Forward pass through policy
|
| 145 |
+
outputs = self.policy(
|
| 146 |
+
input_ids=current_ids,
|
| 147 |
+
attention_mask=current_mask,
|
| 148 |
+
output_hidden_states=True,
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
logits = outputs.logits[:, -1, :] # Next token logits
|
| 152 |
+
hidden_state = outputs.hidden_states[-1][:, -1, :] # Last hidden state
|
| 153 |
+
|
| 154 |
+
# Apply temperature and sampling
|
| 155 |
+
logits = logits / self.config.temperature
|
| 156 |
+
probs = F.softmax(logits, dim=-1)
|
| 157 |
+
|
| 158 |
+
# Top-k and top-p filtering
|
| 159 |
+
if self.config.top_k > 0:
|
| 160 |
+
top_k_probs, top_k_indices = torch.topk(probs, self.config.top_k, dim=-1)
|
| 161 |
+
probs = torch.zeros_like(probs).scatter_(1, top_k_indices, top_k_probs)
|
| 162 |
+
probs = probs / probs.sum(dim=-1, keepdim=True)
|
| 163 |
+
|
| 164 |
+
if self.config.top_p < 1.0:
|
| 165 |
+
sorted_probs, sorted_indices = torch.sort(probs, descending=True, dim=-1)
|
| 166 |
+
cumulative_probs = torch.cumsum(sorted_probs, dim=-1)
|
| 167 |
+
|
| 168 |
+
# Remove tokens with cumulative probability above threshold
|
| 169 |
+
sorted_indices_to_remove = cumulative_probs > self.config.top_p
|
| 170 |
+
sorted_indices_to_remove[:, 1:] = sorted_indices_to_remove[:, :-1].clone()
|
| 171 |
+
sorted_indices_to_remove[:, 0] = False
|
| 172 |
+
|
| 173 |
+
indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
|
| 174 |
+
probs = probs.masked_fill(indices_to_remove, 0.0)
|
| 175 |
+
probs = probs / probs.sum(dim=-1, keepdim=True)
|
| 176 |
+
|
| 177 |
+
# Sample next token
|
| 178 |
+
next_token = torch.multinomial(probs, num_samples=1)
|
| 179 |
+
|
| 180 |
+
# Calculate log probability and entropy
|
| 181 |
+
log_prob = torch.log(probs.gather(1, next_token) + 1e-10)
|
| 182 |
+
entropy = -(probs * torch.log(probs + 1e-10)).sum(dim=-1)
|
| 183 |
+
|
| 184 |
+
# Store step information
|
| 185 |
+
step_embeddings.append(hidden_state.squeeze(0))
|
| 186 |
+
step_logprobs.append(log_prob.squeeze())
|
| 187 |
+
step_entropies.append(entropy.item())
|
| 188 |
+
|
| 189 |
+
# Check for stop token or step completion marker
|
| 190 |
+
if next_token.item() in [self.policy.config.eos_token_id, 198]: # EOS or newline
|
| 191 |
+
break
|
| 192 |
+
|
| 193 |
+
# Update sequence
|
| 194 |
+
current_ids = torch.cat([current_ids, next_token], dim=1)
|
| 195 |
+
current_mask = torch.cat([
|
| 196 |
+
current_mask,
|
| 197 |
+
torch.ones((1, 1), device=self.device)
|
| 198 |
+
], dim=1)
|
| 199 |
+
|
| 200 |
+
return step_embeddings, step_logprobs, step_entropies
|
| 201 |
+
|
| 202 |
+
def compute_advantages(
|
| 203 |
+
self,
|
| 204 |
+
rewards: torch.Tensor,
|
| 205 |
+
values: torch.Tensor,
|
| 206 |
+
dones: torch.Tensor,
|
| 207 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 208 |
+
"""
|
| 209 |
+
Compute Generalized Advantage Estimation (GAE).
|
| 210 |
+
|
| 211 |
+
Args:
|
| 212 |
+
rewards: Step rewards [num_steps]
|
| 213 |
+
values: Value estimates [num_steps]
|
| 214 |
+
dones: Done flags [num_steps]
|
| 215 |
+
|
| 216 |
+
Returns:
|
| 217 |
+
- advantages: Advantage estimates [num_steps]
|
| 218 |
+
- returns: Discounted returns [num_steps]
|
| 219 |
+
"""
|
| 220 |
+
num_steps = len(rewards)
|
| 221 |
+
advantages = torch.zeros_like(rewards)
|
| 222 |
+
returns = torch.zeros_like(rewards)
|
| 223 |
+
|
| 224 |
+
gae = 0
|
| 225 |
+
next_value = 0
|
| 226 |
+
|
| 227 |
+
for t in reversed(range(num_steps)):
|
| 228 |
+
if t == num_steps - 1:
|
| 229 |
+
next_non_terminal = 1.0 - dones[t]
|
| 230 |
+
next_value = 0
|
| 231 |
+
else:
|
| 232 |
+
next_non_terminal = 1.0 - dones[t]
|
| 233 |
+
next_value = values[t + 1]
|
| 234 |
+
|
| 235 |
+
delta = rewards[t] + self.config.gamma * next_value * next_non_terminal - values[t]
|
| 236 |
+
gae = delta + self.config.gamma * self.config.gae_lambda * next_non_terminal * gae
|
| 237 |
+
|
| 238 |
+
advantages[t] = gae
|
| 239 |
+
returns[t] = gae + values[t]
|
| 240 |
+
|
| 241 |
+
return advantages, returns
|
| 242 |
+
|
| 243 |
+
def ppo_update(
|
| 244 |
+
self,
|
| 245 |
+
step_embeddings: List[torch.Tensor],
|
| 246 |
+
old_logprobs: torch.Tensor,
|
| 247 |
+
rewards: torch.Tensor,
|
| 248 |
+
advantages: torch.Tensor,
|
| 249 |
+
returns: torch.Tensor,
|
| 250 |
+
) -> Dict[str, float]:
|
| 251 |
+
"""
|
| 252 |
+
Perform PPO update.
|
| 253 |
+
|
| 254 |
+
Args:
|
| 255 |
+
step_embeddings: Hidden states for each step
|
| 256 |
+
old_logprobs: Old policy log probabilities
|
| 257 |
+
rewards: Step rewards
|
| 258 |
+
advantages: Advantage estimates
|
| 259 |
+
returns: Discounted returns
|
| 260 |
+
|
| 261 |
+
Returns:
|
| 262 |
+
Dictionary of training metrics
|
| 263 |
+
"""
|
| 264 |
+
self.policy.train()
|
| 265 |
+
self.value_head.train()
|
| 266 |
+
|
| 267 |
+
# Stack embeddings
|
| 268 |
+
embeddings = torch.stack(step_embeddings) # [num_steps, hidden_dim]
|
| 269 |
+
|
| 270 |
+
# Normalize advantages
|
| 271 |
+
if self.config.normalize_rewards:
|
| 272 |
+
advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)
|
| 273 |
+
|
| 274 |
+
total_policy_loss = 0
|
| 275 |
+
total_value_loss = 0
|
| 276 |
+
total_entropy_loss = 0
|
| 277 |
+
|
| 278 |
+
for epoch in range(self.config.num_epochs):
|
| 279 |
+
# Compute current policy log probabilities
|
| 280 |
+
# (In practice, you'd recompute through model forward pass)
|
| 281 |
+
# Here we use embeddings for simplification
|
| 282 |
+
|
| 283 |
+
# Compute values
|
| 284 |
+
values = self.value_head(embeddings).squeeze(-1)
|
| 285 |
+
|
| 286 |
+
# Value loss
|
| 287 |
+
value_loss = F.mse_loss(values, returns)
|
| 288 |
+
|
| 289 |
+
# For policy loss, we need to recompute logprobs
|
| 290 |
+
# This is simplified - in practice you'd do full forward pass
|
| 291 |
+
ratio = torch.exp(old_logprobs - old_logprobs) # Would be new/old
|
| 292 |
+
|
| 293 |
+
# PPO clipped objective
|
| 294 |
+
policy_loss_1 = -advantages * ratio
|
| 295 |
+
policy_loss_2 = -advantages * torch.clamp(
|
| 296 |
+
ratio,
|
| 297 |
+
1 - self.config.clip_epsilon,
|
| 298 |
+
1 + self.config.clip_epsilon
|
| 299 |
+
)
|
| 300 |
+
policy_loss = torch.max(policy_loss_1, policy_loss_2).mean()
|
| 301 |
+
|
| 302 |
+
# Entropy bonus for exploration
|
| 303 |
+
entropy_loss = -self.config.entropy_coef * old_logprobs.mean()
|
| 304 |
+
|
| 305 |
+
# Total loss
|
| 306 |
+
loss = (
|
| 307 |
+
policy_loss +
|
| 308 |
+
self.config.value_loss_coef * value_loss +
|
| 309 |
+
entropy_loss
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
# Optimization step
|
| 313 |
+
self.optimizer.zero_grad()
|
| 314 |
+
self.value_optimizer.zero_grad()
|
| 315 |
+
loss.backward()
|
| 316 |
+
|
| 317 |
+
torch.nn.utils.clip_grad_norm_(
|
| 318 |
+
self.policy.parameters(),
|
| 319 |
+
self.config.max_grad_norm
|
| 320 |
+
)
|
| 321 |
+
torch.nn.utils.clip_grad_norm_(
|
| 322 |
+
self.value_head.parameters(),
|
| 323 |
+
self.config.max_grad_norm
|
| 324 |
+
)
|
| 325 |
+
|
| 326 |
+
self.optimizer.step()
|
| 327 |
+
self.value_optimizer.step()
|
| 328 |
+
|
| 329 |
+
total_policy_loss += policy_loss.item()
|
| 330 |
+
total_value_loss += value_loss.item()
|
| 331 |
+
total_entropy_loss += entropy_loss.item()
|
| 332 |
+
|
| 333 |
+
self.global_step += 1
|
| 334 |
+
|
| 335 |
+
return {
|
| 336 |
+
'policy_loss': total_policy_loss / self.config.num_epochs,
|
| 337 |
+
'value_loss': total_value_loss / self.config.num_epochs,
|
| 338 |
+
'entropy_loss': total_entropy_loss / self.config.num_epochs,
|
| 339 |
+
'avg_reward': rewards.mean().item(),
|
| 340 |
+
'avg_advantage': advantages.mean().item(),
|
| 341 |
+
'avg_return': returns.mean().item(),
|
| 342 |
+
}
|
| 343 |
+
|
| 344 |
+
def train_step(
|
| 345 |
+
self,
|
| 346 |
+
batch: Dict[str, torch.Tensor],
|
| 347 |
+
) -> Dict[str, float]:
|
| 348 |
+
"""
|
| 349 |
+
Single RL training step.
|
| 350 |
+
|
| 351 |
+
Args:
|
| 352 |
+
batch: Batch from StepDataset
|
| 353 |
+
|
| 354 |
+
Returns:
|
| 355 |
+
Training metrics
|
| 356 |
+
"""
|
| 357 |
+
# Extract batch data
|
| 358 |
+
image_features = batch['visual_features'].to(self.device) # [batch, vision_dim]
|
| 359 |
+
prompt_ids = batch['prompt_ids'].to(self.device)
|
| 360 |
+
prompt_mask = batch['prompt_mask'].to(self.device)
|
| 361 |
+
|
| 362 |
+
batch_size = image_features.size(0)
|
| 363 |
+
all_metrics = []
|
| 364 |
+
|
| 365 |
+
for b in range(batch_size):
|
| 366 |
+
# Generate reasoning chain with current policy
|
| 367 |
+
step_embeddings, step_logprobs, step_entropies = self.generate_reasoning_chain(
|
| 368 |
+
image_features[b],
|
| 369 |
+
prompt_ids[b],
|
| 370 |
+
prompt_mask[b],
|
| 371 |
+
max_steps=10,
|
| 372 |
+
)
|
| 373 |
+
|
| 374 |
+
if not step_embeddings:
|
| 375 |
+
continue
|
| 376 |
+
|
| 377 |
+
# Get PRM rewards for each step
|
| 378 |
+
step_rewards = []
|
| 379 |
+
for step_emb in step_embeddings:
|
| 380 |
+
reward, _ = self.prm.compute_step_reward(
|
| 381 |
+
image_features[b],
|
| 382 |
+
step_emb,
|
| 383 |
+
previous_steps=torch.stack(step_embeddings[:len(step_rewards)]) if step_rewards else None,
|
| 384 |
+
)
|
| 385 |
+
step_rewards.append(reward * self.config.reward_scale)
|
| 386 |
+
|
| 387 |
+
rewards = torch.tensor(step_rewards, device=self.device)
|
| 388 |
+
logprobs = torch.stack(step_logprobs)
|
| 389 |
+
|
| 390 |
+
# Compute value estimates
|
| 391 |
+
embeddings = torch.stack(step_embeddings)
|
| 392 |
+
with torch.no_grad():
|
| 393 |
+
values = self.value_head(embeddings).squeeze(-1)
|
| 394 |
+
|
| 395 |
+
# Done flags (only last step is done)
|
| 396 |
+
dones = torch.zeros(len(step_embeddings), device=self.device)
|
| 397 |
+
dones[-1] = 1.0
|
| 398 |
+
|
| 399 |
+
# Compute advantages and returns
|
| 400 |
+
advantages, returns = self.compute_advantages(rewards, values, dones)
|
| 401 |
+
|
| 402 |
+
# PPO update
|
| 403 |
+
metrics = self.ppo_update(
|
| 404 |
+
step_embeddings,
|
| 405 |
+
logprobs,
|
| 406 |
+
rewards,
|
| 407 |
+
advantages,
|
| 408 |
+
returns,
|
| 409 |
+
)
|
| 410 |
+
all_metrics.append(metrics)
|
| 411 |
+
|
| 412 |
+
# Average metrics across batch
|
| 413 |
+
if not all_metrics:
|
| 414 |
+
return {}
|
| 415 |
+
|
| 416 |
+
avg_metrics = {
|
| 417 |
+
key: sum(m[key] for m in all_metrics) / len(all_metrics)
|
| 418 |
+
for key in all_metrics[0].keys()
|
| 419 |
+
}
|
| 420 |
+
|
| 421 |
+
return avg_metrics
|
| 422 |
+
|
| 423 |
+
def train(
|
| 424 |
+
self,
|
| 425 |
+
train_dataset: StepDataset,
|
| 426 |
+
num_iterations: int = 1000,
|
| 427 |
+
log_interval: int = 10,
|
| 428 |
+
) -> None:
|
| 429 |
+
"""
|
| 430 |
+
Train the policy with RL.
|
| 431 |
+
|
| 432 |
+
Args:
|
| 433 |
+
train_dataset: Dataset with reasoning chains
|
| 434 |
+
num_iterations: Number of training iterations
|
| 435 |
+
log_interval: Logging frequency
|
| 436 |
+
"""
|
| 437 |
+
dataloader = DataLoader(
|
| 438 |
+
train_dataset,
|
| 439 |
+
batch_size=self.config.batch_size,
|
| 440 |
+
shuffle=True,
|
| 441 |
+
)
|
| 442 |
+
|
| 443 |
+
logger.info(f"Starting RL training for {num_iterations} iterations")
|
| 444 |
+
|
| 445 |
+
for iteration in range(num_iterations):
|
| 446 |
+
for batch in dataloader:
|
| 447 |
+
metrics = self.train_step(batch)
|
| 448 |
+
|
| 449 |
+
if metrics and iteration % log_interval == 0:
|
| 450 |
+
logger.info(
|
| 451 |
+
f"Iteration {iteration}: "
|
| 452 |
+
f"Policy Loss: {metrics['policy_loss']:.4f}, "
|
| 453 |
+
f"Value Loss: {metrics['value_loss']:.4f}, "
|
| 454 |
+
f"Avg Reward: {metrics['avg_reward']:.4f}"
|
| 455 |
+
)
|
| 456 |
+
|
| 457 |
+
logger.info("RL training completed")
|
| 458 |
+
|
| 459 |
+
def save_checkpoint(self, path: str) -> None:
|
| 460 |
+
"""Save training checkpoint."""
|
| 461 |
+
checkpoint = {
|
| 462 |
+
'policy_state_dict': self.policy.state_dict(),
|
| 463 |
+
'value_head_state_dict': self.value_head.state_dict(),
|
| 464 |
+
'optimizer_state_dict': self.optimizer.state_dict(),
|
| 465 |
+
'value_optimizer_state_dict': self.value_optimizer.state_dict(),
|
| 466 |
+
'global_step': self.global_step,
|
| 467 |
+
'config': self.config,
|
| 468 |
+
}
|
| 469 |
+
torch.save(checkpoint, path)
|
| 470 |
+
logger.info(f"Checkpoint saved to {path}")
|
| 471 |
+
|
| 472 |
+
def load_checkpoint(self, path: str) -> None:
|
| 473 |
+
"""Load training checkpoint."""
|
| 474 |
+
checkpoint = torch.load(path, map_location=self.device)
|
| 475 |
+
self.policy.load_state_dict(checkpoint['policy_state_dict'])
|
| 476 |
+
self.value_head.load_state_dict(checkpoint['value_head_state_dict'])
|
| 477 |
+
self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
|
| 478 |
+
self.value_optimizer.load_state_dict(checkpoint['value_optimizer_state_dict'])
|
| 479 |
+
self.global_step = checkpoint['global_step']
|
| 480 |
+
logger.info(f"Checkpoint loaded from {path}")
|
src/reasoning/step_data.py
ADDED
|
@@ -0,0 +1,448 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Data structures for step-level reasoning chains with vision-language annotations.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from dataclasses import dataclass, field
|
| 6 |
+
from typing import Dict, List, Optional, Any
|
| 7 |
+
from enum import Enum
|
| 8 |
+
import torch
|
| 9 |
+
from torch.utils.data import Dataset
|
| 10 |
+
import json
|
| 11 |
+
|
| 12 |
+
# Special tokens for chain-of-thought reasoning
|
| 13 |
+
SPECIAL_TOKENS = [
|
| 14 |
+
# Reasoning structure
|
| 15 |
+
"<|reasoning_start|>", "<|reasoning_end|>",
|
| 16 |
+
|
| 17 |
+
# Step-level markers
|
| 18 |
+
"<|step_start|>", "<|step_end|>",
|
| 19 |
+
"<|step_type|>",
|
| 20 |
+
|
| 21 |
+
# Step components
|
| 22 |
+
"<|description_start|>", "<|description_end|>",
|
| 23 |
+
"<|confidence_start|>", "<|confidence_end|>",
|
| 24 |
+
|
| 25 |
+
# Dependencies
|
| 26 |
+
"<|depends_on|>",
|
| 27 |
+
|
| 28 |
+
# PRM evaluation tokens (from PRM800K)
|
| 29 |
+
"ΠΊΠΈ", # step separator for PRM scoring
|
| 30 |
+
"ΠΊ+ΠΈ", # positive step label
|
| 31 |
+
"ΠΊ-ΠΈ", # negative step label
|
| 32 |
+
|
| 33 |
+
# Final answer
|
| 34 |
+
"<|answer_start|>", "<|answer_end|>",
|
| 35 |
+
|
| 36 |
+
# Region tokens (inspired by Chain-of-Sight for visual grounding)
|
| 37 |
+
"<|region|>", "<|bbox|>", # Region markers and bounding box coordinates
|
| 38 |
+
]
|
| 39 |
+
|
| 40 |
+
SPECIAL_TOKEN_DESCRIPTIONS = [
|
| 41 |
+
"start of reasoning", "end of reasoning",
|
| 42 |
+
"start of step", "end of step",
|
| 43 |
+
"step type indicator",
|
| 44 |
+
"start of description", "end of description",
|
| 45 |
+
"start of confidence", "end of confidence",
|
| 46 |
+
"depends on previous step",
|
| 47 |
+
"step separator for PRM", "positive step", "negative step",
|
| 48 |
+
"start of answer", "end of answer",
|
| 49 |
+
"region reference", "bounding box coordinates",
|
| 50 |
+
]
|
| 51 |
+
|
| 52 |
+
# Generate additional region tokens like <region1>, <region2>, ... <region10>
|
| 53 |
+
# for referring to specific visual regions (Chain-of-Sight style)
|
| 54 |
+
REGION_TOKENS = [f"<region{i}>" for i in range(1, 11)]
|
| 55 |
+
SPECIAL_TOKENS.extend(REGION_TOKENS)
|
| 56 |
+
SPECIAL_TOKEN_DESCRIPTIONS.extend([f"visual region {i}" for i in range(1, 11)])
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
class StepType(Enum):
|
| 60 |
+
"""Types of reasoning steps in vision-language tasks."""
|
| 61 |
+
PERCEPTION = "perception" # Visual understanding
|
| 62 |
+
LOCALIZATION = "localization" # Spatial reasoning
|
| 63 |
+
COMPARISON = "comparison" # Comparing visual elements
|
| 64 |
+
COUNTING = "counting" # Numerical reasoning
|
| 65 |
+
INFERENCE = "inference" # Logical deduction
|
| 66 |
+
COMPOSITION = "composition" # Combining information
|
| 67 |
+
VERIFICATION = "verification" # Checking consistency
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
@dataclass
|
| 71 |
+
class ReasoningStep:
|
| 72 |
+
"""
|
| 73 |
+
A single step in the chain of thought reasoning process.
|
| 74 |
+
|
| 75 |
+
Attributes:
|
| 76 |
+
step_id: Unique identifier for this step
|
| 77 |
+
step_type: Type of reasoning performed
|
| 78 |
+
description: Natural language description of the reasoning
|
| 79 |
+
visual_features: Extracted visual features relevant to this step
|
| 80 |
+
attention_map: Spatial attention weights (H, W)
|
| 81 |
+
confidence: Model's confidence in this step [0, 1]
|
| 82 |
+
reward: Process reward score assigned by PRM [-1, 1]
|
| 83 |
+
dependencies: IDs of previous steps this depends on
|
| 84 |
+
bounding_boxes: List of [x1, y1, x2, y2] bounding boxes (Chain-of-Sight style)
|
| 85 |
+
region_ids: List of region IDs referenced in this step (e.g., [1, 2])
|
| 86 |
+
metadata: Additional step-specific information
|
| 87 |
+
"""
|
| 88 |
+
step_id: int
|
| 89 |
+
step_type: StepType
|
| 90 |
+
description: str
|
| 91 |
+
visual_features: Optional[torch.Tensor] = None
|
| 92 |
+
attention_map: Optional[torch.Tensor] = None
|
| 93 |
+
confidence: float = 0.0
|
| 94 |
+
reward: float = 0.0
|
| 95 |
+
dependencies: List[int] = field(default_factory=list)
|
| 96 |
+
bounding_boxes: List[List[float]] = field(default_factory=list) # [[x1,y1,x2,y2], ...]
|
| 97 |
+
region_ids: List[int] = field(default_factory=list) # [1, 2, 3]
|
| 98 |
+
metadata: Dict[str, Any] = field(default_factory=dict)
|
| 99 |
+
|
| 100 |
+
def to_dict(self) -> Dict[str, Any]:
|
| 101 |
+
"""Convert to dictionary for serialization."""
|
| 102 |
+
return {
|
| 103 |
+
'step_id': self.step_id,
|
| 104 |
+
'step_type': self.step_type.value,
|
| 105 |
+
'description': self.description,
|
| 106 |
+
'confidence': self.confidence,
|
| 107 |
+
'reward': self.reward,
|
| 108 |
+
'dependencies': self.dependencies,
|
| 109 |
+
'metadata': self.metadata,
|
| 110 |
+
'has_visual_features': self.visual_features is not None,
|
| 111 |
+
'has_attention_map': self.attention_map is not None,
|
| 112 |
+
}
|
| 113 |
+
|
| 114 |
+
def format_with_tokens(self) -> str:
|
| 115 |
+
"""Format step with special tokens for training (with Chain-of-Sight region support)."""
|
| 116 |
+
parts = ["<|step_start|>"]
|
| 117 |
+
|
| 118 |
+
# Add step type
|
| 119 |
+
parts.append(f"<|step_type|>{self.step_type.value}")
|
| 120 |
+
|
| 121 |
+
# Add dependencies if any
|
| 122 |
+
if self.dependencies:
|
| 123 |
+
deps_str = ",".join(map(str, self.dependencies))
|
| 124 |
+
parts.append(f"<|depends_on|>{deps_str}")
|
| 125 |
+
|
| 126 |
+
# Add region references (Chain-of-Sight style)
|
| 127 |
+
if self.region_ids:
|
| 128 |
+
region_refs = " ".join([f"<region{rid}>" for rid in self.region_ids])
|
| 129 |
+
parts.append(f"<|region|>{region_refs}")
|
| 130 |
+
|
| 131 |
+
# Add bounding boxes if present
|
| 132 |
+
if self.bounding_boxes:
|
| 133 |
+
bbox_strs = [f"[{b[0]:.3f},{b[1]:.3f},{b[2]:.3f},{b[3]:.3f}]" for b in self.bounding_boxes]
|
| 134 |
+
parts.append(f"<|bbox|>{','.join(bbox_strs)}")
|
| 135 |
+
|
| 136 |
+
# Add description
|
| 137 |
+
parts.append(f"<|description_start|>{self.description}<|description_end|>")
|
| 138 |
+
|
| 139 |
+
# Add confidence
|
| 140 |
+
parts.append(f"<|confidence_start|>{self.confidence:.3f}<|confidence_end|>")
|
| 141 |
+
|
| 142 |
+
# Add PRM separator
|
| 143 |
+
parts.append("ΠΊΠΈ")
|
| 144 |
+
|
| 145 |
+
parts.append("<|step_end|>")
|
| 146 |
+
|
| 147 |
+
return " ".join(parts)
|
| 148 |
+
|
| 149 |
+
@classmethod
|
| 150 |
+
def from_dict(cls, data: Dict[str, Any]) -> 'ReasoningStep':
|
| 151 |
+
"""Create from dictionary."""
|
| 152 |
+
return cls(
|
| 153 |
+
step_id=data['step_id'],
|
| 154 |
+
step_type=StepType(data['step_type']),
|
| 155 |
+
description=data['description'],
|
| 156 |
+
confidence=data.get('confidence', 0.0),
|
| 157 |
+
reward=data.get('reward', 0.0),
|
| 158 |
+
dependencies=data.get('dependencies', []),
|
| 159 |
+
metadata=data.get('metadata', {}),
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
@dataclass
|
| 164 |
+
class ReasoningChain:
|
| 165 |
+
"""
|
| 166 |
+
Complete chain of reasoning steps for a vision-language task.
|
| 167 |
+
|
| 168 |
+
Attributes:
|
| 169 |
+
chain_id: Unique identifier for this reasoning chain
|
| 170 |
+
image_path: Path to the input image
|
| 171 |
+
prompt: Input text prompt/question
|
| 172 |
+
steps: Ordered list of reasoning steps
|
| 173 |
+
final_answer: Final output/answer
|
| 174 |
+
total_reward: Cumulative reward from PRM
|
| 175 |
+
is_correct: Whether final answer is correct
|
| 176 |
+
metadata: Additional chain-level information
|
| 177 |
+
"""
|
| 178 |
+
chain_id: str
|
| 179 |
+
image_path: str
|
| 180 |
+
prompt: str
|
| 181 |
+
steps: List[ReasoningStep]
|
| 182 |
+
final_answer: str
|
| 183 |
+
total_reward: float = 0.0
|
| 184 |
+
is_correct: bool = False
|
| 185 |
+
metadata: Dict[str, Any] = field(default_factory=dict)
|
| 186 |
+
|
| 187 |
+
def __len__(self) -> int:
|
| 188 |
+
"""Number of reasoning steps."""
|
| 189 |
+
return len(self.steps)
|
| 190 |
+
|
| 191 |
+
def get_step_rewards(self) -> List[float]:
|
| 192 |
+
"""Extract reward for each step."""
|
| 193 |
+
return [step.reward for step in self.steps]
|
| 194 |
+
|
| 195 |
+
def get_cumulative_rewards(self) -> List[float]:
|
| 196 |
+
"""Calculate cumulative rewards up to each step."""
|
| 197 |
+
cumulative = []
|
| 198 |
+
total = 0.0
|
| 199 |
+
for step in self.steps:
|
| 200 |
+
total += step.reward
|
| 201 |
+
cumulative.append(total)
|
| 202 |
+
return cumulative
|
| 203 |
+
|
| 204 |
+
def to_dict(self) -> Dict[str, Any]:
|
| 205 |
+
"""Convert to dictionary for serialization."""
|
| 206 |
+
return {
|
| 207 |
+
'chain_id': self.chain_id,
|
| 208 |
+
'image_path': self.image_path,
|
| 209 |
+
'prompt': self.prompt,
|
| 210 |
+
'steps': [step.to_dict() for step in self.steps],
|
| 211 |
+
'final_answer': self.final_answer,
|
| 212 |
+
'total_reward': self.total_reward,
|
| 213 |
+
'is_correct': self.is_correct,
|
| 214 |
+
'metadata': self.metadata
|
| 215 |
+
}
|
| 216 |
+
|
| 217 |
+
def format_with_tokens(self) -> str:
|
| 218 |
+
"""Format entire reasoning chain with special tokens for training."""
|
| 219 |
+
parts = ["<|reasoning_start|>"]
|
| 220 |
+
|
| 221 |
+
# Add all steps
|
| 222 |
+
for step in self.steps:
|
| 223 |
+
parts.append(step.format_with_tokens())
|
| 224 |
+
|
| 225 |
+
parts.append("<|reasoning_end|>")
|
| 226 |
+
|
| 227 |
+
# Add final answer
|
| 228 |
+
parts.append(f"<|answer_start|>{self.final_answer}<|answer_end|>")
|
| 229 |
+
|
| 230 |
+
return " ".join(parts)
|
| 231 |
+
|
| 232 |
+
@classmethod
|
| 233 |
+
def from_dict(cls, data: Dict[str, Any]) -> 'ReasoningChain':
|
| 234 |
+
"""Create from dictionary."""
|
| 235 |
+
return cls(
|
| 236 |
+
chain_id=data['chain_id'],
|
| 237 |
+
image_path=data['image_path'],
|
| 238 |
+
prompt=data['prompt'],
|
| 239 |
+
steps=[ReasoningStep.from_dict(s) for s in data['steps']],
|
| 240 |
+
final_answer=data['final_answer'],
|
| 241 |
+
total_reward=data.get('total_reward', 0.0),
|
| 242 |
+
is_correct=data.get('is_correct', False),
|
| 243 |
+
metadata=data.get('metadata', {}),
|
| 244 |
+
)
|
| 245 |
+
|
| 246 |
+
def save_json(self, path: str) -> None:
|
| 247 |
+
"""Save reasoning chain to JSON file."""
|
| 248 |
+
with open(path, 'w', encoding='utf-8') as f:
|
| 249 |
+
json.dump(self.to_dict(), f, indent=2, ensure_ascii=False)
|
| 250 |
+
|
| 251 |
+
@classmethod
|
| 252 |
+
def load_json(cls, path: str) -> 'ReasoningChain':
|
| 253 |
+
"""Load reasoning chain from JSON file."""
|
| 254 |
+
with open(path, 'r', encoding='utf-8') as f:
|
| 255 |
+
data = json.load(f)
|
| 256 |
+
return cls.from_dict(data)
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
class StepDataset(Dataset):
|
| 260 |
+
"""
|
| 261 |
+
Dataset for training with step-level reasoning chains.
|
| 262 |
+
|
| 263 |
+
Supports both supervised fine-tuning and RL training with process rewards.
|
| 264 |
+
"""
|
| 265 |
+
|
| 266 |
+
def __init__(
|
| 267 |
+
self,
|
| 268 |
+
reasoning_chains: List[ReasoningChain],
|
| 269 |
+
tokenizer: Any,
|
| 270 |
+
max_steps: int = 10,
|
| 271 |
+
include_visual_features: bool = True,
|
| 272 |
+
filter_incorrect: bool = False,
|
| 273 |
+
):
|
| 274 |
+
"""
|
| 275 |
+
Initialize step dataset.
|
| 276 |
+
|
| 277 |
+
Args:
|
| 278 |
+
reasoning_chains: List of annotated reasoning chains
|
| 279 |
+
tokenizer: Tokenizer for text encoding
|
| 280 |
+
max_steps: Maximum number of steps per chain
|
| 281 |
+
include_visual_features: Whether to include visual features
|
| 282 |
+
filter_incorrect: If True, only include correct chains
|
| 283 |
+
"""
|
| 284 |
+
self.tokenizer = tokenizer
|
| 285 |
+
self.max_steps = max_steps
|
| 286 |
+
self.include_visual_features = include_visual_features
|
| 287 |
+
|
| 288 |
+
# Filter chains if needed
|
| 289 |
+
if filter_incorrect:
|
| 290 |
+
self.chains = [c for c in reasoning_chains if c.is_correct]
|
| 291 |
+
else:
|
| 292 |
+
self.chains = reasoning_chains
|
| 293 |
+
|
| 294 |
+
# Pre-compute statistics
|
| 295 |
+
self.avg_steps = sum(len(c) for c in self.chains) / len(self.chains) if self.chains else 0
|
| 296 |
+
self.total_steps = sum(len(c) for c in self.chains)
|
| 297 |
+
|
| 298 |
+
def __len__(self) -> int:
|
| 299 |
+
return len(self.chains)
|
| 300 |
+
|
| 301 |
+
def __getitem__(self, idx: int) -> Dict[str, Any]:
|
| 302 |
+
"""
|
| 303 |
+
Get a single reasoning chain with all steps.
|
| 304 |
+
|
| 305 |
+
Returns:
|
| 306 |
+
Dictionary containing:
|
| 307 |
+
- prompt_ids: Tokenized prompt
|
| 308 |
+
- formatted_input_ids: Tokenized chain with special tokens
|
| 309 |
+
- formatted_text: Raw text with special tokens
|
| 310 |
+
- step_descriptions: List of step descriptions
|
| 311 |
+
- step_types: List of step types
|
| 312 |
+
- step_rewards: List of process rewards
|
| 313 |
+
- visual_features: List of visual features (if enabled)
|
| 314 |
+
- attention_maps: List of attention maps (if available)
|
| 315 |
+
- final_answer: Target answer
|
| 316 |
+
- is_correct: Ground truth correctness
|
| 317 |
+
- chain_metadata: Additional information
|
| 318 |
+
"""
|
| 319 |
+
chain = self.chains[idx]
|
| 320 |
+
|
| 321 |
+
# Format chain with special tokens
|
| 322 |
+
formatted_text = chain.format_with_tokens()
|
| 323 |
+
|
| 324 |
+
# Tokenize formatted text (for training)
|
| 325 |
+
formatted_encoding = self.tokenizer(
|
| 326 |
+
formatted_text,
|
| 327 |
+
truncation=True,
|
| 328 |
+
max_length=2048, # Longer for full reasoning chains
|
| 329 |
+
return_tensors='pt',
|
| 330 |
+
)
|
| 331 |
+
|
| 332 |
+
# Tokenize prompt (for compatibility)
|
| 333 |
+
prompt_encoding = self.tokenizer(
|
| 334 |
+
chain.prompt,
|
| 335 |
+
truncation=True,
|
| 336 |
+
max_length=512,
|
| 337 |
+
return_tensors='pt',
|
| 338 |
+
)
|
| 339 |
+
|
| 340 |
+
# Prepare step data
|
| 341 |
+
step_descriptions = []
|
| 342 |
+
step_types = []
|
| 343 |
+
step_rewards = []
|
| 344 |
+
visual_features = []
|
| 345 |
+
attention_maps = []
|
| 346 |
+
|
| 347 |
+
for step in chain.steps[:self.max_steps]:
|
| 348 |
+
step_descriptions.append(step.description)
|
| 349 |
+
step_types.append(step.step_type.value)
|
| 350 |
+
step_rewards.append(step.reward)
|
| 351 |
+
|
| 352 |
+
if self.include_visual_features:
|
| 353 |
+
if step.visual_features is not None:
|
| 354 |
+
visual_features.append(step.visual_features)
|
| 355 |
+
else:
|
| 356 |
+
visual_features.append(torch.zeros(768)) # Default dimension
|
| 357 |
+
|
| 358 |
+
if step.attention_map is not None:
|
| 359 |
+
attention_maps.append(step.attention_map)
|
| 360 |
+
|
| 361 |
+
# Tokenize final answer
|
| 362 |
+
answer_encoding = self.tokenizer(
|
| 363 |
+
chain.final_answer,
|
| 364 |
+
truncation=True,
|
| 365 |
+
max_length=256,
|
| 366 |
+
return_tensors='pt',
|
| 367 |
+
)
|
| 368 |
+
|
| 369 |
+
result = {
|
| 370 |
+
'prompt_ids': prompt_encoding['input_ids'].squeeze(0),
|
| 371 |
+
'prompt_mask': prompt_encoding['attention_mask'].squeeze(0),
|
| 372 |
+
'formatted_input_ids': formatted_encoding['input_ids'].squeeze(0),
|
| 373 |
+
'formatted_attention_mask': formatted_encoding['attention_mask'].squeeze(0),
|
| 374 |
+
'formatted_text': formatted_text,
|
| 375 |
+
'step_descriptions': step_descriptions,
|
| 376 |
+
'step_types': step_types,
|
| 377 |
+
'step_rewards': torch.tensor(step_rewards, dtype=torch.float32),
|
| 378 |
+
'answer_ids': answer_encoding['input_ids'].squeeze(0),
|
| 379 |
+
'answer_mask': answer_encoding['attention_mask'].squeeze(0),
|
| 380 |
+
'is_correct': torch.tensor(chain.is_correct, dtype=torch.bool),
|
| 381 |
+
'total_reward': torch.tensor(chain.total_reward, dtype=torch.float32),
|
| 382 |
+
'num_steps': len(chain.steps),
|
| 383 |
+
'chain_id': chain.chain_id,
|
| 384 |
+
'image_path': chain.image_path,
|
| 385 |
+
}
|
| 386 |
+
|
| 387 |
+
if self.include_visual_features and visual_features:
|
| 388 |
+
result['visual_features'] = torch.stack(visual_features)
|
| 389 |
+
|
| 390 |
+
if attention_maps:
|
| 391 |
+
result['attention_maps'] = attention_maps
|
| 392 |
+
|
| 393 |
+
return result
|
| 394 |
+
|
| 395 |
+
@classmethod
|
| 396 |
+
def from_json_dir(
|
| 397 |
+
cls,
|
| 398 |
+
json_dir: str,
|
| 399 |
+
tokenizer: Any,
|
| 400 |
+
**kwargs
|
| 401 |
+
) -> 'StepDataset':
|
| 402 |
+
"""
|
| 403 |
+
Load dataset from directory of JSON files.
|
| 404 |
+
|
| 405 |
+
Args:
|
| 406 |
+
json_dir: Directory containing reasoning chain JSON files
|
| 407 |
+
tokenizer: Tokenizer for text encoding
|
| 408 |
+
**kwargs: Additional arguments for StepDataset
|
| 409 |
+
"""
|
| 410 |
+
from pathlib import Path
|
| 411 |
+
|
| 412 |
+
json_path = Path(json_dir)
|
| 413 |
+
chains = []
|
| 414 |
+
|
| 415 |
+
for json_file in json_path.glob('*.json'):
|
| 416 |
+
try:
|
| 417 |
+
chain = ReasoningChain.load_json(str(json_file))
|
| 418 |
+
chains.append(chain)
|
| 419 |
+
except Exception as e:
|
| 420 |
+
print(f"Error loading {json_file}: {e}")
|
| 421 |
+
|
| 422 |
+
return cls(chains, tokenizer, **kwargs)
|
| 423 |
+
|
| 424 |
+
def get_statistics(self) -> Dict[str, Any]:
|
| 425 |
+
"""Calculate dataset statistics."""
|
| 426 |
+
step_types = {}
|
| 427 |
+
rewards = []
|
| 428 |
+
correct_count = 0
|
| 429 |
+
|
| 430 |
+
for chain in self.chains:
|
| 431 |
+
if chain.is_correct:
|
| 432 |
+
correct_count += 1
|
| 433 |
+
|
| 434 |
+
for step in chain.steps:
|
| 435 |
+
step_type = step.step_type.value
|
| 436 |
+
step_types[step_type] = step_types.get(step_type, 0) + 1
|
| 437 |
+
rewards.append(step.reward)
|
| 438 |
+
|
| 439 |
+
return {
|
| 440 |
+
'num_chains': len(self.chains),
|
| 441 |
+
'total_steps': self.total_steps,
|
| 442 |
+
'avg_steps_per_chain': self.avg_steps,
|
| 443 |
+
'accuracy': correct_count / len(self.chains) if self.chains else 0,
|
| 444 |
+
'step_type_distribution': step_types,
|
| 445 |
+
'avg_step_reward': sum(rewards) / len(rewards) if rewards else 0,
|
| 446 |
+
'min_step_reward': min(rewards) if rewards else 0,
|
| 447 |
+
'max_step_reward': max(rewards) if rewards else 0,
|
| 448 |
+
}
|
src/reasoning/step_level_cot.py
ADDED
|
@@ -0,0 +1,477 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Step-Level Chain of Thought Trainer for Vision-Language Models.
|
| 3 |
+
|
| 4 |
+
Integrates PRM, RL training, and inference-time scaling for fine-grained reasoning.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import argparse
|
| 8 |
+
import logging
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import Dict, Optional, Any
|
| 11 |
+
import torch
|
| 12 |
+
import yaml
|
| 13 |
+
from accelerate import Accelerator
|
| 14 |
+
from accelerate.logging import get_logger
|
| 15 |
+
from transformers import AutoTokenizer
|
| 16 |
+
from diffusers import QwenImagePipeline, QwenImageTransformer2DModel
|
| 17 |
+
from omegaconf import OmegaConf
|
| 18 |
+
from tqdm.auto import tqdm
|
| 19 |
+
|
| 20 |
+
from .step_data import StepDataset, ReasoningChain
|
| 21 |
+
from .prm import ProcessRewardModel, PRMTrainer
|
| 22 |
+
from .rl_trainer import RLReasoningTrainer, RLConfig
|
| 23 |
+
from .inference_scaling import InferenceTimeScaling, InferenceConfig
|
| 24 |
+
|
| 25 |
+
logger = get_logger(__name__, log_level="INFO")
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class StepLevelCoTTrainer:
|
| 29 |
+
"""
|
| 30 |
+
Complete training pipeline for step-level chain-of-thought reasoning.
|
| 31 |
+
|
| 32 |
+
Training phases:
|
| 33 |
+
1. Supervised Fine-tuning (SFT) on reasoning chains
|
| 34 |
+
2. PRM training on step-level rewards
|
| 35 |
+
3. RL fine-tuning with PRM rewards
|
| 36 |
+
4. Inference-time scaling evaluation
|
| 37 |
+
"""
|
| 38 |
+
|
| 39 |
+
def __init__(
|
| 40 |
+
self,
|
| 41 |
+
config_path: str,
|
| 42 |
+
device: str = "cuda",
|
| 43 |
+
):
|
| 44 |
+
"""
|
| 45 |
+
Initialize trainer.
|
| 46 |
+
|
| 47 |
+
Args:
|
| 48 |
+
config_path: Path to training configuration YAML
|
| 49 |
+
device: Device for training
|
| 50 |
+
"""
|
| 51 |
+
self.device = device
|
| 52 |
+
|
| 53 |
+
# Load configuration
|
| 54 |
+
with open(config_path, 'r') as f:
|
| 55 |
+
config_dict = yaml.safe_load(f)
|
| 56 |
+
self.config = OmegaConf.create(config_dict)
|
| 57 |
+
|
| 58 |
+
# Initialize accelerator
|
| 59 |
+
self.accelerator = Accelerator(
|
| 60 |
+
gradient_accumulation_steps=self.config.gradient_accumulation_steps,
|
| 61 |
+
mixed_precision=self.config.mixed_precision,
|
| 62 |
+
log_with=self.config.report_to,
|
| 63 |
+
project_dir=self.config.output_dir,
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
# Load model and tokenizer
|
| 67 |
+
self._load_model()
|
| 68 |
+
|
| 69 |
+
# Initialize PRM
|
| 70 |
+
self.prm = ProcessRewardModel(
|
| 71 |
+
vision_dim=self.config.vision_dim,
|
| 72 |
+
text_dim=self.config.text_dim,
|
| 73 |
+
hidden_dim=self.config.prm_hidden_dim,
|
| 74 |
+
num_heads=self.config.prm_num_heads,
|
| 75 |
+
dropout=self.config.prm_dropout,
|
| 76 |
+
max_steps=self.config.max_reasoning_steps,
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
# Initialize RL trainer
|
| 80 |
+
rl_config = RLConfig(
|
| 81 |
+
clip_epsilon=self.config.ppo_clip_epsilon,
|
| 82 |
+
value_loss_coef=self.config.value_loss_coef,
|
| 83 |
+
entropy_coef=self.config.entropy_coef,
|
| 84 |
+
gamma=self.config.gamma,
|
| 85 |
+
gae_lambda=self.config.gae_lambda,
|
| 86 |
+
learning_rate=self.config.rl_learning_rate,
|
| 87 |
+
batch_size=self.config.train_batch_size,
|
| 88 |
+
use_prm_rewards=True,
|
| 89 |
+
normalize_rewards=True,
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
self.rl_trainer = RLReasoningTrainer(
|
| 93 |
+
policy_model=self.model,
|
| 94 |
+
prm_model=self.prm,
|
| 95 |
+
config=rl_config,
|
| 96 |
+
device=device,
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
# Initialize inference scaler
|
| 100 |
+
inference_config = InferenceConfig(
|
| 101 |
+
num_samples=self.config.num_inference_samples,
|
| 102 |
+
temperature=self.config.inference_temperature,
|
| 103 |
+
aggregation=self.config.aggregation_method,
|
| 104 |
+
use_prm_scores=True,
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
self.inference_scaler = InferenceTimeScaling(
|
| 108 |
+
model=self.model,
|
| 109 |
+
prm=self.prm,
|
| 110 |
+
config=inference_config,
|
| 111 |
+
device=device,
|
| 112 |
+
)
|
| 113 |
+
|
| 114 |
+
logger.info("StepLevelCoTTrainer initialized")
|
| 115 |
+
|
| 116 |
+
def _load_model(self) -> None:
|
| 117 |
+
"""Load vision-language model."""
|
| 118 |
+
logger.info(f"Loading model: {self.config.pretrained_model_name_or_path}")
|
| 119 |
+
|
| 120 |
+
# Load Qwen-Image model
|
| 121 |
+
self.pipeline = QwenImagePipeline.from_pretrained(
|
| 122 |
+
self.config.pretrained_model_name_or_path,
|
| 123 |
+
torch_dtype=torch.float16 if self.config.mixed_precision == "fp16" else torch.float32,
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
self.model = self.pipeline.transformer
|
| 127 |
+
self.vae = self.pipeline.vae
|
| 128 |
+
self.text_encoder = self.pipeline.text_encoder
|
| 129 |
+
self.tokenizer = self.pipeline.tokenizer
|
| 130 |
+
|
| 131 |
+
# Add special tokens for chain-of-thought reasoning
|
| 132 |
+
self._add_special_tokens()
|
| 133 |
+
|
| 134 |
+
# Enable gradient checkpointing
|
| 135 |
+
if self.config.gradient_checkpointing:
|
| 136 |
+
self.model.enable_gradient_checkpointing()
|
| 137 |
+
|
| 138 |
+
logger.info("Model loaded successfully")
|
| 139 |
+
|
| 140 |
+
def _add_special_tokens(self) -> None:
|
| 141 |
+
"""Add special tokens for chain-of-thought reasoning."""
|
| 142 |
+
from .step_data import SPECIAL_TOKENS, SPECIAL_TOKEN_DESCRIPTIONS
|
| 143 |
+
|
| 144 |
+
logger.info(f"Adding {len(SPECIAL_TOKENS)} special tokens for chain-of-thought reasoning...")
|
| 145 |
+
|
| 146 |
+
# Add special tokens to tokenizer
|
| 147 |
+
num_added_toks = self.tokenizer.add_special_tokens(
|
| 148 |
+
{"additional_special_tokens": SPECIAL_TOKENS}
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
if num_added_toks > 0:
|
| 152 |
+
logger.info(f"Added {num_added_toks} new tokens. Resizing model embeddings...")
|
| 153 |
+
|
| 154 |
+
# Get current embedding size
|
| 155 |
+
old_size = self.model.get_input_embeddings().weight.size(0)
|
| 156 |
+
|
| 157 |
+
# Resize token embeddings
|
| 158 |
+
self.model.resize_token_embeddings(len(self.tokenizer))
|
| 159 |
+
|
| 160 |
+
# Initialize new token embeddings with semantic initialization
|
| 161 |
+
self._initialize_new_token_embeddings(num_added_toks, SPECIAL_TOKEN_DESCRIPTIONS)
|
| 162 |
+
|
| 163 |
+
logger.info(f"Resized embeddings from {old_size} to {len(self.tokenizer)}")
|
| 164 |
+
else:
|
| 165 |
+
logger.info("Special tokens already present in tokenizer.")
|
| 166 |
+
|
| 167 |
+
def _initialize_new_token_embeddings(self, num_added_toks: int, descriptions: list) -> None:
|
| 168 |
+
"""Initialize newly added token embeddings with semantic information."""
|
| 169 |
+
with torch.no_grad():
|
| 170 |
+
input_embeddings = self.model.get_input_embeddings()
|
| 171 |
+
|
| 172 |
+
# Check if model has language model head
|
| 173 |
+
has_lm_head = hasattr(self.model, 'lm_head')
|
| 174 |
+
if has_lm_head:
|
| 175 |
+
lm_head = self.model.lm_head
|
| 176 |
+
|
| 177 |
+
for i, description in enumerate(reversed(descriptions), start=1):
|
| 178 |
+
# Tokenize the description
|
| 179 |
+
tokenized = self.tokenizer.tokenize(description)
|
| 180 |
+
if not tokenized:
|
| 181 |
+
continue
|
| 182 |
+
|
| 183 |
+
tokenized_ids = self.tokenizer.convert_tokens_to_ids(tokenized)
|
| 184 |
+
|
| 185 |
+
# Average embeddings of description tokens
|
| 186 |
+
new_embedding = input_embeddings.weight[tokenized_ids].mean(dim=0)
|
| 187 |
+
input_embeddings.weight[-i, :] = new_embedding.clone().detach()
|
| 188 |
+
|
| 189 |
+
# Initialize lm_head if it exists
|
| 190 |
+
if has_lm_head:
|
| 191 |
+
new_lm_head = lm_head.weight[tokenized_ids].mean(dim=0)
|
| 192 |
+
lm_head.weight[-i, :] = new_lm_head.clone().detach()
|
| 193 |
+
|
| 194 |
+
logger.info(f"Initialized {num_added_toks} new token embeddings with semantic information.")
|
| 195 |
+
|
| 196 |
+
def load_step_dataset(
|
| 197 |
+
self,
|
| 198 |
+
data_dir: str,
|
| 199 |
+
split: str = "train",
|
| 200 |
+
) -> StepDataset:
|
| 201 |
+
"""
|
| 202 |
+
Load step-level reasoning dataset.
|
| 203 |
+
|
| 204 |
+
Args:
|
| 205 |
+
data_dir: Directory containing reasoning chain JSON files
|
| 206 |
+
split: Dataset split (train/val/test)
|
| 207 |
+
|
| 208 |
+
Returns:
|
| 209 |
+
StepDataset instance
|
| 210 |
+
"""
|
| 211 |
+
data_path = Path(data_dir) / split
|
| 212 |
+
|
| 213 |
+
logger.info(f"Loading dataset from {data_path}")
|
| 214 |
+
|
| 215 |
+
dataset = StepDataset.from_json_dir(
|
| 216 |
+
str(data_path),
|
| 217 |
+
tokenizer=self.tokenizer,
|
| 218 |
+
max_steps=self.config.max_reasoning_steps,
|
| 219 |
+
include_visual_features=True,
|
| 220 |
+
filter_incorrect=False, # Include all chains for learning
|
| 221 |
+
)
|
| 222 |
+
|
| 223 |
+
stats = dataset.get_statistics()
|
| 224 |
+
logger.info(f"Dataset loaded: {stats}")
|
| 225 |
+
|
| 226 |
+
return dataset
|
| 227 |
+
|
| 228 |
+
def train_prm(
|
| 229 |
+
self,
|
| 230 |
+
train_dataset: StepDataset,
|
| 231 |
+
val_dataset: Optional[StepDataset] = None,
|
| 232 |
+
num_epochs: int = 10,
|
| 233 |
+
) -> None:
|
| 234 |
+
"""
|
| 235 |
+
Train Process Reward Model.
|
| 236 |
+
|
| 237 |
+
Args:
|
| 238 |
+
train_dataset: Training dataset with step-level rewards
|
| 239 |
+
val_dataset: Optional validation dataset
|
| 240 |
+
num_epochs: Number of training epochs
|
| 241 |
+
"""
|
| 242 |
+
logger.info("Starting PRM training")
|
| 243 |
+
|
| 244 |
+
prm_trainer = PRMTrainer(
|
| 245 |
+
model=self.prm,
|
| 246 |
+
learning_rate=self.config.prm_learning_rate,
|
| 247 |
+
weight_decay=self.config.weight_decay,
|
| 248 |
+
warmup_steps=self.config.warmup_steps,
|
| 249 |
+
)
|
| 250 |
+
|
| 251 |
+
train_dataloader = torch.utils.data.DataLoader(
|
| 252 |
+
train_dataset,
|
| 253 |
+
batch_size=self.config.train_batch_size,
|
| 254 |
+
shuffle=True,
|
| 255 |
+
num_workers=self.config.dataloader_num_workers,
|
| 256 |
+
)
|
| 257 |
+
|
| 258 |
+
# Prepare with accelerator
|
| 259 |
+
prm_trainer.model, prm_trainer.optimizer, train_dataloader = self.accelerator.prepare(
|
| 260 |
+
prm_trainer.model, prm_trainer.optimizer, train_dataloader
|
| 261 |
+
)
|
| 262 |
+
|
| 263 |
+
best_val_loss = float('inf')
|
| 264 |
+
|
| 265 |
+
for epoch in range(num_epochs):
|
| 266 |
+
# Training loop
|
| 267 |
+
prm_trainer.model.train()
|
| 268 |
+
total_loss = 0
|
| 269 |
+
|
| 270 |
+
progress_bar = tqdm(
|
| 271 |
+
train_dataloader,
|
| 272 |
+
desc=f"PRM Epoch {epoch + 1}/{num_epochs}",
|
| 273 |
+
disable=not self.accelerator.is_local_main_process,
|
| 274 |
+
)
|
| 275 |
+
|
| 276 |
+
for batch in progress_bar:
|
| 277 |
+
# Extract batch data
|
| 278 |
+
vision_features = batch['visual_features']
|
| 279 |
+
step_embeddings = batch['step_descriptions'] # Need to encode these
|
| 280 |
+
target_rewards = batch['step_rewards']
|
| 281 |
+
|
| 282 |
+
# Encode step descriptions
|
| 283 |
+
# (In practice, you'd encode text properly)
|
| 284 |
+
|
| 285 |
+
metrics = prm_trainer.train_step(
|
| 286 |
+
vision_features,
|
| 287 |
+
step_embeddings,
|
| 288 |
+
target_rewards,
|
| 289 |
+
)
|
| 290 |
+
|
| 291 |
+
total_loss += metrics['loss']
|
| 292 |
+
progress_bar.set_postfix({'loss': metrics['loss']})
|
| 293 |
+
|
| 294 |
+
avg_loss = total_loss / len(train_dataloader)
|
| 295 |
+
logger.info(f"Epoch {epoch + 1} - Avg Loss: {avg_loss:.4f}")
|
| 296 |
+
|
| 297 |
+
# Validation
|
| 298 |
+
if val_dataset is not None:
|
| 299 |
+
val_metrics = self._evaluate_prm(prm_trainer, val_dataset)
|
| 300 |
+
logger.info(f"Validation: {val_metrics}")
|
| 301 |
+
|
| 302 |
+
# Save best model
|
| 303 |
+
if val_metrics['mse'] < best_val_loss:
|
| 304 |
+
best_val_loss = val_metrics['mse']
|
| 305 |
+
self._save_prm(epoch)
|
| 306 |
+
|
| 307 |
+
logger.info("PRM training completed")
|
| 308 |
+
|
| 309 |
+
def train_rl(
|
| 310 |
+
self,
|
| 311 |
+
train_dataset: StepDataset,
|
| 312 |
+
num_iterations: int = 1000,
|
| 313 |
+
) -> None:
|
| 314 |
+
"""
|
| 315 |
+
Train policy with reinforcement learning.
|
| 316 |
+
|
| 317 |
+
Args:
|
| 318 |
+
train_dataset: Training dataset
|
| 319 |
+
num_iterations: Number of RL iterations
|
| 320 |
+
"""
|
| 321 |
+
logger.info("Starting RL training")
|
| 322 |
+
|
| 323 |
+
self.rl_trainer.train(
|
| 324 |
+
train_dataset=train_dataset,
|
| 325 |
+
num_iterations=num_iterations,
|
| 326 |
+
log_interval=self.config.logging_steps,
|
| 327 |
+
)
|
| 328 |
+
|
| 329 |
+
logger.info("RL training completed")
|
| 330 |
+
|
| 331 |
+
def evaluate_inference_scaling(
|
| 332 |
+
self,
|
| 333 |
+
test_dataset: StepDataset,
|
| 334 |
+
num_samples: int = 10,
|
| 335 |
+
) -> Dict[str, Any]:
|
| 336 |
+
"""
|
| 337 |
+
Evaluate inference-time scaling performance.
|
| 338 |
+
|
| 339 |
+
Args:
|
| 340 |
+
test_dataset: Test dataset
|
| 341 |
+
num_samples: Number of test samples
|
| 342 |
+
|
| 343 |
+
Returns:
|
| 344 |
+
Evaluation metrics
|
| 345 |
+
"""
|
| 346 |
+
logger.info("Evaluating inference-time scaling")
|
| 347 |
+
|
| 348 |
+
results = []
|
| 349 |
+
|
| 350 |
+
for idx in range(min(num_samples, len(test_dataset))):
|
| 351 |
+
sample = test_dataset[idx]
|
| 352 |
+
|
| 353 |
+
# Run inference with scaling
|
| 354 |
+
best_chain, all_chains = self.inference_scaler(
|
| 355 |
+
image_features=sample['visual_features'][0],
|
| 356 |
+
prompt_ids=sample['prompt_ids'],
|
| 357 |
+
prompt_mask=sample['prompt_mask'],
|
| 358 |
+
image_path=sample['image_path'],
|
| 359 |
+
prompt_text="", # Would decode from IDs
|
| 360 |
+
)
|
| 361 |
+
|
| 362 |
+
results.append({
|
| 363 |
+
'chain_id': sample['chain_id'],
|
| 364 |
+
'best_reward': best_chain.total_reward,
|
| 365 |
+
'num_steps': len(best_chain.steps),
|
| 366 |
+
'avg_reward_all': sum(c.total_reward for c in all_chains) / len(all_chains),
|
| 367 |
+
'std_reward_all': torch.tensor([c.total_reward for c in all_chains]).std().item(),
|
| 368 |
+
})
|
| 369 |
+
|
| 370 |
+
# Aggregate metrics
|
| 371 |
+
metrics = {
|
| 372 |
+
'avg_best_reward': sum(r['best_reward'] for r in results) / len(results),
|
| 373 |
+
'avg_num_steps': sum(r['num_steps'] for r in results) / len(results),
|
| 374 |
+
'improvement_over_mean': sum(
|
| 375 |
+
(r['best_reward'] - r['avg_reward_all']) / (r['std_reward_all'] + 1e-8)
|
| 376 |
+
for r in results
|
| 377 |
+
) / len(results),
|
| 378 |
+
}
|
| 379 |
+
|
| 380 |
+
logger.info(f"Inference scaling metrics: {metrics}")
|
| 381 |
+
|
| 382 |
+
return metrics
|
| 383 |
+
|
| 384 |
+
def _evaluate_prm(
|
| 385 |
+
self,
|
| 386 |
+
prm_trainer: PRMTrainer,
|
| 387 |
+
val_dataset: StepDataset,
|
| 388 |
+
) -> Dict[str, float]:
|
| 389 |
+
"""Evaluate PRM on validation set."""
|
| 390 |
+
val_dataloader = torch.utils.data.DataLoader(
|
| 391 |
+
val_dataset,
|
| 392 |
+
batch_size=self.config.train_batch_size,
|
| 393 |
+
shuffle=False,
|
| 394 |
+
)
|
| 395 |
+
|
| 396 |
+
total_metrics = {'mse': 0, 'mae': 0, 'correlation': 0}
|
| 397 |
+
|
| 398 |
+
for batch in val_dataloader:
|
| 399 |
+
vision_features = batch['visual_features']
|
| 400 |
+
step_embeddings = batch['step_descriptions']
|
| 401 |
+
target_rewards = batch['step_rewards']
|
| 402 |
+
|
| 403 |
+
metrics = prm_trainer.evaluate(
|
| 404 |
+
vision_features,
|
| 405 |
+
step_embeddings,
|
| 406 |
+
target_rewards,
|
| 407 |
+
)
|
| 408 |
+
|
| 409 |
+
for key in total_metrics:
|
| 410 |
+
total_metrics[key] += metrics[key]
|
| 411 |
+
|
| 412 |
+
# Average
|
| 413 |
+
for key in total_metrics:
|
| 414 |
+
total_metrics[key] /= len(val_dataloader)
|
| 415 |
+
|
| 416 |
+
return total_metrics
|
| 417 |
+
|
| 418 |
+
def _save_prm(self, epoch: int) -> None:
|
| 419 |
+
"""Save PRM checkpoint."""
|
| 420 |
+
save_path = Path(self.config.output_dir) / "prm" / f"epoch_{epoch}"
|
| 421 |
+
save_path.mkdir(parents=True, exist_ok=True)
|
| 422 |
+
|
| 423 |
+
torch.save(
|
| 424 |
+
self.prm.state_dict(),
|
| 425 |
+
save_path / "prm_model.pt",
|
| 426 |
+
)
|
| 427 |
+
logger.info(f"PRM saved to {save_path}")
|
| 428 |
+
|
| 429 |
+
def save_checkpoint(self, step: int) -> None:
|
| 430 |
+
"""Save complete training checkpoint."""
|
| 431 |
+
save_path = Path(self.config.output_dir) / f"checkpoint-{step}"
|
| 432 |
+
save_path.mkdir(parents=True, exist_ok=True)
|
| 433 |
+
|
| 434 |
+
# Save policy
|
| 435 |
+
self.accelerator.save_state(str(save_path / "policy"))
|
| 436 |
+
|
| 437 |
+
# Save PRM
|
| 438 |
+
torch.save(self.prm.state_dict(), save_path / "prm.pt")
|
| 439 |
+
|
| 440 |
+
# Save RL trainer state
|
| 441 |
+
self.rl_trainer.save_checkpoint(str(save_path / "rl_trainer.pt"))
|
| 442 |
+
|
| 443 |
+
logger.info(f"Checkpoint saved to {save_path}")
|
| 444 |
+
|
| 445 |
+
|
| 446 |
+
def main():
|
| 447 |
+
"""Main training entry point."""
|
| 448 |
+
parser = argparse.ArgumentParser(description="Train step-level CoT reasoning for vision models")
|
| 449 |
+
parser.add_argument("--config", type=str, required=True, help="Path to config YAML")
|
| 450 |
+
parser.add_argument("--phase", type=str, choices=["prm", "rl", "inference", "all"], default="all")
|
| 451 |
+
parser.add_argument("--data_dir", type=str, required=True, help="Path to reasoning chain data")
|
| 452 |
+
args = parser.parse_args()
|
| 453 |
+
|
| 454 |
+
# Initialize trainer
|
| 455 |
+
trainer = StepLevelCoTTrainer(
|
| 456 |
+
config_path=args.config,
|
| 457 |
+
device="cuda" if torch.cuda.is_available() else "cpu",
|
| 458 |
+
)
|
| 459 |
+
|
| 460 |
+
# Load datasets
|
| 461 |
+
train_dataset = trainer.load_step_dataset(args.data_dir, split="train")
|
| 462 |
+
val_dataset = trainer.load_step_dataset(args.data_dir, split="val")
|
| 463 |
+
test_dataset = trainer.load_step_dataset(args.data_dir, split="test")
|
| 464 |
+
|
| 465 |
+
# Run training phases
|
| 466 |
+
if args.phase in ["prm", "all"]:
|
| 467 |
+
trainer.train_prm(train_dataset, val_dataset)
|
| 468 |
+
|
| 469 |
+
if args.phase in ["rl", "all"]:
|
| 470 |
+
trainer.train_rl(train_dataset)
|
| 471 |
+
|
| 472 |
+
if args.phase in ["inference", "all"]:
|
| 473 |
+
trainer.evaluate_inference_scaling(test_dataset)
|
| 474 |
+
|
| 475 |
+
|
| 476 |
+
if __name__ == "__main__":
|
| 477 |
+
main()
|
train_configs/train_full_qwen_image.yaml
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Full Training Configuration for Qwen Image Model
|
| 2 |
+
pretrained_model_name_or_path: Qwen/Qwen-Image
|
| 3 |
+
|
| 4 |
+
# Data configuration
|
| 5 |
+
data_config:
|
| 6 |
+
train_batch_size: 1
|
| 7 |
+
num_workers: 4
|
| 8 |
+
img_size: 1024
|
| 9 |
+
caption_dropout_rate: 0.1
|
| 10 |
+
img_dir: /path/to/your/dataset
|
| 11 |
+
random_ratio: false
|
| 12 |
+
caption_type: txt
|
| 13 |
+
|
| 14 |
+
# Training parameters
|
| 15 |
+
train_batch_size: 1
|
| 16 |
+
output_dir: ./output_full_training
|
| 17 |
+
max_train_steps: 30000
|
| 18 |
+
num_train_epochs: 1
|
| 19 |
+
|
| 20 |
+
# Optimizer settings
|
| 21 |
+
learning_rate: 6e-4 # Lower learning rate for full training
|
| 22 |
+
use_8bit_adam: true # Use 8-bit Adam to save memory
|
| 23 |
+
adam_beta1: 0.9
|
| 24 |
+
adam_beta2: 0.999
|
| 25 |
+
adam_weight_decay: 0.01
|
| 26 |
+
adam_epsilon: 1e-8
|
| 27 |
+
|
| 28 |
+
# Learning rate scheduler
|
| 29 |
+
lr_scheduler: cosine_with_restarts
|
| 30 |
+
lr_warmup_steps: 1000
|
| 31 |
+
|
| 32 |
+
# Gradient settings
|
| 33 |
+
max_grad_norm: 1.0
|
| 34 |
+
gradient_accumulation_steps: 4 # Increase for effective batch size
|
| 35 |
+
|
| 36 |
+
# Precision and memory
|
| 37 |
+
mixed_precision: "bf16" # Use bfloat16 for better stability
|
| 38 |
+
freeze_text_encoder: true # Keep text encoder frozen to save memory
|
| 39 |
+
|
| 40 |
+
# Logging and checkpointing
|
| 41 |
+
logging_dir: logs
|
| 42 |
+
report_to: null # Can be "wandb", "tensorboard", etc.
|
| 43 |
+
checkpointing_steps: 1000
|
| 44 |
+
checkpoints_total_limit: 2
|
| 45 |
+
tracker_project_name: qwen_image_full_training
|
| 46 |
+
|
| 47 |
+
# Resume training
|
| 48 |
+
resume_from_checkpoint: latest
|
train_configs/train_lora.yaml
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pretrained_model_name_or_path: Qwen/Qwen-Image
|
| 2 |
+
data_config:
|
| 3 |
+
train_batch_size: 1
|
| 4 |
+
num_workers: 4
|
| 5 |
+
img_size: 1024
|
| 6 |
+
caption_dropout_rate: 0.05 # Lower dropout for better brand consistency
|
| 7 |
+
img_dir: ./brand_logo_dataset
|
| 8 |
+
random_ratio: false # Maintain consistent aspect ratios for logos
|
| 9 |
+
caption_type: txt
|
| 10 |
+
report_to: null
|
| 11 |
+
train_batch_size: 1
|
| 12 |
+
output_dir: ./brand_logo_output
|
| 13 |
+
max_train_steps: 5000 # Increased for better logo quality
|
| 14 |
+
learning_rate: 8e-5 # Lower learning rate for professional results
|
| 15 |
+
lr_scheduler: cosine_with_restarts
|
| 16 |
+
lr_warmup_steps: 200
|
| 17 |
+
adam_beta1: 0.9
|
| 18 |
+
adam_beta2: 0.999
|
| 19 |
+
adam_weight_decay: 0.01
|
| 20 |
+
adam_epsilon: 1e-8
|
| 21 |
+
max_grad_norm: 1.0
|
| 22 |
+
logging_dir: logs
|
| 23 |
+
mixed_precision: "bf16"
|
| 24 |
+
checkpointing_steps: 500
|
| 25 |
+
checkpoints_total_limit: 5
|
| 26 |
+
tracker_project_name: brand_logo_training
|
| 27 |
+
resume_from_checkpoint: latest
|
| 28 |
+
gradient_accumulation_steps: 1
|
| 29 |
+
rank: 32 # Higher rank for better detail in logos
|
train_configs/train_lora_4090.yaml
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pretrained_model_name_or_path: Qwen/Qwen-Image
|
| 2 |
+
data_config:
|
| 3 |
+
dataset_name: HuggingFaceM4/FineVision
|
| 4 |
+
subset: <your_subset_name> # e.g. "default" or the first from get_dataset_config_names
|
| 5 |
+
split: train
|
| 6 |
+
streaming: true
|
| 7 |
+
image_key: image
|
| 8 |
+
prompt_key: text
|
| 9 |
+
train_batch_size: 1
|
| 10 |
+
num_workers: 4
|
| 11 |
+
img_size: 1024
|
| 12 |
+
caption_dropout_rate: 0.1
|
| 13 |
+
# img_dir: ./your_lora_dataset
|
| 14 |
+
random_ratio: false # support multi crop preprocessing
|
| 15 |
+
caption_type: txt
|
| 16 |
+
report_to: null
|
| 17 |
+
train_batch_size: 1
|
| 18 |
+
output_dir: ./test_lora_saves
|
| 19 |
+
max_train_steps: 3000
|
| 20 |
+
learning_rate: 3e-4
|
| 21 |
+
lr_scheduler: constant
|
| 22 |
+
lr_warmup_steps: 10
|
| 23 |
+
adam_beta1: 0.9
|
| 24 |
+
adam_beta2: 0.999
|
| 25 |
+
adam_weight_decay: 0.01
|
| 26 |
+
adam_epsilon: 1e-8
|
| 27 |
+
max_grad_norm: 1.0
|
| 28 |
+
logging_dir: logs
|
| 29 |
+
mixed_precision: "fp16"
|
| 30 |
+
checkpointing_steps: 250
|
| 31 |
+
checkpoints_total_limit: 10
|
| 32 |
+
tracker_project_name: lora_test
|
| 33 |
+
resume_from_checkpoint: latest
|
| 34 |
+
gradient_accumulation_steps: 1
|
| 35 |
+
rank: 16
|
| 36 |
+
precompute_text_embeddings: true
|
| 37 |
+
precompute_image_embeddings: true
|
| 38 |
+
quantize: true
|
| 39 |
+
adam8bit: true
|
| 40 |
+
save_cache_on_disk: true
|
train_configs/train_lora_qwen_edit.yaml
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pretrained_model_name_or_path: Qwen/Qwen-Image-Edit
|
| 2 |
+
data_config:
|
| 3 |
+
train_batch_size: 1
|
| 4 |
+
num_workers: 4
|
| 5 |
+
img_size: 1024
|
| 6 |
+
caption_dropout_rate: 0.05 # Lower dropout for brand consistency
|
| 7 |
+
img_dir: ./brand_integration_dataset/images
|
| 8 |
+
control_dir: ./brand_integration_dataset/base_images
|
| 9 |
+
random_ratio: false # Maintain consistency for brand integration
|
| 10 |
+
caption_type: txt
|
| 11 |
+
report_to: null
|
| 12 |
+
train_batch_size: 1
|
| 13 |
+
output_dir: ./brand_integration_output
|
| 14 |
+
max_train_steps: 4000
|
| 15 |
+
learning_rate: 1e-4
|
| 16 |
+
lr_scheduler: cosine_with_restarts
|
| 17 |
+
lr_warmup_steps: 100
|
| 18 |
+
adam_beta1: 0.9
|
| 19 |
+
adam_beta2: 0.999
|
| 20 |
+
adam_weight_decay: 0.01
|
| 21 |
+
adam_epsilon: 1e-8
|
| 22 |
+
max_grad_norm: 1.0
|
| 23 |
+
logging_dir: logs
|
| 24 |
+
mixed_precision: "bf16"
|
| 25 |
+
checkpointing_steps: 400
|
| 26 |
+
checkpoints_total_limit: 5
|
| 27 |
+
tracker_project_name: brand_integration_training
|
| 28 |
+
resume_from_checkpoint: latest
|
| 29 |
+
gradient_accumulation_steps: 1
|
| 30 |
+
rank: 24 # Higher rank for detailed brand integration
|
| 31 |
+
precompute_text_embeddings: true
|
| 32 |
+
precompute_image_embeddings: true
|
| 33 |
+
quantize: false
|
| 34 |
+
adam8bit: false
|
| 35 |
+
save_cache_on_disk: true
|
| 36 |
+
|
train_configs/train_qwen_cot_dual.yaml
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Training Configuration for Step-Level CoT Vision Model
|
| 2 |
+
# Using FLUX-Reason-6M + FineVision datasets with Qwen-Image base model
|
| 3 |
+
|
| 4 |
+
# Model Configuration
|
| 5 |
+
pretrained_model_name_or_path: Qwen/Qwen-Image
|
| 6 |
+
model_type: qwen_image # or internvl2, flux, etc.
|
| 7 |
+
|
| 8 |
+
# Special Tokens - Automatically added by StepLevelCoTTrainer
|
| 9 |
+
# 27 tokens total: 15 base reasoning + 12 Chain-of-Sight region tokens
|
| 10 |
+
add_special_tokens: true
|
| 11 |
+
special_tokens_path: null # Leave null for automatic token addition
|
| 12 |
+
|
| 13 |
+
# Output Configuration
|
| 14 |
+
output_dir: ./qwen_cot_checkpoints
|
| 15 |
+
logging_dir: ./qwen_cot_logs
|
| 16 |
+
tracker_project_name: qwen_step_cot_training
|
| 17 |
+
report_to: tensorboard # or "wandb" for Weights & Biases
|
| 18 |
+
|
| 19 |
+
# Dataset Configuration
|
| 20 |
+
datasets:
|
| 21 |
+
# FLUX-Reason-6M: Primary reasoning dataset with step-by-step annotations
|
| 22 |
+
- name: flux_reason
|
| 23 |
+
type: flux_reason_6m
|
| 24 |
+
dataset_name: LucasFang/FLUX-Reason-6M
|
| 25 |
+
split: train
|
| 26 |
+
streaming: true
|
| 27 |
+
max_samples: null # Use all available samples
|
| 28 |
+
weight: 0.7 # 70% of batches from FLUX-Reason
|
| 29 |
+
image_key: image
|
| 30 |
+
reasoning_key: reasoning
|
| 31 |
+
question_key: question
|
| 32 |
+
answer_key: answer
|
| 33 |
+
|
| 34 |
+
# FineVision: Vision-language understanding with synthetic reasoning
|
| 35 |
+
- name: finevision
|
| 36 |
+
type: finevision
|
| 37 |
+
dataset_name: HuggingFaceM4/FineVision
|
| 38 |
+
subset: null # Specify subset if needed
|
| 39 |
+
split: train
|
| 40 |
+
streaming: true
|
| 41 |
+
max_samples: null
|
| 42 |
+
weight: 0.3 # 30% of batches from FineVision
|
| 43 |
+
image_key: image
|
| 44 |
+
text_key: text
|
| 45 |
+
generate_synthetic_reasoning: true
|
| 46 |
+
num_synthetic_steps: 3
|
| 47 |
+
|
| 48 |
+
# Data Processing
|
| 49 |
+
data_config:
|
| 50 |
+
train_batch_size: 2 # Per device
|
| 51 |
+
gradient_accumulation_steps: 8 # Effective batch size: 2 * 8 = 16
|
| 52 |
+
num_workers: 4
|
| 53 |
+
max_sequence_length: 2048 # Longer for reasoning chains
|
| 54 |
+
img_size: 1024
|
| 55 |
+
caption_dropout_rate: 0.1
|
| 56 |
+
random_ratio: false
|
| 57 |
+
prefetch_factor: 2
|
| 58 |
+
|
| 59 |
+
# Training Phases
|
| 60 |
+
training_phases:
|
| 61 |
+
# Phase 1: Supervised Fine-Tuning with reasoning chains
|
| 62 |
+
- phase: sft
|
| 63 |
+
enabled: true
|
| 64 |
+
max_train_steps: 10000
|
| 65 |
+
learning_rate: 2e-5
|
| 66 |
+
warmup_steps: 500
|
| 67 |
+
save_steps: 1000
|
| 68 |
+
eval_steps: 500
|
| 69 |
+
|
| 70 |
+
# Phase 2: Process Reward Model (PRM) training
|
| 71 |
+
- phase: prm
|
| 72 |
+
enabled: true
|
| 73 |
+
max_train_steps: 5000
|
| 74 |
+
learning_rate: 1e-5
|
| 75 |
+
warmup_steps: 250
|
| 76 |
+
save_steps: 500
|
| 77 |
+
eval_steps: 250
|
| 78 |
+
prm_config:
|
| 79 |
+
hidden_dim: 768
|
| 80 |
+
num_layers: 2
|
| 81 |
+
dropout: 0.1
|
| 82 |
+
reward_scale: 1.0
|
| 83 |
+
|
| 84 |
+
# Phase 3: Reinforcement Learning (PPO) with PRM rewards
|
| 85 |
+
- phase: rl
|
| 86 |
+
enabled: true
|
| 87 |
+
max_train_steps: 8000
|
| 88 |
+
learning_rate: 5e-6
|
| 89 |
+
warmup_steps: 200
|
| 90 |
+
save_steps: 500
|
| 91 |
+
eval_steps: 250
|
| 92 |
+
ppo_config:
|
| 93 |
+
gamma: 0.99
|
| 94 |
+
lam: 0.95
|
| 95 |
+
cliprange: 0.2
|
| 96 |
+
cliprange_value: 0.2
|
| 97 |
+
vf_coef: 0.5
|
| 98 |
+
ent_coef: 0.01
|
| 99 |
+
max_grad_norm: 1.0
|
| 100 |
+
minibatch_size: 4
|
| 101 |
+
ppo_epochs: 4
|
| 102 |
+
|
| 103 |
+
# Optimizer Configuration
|
| 104 |
+
learning_rate: 2e-5 # Default for SFT phase
|
| 105 |
+
lr_scheduler: cosine_with_restarts
|
| 106 |
+
lr_warmup_steps: 500
|
| 107 |
+
adam_beta1: 0.9
|
| 108 |
+
adam_beta2: 0.999
|
| 109 |
+
adam_weight_decay: 0.01
|
| 110 |
+
adam_epsilon: 1e-8
|
| 111 |
+
adam8bit: true # Use 8-bit Adam for memory efficiency
|
| 112 |
+
max_grad_norm: 1.0
|
| 113 |
+
|
| 114 |
+
# LoRA Configuration (for memory-efficient fine-tuning)
|
| 115 |
+
use_lora: true
|
| 116 |
+
lora_config:
|
| 117 |
+
rank: 32
|
| 118 |
+
alpha: 64
|
| 119 |
+
dropout: 0.1
|
| 120 |
+
target_modules:
|
| 121 |
+
- q_proj
|
| 122 |
+
- k_proj
|
| 123 |
+
- v_proj
|
| 124 |
+
- o_proj
|
| 125 |
+
- gate_proj
|
| 126 |
+
- up_proj
|
| 127 |
+
- down_proj
|
| 128 |
+
bias: none
|
| 129 |
+
task_type: CAUSAL_LM
|
| 130 |
+
|
| 131 |
+
# Memory Optimization
|
| 132 |
+
mixed_precision: bf16 # Use bfloat16 for better stability than fp16
|
| 133 |
+
quantize: false # Set to true for 8-bit quantization (if memory is tight)
|
| 134 |
+
gradient_checkpointing: true
|
| 135 |
+
precompute_text_embeddings: false # Dynamic computation for reasoning chains
|
| 136 |
+
precompute_image_embeddings: false
|
| 137 |
+
save_cache_on_disk: false # Streaming mode doesn't need cache
|
| 138 |
+
|
| 139 |
+
# Checkpointing
|
| 140 |
+
checkpointing_steps: 1000
|
| 141 |
+
checkpoints_total_limit: 5
|
| 142 |
+
resume_from_checkpoint: latest # Resume from last checkpoint if available
|
| 143 |
+
save_safetensors: true
|
| 144 |
+
|
| 145 |
+
# Chain-of-Sight Configuration
|
| 146 |
+
chain_of_sight:
|
| 147 |
+
enabled: true # Enable region tokens and bounding boxes
|
| 148 |
+
num_regions: 10 # Number of region tokens (<region1> to <region10>)
|
| 149 |
+
use_bounding_boxes: true
|
| 150 |
+
spatial_reward_weight: 0.3 # Weight for spatial accuracy in RL
|
| 151 |
+
|
| 152 |
+
# Reasoning Configuration
|
| 153 |
+
reasoning_config:
|
| 154 |
+
max_steps: 10 # Maximum reasoning steps per chain
|
| 155 |
+
step_types:
|
| 156 |
+
- perception
|
| 157 |
+
- localization
|
| 158 |
+
- comparison
|
| 159 |
+
- counting
|
| 160 |
+
- inference
|
| 161 |
+
- composition
|
| 162 |
+
- verification
|
| 163 |
+
include_visual_features: false # Extract visual features per step (memory intensive)
|
| 164 |
+
filter_incorrect: false # Keep all samples during training
|
| 165 |
+
confidence_threshold: 0.5 # Minimum confidence for step acceptance
|
| 166 |
+
|
| 167 |
+
# Inference-Time Scaling
|
| 168 |
+
inference_config:
|
| 169 |
+
enable_best_of_n: true
|
| 170 |
+
n_samples: 8 # Generate 8 chains and select best
|
| 171 |
+
enable_voting: true
|
| 172 |
+
voting_strategy: prm_weighted # Use PRM scores for voting
|
| 173 |
+
temperature: 0.7
|
| 174 |
+
top_p: 0.9
|
| 175 |
+
max_new_tokens: 1024
|
| 176 |
+
|
| 177 |
+
# Validation
|
| 178 |
+
validation:
|
| 179 |
+
enabled: true
|
| 180 |
+
val_split: 0.05 # Use 5% of data for validation
|
| 181 |
+
metrics:
|
| 182 |
+
- step_accuracy # Per-step correctness
|
| 183 |
+
- chain_accuracy # Full chain correctness
|
| 184 |
+
- final_answer_accuracy # Answer-only accuracy
|
| 185 |
+
- average_confidence # Model confidence
|
| 186 |
+
- average_reward # PRM reward scores
|
| 187 |
+
- reasoning_length # Average number of steps
|
| 188 |
+
|
| 189 |
+
# Hardware Configuration
|
| 190 |
+
device: cuda
|
| 191 |
+
num_gpus: 1 # Single GPU training (adjust for multi-GPU)
|
| 192 |
+
dataloader_pin_memory: true
|
| 193 |
+
dataloader_persistent_workers: true
|
| 194 |
+
|
| 195 |
+
# Logging and Monitoring
|
| 196 |
+
logging_steps: 50
|
| 197 |
+
eval_logging_steps: 10
|
| 198 |
+
save_logs: true
|
| 199 |
+
log_level: info
|
| 200 |
+
disable_tqdm: false
|
| 201 |
+
|
| 202 |
+
# Experiment Tracking
|
| 203 |
+
experiment_name: qwen_step_cot_dual_dataset
|
| 204 |
+
experiment_tags:
|
| 205 |
+
- step-level-cot
|
| 206 |
+
- chain-of-sight
|
| 207 |
+
- flux-reason
|
| 208 |
+
- finevision
|
| 209 |
+
- qwen-image
|
| 210 |
+
- dual-purpose # Document processing + image generation
|
| 211 |
+
|
| 212 |
+
# Reproducibility
|
| 213 |
+
seed: 42
|
| 214 |
+
deterministic: false # Set to true for full reproducibility (slower)
|
| 215 |
+
|
| 216 |
+
# Advanced Training Options
|
| 217 |
+
compile_model: false # PyTorch 2.0+ compilation (may not work with all models)
|
| 218 |
+
tf32: true # Use TF32 for better performance on Ampere+ GPUs
|
| 219 |
+
fsdp: false # Fully Sharded Data Parallel (for multi-GPU)
|
| 220 |
+
deepspeed: null # Path to DeepSpeed config (alternative to FSDP)
|
| 221 |
+
|
| 222 |
+
# Early Stopping
|
| 223 |
+
early_stopping:
|
| 224 |
+
enabled: true
|
| 225 |
+
patience: 3 # Stop if no improvement for 3 evaluations
|
| 226 |
+
metric: chain_accuracy
|
| 227 |
+
mode: max # Maximize chain accuracy
|
| 228 |
+
|
| 229 |
+
# Data Augmentation
|
| 230 |
+
augmentation:
|
| 231 |
+
enabled: true
|
| 232 |
+
random_horizontal_flip: 0.5
|
| 233 |
+
random_rotation: 15 # degrees
|
| 234 |
+
color_jitter: 0.2
|
| 235 |
+
random_crop: false # May interfere with spatial reasoning
|
| 236 |
+
|
| 237 |
+
# Document Processing Specific
|
| 238 |
+
document_processing:
|
| 239 |
+
enabled: true
|
| 240 |
+
ocr_integration: true # Extract text from images during inference
|
| 241 |
+
layout_analysis: true # Analyze document structure
|
| 242 |
+
table_detection: true # Detect and reason about tables
|
| 243 |
+
chart_understanding: true # Interpret charts and graphs
|
| 244 |
+
|
| 245 |
+
# Image Generation Specific
|
| 246 |
+
image_generation:
|
| 247 |
+
enabled: true
|
| 248 |
+
reasoning_guided: true # Use CoT reasoning to guide generation
|
| 249 |
+
progressive_generation: true # Generate step-by-step with intermediate feedback
|
| 250 |
+
quality_scoring: true # Use PRM to score generation quality
|
train_configs/train_step_cot.yaml
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Step-Level CoT Training Configuration
|
| 2 |
+
# Configuration for training vision-language models with fine-grained chain-of-thought reasoning
|
| 3 |
+
|
| 4 |
+
# Model settings
|
| 5 |
+
pretrained_model_name_or_path: "Qwen/Qwen-Image"
|
| 6 |
+
vision_dim: 768
|
| 7 |
+
text_dim: 768
|
| 8 |
+
|
| 9 |
+
# Output settings
|
| 10 |
+
output_dir: "./outputs/step_level_cot"
|
| 11 |
+
report_to: "tensorboard" # Options: tensorboard, wandb
|
| 12 |
+
|
| 13 |
+
# Training settings
|
| 14 |
+
train_batch_size: 4
|
| 15 |
+
gradient_accumulation_steps: 4
|
| 16 |
+
learning_rate: 5.0e-5
|
| 17 |
+
rl_learning_rate: 1.0e-5
|
| 18 |
+
prm_learning_rate: 3.0e-5
|
| 19 |
+
weight_decay: 0.01
|
| 20 |
+
warmup_steps: 1000
|
| 21 |
+
max_train_steps: 50000
|
| 22 |
+
mixed_precision: "fp16" # Options: no, fp16, bf16
|
| 23 |
+
gradient_checkpointing: true
|
| 24 |
+
|
| 25 |
+
# Process Reward Model (PRM) settings
|
| 26 |
+
prm_hidden_dim: 512
|
| 27 |
+
prm_num_heads: 8
|
| 28 |
+
prm_dropout: 0.1
|
| 29 |
+
max_reasoning_steps: 10
|
| 30 |
+
|
| 31 |
+
# Reinforcement Learning settings (PPO)
|
| 32 |
+
ppo_clip_epsilon: 0.2
|
| 33 |
+
value_loss_coef: 0.5
|
| 34 |
+
entropy_coef: 0.01
|
| 35 |
+
gamma: 0.99 # Discount factor
|
| 36 |
+
gae_lambda: 0.95 # GAE lambda
|
| 37 |
+
|
| 38 |
+
# Inference-time scaling settings
|
| 39 |
+
num_inference_samples: 8
|
| 40 |
+
inference_temperature: 0.7
|
| 41 |
+
aggregation_method: "best_of_n" # Options: best_of_n, majority_vote, weighted_vote
|
| 42 |
+
|
| 43 |
+
# Data settings
|
| 44 |
+
dataloader_num_workers: 4
|
| 45 |
+
max_seq_length: 512
|
| 46 |
+
|
| 47 |
+
# Logging settings
|
| 48 |
+
logging_steps: 10
|
| 49 |
+
save_steps: 1000
|
| 50 |
+
eval_steps: 500
|
| 51 |
+
save_total_limit: 5
|
| 52 |
+
|
| 53 |
+
# Evaluation settings
|
| 54 |
+
run_evaluation: true
|
| 55 |
+
eval_batch_size: 8
|
| 56 |
+
num_eval_samples: 100
|
| 57 |
+
|
| 58 |
+
# Ablation study settings
|
| 59 |
+
run_ablation: false
|
| 60 |
+
ablation_components:
|
| 61 |
+
- "prm"
|
| 62 |
+
- "rl"
|
| 63 |
+
- "inference_scaling"
|
| 64 |
+
- "full_system"
|
| 65 |
+
|
| 66 |
+
# Step-level reasoning data format
|
| 67 |
+
# Expected structure:
|
| 68 |
+
# data_dir/
|
| 69 |
+
# train/
|
| 70 |
+
# chain_0001.json
|
| 71 |
+
# chain_0002.json
|
| 72 |
+
# ...
|
| 73 |
+
# val/
|
| 74 |
+
# chain_0001.json
|
| 75 |
+
# ...
|
| 76 |
+
# test/
|
| 77 |
+
# chain_0001.json
|
| 78 |
+
# ...
|
| 79 |
+
#
|
| 80 |
+
# Each JSON file contains a ReasoningChain with:
|
| 81 |
+
# {
|
| 82 |
+
# "chain_id": "unique_id",
|
| 83 |
+
# "image_path": "path/to/image.jpg",
|
| 84 |
+
# "prompt": "What is shown in the image?",
|
| 85 |
+
# "steps": [
|
| 86 |
+
# {
|
| 87 |
+
# "step_id": 0,
|
| 88 |
+
# "step_type": "perception",
|
| 89 |
+
# "description": "I observe a red ball...",
|
| 90 |
+
# "confidence": 0.9,
|
| 91 |
+
# "reward": 0.8,
|
| 92 |
+
# "dependencies": []
|
| 93 |
+
# },
|
| 94 |
+
# ...
|
| 95 |
+
# ],
|
| 96 |
+
# "final_answer": "The image shows a red ball.",
|
| 97 |
+
# "total_reward": 2.4,
|
| 98 |
+
# "is_correct": true
|
| 99 |
+
# }
|
train_qwen_cot_dual.py
ADDED
|
@@ -0,0 +1,357 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Comprehensive training launcher for step-level CoT vision model.
|
| 3 |
+
|
| 4 |
+
This script orchestrates the full training pipeline:
|
| 5 |
+
1. Add special tokens to base model
|
| 6 |
+
2. Prepare datasets (FLUX-Reason-6M + FineVision)
|
| 7 |
+
3. Run SFT + PRM + RL training phases
|
| 8 |
+
4. Validate and save final model
|
| 9 |
+
|
| 10 |
+
Usage:
|
| 11 |
+
python train_qwen_cot_dual.py --config train_configs/train_qwen_cot_dual.yaml
|
| 12 |
+
|
| 13 |
+
For specific phases only:
|
| 14 |
+
python train_qwen_cot_dual.py --config CONFIG --phases sft prm
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
import argparse
|
| 18 |
+
import os
|
| 19 |
+
import sys
|
| 20 |
+
from pathlib import Path
|
| 21 |
+
import yaml
|
| 22 |
+
import torch
|
| 23 |
+
from typing import Dict, List, Optional, Any
|
| 24 |
+
import logging
|
| 25 |
+
|
| 26 |
+
# Add src to path
|
| 27 |
+
sys.path.insert(0, str(Path(__file__).parent))
|
| 28 |
+
|
| 29 |
+
from src.reasoning.step_level_cot import StepLevelCoTTrainer
|
| 30 |
+
from src.reasoning.flux_reason_dataset import FLUXReasonDataset, create_flux_reason_dataloader
|
| 31 |
+
from src.reasoning.finevision_dataset import FineVisionDataset, create_finevision_dataloader
|
| 32 |
+
from src.reasoning.step_data import ReasoningChain
|
| 33 |
+
from utils.add_special_tokens import add_special_tokens_to_model
|
| 34 |
+
|
| 35 |
+
# Set up logging
|
| 36 |
+
logging.basicConfig(
|
| 37 |
+
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
| 38 |
+
level=logging.INFO
|
| 39 |
+
)
|
| 40 |
+
logger = logging.getLogger(__name__)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def load_config(config_path: str) -> Dict[str, Any]:
|
| 44 |
+
"""Load training configuration from YAML file."""
|
| 45 |
+
with open(config_path, 'r') as f:
|
| 46 |
+
config = yaml.safe_load(f)
|
| 47 |
+
return config
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def prepare_model_with_tokens(config: Dict[str, Any]) -> str:
|
| 51 |
+
"""
|
| 52 |
+
Add special tokens to the base model and save to output directory.
|
| 53 |
+
|
| 54 |
+
Args:
|
| 55 |
+
config: Training configuration
|
| 56 |
+
|
| 57 |
+
Returns:
|
| 58 |
+
Path to the model with special tokens added
|
| 59 |
+
"""
|
| 60 |
+
logger.info("=" * 60)
|
| 61 |
+
logger.info("STEP 1: Adding Special Tokens to Base Model")
|
| 62 |
+
logger.info("=" * 60)
|
| 63 |
+
|
| 64 |
+
model_path = config['pretrained_model_name_or_path']
|
| 65 |
+
output_dir = Path(config['output_dir']) / 'model_with_tokens'
|
| 66 |
+
|
| 67 |
+
if output_dir.exists() and (output_dir / 'config.json').exists():
|
| 68 |
+
logger.info(f"Model with tokens already exists at {output_dir}")
|
| 69 |
+
logger.info("Skipping token addition step.")
|
| 70 |
+
return str(output_dir)
|
| 71 |
+
|
| 72 |
+
logger.info(f"Base model: {model_path}")
|
| 73 |
+
logger.info(f"Output directory: {output_dir}")
|
| 74 |
+
logger.info("Adding 27 special tokens (15 base + 12 region tokens)...")
|
| 75 |
+
|
| 76 |
+
try:
|
| 77 |
+
add_special_tokens_to_model(
|
| 78 |
+
model_path=model_path,
|
| 79 |
+
output_dir=str(output_dir),
|
| 80 |
+
prm_only=False, # Add all tokens
|
| 81 |
+
)
|
| 82 |
+
logger.info("β Successfully added special tokens!")
|
| 83 |
+
return str(output_dir)
|
| 84 |
+
except Exception as e:
|
| 85 |
+
logger.error(f"β Failed to add special tokens: {e}")
|
| 86 |
+
logger.info("Continuing with base model (tokens will be added by trainer)")
|
| 87 |
+
return model_path
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def prepare_datasets(config: Dict[str, Any], tokenizer: Any) -> Dict[str, Any]:
|
| 91 |
+
"""
|
| 92 |
+
Prepare FLUX-Reason-6M and FineVision datasets.
|
| 93 |
+
|
| 94 |
+
Args:
|
| 95 |
+
config: Training configuration
|
| 96 |
+
tokenizer: Model tokenizer
|
| 97 |
+
|
| 98 |
+
Returns:
|
| 99 |
+
Dictionary with dataloaders for each dataset
|
| 100 |
+
"""
|
| 101 |
+
logger.info("=" * 60)
|
| 102 |
+
logger.info("STEP 2: Preparing Datasets")
|
| 103 |
+
logger.info("=" * 60)
|
| 104 |
+
|
| 105 |
+
datasets_config = config.get('datasets', [])
|
| 106 |
+
data_config = config.get('data_config', {})
|
| 107 |
+
|
| 108 |
+
dataloaders = {}
|
| 109 |
+
dataset_weights = {}
|
| 110 |
+
|
| 111 |
+
for dataset_cfg in datasets_config:
|
| 112 |
+
dataset_name = dataset_cfg['name']
|
| 113 |
+
dataset_type = dataset_cfg['type']
|
| 114 |
+
weight = dataset_cfg.get('weight', 1.0)
|
| 115 |
+
|
| 116 |
+
logger.info(f"\nLoading {dataset_name} ({dataset_type})...")
|
| 117 |
+
logger.info(f" Weight: {weight}")
|
| 118 |
+
|
| 119 |
+
try:
|
| 120 |
+
if dataset_type == 'flux_reason_6m':
|
| 121 |
+
dataloader = create_flux_reason_dataloader(
|
| 122 |
+
tokenizer=tokenizer,
|
| 123 |
+
batch_size=data_config.get('train_batch_size', 2),
|
| 124 |
+
num_workers=data_config.get('num_workers', 4),
|
| 125 |
+
streaming=dataset_cfg.get('streaming', True),
|
| 126 |
+
max_samples=dataset_cfg.get('max_samples'),
|
| 127 |
+
dataset_name=dataset_cfg['dataset_name'],
|
| 128 |
+
split=dataset_cfg.get('split', 'train'),
|
| 129 |
+
image_key=dataset_cfg.get('image_key', 'image'),
|
| 130 |
+
reasoning_key=dataset_cfg.get('reasoning_key', 'reasoning'),
|
| 131 |
+
question_key=dataset_cfg.get('question_key', 'question'),
|
| 132 |
+
answer_key=dataset_cfg.get('answer_key', 'answer'),
|
| 133 |
+
)
|
| 134 |
+
dataloaders[dataset_name] = dataloader
|
| 135 |
+
dataset_weights[dataset_name] = weight
|
| 136 |
+
logger.info(f"β Loaded {dataset_name}")
|
| 137 |
+
|
| 138 |
+
elif dataset_type == 'finevision':
|
| 139 |
+
dataloader = create_finevision_dataloader(
|
| 140 |
+
tokenizer=tokenizer,
|
| 141 |
+
batch_size=data_config.get('train_batch_size', 2),
|
| 142 |
+
num_workers=data_config.get('num_workers', 4),
|
| 143 |
+
streaming=dataset_cfg.get('streaming', True),
|
| 144 |
+
max_samples=dataset_cfg.get('max_samples'),
|
| 145 |
+
dataset_name=dataset_cfg['dataset_name'],
|
| 146 |
+
subset=dataset_cfg.get('subset'),
|
| 147 |
+
split=dataset_cfg.get('split', 'train'),
|
| 148 |
+
image_key=dataset_cfg.get('image_key', 'image'),
|
| 149 |
+
text_key=dataset_cfg.get('text_key', 'text'),
|
| 150 |
+
generate_synthetic_reasoning=dataset_cfg.get('generate_synthetic_reasoning', True),
|
| 151 |
+
num_synthetic_steps=dataset_cfg.get('num_synthetic_steps', 3),
|
| 152 |
+
)
|
| 153 |
+
dataloaders[dataset_name] = dataloader
|
| 154 |
+
dataset_weights[dataset_name] = weight
|
| 155 |
+
logger.info(f"β Loaded {dataset_name}")
|
| 156 |
+
else:
|
| 157 |
+
logger.warning(f"Unknown dataset type: {dataset_type}")
|
| 158 |
+
|
| 159 |
+
except Exception as e:
|
| 160 |
+
logger.error(f"β Failed to load {dataset_name}: {e}")
|
| 161 |
+
logger.info("Continuing without this dataset...")
|
| 162 |
+
|
| 163 |
+
logger.info(f"\nβ Prepared {len(dataloaders)} datasets")
|
| 164 |
+
return {'dataloaders': dataloaders, 'weights': dataset_weights}
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def run_training_phase(
|
| 168 |
+
trainer: StepLevelCoTTrainer,
|
| 169 |
+
phase: str,
|
| 170 |
+
phase_config: Dict[str, Any],
|
| 171 |
+
dataloaders: Dict[str, Any],
|
| 172 |
+
) -> None:
|
| 173 |
+
"""
|
| 174 |
+
Run a specific training phase (SFT, PRM, or RL).
|
| 175 |
+
|
| 176 |
+
Args:
|
| 177 |
+
trainer: StepLevelCoTTrainer instance
|
| 178 |
+
phase: Training phase name ('sft', 'prm', or 'rl')
|
| 179 |
+
phase_config: Configuration for this phase
|
| 180 |
+
dataloaders: Dataset dataloaders
|
| 181 |
+
"""
|
| 182 |
+
logger.info("=" * 60)
|
| 183 |
+
logger.info(f"STEP 3: Running {phase.upper()} Training Phase")
|
| 184 |
+
logger.info("=" * 60)
|
| 185 |
+
|
| 186 |
+
if not phase_config.get('enabled', False):
|
| 187 |
+
logger.info(f"{phase.upper()} phase is disabled in config. Skipping...")
|
| 188 |
+
return
|
| 189 |
+
|
| 190 |
+
max_steps = phase_config.get('max_train_steps', 5000)
|
| 191 |
+
learning_rate = phase_config.get('learning_rate', 1e-5)
|
| 192 |
+
|
| 193 |
+
logger.info(f"Phase: {phase.upper()}")
|
| 194 |
+
logger.info(f"Max steps: {max_steps}")
|
| 195 |
+
logger.info(f"Learning rate: {learning_rate}")
|
| 196 |
+
|
| 197 |
+
try:
|
| 198 |
+
if phase == 'sft':
|
| 199 |
+
logger.info("\nRunning Supervised Fine-Tuning...")
|
| 200 |
+
trainer.train_sft(
|
| 201 |
+
max_steps=max_steps,
|
| 202 |
+
learning_rate=learning_rate,
|
| 203 |
+
)
|
| 204 |
+
logger.info("β SFT phase complete!")
|
| 205 |
+
|
| 206 |
+
elif phase == 'prm':
|
| 207 |
+
logger.info("\nTraining Process Reward Model...")
|
| 208 |
+
prm_config = phase_config.get('prm_config', {})
|
| 209 |
+
trainer.train_prm(
|
| 210 |
+
max_steps=max_steps,
|
| 211 |
+
learning_rate=learning_rate,
|
| 212 |
+
**prm_config,
|
| 213 |
+
)
|
| 214 |
+
logger.info("β PRM training complete!")
|
| 215 |
+
|
| 216 |
+
elif phase == 'rl':
|
| 217 |
+
logger.info("\nRunning Reinforcement Learning with PPO...")
|
| 218 |
+
ppo_config = phase_config.get('ppo_config', {})
|
| 219 |
+
trainer.train_rl(
|
| 220 |
+
max_steps=max_steps,
|
| 221 |
+
learning_rate=learning_rate,
|
| 222 |
+
**ppo_config,
|
| 223 |
+
)
|
| 224 |
+
logger.info("β RL training complete!")
|
| 225 |
+
|
| 226 |
+
else:
|
| 227 |
+
logger.warning(f"Unknown phase: {phase}")
|
| 228 |
+
|
| 229 |
+
except Exception as e:
|
| 230 |
+
logger.error(f"β {phase.upper()} phase failed: {e}")
|
| 231 |
+
logger.error("Stopping training pipeline.")
|
| 232 |
+
raise
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
def main():
|
| 236 |
+
parser = argparse.ArgumentParser(description="Train step-level CoT vision model")
|
| 237 |
+
parser.add_argument(
|
| 238 |
+
'--config',
|
| 239 |
+
type=str,
|
| 240 |
+
required=True,
|
| 241 |
+
help='Path to training configuration YAML file'
|
| 242 |
+
)
|
| 243 |
+
parser.add_argument(
|
| 244 |
+
'--phases',
|
| 245 |
+
nargs='+',
|
| 246 |
+
choices=['sft', 'prm', 'rl', 'all'],
|
| 247 |
+
default=['all'],
|
| 248 |
+
help='Training phases to run (default: all)'
|
| 249 |
+
)
|
| 250 |
+
parser.add_argument(
|
| 251 |
+
'--skip_token_addition',
|
| 252 |
+
action='store_true',
|
| 253 |
+
help='Skip special token addition step (if already done)'
|
| 254 |
+
)
|
| 255 |
+
parser.add_argument(
|
| 256 |
+
'--resume',
|
| 257 |
+
type=str,
|
| 258 |
+
default=None,
|
| 259 |
+
help='Resume from checkpoint directory'
|
| 260 |
+
)
|
| 261 |
+
|
| 262 |
+
args = parser.parse_args()
|
| 263 |
+
|
| 264 |
+
# Load configuration
|
| 265 |
+
logger.info("=" * 60)
|
| 266 |
+
logger.info("QWEN STEP-LEVEL COT TRAINING PIPELINE")
|
| 267 |
+
logger.info("=" * 60)
|
| 268 |
+
logger.info(f"Config: {args.config}")
|
| 269 |
+
logger.info(f"Phases: {args.phases}")
|
| 270 |
+
|
| 271 |
+
config = load_config(args.config)
|
| 272 |
+
|
| 273 |
+
# Determine which phases to run
|
| 274 |
+
if 'all' in args.phases:
|
| 275 |
+
phases_to_run = ['sft', 'prm', 'rl']
|
| 276 |
+
else:
|
| 277 |
+
phases_to_run = args.phases
|
| 278 |
+
|
| 279 |
+
# Step 1: Add special tokens (optional, can be skipped)
|
| 280 |
+
if args.skip_token_addition:
|
| 281 |
+
logger.info("\nSkipping special token addition (--skip_token_addition)")
|
| 282 |
+
model_path = config['pretrained_model_name_or_path']
|
| 283 |
+
else:
|
| 284 |
+
model_path = prepare_model_with_tokens(config)
|
| 285 |
+
|
| 286 |
+
# Update config with tokenized model path
|
| 287 |
+
config['pretrained_model_name_or_path'] = model_path
|
| 288 |
+
|
| 289 |
+
# Step 2: Initialize trainer
|
| 290 |
+
logger.info("\n" + "=" * 60)
|
| 291 |
+
logger.info("INITIALIZING TRAINER")
|
| 292 |
+
logger.info("=" * 60)
|
| 293 |
+
|
| 294 |
+
try:
|
| 295 |
+
trainer = StepLevelCoTTrainer(config)
|
| 296 |
+
logger.info("β Trainer initialized successfully!")
|
| 297 |
+
except Exception as e:
|
| 298 |
+
logger.error(f"β Failed to initialize trainer: {e}")
|
| 299 |
+
return 1
|
| 300 |
+
|
| 301 |
+
# Step 3: Prepare datasets
|
| 302 |
+
dataset_info = prepare_datasets(config, trainer.tokenizer)
|
| 303 |
+
|
| 304 |
+
# Attach datasets to trainer
|
| 305 |
+
trainer.dataloaders = dataset_info['dataloaders']
|
| 306 |
+
trainer.dataset_weights = dataset_info['weights']
|
| 307 |
+
|
| 308 |
+
# Step 4: Run training phases
|
| 309 |
+
training_phases = config.get('training_phases', [])
|
| 310 |
+
|
| 311 |
+
for phase_info in training_phases:
|
| 312 |
+
phase_name = phase_info['phase']
|
| 313 |
+
|
| 314 |
+
if phase_name not in phases_to_run:
|
| 315 |
+
logger.info(f"\nSkipping {phase_name.upper()} phase (not in --phases)")
|
| 316 |
+
continue
|
| 317 |
+
|
| 318 |
+
run_training_phase(
|
| 319 |
+
trainer=trainer,
|
| 320 |
+
phase=phase_name,
|
| 321 |
+
phase_config=phase_info,
|
| 322 |
+
dataloaders=dataset_info['dataloaders'],
|
| 323 |
+
)
|
| 324 |
+
|
| 325 |
+
# Step 5: Final model save
|
| 326 |
+
logger.info("\n" + "=" * 60)
|
| 327 |
+
logger.info("STEP 4: Saving Final Model")
|
| 328 |
+
logger.info("=" * 60)
|
| 329 |
+
|
| 330 |
+
final_output_dir = Path(config['output_dir']) / 'final_model'
|
| 331 |
+
final_output_dir.mkdir(parents=True, exist_ok=True)
|
| 332 |
+
|
| 333 |
+
try:
|
| 334 |
+
trainer.save_checkpoint(str(final_output_dir))
|
| 335 |
+
logger.info(f"β Final model saved to {final_output_dir}")
|
| 336 |
+
except Exception as e:
|
| 337 |
+
logger.error(f"β Failed to save final model: {e}")
|
| 338 |
+
|
| 339 |
+
# Summary
|
| 340 |
+
logger.info("\n" + "=" * 60)
|
| 341 |
+
logger.info("TRAINING PIPELINE COMPLETE!")
|
| 342 |
+
logger.info("=" * 60)
|
| 343 |
+
logger.info(f"Model: {config['pretrained_model_name_or_path']}")
|
| 344 |
+
logger.info(f"Phases completed: {phases_to_run}")
|
| 345 |
+
logger.info(f"Output directory: {config['output_dir']}")
|
| 346 |
+
logger.info(f"Final model: {final_output_dir}")
|
| 347 |
+
logger.info("\nNext steps:")
|
| 348 |
+
logger.info(" 1. Test inference: python inference_document_processor.py")
|
| 349 |
+
logger.info(" 2. Test generation: python inference_image_generator.py")
|
| 350 |
+
logger.info(" 3. Evaluate on benchmarks: python evaluate_reasoning.py")
|
| 351 |
+
|
| 352 |
+
return 0
|
| 353 |
+
|
| 354 |
+
|
| 355 |
+
if __name__ == '__main__':
|
| 356 |
+
exit_code = main()
|
| 357 |
+
sys.exit(exit_code)
|