Download src/reasoning/inference_scaling.py from Mulebot/dei-model: direct link, hf CLI and curl.
- Browser
- Download file 15.4 kB
-
https://huggingface.co/Mulebot/dei-model/resolve/main/src/reasoning/inference_scaling.py
- Command line
-
hf download hf://Mulebot/dei-model/src/reasoning/inference_scaling.py
-
curl -L -o inference_scaling.py https://huggingface.co/Mulebot/dei-model/resolve/main/src/reasoning/inference_scaling.py
15.4 kB
| """ | |
| Inference-time scaling mechanisms for step-level reasoning. | |
| Implements best-of-N sampling, majority voting, and self-consistency | |
| to improve reasoning quality through test-time compute. | |
| """ | |
| from typing import Dict, List, Optional, Tuple, Any, Callable | |
| import torch | |
| import torch.nn as nn | |
| from collections import Counter | |
| import logging | |
| from dataclasses import dataclass | |
| from .prm import ProcessRewardModel | |
| from .step_data import ReasoningChain, ReasoningStep | |
| logger = logging.getLogger(__name__) | |
| class InferenceConfig: | |
| """Configuration for inference-time scaling.""" | |
| num_samples: int = 8 # Number of reasoning chains to generate | |
| temperature: float = 0.7 # Sampling temperature | |
| top_k: int = 50 | |
| top_p: float = 0.95 | |
| max_steps: int = 10 # Maximum reasoning steps | |
| # Aggregation method | |
| aggregation: str = "best_of_n" # Options: best_of_n, majority_vote, weighted_vote | |
| # PRM-based selection | |
| use_prm_scores: bool = True | |
| min_prm_score: float = 0.3 # Minimum acceptable PRM score | |
| # Self-consistency | |
| consistency_threshold: float = 0.6 # Required agreement for majority vote | |
| class InferenceTimeScaling: | |
| """ | |
| Inference-time scaling for improved reasoning quality. | |
| Generates multiple reasoning chains and selects the best one using | |
| various aggregation strategies. | |
| """ | |
| def __init__( | |
| self, | |
| model: nn.Module, | |
| prm: Optional[ProcessRewardModel] = None, | |
| config: Optional[InferenceConfig] = None, | |
| device: str = "cuda", | |
| ): | |
| """ | |
| Initialize inference scaler. | |
| Args: | |
| model: Vision-language model | |
| prm: Process Reward Model for evaluation | |
| config: Inference configuration | |
| device: Device for inference | |
| """ | |
| self.model = model | |
| self.prm = prm | |
| self.config = config or InferenceConfig() | |
| self.device = device | |
| # Move to device only if not already quantized with device_map | |
| if not (hasattr(self.model, 'hf_device_map') or | |
| getattr(self.model, 'is_quantized', False)): | |
| self.model.to(device) | |
| self.model.eval() | |
| if self.prm: | |
| if not (hasattr(self.prm, 'hf_device_map') or | |
| getattr(self.prm, 'is_quantized', False)): | |
| self.prm.to(device) | |
| self.prm.eval() | |
| def generate_multiple_chains( | |
| self, | |
| image_features: torch.Tensor, | |
| prompt_ids: torch.Tensor, | |
| prompt_mask: torch.Tensor, | |
| image_path: str = "", | |
| prompt_text: str = "", | |
| ) -> List[ReasoningChain]: | |
| """ | |
| Generate multiple reasoning chains with different samples. | |
| Args: | |
| image_features: Visual features | |
| prompt_ids: Tokenized prompt | |
| prompt_mask: Attention mask | |
| image_path: Path to image (for metadata) | |
| prompt_text: Original prompt text | |
| Returns: | |
| List of reasoning chains | |
| """ | |
| chains = [] | |
| for sample_idx in range(self.config.num_samples): | |
| steps = [] | |
| current_ids = prompt_ids.unsqueeze(0).to(self.device) | |
| current_mask = prompt_mask.unsqueeze(0).to(self.device) | |
| step_embeddings = [] | |
| for step_id in range(self.config.max_steps): | |
| # Generate next step | |
| outputs = self.model( | |
| input_ids=current_ids, | |
| attention_mask=current_mask, | |
| output_hidden_states=True, | |
| ) | |
| logits = outputs.logits[:, -1, :] / self.config.temperature | |
| probs = torch.softmax(logits, dim=-1) | |
| # Top-k sampling | |
| if self.config.top_k > 0: | |
| top_k_probs, top_k_indices = torch.topk(probs, self.config.top_k) | |
| next_token = top_k_indices[0, torch.multinomial(top_k_probs[0], 1)] | |
| else: | |
| next_token = torch.multinomial(probs[0], 1) | |
| # Get hidden state for this step | |
| hidden_state = outputs.hidden_states[-1][:, -1, :].squeeze(0) | |
| step_embeddings.append(hidden_state) | |
| # Compute PRM reward if available | |
| reward = 0.0 | |
| if self.prm: | |
| reward, metrics = self.prm.compute_step_reward( | |
| image_features, | |
| hidden_state, | |
| previous_steps=torch.stack(step_embeddings[:-1]) if len(step_embeddings) > 1 else None, | |
| return_metrics=True, | |
| ) | |
| # Decode step text | |
| step_text = self.model.config.tokenizer.decode( | |
| next_token.unsqueeze(0), | |
| skip_special_tokens=True, | |
| ) | |
| # Create reasoning step | |
| step = ReasoningStep( | |
| step_id=step_id, | |
| step_type="inference", # Would be classified in practice | |
| description=step_text, | |
| visual_features=hidden_state.cpu(), | |
| confidence=metrics.confidence if self.prm and metrics else 0.5, | |
| reward=reward, | |
| ) | |
| steps.append(step) | |
| # Update sequence | |
| current_ids = torch.cat([current_ids, next_token.unsqueeze(0).unsqueeze(0)], dim=1) | |
| current_mask = torch.cat([ | |
| current_mask, | |
| torch.ones((1, 1), device=self.device) | |
| ], dim=1) | |
| # Check for completion | |
| if next_token.item() == self.model.config.eos_token_id: | |
| break | |
| # Create reasoning chain | |
| final_answer = self.model.config.tokenizer.decode( | |
| current_ids[0, len(prompt_ids):], | |
| skip_special_tokens=True, | |
| ) | |
| total_reward = sum(s.reward for s in steps) | |
| chain = ReasoningChain( | |
| chain_id=f"sample_{sample_idx}", | |
| image_path=image_path, | |
| prompt=prompt_text, | |
| steps=steps, | |
| final_answer=final_answer, | |
| total_reward=total_reward, | |
| ) | |
| chains.append(chain) | |
| return chains | |
| def best_of_n_selection( | |
| self, | |
| chains: List[ReasoningChain], | |
| ) -> ReasoningChain: | |
| """ | |
| Select best chain based on PRM scores. | |
| Args: | |
| chains: List of reasoning chains | |
| Returns: | |
| Chain with highest total reward | |
| """ | |
| if not chains: | |
| raise ValueError("No chains to select from") | |
| # Filter by minimum PRM score if required | |
| if self.config.use_prm_scores and self.prm: | |
| filtered_chains = [ | |
| c for c in chains | |
| if c.total_reward >= self.config.min_prm_score * len(c.steps) | |
| ] | |
| if filtered_chains: | |
| chains = filtered_chains | |
| # Select chain with highest total reward | |
| best_chain = max(chains, key=lambda c: c.total_reward) | |
| logger.info( | |
| f"Best-of-N: Selected chain with reward {best_chain.total_reward:.3f} " | |
| f"({len(best_chain.steps)} steps)" | |
| ) | |
| return best_chain | |
| def majority_voting( | |
| self, | |
| chains: List[ReasoningChain], | |
| ) -> ReasoningChain: | |
| """ | |
| Select answer by majority voting with self-consistency. | |
| Args: | |
| chains: List of reasoning chains | |
| Returns: | |
| Chain corresponding to majority answer | |
| """ | |
| if not chains: | |
| raise ValueError("No chains to vote on") | |
| # Extract final answers | |
| answers = [c.final_answer.strip().lower() for c in chains] | |
| # Count votes | |
| vote_counts = Counter(answers) | |
| majority_answer, majority_count = vote_counts.most_common(1)[0] | |
| # Check consistency threshold | |
| consistency = majority_count / len(chains) | |
| if consistency < self.config.consistency_threshold: | |
| logger.warning( | |
| f"Low consistency: {consistency:.2f} < {self.config.consistency_threshold:.2f}" | |
| ) | |
| # Find chain with majority answer and highest PRM score | |
| majority_chains = [ | |
| c for c in chains | |
| if c.final_answer.strip().lower() == majority_answer | |
| ] | |
| best_chain = max(majority_chains, key=lambda c: c.total_reward) | |
| logger.info( | |
| f"Majority Vote: '{majority_answer}' ({majority_count}/{len(chains)}) " | |
| f"with consistency {consistency:.2f}" | |
| ) | |
| return best_chain | |
| def weighted_voting( | |
| self, | |
| chains: List[ReasoningChain], | |
| ) -> ReasoningChain: | |
| """ | |
| Weighted voting using PRM scores as weights. | |
| Args: | |
| chains: List of reasoning chains | |
| Returns: | |
| Chain with highest weighted vote | |
| """ | |
| if not chains or not self.prm: | |
| return self.best_of_n_selection(chains) | |
| # Group chains by answer | |
| answer_groups: Dict[str, List[ReasoningChain]] = {} | |
| for chain in chains: | |
| answer_key = chain.final_answer.strip().lower() | |
| if answer_key not in answer_groups: | |
| answer_groups[answer_key] = [] | |
| answer_groups[answer_key].append(chain) | |
| # Calculate weighted votes | |
| weighted_votes = {} | |
| for answer, group_chains in answer_groups.items(): | |
| # Use exponential of reward as weight (emphasizes high-quality chains) | |
| weight = sum(torch.exp(torch.tensor(c.total_reward / len(c.steps))).item() | |
| for c in group_chains) | |
| weighted_votes[answer] = weight | |
| # Select answer with highest weighted vote | |
| best_answer = max(weighted_votes.items(), key=lambda x: x[1])[0] | |
| # Return best chain for that answer | |
| best_chains = answer_groups[best_answer] | |
| best_chain = max(best_chains, key=lambda c: c.total_reward) | |
| logger.info( | |
| f"Weighted Vote: '{best_answer}' with weight {weighted_votes[best_answer]:.3f}" | |
| ) | |
| return best_chain | |
| def __call__( | |
| self, | |
| image_features: torch.Tensor, | |
| prompt_ids: torch.Tensor, | |
| prompt_mask: torch.Tensor, | |
| image_path: str = "", | |
| prompt_text: str = "", | |
| ) -> Tuple[ReasoningChain, List[ReasoningChain]]: | |
| """ | |
| Perform inference with scaling. | |
| Args: | |
| image_features: Visual features | |
| prompt_ids: Tokenized prompt | |
| prompt_mask: Attention mask | |
| image_path: Path to image | |
| prompt_text: Original prompt | |
| Returns: | |
| - best_chain: Selected reasoning chain | |
| - all_chains: All generated chains for analysis | |
| """ | |
| # Generate multiple chains | |
| chains = self.generate_multiple_chains( | |
| image_features, | |
| prompt_ids, | |
| prompt_mask, | |
| image_path, | |
| prompt_text, | |
| ) | |
| # Select best chain based on aggregation method | |
| if self.config.aggregation == "best_of_n": | |
| best_chain = self.best_of_n_selection(chains) | |
| elif self.config.aggregation == "majority_vote": | |
| best_chain = self.majority_voting(chains) | |
| elif self.config.aggregation == "weighted_vote": | |
| best_chain = self.weighted_voting(chains) | |
| else: | |
| raise ValueError(f"Unknown aggregation method: {self.config.aggregation}") | |
| return best_chain, chains | |
| class MajorityVoting: | |
| """ | |
| Standalone majority voting for answer aggregation. | |
| Useful for post-processing multiple model predictions. | |
| """ | |
| def vote( | |
| answers: List[str], | |
| weights: Optional[List[float]] = None, | |
| threshold: float = 0.5, | |
| ) -> Tuple[str, float]: | |
| """ | |
| Perform majority voting on answers. | |
| Args: | |
| answers: List of answer strings | |
| weights: Optional weights for each answer | |
| threshold: Minimum agreement threshold | |
| Returns: | |
| - majority_answer: Most common answer | |
| - confidence: Voting confidence [0, 1] | |
| """ | |
| if not answers: | |
| return "", 0.0 | |
| if weights is None: | |
| weights = [1.0] * len(answers) | |
| # Normalize answers | |
| normalized = [a.strip().lower() for a in answers] | |
| # Weighted voting | |
| vote_dict: Dict[str, float] = {} | |
| for answer, weight in zip(normalized, weights): | |
| vote_dict[answer] = vote_dict.get(answer, 0.0) + weight | |
| # Find majority | |
| total_weight = sum(weights) | |
| majority_answer = max(vote_dict.items(), key=lambda x: x[1])[0] | |
| confidence = vote_dict[majority_answer] / total_weight | |
| # Check threshold | |
| if confidence < threshold: | |
| logger.warning( | |
| f"Majority voting confidence {confidence:.2f} below threshold {threshold}" | |
| ) | |
| return majority_answer, confidence | |
| def self_consistency( | |
| chains: List[ReasoningChain], | |
| min_agreement: float = 0.6, | |
| ) -> Tuple[Optional[str], float, List[ReasoningChain]]: | |
| """ | |
| Self-consistency checking across reasoning chains. | |
| Args: | |
| chains: List of reasoning chains | |
| min_agreement: Minimum agreement required | |
| Returns: | |
| - consensus_answer: Answer if consensus reached, else None | |
| - agreement: Agreement ratio [0, 1] | |
| - supporting_chains: Chains supporting consensus | |
| """ | |
| if not chains: | |
| return None, 0.0, [] | |
| # Extract answers | |
| answers = [c.final_answer.strip().lower() for c in chains] | |
| # Count occurrences | |
| answer_counts = Counter(answers) | |
| most_common_answer, count = answer_counts.most_common(1)[0] | |
| # Calculate agreement | |
| agreement = count / len(chains) | |
| # Check if consensus reached | |
| consensus = most_common_answer if agreement >= min_agreement else None | |
| # Find supporting chains | |
| supporting_chains = [ | |
| c for c in chains | |
| if c.final_answer.strip().lower() == most_common_answer | |
| ] | |
| logger.info( | |
| f"Self-consistency: {agreement:.1%} agreement " | |
| f"({'consensus' if consensus else 'no consensus'})" | |
| ) | |
| return consensus, agreement, supporting_chains | |