Download Reward_sdxl_idealized/gradient_ascent_utils.py from aryadomain/all_code_base: direct link, hf CLI and curl.
- Browser
- Download file 13.3 kB
-
https://huggingface.co/aryadomain/all_code_base/resolve/main/Reward_sdxl_idealized/gradient_ascent_utils.py
- Command line
-
hf download hf://aryadomain/all_code_base/Reward_sdxl_idealized/gradient_ascent_utils.py
-
curl -L -o gradient_ascent_utils.py https://huggingface.co/aryadomain/all_code_base/resolve/main/Reward_sdxl_idealized/gradient_ascent_utils.py
13.3 kB
| """ | |
| Gradient Ascent utilities for reward-guided diffusion generation. | |
| This module implements gradient ascent on the LRM reward score to guide | |
| the diffusion process toward higher preference scores. | |
| """ | |
| import torch | |
| import torch.nn.functional as F | |
| from typing import Optional, Tuple, List, Literal | |
| from tqdm import tqdm | |
| from lr_scheduler import create_lr_scheduler, LRScheduler | |
| class RewardGuidedDiffusion: | |
| """ | |
| Implements reward-guided generation using gradient ascent. | |
| During denoising, at specified timesteps, we: | |
| 1. Compute the reward score for current latents | |
| 2. Calculate gradients of reward w.r.t. latents | |
| 3. Update latents in the direction that increases reward | |
| This guides generation toward higher preference scores. | |
| """ | |
| def __init__( | |
| self, | |
| reward_model, | |
| grad_scale: float = 1.0, | |
| grad_timestep_range: Optional[Tuple[int, int]] = None, | |
| num_grad_steps: int = 5, | |
| grad_step_size: float = 0.1, | |
| gradient_checkpoint: bool = False, | |
| # LR Scheduling | |
| lr_scheduler_type: Literal["constant", "linear", "cosine", "exponential", "step"] = "constant", | |
| lr_scheduler_kwargs: Optional[dict] = None, | |
| # Momentum | |
| use_momentum: bool = False, | |
| momentum: float = 0.9, | |
| use_nesterov: bool = False, | |
| use_iso_projection: bool = False | |
| ): | |
| """ | |
| Initialize reward-guided diffusion. | |
| Args: | |
| reward_model: LRM reward model for computing preference scores | |
| grad_scale: Scale factor for gradient updates (default: 1.0) | |
| grad_timestep_range: Tuple of (min_t, max_t) for gradient ascent. | |
| If None, applies to all timesteps. | |
| num_grad_steps: Number of gradient ascent steps per timestep | |
| grad_step_size: Step size for each gradient update (initial LR) | |
| gradient_checkpoint: Whether to use gradient checkpointing | |
| lr_scheduler_type: Type of LR scheduler ("constant", "linear", "cosine", "exponential", "step") | |
| lr_scheduler_kwargs: Additional kwargs for LR scheduler (e.g., end_lr, min_lr, warmup_steps) | |
| use_momentum: Whether to use momentum in gradient updates | |
| momentum: Momentum coefficient (typically 0.9) | |
| use_nesterov: Whether to use Nesterov momentum | |
| use_iso_projection: Whether to use Iso Projection | |
| """ | |
| self.reward_model = reward_model | |
| self.grad_scale = grad_scale | |
| self.grad_timestep_range = grad_timestep_range | |
| self.num_grad_steps = num_grad_steps | |
| self.grad_step_size = grad_step_size | |
| self.gradient_checkpoint = gradient_checkpoint | |
| # LR Scheduler | |
| self.lr_scheduler_type = lr_scheduler_type | |
| self.lr_scheduler_kwargs = lr_scheduler_kwargs or {} | |
| self.lr_scheduler: Optional[LRScheduler] = None | |
| self.global_lr_scheduler: Optional[LRScheduler] = None # Scheduler across denoising timesteps | |
| # Momentum | |
| self.use_momentum = use_momentum | |
| self.momentum = momentum | |
| self.use_nesterov = use_nesterov | |
| self.velocity = None # Will be initialized per optimization | |
| self.use_iso_projection = use_iso_projection | |
| # Statistics | |
| self.grad_stats = [] | |
| self.timestep_counter = 0 # Track which timestep we're on | |
| def should_apply_gradient(self, timestep: int) -> bool: | |
| """Check if gradient ascent should be applied at this timestep.""" | |
| if self.grad_timestep_range is None: | |
| return False | |
| min_t, max_t = self.grad_timestep_range | |
| return min_t <= timestep <= max_t | |
| def compute_reward_gradient( | |
| self, | |
| latents: torch.Tensor, | |
| prompt: str, | |
| timestep: int, | |
| ) -> Tuple[torch.Tensor, float]: | |
| """ | |
| Compute gradient of reward score w.r.t. latents in FP32 to prevent underflow. | |
| """ | |
| # 1. Cast to FP32 and ensure we are detached from previous iterations | |
| latents_fp32 = latents.detach().to(torch.float32).clone() | |
| latents_fp32.requires_grad_(True) | |
| # 2. Compute reward score | |
| # Note: Even if the model internally uses fp16/bf16, autograd will | |
| # safely accumulate the gradient in fp32 for our leaf node. | |
| reward_score = self.reward_model.get_reward_score( | |
| latents_fp32, | |
| prompt, | |
| timestep, | |
| enable_grad=True | |
| ) | |
| if reward_score.numel() > 1: | |
| reward_score = reward_score.mean() | |
| # 3. Extract gradient | |
| # CRITICAL: retain_graph=True prevents the graph from dying across multiple | |
| # gradient steps if your reward model relies on cached text embeddings. | |
| grad = torch.autograd.grad( | |
| outputs=reward_score, | |
| inputs=latents_fp32, | |
| create_graph=False, | |
| retain_graph=True, # Keeps the graph alive for the next step! | |
| allow_unused=True, | |
| )[0] | |
| # 4. Handle None gradients and cast back to the pipeline's original dtype | |
| if grad is None: | |
| grad = torch.zeros_like(latents) | |
| else: | |
| grad = grad.to(latents.dtype) | |
| return grad, reward_score.item() | |
| def apply_gradient_ascent( | |
| self, | |
| latents: torch.Tensor, | |
| prompt: str, | |
| timestep: int, | |
| base_noise: Optional[torch.Tensor] = None, # Required for Iso-Marginal projection | |
| verbose: bool = True, | |
| total_denoising_steps: Optional[int] = None, | |
| ) -> Tuple[torch.Tensor, dict]: | |
| # 1. UPCAST TO FP32 AND SETUP OPTIMIZER (Targeting Latents) | |
| original_latents = latents.detach().clone().to(torch.float32) | |
| current_latents = torch.nn.Parameter(original_latents.clone()) | |
| self.reward_model.unet.conv_in.weight.requires_grad_(True) | |
| # Initial reward tracking | |
| with torch.no_grad(): | |
| initial_reward = self.reward_model.get_reward_score( | |
| latents, | |
| prompt, | |
| timestep | |
| ) | |
| initial_reward_val = initial_reward.item() if initial_reward.numel() == 1 else initial_reward.mean().item() | |
| # Initialize tracking lists | |
| grad_norms = [] | |
| reward_history = [initial_reward_val] | |
| lr_history = [] | |
| # 2. FORWARD PASS (downcast to FP16 just for the model forward pass) | |
| reward = self.reward_model.get_reward_score( | |
| current_latents.to(latents.dtype), | |
| prompt, | |
| timestep, | |
| enable_grad=True | |
| ) | |
| loss = -reward.mean() | |
| loss.backward() | |
| # Extract latent gradient | |
| raw_grad = current_latents.grad | |
| reward_history.append(reward.mean().item()) | |
| # 3. ISO-MARGINAL PROJECTION WITH ASYMMETRIC INCLUSION | |
| if raw_grad is not None and base_noise is not None and self.use_iso_projection: | |
| gamma = 1e-8 | |
| B = raw_grad.shape[0] | |
| grad_flat = raw_grad.view(B, -1) | |
| noise_flat = base_noise.view(B, -1).to(torch.float32) | |
| # Compute projection scalar for raw_grad (which is -?R) | |
| dot_product = (grad_flat * noise_flat).sum(dim=1, keepdim=True) | |
| noise_norm_sq = (noise_flat * noise_flat).sum(dim=1, keepdim=True) | |
| proj_scalar = dot_product / (noise_norm_sq + gamma) | |
| proj_scalar = proj_scalar.view(B, 1, 1, 1) | |
| # 1. Decompose | |
| grad_parallel = proj_scalar * base_noise.to(torch.float32) | |
| grad_perp = raw_grad - grad_parallel | |
| # 2. Asymmetric Inclusion | |
| # proj_scalar > 0 means the applied step (+?R) points toward -epsilon (Denoising. GOOD.) | |
| # proj_scalar < 0 means the applied step (+?R) points toward +epsilon (Noising. BAD.) | |
| safe_proj_scalar = torch.clamp(proj_scalar, min=0.0) | |
| beta = 1.0 # Retention factor for the safe parallel gradient | |
| safe_grad_parallel = beta * (safe_proj_scalar * base_noise.to(torch.float32)) | |
| # 3. Recombine | |
| #grad_perp = grad_perp + safe_grad_parallel | |
| else: | |
| grad_perp = raw_grad | |
| if base_noise is None: | |
| print("?? WARNING: base_noise missing. Skipping Iso-Marginal projection.") | |
| # 4. KINETIC RECTIFICATION (Applied to the projected latent gradient) | |
| if grad_perp is not None: | |
| max_grad = grad_perp.norm().item() | |
| if max_grad > 0: | |
| kinetic_direction = grad_perp / (max_grad + 1e-8) | |
| # Because the max element is 1.0, alpha is the EXACT float32 change applied. | |
| alpha = self.grad_step_size | |
| with torch.no_grad(): | |
| rectified_latents = original_latents - (alpha * kinetic_direction) | |
| else: | |
| print("?? WARNING: Gradient exists but max value is 0.0") | |
| rectified_latents = original_latents.clone() | |
| alpha = 0.0 | |
| else: | |
| print("?? FATAL: PyTorch completely dropped the latent gradient!") | |
| rectified_latents = original_latents.clone() | |
| max_grad = 0.0 | |
| alpha = 0.0 | |
| if verbose: | |
| print(f" Grad step | LR: {alpha:.6f} | Reward: {reward.mean().item():.4f} | Max Grad: {max_grad:.4f}") | |
| # 5. DOWNCAST AND RETURN | |
| final_latents = rectified_latents.detach().to(latents.dtype) | |
| with torch.no_grad(): | |
| final_reward = self.reward_model.get_reward_score( | |
| final_latents, prompt, timestep | |
| ) | |
| final_reward_val = final_reward.item() if final_reward.numel() == 1 else final_reward.mean().item() | |
| stats = { | |
| 'timestep': timestep, | |
| 'initial_reward': initial_reward_val, | |
| 'final_reward': final_reward_val, | |
| 'reward_improvement': final_reward_val - initial_reward_val, | |
| 'grad_norms': [max_grad], | |
| 'reward_history': reward_history, | |
| 'lr_history': [alpha], # Kept for plotting logic | |
| 'latent_change': (final_latents - original_latents.to(latents.dtype)).norm().item(), | |
| } | |
| self.grad_stats.append(stats) | |
| return final_latents, stats | |
| def get_statistics(self) -> dict: | |
| """Get aggregated statistics across all gradient ascent applications.""" | |
| if not self.grad_stats: | |
| return {} | |
| total_improvement = sum(s['reward_improvement'] for s in self.grad_stats) | |
| avg_improvement = total_improvement / len(self.grad_stats) | |
| all_grad_norms = [n for s in self.grad_stats for n in s['grad_norms']] | |
| return { | |
| 'num_applications': len(self.grad_stats), | |
| 'total_reward_improvement': total_improvement, | |
| 'avg_reward_improvement': avg_improvement, | |
| 'avg_grad_norm': sum(all_grad_norms) / len(all_grad_norms) if all_grad_norms else 0, | |
| 'max_grad_norm': max(all_grad_norms) if all_grad_norms else 0, | |
| 'detailed_stats': self.grad_stats, | |
| } | |
| def reset_statistics(self): | |
| """Reset statistics and global scheduler.""" | |
| self.grad_stats = [] | |
| self.global_lr_scheduler = None | |
| self.timestep_counter = 0 | |
| def create_reward_guided_generator( | |
| reward_model, | |
| grad_timestep_range: Tuple[int, int] = (500, 700), | |
| grad_scale: float = 1.0, | |
| num_grad_steps: int = 5, | |
| grad_step_size: float = 0.1, | |
| lr_scheduler_type: str = "constant", | |
| lr_scheduler_kwargs: Optional[dict] = None, | |
| use_momentum: bool = False, | |
| momentum: float = 0.9, | |
| use_nesterov: bool = False, | |
| use_iso_projection: bool = False | |
| ) -> RewardGuidedDiffusion: | |
| """ | |
| Convenience function to create a reward-guided diffusion generator. | |
| Args: | |
| reward_model: LRM reward model | |
| grad_timestep_range: Tuple of (min_t, max_t) for applying gradients | |
| grad_scale: Scale factor for gradient magnitude | |
| num_grad_steps: Number of gradient ascent iterations per timestep | |
| grad_step_size: Step size for each gradient update (initial LR) | |
| lr_scheduler_type: Type of LR scheduler | |
| lr_scheduler_kwargs: Additional kwargs for LR scheduler | |
| use_momentum: Whether to use momentum | |
| momentum: Momentum coefficient | |
| use_nesterov: Whether to use Nesterov momentum | |
| use_iso_projection: Whether to use Iso Projection | |
| Returns: | |
| RewardGuidedDiffusion instance | |
| """ | |
| return RewardGuidedDiffusion( | |
| reward_model=reward_model, | |
| grad_scale=grad_scale, | |
| grad_timestep_range=grad_timestep_range, | |
| num_grad_steps=num_grad_steps, | |
| grad_step_size=grad_step_size, | |
| lr_scheduler_type=lr_scheduler_type, | |
| lr_scheduler_kwargs=lr_scheduler_kwargs, | |
| use_momentum=use_momentum, | |
| momentum=momentum, | |
| use_nesterov=use_nesterov, | |
| use_iso_projection= False | |
| ) | |