mhsnar's picture
Add files using upload-large-folder tool
7985e7e verified
Raw History Blame Contribute Delete
6.04 kB
"""
ClassifierGuidancePolicy: standard classifier guidance during DDPM reverse process.
Evaluates the value function directly on the noisy x_t and shifts the DDPM
posterior mean by guidance_lr * posterior_var * grad(value(x_t)).
Single gradient evaluation per step (no inner loop).
Fully batched on GPU.
"""
import math
import time
from collections import namedtuple
import numpy as np
import torch
from torch import nn
from model import CosineScheduleDiffusion, apply_conditioning
from utils.arrays import to_np, to_torch, apply_dict
Trajectories = namedtuple("Trajectories", "actions observations values")
def _zero_conditioned_grad(g, conditions, action_dim):
"""Zero out gradient at conditioned observation positions."""
for t, val in conditions.items():
g[:, t, action_dim:] = 0.0
return g
class ClassifierGuidancePolicy(nn.Module):
"""DDPM sampling with classifier guidance.
Reverse-diffuses from t=1 to t=0, shifting the posterior mean by
the gradient of the value function evaluated on the noisy x_t,
scaled by the posterior variance.
"""
def __init__(
self,
model,
normalizer,
value_model,
action_dim,
state_dim,
horizon,
n_sampling_steps=10,
guidance_lr=0.05,
gradient_start=0.0,
device="cuda:0",
):
super().__init__()
self.model = model
self.normalizer = normalizer
self.value_model = value_model
self.action_dim = action_dim
self.state_dim = state_dim
self.horizon = horizon
self.n_sampling_steps = n_sampling_steps
self.guidance_lr = guidance_lr
self.gradient_start = gradient_start
self.device = device
self.diffusion = CosineScheduleDiffusion(model, action_dim)
def __call__(self, conditions, batch_size=1):
"""Sample trajectories with classifier guidance.
Same interface as UnconditionalPolicy.__call__.
"""
conditions = apply_dict(
self.normalizer.normalize, conditions, "observations"
)
t_start = time.time()
transition_dim = self.action_dim + self.state_dim
conditions_torch = to_torch(conditions, device=self.device)
# start from pure noise
x = torch.randn(
batch_size, self.horizon, transition_dim, device=self.device
)
x = apply_conditioning(x, conditions_torch, self.action_dim)
# time grid: t=1 (noisy) to t=0 (clean)
timesteps = torch.linspace(1.0, 0.0, self.n_sampling_steps + 1, device=self.device)
save_chain = (batch_size == 1)
if save_chain:
denoising_chain = [x.clone()]
x0_estimates = []
for i in range(self.n_sampling_steps):
t_now = timesteps[i]
t_next = timesteps[i + 1]
is_last_step = (i == self.n_sampling_steps - 1)
# --- standard DDPM x0 prediction ---
abar_now = self.diffusion.alpha_bar(t_now)
abar_next = self.diffusion.alpha_bar(t_next)
t_batch = t_now.expand(batch_size)
x0_pred = self.model(x, t_batch)
x0_pred = apply_conditioning(x0_pred, conditions_torch, self.action_dim)
if save_chain:
x0_estimates.append(x0_pred.detach().clone())
# --- DDPM posterior ---
beta = 1.0 - abar_now / abar_next
coeff_x0 = math.sqrt(abar_next) * beta / (1.0 - abar_now)
coeff_xt = math.sqrt(1.0 - beta) * (1.0 - abar_next) / (1.0 - abar_now)
posterior_mean = coeff_x0 * x0_pred + coeff_xt * x
posterior_var = max((1.0 - abar_next) / (1.0 - abar_now) * beta, 0.0)
# --- classifier guidance: shift posterior mean ---
frac_done = i / self.n_sampling_steps
if frac_done >= self.gradient_start and posterior_var > 0:
x_in = x.detach().clone().requires_grad_(True)
value = self.value_model(x_in) # evaluate on noisy x_t
loss = -value.sum()
g = torch.autograd.grad(loss, x_in)[0]
g = _zero_conditioned_grad(g, conditions_torch, self.action_dim)
posterior_mean = posterior_mean + self.guidance_lr * posterior_var * g
# --- sample ---
if is_last_step:
x = posterior_mean
else:
noise = torch.randn_like(x)
x = posterior_mean + math.sqrt(posterior_var) * noise
x = apply_conditioning(x, conditions_torch, self.action_dim)
x = x.detach()
if save_chain:
denoising_chain.append(x.clone())
# unnormalize
normed_actions = x[:, :, :self.action_dim]
actions = self.normalizer.unnormalize(to_np(normed_actions), "actions")
normed_observations = x[:, :, self.action_dim:]
observations = self.normalizer.unnormalize(to_np(normed_observations), "observations")
trajectories = Trajectories(actions, observations, np.zeros((batch_size, 1)))
t_end = time.time()
info = {"computation_time": t_end - t_start}
if save_chain:
chain = torch.stack(denoising_chain, dim=1)
chain = self._unnormalize_chain(chain)
x0_est = torch.stack(x0_estimates, dim=1)
x0_est = self._unnormalize_chain(x0_est)
return actions[:, 0], trajectories, chain, x0_est, info
else:
return actions[:, 0], trajectories, None, None, info
def _unnormalize_chain(self, chain):
"""Unnormalize a chain of shape (batch, steps, horizon, transition_dim)."""
obs_chain = chain[..., self.action_dim:]
obs_unnorm = self.normalizer.unnormalize(to_np(obs_chain), "observations")
act_chain = chain[..., :self.action_dim]
act_unnorm = self.normalizer.unnormalize(to_np(act_chain), "actions")
return np.concatenate([act_unnorm, obs_unnorm], axis=-1)