onnx / tests /test_moss_image_models.py
trixlyaisolutions's picture
Upload folder using huggingface_hub (part 2)
e0cba20 verified
Raw History Blame Contribute Delete
14.4 kB
import pytest
import torch
from torch import nn
from torch.nn import functional as F
from library import flux2_vae, moss_image_models, moss_image_train_utils, moss_image_utils
# region reference implementation (per-sample loop, copied from the official pipeline_moss_image.py, Apache-2.0)
class _RMSNorm(nn.Module):
def __init__(self, dim):
super().__init__()
self.weight = nn.Parameter(torch.ones(dim))
def forward(self, x):
y = x.float() * torch.rsqrt(x.float().square().mean(-1, keepdim=True) + 1e-6)
return y.to(x.dtype) * self.weight
def _situ(hidden, gate):
return (4.0 * torch.tanh(gate / 4.0) * torch.sigmoid(gate)) * (25.0 * torch.tanh(hidden / 25.0))
def _rotate(x, rope):
sin, cos = rope.chunk(2, -1)
sin = torch.cat((sin, sin), -1).unsqueeze(-2)
cos = torch.cat((cos, cos), -1).unsqueeze(-2)
a, b = x.chunk(2, -1)
return x * cos + torch.cat((-b, a), -1) * sin
def _attention(q, k, v, rope, heads):
shape = (*q.shape[:2], heads, q.shape[-1] // heads)
q = _rotate(q.reshape(shape), rope).transpose(1, 2)
k = _rotate(k.reshape(shape), rope).transpose(1, 2)
v = v.reshape(shape).transpose(1, 2)
return F.scaled_dot_product_attention(q.to(v.dtype), k.to(v.dtype), v).transpose(1, 2).flatten(2)
class _TextBlock(nn.Module):
def __init__(self, dim, ffn_dim, heads):
super().__init__()
self.heads = heads
self.norm1, self.norm2 = _RMSNorm(dim), _RMSNorm(dim)
self.q_norm, self.k_norm = _RMSNorm(dim // heads), _RMSNorm(dim // heads)
self.qkv = nn.Linear(dim, 3 * dim)
self.proj = nn.Linear(dim, dim)
self.ffn_gate, self.ffn_up = nn.Linear(dim, ffn_dim), nn.Linear(dim, ffn_dim)
self.ffn_down = nn.Linear(ffn_dim, dim)
def forward(self, x, rope):
q, k, v = self.qkv(self.norm1(x)).chunk(3, -1)
q = self.q_norm(q.unflatten(-1, (self.heads, -1))).flatten(2)
k = self.k_norm(k.unflatten(-1, (self.heads, -1))).flatten(2)
x = x + self.proj(_attention(q, k, v, rope, self.heads))
y = self.norm2(x)
return x + self.ffn_down(_situ(self.ffn_up(y), self.ffn_gate(y)))
class _TextProjection(nn.Module):
def __init__(self, text_dim, dim, ffn_dim, heads):
super().__init__()
self.proj = nn.Linear(text_dim, dim)
self.blocks = nn.ModuleList([_TextBlock(dim, ffn_dim, heads) for _ in range(2)])
def forward(self, x, rope):
x = self.proj(x.to(self.proj.weight.dtype))
for block in self.blocks:
x = block(x, rope)
return x
class _Block(nn.Module):
def __init__(self, dim, ffn_dim, heads):
super().__init__()
self.heads = heads
self.norm1, self.norm2 = _RMSNorm(dim), _RMSNorm(dim)
self.q_norm, self.k_norm = _RMSNorm(dim // heads), _RMSNorm(dim // heads)
self.proj = nn.ModuleDict({name: nn.Linear(dim, dim, bias=False) for name in ("q", "k", "v", "o")})
self.proj["ffn_up_gate"] = nn.Linear(dim, 2 * ffn_dim, bias=False)
self.proj["ffn_down"] = nn.Linear(ffn_dim, dim, bias=False)
def forward(self, x, rope, modulation):
shift_a, scale_a, gate_a, shift_f, scale_f, gate_f = modulation.unbind(1)
y = self.norm1(x) * (1 + scale_a[:, None]) + shift_a[:, None]
dtype = self.proj["q"].weight.dtype
y = y.to(dtype)
q = self.q_norm(self.proj["q"](y).unflatten(-1, (self.heads, -1))).flatten(2)
k = self.k_norm(self.proj["k"](y).unflatten(-1, (self.heads, -1))).flatten(2)
x = x + self.proj["o"](_attention(q, k, self.proj["v"](y), rope, self.heads)) * gate_a[:, None]
y = self.norm2(x) * (1 + scale_f[:, None]) + shift_f[:, None]
hidden, gate = self.proj["ffn_up_gate"](y.to(dtype)).chunk(2, -1)
return x + self.proj["ffn_down"](_situ(hidden, gate)) * gate_f[:, None]
class _RoPE(nn.Module):
def __init__(self, head_dim, base):
super().__init__()
self.num_bands = head_dim // 8
self.base = base
exponent = torch.arange(self.num_bands, dtype=torch.float32) / self.num_bands
self.register_buffer("bands", 1 / (self.base**exponent))
def forward(self, height, width, text_length, device):
h, w = torch.meshgrid(torch.arange(height, device=device), torch.arange(width, device=device), indexing="ij")
image = torch.stack((torch.zeros_like(h), torch.full_like(h, text_length), h, w), -1).flatten(0, 1).float()
image[:, 2] -= (height - 1) / 2
image[:, 3] -= (width - 1) / 2
text = torch.zeros(text_length, 4, device=device, dtype=torch.float32)
text[:, 1] = torch.arange(text_length, device=device)
phase = torch.cat((image, text), 0).unsqueeze(-1) * self.bands
return torch.cat((phase.sin(), phase.cos()), -2).flatten(-2).unsqueeze(0)
class ReferenceMossImage(nn.Module):
def __init__(self, dim, ffn_dim, num_heads, num_layers, text_dim, freq_dim, latent_channels, rope_base):
super().__init__()
self.freq_dim = freq_dim
self.video_embedder = nn.Linear(latent_channels, dim)
self.text_embedder = _TextProjection(text_dim, dim, ffn_dim, num_heads)
self.rope = _RoPE(dim // num_heads, rope_base)
self.time_embedding = nn.Sequential(nn.Linear(freq_dim, dim), nn.SiLU(), nn.Linear(dim, dim))
self.shared_block_modulation = nn.Linear(dim, 6 * dim)
self.layers = nn.ModuleList([_Block(dim, ffn_dim, num_heads) for _ in range(num_layers)])
self.final_norm_video = _RMSNorm(dim)
self.final_linear_video = nn.Linear(dim, latent_channels, bias=False)
self.video_head_modulation = nn.Linear(dim, 2 * dim)
def forward(self, hidden_states, timestep, encoder_hidden_states, mask, skip_layers=()):
skip_layers = frozenset(skip_layers)
batch, _, height, width = hidden_states.shape
times = torch.as_tensor(timestep, device=hidden_states.device, dtype=torch.float32).reshape(-1).expand(batch)
half = self.freq_dim // 2
freq = 10000 ** (-torch.arange(half, device=times.device, dtype=torch.float64) / half)
phase = times.double()[:, None] * freq
embedding = torch.cat((phase.cos(), phase.sin()), -1).float()
time = self.time_embedding(embedding).float()
mod = self.shared_block_modulation(time).view(batch, 6, -1)
head = self.video_head_modulation(time).view(batch, 2, -1)
outputs = []
for index in range(batch):
context = encoder_hidden_states[index : index + 1, mask[index].bool()]
video = hidden_states[index : index + 1].flatten(2).transpose(1, 2)
video = self.video_embedder(video.contiguous())
rope = self.rope(height, width, context.shape[1], hidden_states.device).to(video.dtype)
text = self.text_embedder(context, rope[:, height * width :])
x = torch.cat((video, text), 1)
for layer_index, layer in enumerate(self.layers):
if layer_index not in skip_layers:
x = layer(x, rope, mod[index : index + 1])
x = self.final_norm_video(x[:, : height * width])
x = x * (1 + head[index : index + 1, 1, None]) + head[index : index + 1, 0, None]
x = self.final_linear_video(x)
outputs.append(x.transpose(1, 2).reshape(1, -1, height, width))
return torch.cat(outputs)
# endregion
SMALL_CONFIG = dict(dim=64, ffn_dim=96, num_heads=4, num_layers=3, text_dim=24, freq_dim=16, latent_channels=8, rope_base=10000.0)
def _make_models(split_attn: bool):
torch.manual_seed(0)
reference = ReferenceMossImage(**SMALL_CONFIG).eval()
model = moss_image_models.MossImage(**SMALL_CONFIG, attn_mode="torch", split_attn=split_attn).eval()
missing, unexpected = model.load_state_dict(reference.state_dict(), strict=True), None
return reference, model
def _make_inputs():
torch.manual_seed(1)
batch, height, width = 2, 3, 5
lens = [4, 7]
x = torch.randn(batch, SMALL_CONFIG["latent_channels"], height, width)
context = torch.randn(batch, max(lens), SMALL_CONFIG["text_dim"])
mask = torch.zeros(batch, max(lens), dtype=torch.int32)
for i, l in enumerate(lens):
mask[i, :l] = 1
context = context * mask[..., None] # padded positions are zero like the official pipeline
timesteps = torch.tensor([750.0, 120.0])
return x, timesteps, context, mask
@pytest.mark.parametrize("split_attn", [False, True])
def test_batched_forward_matches_reference_loop(split_attn):
reference, model = _make_models(split_attn)
x, timesteps, context, mask = _make_inputs()
with torch.no_grad():
expected = reference(x, timesteps, context, mask)
actual = model(x, timesteps, context, mask)
assert actual.shape == expected.shape
torch.testing.assert_close(actual, expected, atol=1e-5, rtol=1e-5)
def test_batched_forward_matches_reference_with_skip_layers():
reference, model = _make_models(False)
x, timesteps, context, mask = _make_inputs()
with torch.no_grad():
expected = reference(x, timesteps, context, mask, skip_layers=(1,))
actual = model(x, timesteps, context, mask, skip_layers=(1,))
without_skip = model(x, timesteps, context, mask)
torch.testing.assert_close(actual, expected, atol=1e-5, rtol=1e-5)
assert not torch.allclose(actual, without_skip)
def test_forward_without_mask_is_full_length():
reference, model = _make_models(False)
x, timesteps, context, mask = _make_inputs()
full_mask = torch.ones_like(mask)
with torch.no_grad():
expected = reference(x, timesteps, context, full_mask)
actual = model(x, timesteps, context, None)
torch.testing.assert_close(actual, expected, atol=1e-5, rtol=1e-5)
def test_gradient_checkpointing_matches_plain_forward():
_, model = _make_models(False)
x, timesteps, context, mask = _make_inputs()
with torch.no_grad():
expected = model(x, timesteps, context, mask)
model.train()
model.enable_gradient_checkpointing()
x_grad = x.clone().requires_grad_(True)
out = model(x_grad, timesteps, context, mask)
out.sum().backward()
torch.testing.assert_close(out.detach(), expected, atol=1e-5, rtol=1e-5)
assert x_grad.grad is not None and torch.isfinite(x_grad.grad).all()
def test_split_fused_qkv_hook():
dim = 3072
fused = torch.randn(3 * dim, dim)
keys, _ = moss_image_utils.split_fused_qkv("layers.3.proj.qkv.weight", None, dim)
assert keys == ["layers.3.proj.q.weight", "layers.3.proj.k.weight", "layers.3.proj.v.weight"]
keys, tensors = moss_image_utils.split_fused_qkv("model.diffusion_model.layers.3.proj.qkv.weight", fused, dim)
assert keys == ["layers.3.proj.q.weight", "layers.3.proj.k.weight", "layers.3.proj.v.weight"]
assert torch.equal(torch.cat(tensors, 0), fused)
assert moss_image_utils.split_fused_qkv("layers.3.proj.q.weight", fused, dim) == (None, None)
assert moss_image_utils.split_fused_qkv("text_embedder.blocks.0.qkv.weight", fused, dim) == (None, None)
def test_sampling_sigmas_match_official_scheduler():
# values computed with the official MOSSImageScheduler (shift 5.0, training_shift 1.0, 1000 steps)
sigmas = moss_image_train_utils.get_sampling_sigmas(50, 5.0)
assert sigmas[0] == 1.0 and sigmas[-1] == 0.0
assert torch.all(sigmas[:-1] > sigmas[1:]) # strictly descending (duplicates removed)
training = torch.linspace(1, 1 / 1000, 1000)
# every sigma (except the trailing 0) lies on the training grid
for s in sigmas[:-1]:
assert torch.isclose(training, s).any()
# the number of steps is reduced by snapping near sigma 0 where the shifted grid is denser than the training grid
assert 40 <= len(sigmas) - 1 <= 50
def test_pad_text_embeds():
a = torch.randn(3, 4)
b = torch.randn(5, 4)
embeds, mask = moss_image_utils.pad_text_embeds([a, b])
assert embeds.shape == (2, 5, 4) and mask.shape == (2, 5)
assert torch.equal(embeds[0, :3], a) and torch.equal(embeds[0, 3:], torch.zeros(2, 4))
assert mask.tolist() == [[1, 1, 1, 0, 0], [1, 1, 1, 1, 1]]
def test_latent_patchify_roundtrip():
z = torch.randn(2, 32, 8, 6)
p = flux2_vae.patchify_latents(z)
assert p.shape == (2, 128, 4, 3)
assert torch.equal(flux2_vae.unpatchify_latents(p), z)
# channel order matches the official pipeline: z.view(B, C, H/2, 2, W/2, 2).permute(0, 1, 3, 5, 2, 4)
expected = z.view(2, 32, 4, 2, 3, 2).permute(0, 1, 3, 5, 2, 4).reshape(2, 128, 4, 3)
assert torch.equal(p, expected)
mean, var = torch.randn(128), torch.rand(128) + 0.5
n = flux2_vae.normalize_latents(p, mean, var)
torch.testing.assert_close(flux2_vae.denormalize_latents(n, mean, var), p, atol=1e-5, rtol=1e-5)
def test_flux2_vae_diffusers_conversion():
diffusers = pytest.importorskip("diffusers")
from diffusers import AutoencoderKLFlux2
torch.manual_seed(0)
ref = AutoencoderKLFlux2(
in_channels=3,
out_channels=3,
down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"],
up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"],
block_out_channels=[64, 128],
decoder_block_out_channels=[32, 64],
layers_per_block=2,
latent_channels=4,
norm_num_groups=32,
sample_size=32,
mid_block_add_attention=True,
use_quant_conv=True,
use_post_quant_conv=True,
batch_norm_eps=1e-4,
patch_size=[2, 2],
force_upcast=True,
).eval()
sd = flux2_vae._convert_diffusers_vae_state_dict(ref.state_dict())
params = flux2_vae.AutoEncoderParams(ch=64, ch_mult=[1, 2], z_channels=4, decoder_ch=32)
ae = flux2_vae.AutoEncoder(params).eval()
ae.load_state_dict(sd, strict=True)
pixels = torch.randn(1, 3, 32, 32)
latents = torch.randn(1, 4, 16, 16)
with torch.no_grad():
expected_latents = ref.encode(pixels, return_dict=False)[0].mode()
actual_latents = ae.encode(pixels)
expected_pixels = ref.decode(latents, return_dict=False)[0]
actual_pixels = ae.decode(latents)
torch.testing.assert_close(actual_latents, expected_latents, atol=1e-5, rtol=1e-5)
torch.testing.assert_close(actual_pixels, expected_pixels, atol=1e-5, rtol=1e-5)