Download tests/test_moss_image_models.py from trixlyaisolutions/onnx: direct link, hf CLI and curl.
- Browser
- Download file 14.4 kB
-
https://huggingface.co/trixlyaisolutions/onnx/resolve/main/tests/test_moss_image_models.py
- Command line
-
hf download hf://trixlyaisolutions/onnx/tests/test_moss_image_models.py
-
curl -L -o test_moss_image_models.py https://huggingface.co/trixlyaisolutions/onnx/resolve/main/tests/test_moss_image_models.py
14.4 kB
| import pytest | |
| import torch | |
| from torch import nn | |
| from torch.nn import functional as F | |
| from library import flux2_vae, moss_image_models, moss_image_train_utils, moss_image_utils | |
| # region reference implementation (per-sample loop, copied from the official pipeline_moss_image.py, Apache-2.0) | |
| class _RMSNorm(nn.Module): | |
| def __init__(self, dim): | |
| super().__init__() | |
| self.weight = nn.Parameter(torch.ones(dim)) | |
| def forward(self, x): | |
| y = x.float() * torch.rsqrt(x.float().square().mean(-1, keepdim=True) + 1e-6) | |
| return y.to(x.dtype) * self.weight | |
| def _situ(hidden, gate): | |
| return (4.0 * torch.tanh(gate / 4.0) * torch.sigmoid(gate)) * (25.0 * torch.tanh(hidden / 25.0)) | |
| def _rotate(x, rope): | |
| sin, cos = rope.chunk(2, -1) | |
| sin = torch.cat((sin, sin), -1).unsqueeze(-2) | |
| cos = torch.cat((cos, cos), -1).unsqueeze(-2) | |
| a, b = x.chunk(2, -1) | |
| return x * cos + torch.cat((-b, a), -1) * sin | |
| def _attention(q, k, v, rope, heads): | |
| shape = (*q.shape[:2], heads, q.shape[-1] // heads) | |
| q = _rotate(q.reshape(shape), rope).transpose(1, 2) | |
| k = _rotate(k.reshape(shape), rope).transpose(1, 2) | |
| v = v.reshape(shape).transpose(1, 2) | |
| return F.scaled_dot_product_attention(q.to(v.dtype), k.to(v.dtype), v).transpose(1, 2).flatten(2) | |
| class _TextBlock(nn.Module): | |
| def __init__(self, dim, ffn_dim, heads): | |
| super().__init__() | |
| self.heads = heads | |
| self.norm1, self.norm2 = _RMSNorm(dim), _RMSNorm(dim) | |
| self.q_norm, self.k_norm = _RMSNorm(dim // heads), _RMSNorm(dim // heads) | |
| self.qkv = nn.Linear(dim, 3 * dim) | |
| self.proj = nn.Linear(dim, dim) | |
| self.ffn_gate, self.ffn_up = nn.Linear(dim, ffn_dim), nn.Linear(dim, ffn_dim) | |
| self.ffn_down = nn.Linear(ffn_dim, dim) | |
| def forward(self, x, rope): | |
| q, k, v = self.qkv(self.norm1(x)).chunk(3, -1) | |
| q = self.q_norm(q.unflatten(-1, (self.heads, -1))).flatten(2) | |
| k = self.k_norm(k.unflatten(-1, (self.heads, -1))).flatten(2) | |
| x = x + self.proj(_attention(q, k, v, rope, self.heads)) | |
| y = self.norm2(x) | |
| return x + self.ffn_down(_situ(self.ffn_up(y), self.ffn_gate(y))) | |
| class _TextProjection(nn.Module): | |
| def __init__(self, text_dim, dim, ffn_dim, heads): | |
| super().__init__() | |
| self.proj = nn.Linear(text_dim, dim) | |
| self.blocks = nn.ModuleList([_TextBlock(dim, ffn_dim, heads) for _ in range(2)]) | |
| def forward(self, x, rope): | |
| x = self.proj(x.to(self.proj.weight.dtype)) | |
| for block in self.blocks: | |
| x = block(x, rope) | |
| return x | |
| class _Block(nn.Module): | |
| def __init__(self, dim, ffn_dim, heads): | |
| super().__init__() | |
| self.heads = heads | |
| self.norm1, self.norm2 = _RMSNorm(dim), _RMSNorm(dim) | |
| self.q_norm, self.k_norm = _RMSNorm(dim // heads), _RMSNorm(dim // heads) | |
| self.proj = nn.ModuleDict({name: nn.Linear(dim, dim, bias=False) for name in ("q", "k", "v", "o")}) | |
| self.proj["ffn_up_gate"] = nn.Linear(dim, 2 * ffn_dim, bias=False) | |
| self.proj["ffn_down"] = nn.Linear(ffn_dim, dim, bias=False) | |
| def forward(self, x, rope, modulation): | |
| shift_a, scale_a, gate_a, shift_f, scale_f, gate_f = modulation.unbind(1) | |
| y = self.norm1(x) * (1 + scale_a[:, None]) + shift_a[:, None] | |
| dtype = self.proj["q"].weight.dtype | |
| y = y.to(dtype) | |
| q = self.q_norm(self.proj["q"](y).unflatten(-1, (self.heads, -1))).flatten(2) | |
| k = self.k_norm(self.proj["k"](y).unflatten(-1, (self.heads, -1))).flatten(2) | |
| x = x + self.proj["o"](_attention(q, k, self.proj["v"](y), rope, self.heads)) * gate_a[:, None] | |
| y = self.norm2(x) * (1 + scale_f[:, None]) + shift_f[:, None] | |
| hidden, gate = self.proj["ffn_up_gate"](y.to(dtype)).chunk(2, -1) | |
| return x + self.proj["ffn_down"](_situ(hidden, gate)) * gate_f[:, None] | |
| class _RoPE(nn.Module): | |
| def __init__(self, head_dim, base): | |
| super().__init__() | |
| self.num_bands = head_dim // 8 | |
| self.base = base | |
| exponent = torch.arange(self.num_bands, dtype=torch.float32) / self.num_bands | |
| self.register_buffer("bands", 1 / (self.base**exponent)) | |
| def forward(self, height, width, text_length, device): | |
| h, w = torch.meshgrid(torch.arange(height, device=device), torch.arange(width, device=device), indexing="ij") | |
| image = torch.stack((torch.zeros_like(h), torch.full_like(h, text_length), h, w), -1).flatten(0, 1).float() | |
| image[:, 2] -= (height - 1) / 2 | |
| image[:, 3] -= (width - 1) / 2 | |
| text = torch.zeros(text_length, 4, device=device, dtype=torch.float32) | |
| text[:, 1] = torch.arange(text_length, device=device) | |
| phase = torch.cat((image, text), 0).unsqueeze(-1) * self.bands | |
| return torch.cat((phase.sin(), phase.cos()), -2).flatten(-2).unsqueeze(0) | |
| class ReferenceMossImage(nn.Module): | |
| def __init__(self, dim, ffn_dim, num_heads, num_layers, text_dim, freq_dim, latent_channels, rope_base): | |
| super().__init__() | |
| self.freq_dim = freq_dim | |
| self.video_embedder = nn.Linear(latent_channels, dim) | |
| self.text_embedder = _TextProjection(text_dim, dim, ffn_dim, num_heads) | |
| self.rope = _RoPE(dim // num_heads, rope_base) | |
| self.time_embedding = nn.Sequential(nn.Linear(freq_dim, dim), nn.SiLU(), nn.Linear(dim, dim)) | |
| self.shared_block_modulation = nn.Linear(dim, 6 * dim) | |
| self.layers = nn.ModuleList([_Block(dim, ffn_dim, num_heads) for _ in range(num_layers)]) | |
| self.final_norm_video = _RMSNorm(dim) | |
| self.final_linear_video = nn.Linear(dim, latent_channels, bias=False) | |
| self.video_head_modulation = nn.Linear(dim, 2 * dim) | |
| def forward(self, hidden_states, timestep, encoder_hidden_states, mask, skip_layers=()): | |
| skip_layers = frozenset(skip_layers) | |
| batch, _, height, width = hidden_states.shape | |
| times = torch.as_tensor(timestep, device=hidden_states.device, dtype=torch.float32).reshape(-1).expand(batch) | |
| half = self.freq_dim // 2 | |
| freq = 10000 ** (-torch.arange(half, device=times.device, dtype=torch.float64) / half) | |
| phase = times.double()[:, None] * freq | |
| embedding = torch.cat((phase.cos(), phase.sin()), -1).float() | |
| time = self.time_embedding(embedding).float() | |
| mod = self.shared_block_modulation(time).view(batch, 6, -1) | |
| head = self.video_head_modulation(time).view(batch, 2, -1) | |
| outputs = [] | |
| for index in range(batch): | |
| context = encoder_hidden_states[index : index + 1, mask[index].bool()] | |
| video = hidden_states[index : index + 1].flatten(2).transpose(1, 2) | |
| video = self.video_embedder(video.contiguous()) | |
| rope = self.rope(height, width, context.shape[1], hidden_states.device).to(video.dtype) | |
| text = self.text_embedder(context, rope[:, height * width :]) | |
| x = torch.cat((video, text), 1) | |
| for layer_index, layer in enumerate(self.layers): | |
| if layer_index not in skip_layers: | |
| x = layer(x, rope, mod[index : index + 1]) | |
| x = self.final_norm_video(x[:, : height * width]) | |
| x = x * (1 + head[index : index + 1, 1, None]) + head[index : index + 1, 0, None] | |
| x = self.final_linear_video(x) | |
| outputs.append(x.transpose(1, 2).reshape(1, -1, height, width)) | |
| return torch.cat(outputs) | |
| # endregion | |
| SMALL_CONFIG = dict(dim=64, ffn_dim=96, num_heads=4, num_layers=3, text_dim=24, freq_dim=16, latent_channels=8, rope_base=10000.0) | |
| def _make_models(split_attn: bool): | |
| torch.manual_seed(0) | |
| reference = ReferenceMossImage(**SMALL_CONFIG).eval() | |
| model = moss_image_models.MossImage(**SMALL_CONFIG, attn_mode="torch", split_attn=split_attn).eval() | |
| missing, unexpected = model.load_state_dict(reference.state_dict(), strict=True), None | |
| return reference, model | |
| def _make_inputs(): | |
| torch.manual_seed(1) | |
| batch, height, width = 2, 3, 5 | |
| lens = [4, 7] | |
| x = torch.randn(batch, SMALL_CONFIG["latent_channels"], height, width) | |
| context = torch.randn(batch, max(lens), SMALL_CONFIG["text_dim"]) | |
| mask = torch.zeros(batch, max(lens), dtype=torch.int32) | |
| for i, l in enumerate(lens): | |
| mask[i, :l] = 1 | |
| context = context * mask[..., None] # padded positions are zero like the official pipeline | |
| timesteps = torch.tensor([750.0, 120.0]) | |
| return x, timesteps, context, mask | |
| def test_batched_forward_matches_reference_loop(split_attn): | |
| reference, model = _make_models(split_attn) | |
| x, timesteps, context, mask = _make_inputs() | |
| with torch.no_grad(): | |
| expected = reference(x, timesteps, context, mask) | |
| actual = model(x, timesteps, context, mask) | |
| assert actual.shape == expected.shape | |
| torch.testing.assert_close(actual, expected, atol=1e-5, rtol=1e-5) | |
| def test_batched_forward_matches_reference_with_skip_layers(): | |
| reference, model = _make_models(False) | |
| x, timesteps, context, mask = _make_inputs() | |
| with torch.no_grad(): | |
| expected = reference(x, timesteps, context, mask, skip_layers=(1,)) | |
| actual = model(x, timesteps, context, mask, skip_layers=(1,)) | |
| without_skip = model(x, timesteps, context, mask) | |
| torch.testing.assert_close(actual, expected, atol=1e-5, rtol=1e-5) | |
| assert not torch.allclose(actual, without_skip) | |
| def test_forward_without_mask_is_full_length(): | |
| reference, model = _make_models(False) | |
| x, timesteps, context, mask = _make_inputs() | |
| full_mask = torch.ones_like(mask) | |
| with torch.no_grad(): | |
| expected = reference(x, timesteps, context, full_mask) | |
| actual = model(x, timesteps, context, None) | |
| torch.testing.assert_close(actual, expected, atol=1e-5, rtol=1e-5) | |
| def test_gradient_checkpointing_matches_plain_forward(): | |
| _, model = _make_models(False) | |
| x, timesteps, context, mask = _make_inputs() | |
| with torch.no_grad(): | |
| expected = model(x, timesteps, context, mask) | |
| model.train() | |
| model.enable_gradient_checkpointing() | |
| x_grad = x.clone().requires_grad_(True) | |
| out = model(x_grad, timesteps, context, mask) | |
| out.sum().backward() | |
| torch.testing.assert_close(out.detach(), expected, atol=1e-5, rtol=1e-5) | |
| assert x_grad.grad is not None and torch.isfinite(x_grad.grad).all() | |
| def test_split_fused_qkv_hook(): | |
| dim = 3072 | |
| fused = torch.randn(3 * dim, dim) | |
| keys, _ = moss_image_utils.split_fused_qkv("layers.3.proj.qkv.weight", None, dim) | |
| assert keys == ["layers.3.proj.q.weight", "layers.3.proj.k.weight", "layers.3.proj.v.weight"] | |
| keys, tensors = moss_image_utils.split_fused_qkv("model.diffusion_model.layers.3.proj.qkv.weight", fused, dim) | |
| assert keys == ["layers.3.proj.q.weight", "layers.3.proj.k.weight", "layers.3.proj.v.weight"] | |
| assert torch.equal(torch.cat(tensors, 0), fused) | |
| assert moss_image_utils.split_fused_qkv("layers.3.proj.q.weight", fused, dim) == (None, None) | |
| assert moss_image_utils.split_fused_qkv("text_embedder.blocks.0.qkv.weight", fused, dim) == (None, None) | |
| def test_sampling_sigmas_match_official_scheduler(): | |
| # values computed with the official MOSSImageScheduler (shift 5.0, training_shift 1.0, 1000 steps) | |
| sigmas = moss_image_train_utils.get_sampling_sigmas(50, 5.0) | |
| assert sigmas[0] == 1.0 and sigmas[-1] == 0.0 | |
| assert torch.all(sigmas[:-1] > sigmas[1:]) # strictly descending (duplicates removed) | |
| training = torch.linspace(1, 1 / 1000, 1000) | |
| # every sigma (except the trailing 0) lies on the training grid | |
| for s in sigmas[:-1]: | |
| assert torch.isclose(training, s).any() | |
| # the number of steps is reduced by snapping near sigma 0 where the shifted grid is denser than the training grid | |
| assert 40 <= len(sigmas) - 1 <= 50 | |
| def test_pad_text_embeds(): | |
| a = torch.randn(3, 4) | |
| b = torch.randn(5, 4) | |
| embeds, mask = moss_image_utils.pad_text_embeds([a, b]) | |
| assert embeds.shape == (2, 5, 4) and mask.shape == (2, 5) | |
| assert torch.equal(embeds[0, :3], a) and torch.equal(embeds[0, 3:], torch.zeros(2, 4)) | |
| assert mask.tolist() == [[1, 1, 1, 0, 0], [1, 1, 1, 1, 1]] | |
| def test_latent_patchify_roundtrip(): | |
| z = torch.randn(2, 32, 8, 6) | |
| p = flux2_vae.patchify_latents(z) | |
| assert p.shape == (2, 128, 4, 3) | |
| assert torch.equal(flux2_vae.unpatchify_latents(p), z) | |
| # channel order matches the official pipeline: z.view(B, C, H/2, 2, W/2, 2).permute(0, 1, 3, 5, 2, 4) | |
| expected = z.view(2, 32, 4, 2, 3, 2).permute(0, 1, 3, 5, 2, 4).reshape(2, 128, 4, 3) | |
| assert torch.equal(p, expected) | |
| mean, var = torch.randn(128), torch.rand(128) + 0.5 | |
| n = flux2_vae.normalize_latents(p, mean, var) | |
| torch.testing.assert_close(flux2_vae.denormalize_latents(n, mean, var), p, atol=1e-5, rtol=1e-5) | |
| def test_flux2_vae_diffusers_conversion(): | |
| diffusers = pytest.importorskip("diffusers") | |
| from diffusers import AutoencoderKLFlux2 | |
| torch.manual_seed(0) | |
| ref = AutoencoderKLFlux2( | |
| in_channels=3, | |
| out_channels=3, | |
| down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], | |
| up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], | |
| block_out_channels=[64, 128], | |
| decoder_block_out_channels=[32, 64], | |
| layers_per_block=2, | |
| latent_channels=4, | |
| norm_num_groups=32, | |
| sample_size=32, | |
| mid_block_add_attention=True, | |
| use_quant_conv=True, | |
| use_post_quant_conv=True, | |
| batch_norm_eps=1e-4, | |
| patch_size=[2, 2], | |
| force_upcast=True, | |
| ).eval() | |
| sd = flux2_vae._convert_diffusers_vae_state_dict(ref.state_dict()) | |
| params = flux2_vae.AutoEncoderParams(ch=64, ch_mult=[1, 2], z_channels=4, decoder_ch=32) | |
| ae = flux2_vae.AutoEncoder(params).eval() | |
| ae.load_state_dict(sd, strict=True) | |
| pixels = torch.randn(1, 3, 32, 32) | |
| latents = torch.randn(1, 4, 16, 16) | |
| with torch.no_grad(): | |
| expected_latents = ref.encode(pixels, return_dict=False)[0].mode() | |
| actual_latents = ae.encode(pixels) | |
| expected_pixels = ref.decode(latents, return_dict=False)[0] | |
| actual_pixels = ae.decode(latents) | |
| torch.testing.assert_close(actual_latents, expected_latents, atol=1e-5, rtol=1e-5) | |
| torch.testing.assert_close(actual_pixels, expected_pixels, atol=1e-5, rtol=1e-5) | |