Blame for diffusers/modular_pipelines/flux3/denoise.py

# Copyright 2026 Black Forest Labs and The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import inspect
from typing import Any

import torch

from ...configuration_utils import FrozenDict
from ...guiders import ClassifierFreeGuidance
from ...models import Flux3Transformer2DModel
from ...schedulers import FlowMatchEulerDiscreteScheduler
from ...utils import is_torch_xla_available, logging
from ..modular_pipeline import (
    BlockState,
    LoopSequentialPipelineBlocks,
    ModularPipelineBlocks,
    PipelineState,
)
from ..modular_pipeline_utils import ComponentSpec, InputParam, OutputParam
from .modular_pipeline import Flux3ModularPipeline


if is_torch_xla_available():
    import torch_xla.core.xla_model as xm

    XLA_AVAILABLE = True
else:
    XLA_AVAILABLE = False

logger = logging.get_logger(__name__)  # pylint: disable=invalid-name


class Flux3LoopDenoiser(ModularPipelineBlocks):
    model_name = "flux3"

    def __init__(self, guidance_scale: float = 3.5):
        self.guidance_scale = guidance_scale
        super().__init__()

    @property
    def expected_components(self) -> list[ComponentSpec]:
        return [
            ComponentSpec("transformer", Flux3Transformer2DModel),
            ComponentSpec(
                "guider",
                ClassifierFreeGuidance,
                config=FrozenDict({"guidance_scale": self.guidance_scale}),
                default_creation_method="from_config",
            ),
        ]

    @property
    def description(self) -> str:
        return (
            "Step within the denoising loop that predicts the velocity of the image tokens. The static conditioning "
            "fields (`img_ids`, the reference image tokens and their ids) arrive via `denoiser_input_fields`; "
            "everything matching the transformer signature is forwarded. "
            "This block should be used to compose the `sub_blocks` attribute of a `LoopSequentialPipelineBlocks` "
            "object (e.g. `Flux3DenoiseStep`)"
        )

    @property
    def inputs(self) -> list[tuple[str, Any]]:
        return [
            InputParam(
                "latents",
                required=True,
                type_hint=torch.Tensor,
                description="The image latent tokens to denoise. Shape: (B, seq_len, C)",
            ),
            InputParam.template("denoiser_input_fields"),
        ]

    @property
    def intermediate_outputs(self) -> list[OutputParam]:
        return [OutputParam("noise_pred", type_hint=torch.Tensor, description="The predicted velocity for this step")]

    @torch.no_grad()
    def __call__(
        self, components: Flux3ModularPipeline, block_state: BlockState, i: int, t: torch.Tensor
    ) -> PipelineState:
        denoiser_input_fields = block_state.denoiser_input_fields
        guider_inputs = {
            "encoder_hidden_states": (
                denoiser_input_fields.get("prompt_embeds"),
                denoiser_input_fields.get("negative_prompt_embeds"),
            ),
            "txt_ids": (
                denoiser_input_fields.get("txt_ids"),
                denoiser_input_fields.get("negative_txt_ids"),
            ),
        }

        # transformer kwargs = every collected static field matching the forward signature (`img_ids`, and the
        # `image_cond_*` fields of image-to-image generation)
        transformer_args = set(inspect.signature(components.transformer.forward).parameters.keys())
        transformer_kwargs = {
            field_name: field_value
            for field_name, field_value in denoiser_input_fields.items()
            if field_name in transformer_args and field_name not in guider_inputs
        }

        latents = block_state.latents.to(components.transformer.dtype)
        # the flow timestep in [0, 1] (scheduler timesteps are sigmas * 1000); the transformer applies it to every
        # image token while the reference image tokens and the text context stay at 0
        timestep = (t / 1000).expand(latents.shape[0]).to(latents.dtype)

        components.guider.set_state(step=i, num_inference_steps=block_state.num_inference_steps, timestep=t)
        guider_state = components.guider.prepare_inputs(guider_inputs)

        for guider_state_batch in guider_state:
            components.guider.prepare_models(components.transformer)
            cond_kwargs = {input_name: getattr(guider_state_batch, input_name) for input_name in guider_inputs.keys()}

            guider_state_batch.noise_pred = components.transformer(
                hidden_states=latents,
                timestep=timestep,
                return_dict=False,
                **transformer_kwargs,
                **cond_kwargs,
            )[0]
            components.guider.cleanup_models(components.transformer)

        block_state.noise_pred = components.guider(guider_state)[0]

        return components, block_state


class Flux3DistilledLoopDenoiser(ModularPipelineBlocks):
    model_name = "flux3-distilled"

    @property
    def expected_components(self) -> list[ComponentSpec]:
        return [ComponentSpec("transformer", Flux3Transformer2DModel)]

    @property
    def description(self) -> str:
        return (
            "Step within the denoising loop that predicts the velocity of the image tokens with a step-distilled "
            "(flow-map) FLUX.3 transformer: no classifier-free guidance, and each step also passes the transformer the "
            "timestep it jumps to. The static conditioning fields (`prompt_embeds`, `txt_ids`, `img_ids`, the "
            "reference image tokens and their ids) arrive via `denoiser_input_fields`. "
            "This block should be used to compose the `sub_blocks` attribute of a `LoopSequentialPipelineBlocks` "
            "object (e.g. `Flux3DistilledDenoiseStep`)"
        )

    @property
    def inputs(self) -> list[tuple[str, Any]]:
        return [
            InputParam(
                "latents",
                required=True,
                type_hint=torch.Tensor,
                description="The image latent tokens to denoise. Shape: (B, seq_len, C)",
            ),
            InputParam.template("denoiser_input_fields"),
        ]

    @property
    def intermediate_outputs(self) -> list[OutputParam]:
        return [OutputParam("noise_pred", type_hint=torch.Tensor, description="The predicted velocity for this step")]

    @torch.no_grad()
    def __call__(
        self, components: Flux3ModularPipeline, block_state: BlockState, i: int, t: torch.Tensor
    ) -> PipelineState:
        denoiser_input_fields = block_state.denoiser_input_fields

        # transformer kwargs = every collected static field matching the forward signature (`img_ids`, `txt_ids`, and
        # the `image_cond_*` fields of image-to-image generation)
        transformer_args = set(inspect.signature(components.transformer.forward).parameters.keys())
        transformer_kwargs = {
            field_name: field_value
            for field_name, field_value in denoiser_input_fields.items()
            if field_name in transformer_args
        }

        latents = block_state.latents.to(components.transformer.dtype)
        batch_size = latents.shape[0]
        # the flow timestep in [0, 1] (scheduler timesteps are sigmas * 1000), and the one this step jumps to: the next
        # scheduler timestep, 0 after the last
        timesteps = block_state.timesteps
        next_t = timesteps[i + 1] if i + 1 < len(timesteps) else torch.zeros_like(t)
        timestep = (t / 1000).expand(batch_size).to(latents.dtype)
        target_timestep = (next_t / 1000).expand(batch_size).to(latents.dtype)

        block_state.noise_pred = components.transformer(
            hidden_states=latents,
            encoder_hidden_states=denoiser_input_fields["prompt_embeds"],
            timestep=timestep,
            target_timestep=target_timestep,
            return_dict=False,
            **transformer_kwargs,
        )[0]

        return components, block_state


class Flux3LoopAfterDenoiser(ModularPipelineBlocks):
    model_name = "flux3"

    @property
    def expected_components(self) -> list[ComponentSpec]:
        # config-less: the scheduler config is owned by `Flux3SetTimestepsStep`, whose spec is combined first
        return [ComponentSpec("scheduler", FlowMatchEulerDiscreteScheduler, default_creation_method="from_config")]

    @property
    def description(self) -> str:
        return (
            "Step within the denoising loop that updates the latents after denoising. "
            "This block should be used to compose the `sub_blocks` attribute of a `LoopSequentialPipelineBlocks` "
            "object (e.g. `Flux3DenoiseLoopWrapper`)"
        )

    @property
    def intermediate_outputs(self) -> list[OutputParam]:
        return [OutputParam("latents", type_hint=torch.Tensor, description="The denoised latents")]

    @torch.no_grad()
    def __call__(self, components: Flux3ModularPipeline, block_state: BlockState, i: int, t: torch.Tensor):
        latents_dtype = block_state.latents.dtype
        block_state.latents = components.scheduler.step(
            block_state.noise_pred,
            t,
            block_state.latents,
            return_dict=False,
        )[0]

        if block_state.latents.dtype != latents_dtype:
            if torch.backends.mps.is_available():
                block_state.latents = block_state.latents.to(latents_dtype)

        return components, block_state


class Flux3DenoiseLoopWrapper(LoopSequentialPipelineBlocks):
    model_name = "flux3"

    @property
    def description(self) -> str:
        return (
            "Pipeline block that iteratively denoises the latents over `timesteps`. "
            "The specific steps within each iteration can be customized with `sub_blocks` attribute"
        )

    @property
    def loop_expected_components(self) -> list[ComponentSpec]:
        return [
            # config-less: the scheduler config is owned by `Flux3SetTimestepsStep`, whose spec is combined first
            ComponentSpec("scheduler", FlowMatchEulerDiscreteScheduler, default_creation_method="from_config"),
            ComponentSpec("transformer", Flux3Transformer2DModel),
        ]

    @property
    def loop_inputs(self) -> list[InputParam]:
        return [
            InputParam(
                "timesteps",
                required=True,
                type_hint=torch.Tensor,
                description="The timesteps to use for the denoising process.",
            ),
            InputParam(
                "num_inference_steps",
                required=True,
                type_hint=int,
                description="The number of inference steps to use for the denoising process.",
            ),
        ]

    @torch.no_grad()
    def __call__(self, components: Flux3ModularPipeline, state: PipelineState) -> PipelineState:
        block_state = self.get_block_state(state)

        block_state.num_warmup_steps = max(
            len(block_state.timesteps) - block_state.num_inference_steps * components.scheduler.order, 0
        )

        with self.progress_bar(total=block_state.num_inference_steps) as progress_bar:
            for i, t in enumerate(block_state.timesteps):
                components, block_state = self.loop_step(components, block_state, i=i, t=t)

                if i == len(block_state.timesteps) - 1 or (
                    (i + 1) > block_state.num_warmup_steps and (i + 1) % components.scheduler.order == 0
                ):
                    progress_bar.update()

                if XLA_AVAILABLE:
                    xm.mark_step()

        self.set_block_state(state, block_state)
        return components, state


# auto_docstring
class Flux3DenoiseStep(Flux3DenoiseLoopWrapper):
    """
    Denoise step that iteratively denoises the latents for FLUX.3.
      Its loop logic is defined in `Flux3DenoiseLoopWrapper.__call__` method At each iteration, it runs blocks defined
      in `sub_blocks` sequentially:
       - `Flux3LoopDenoiser`
       - `Flux3LoopAfterDenoiser`
      This block supports text-to-image and image-to-image generation.

      Components:
          transformer (`Flux3Transformer2DModel`) guider (`ClassifierFreeGuidance`) scheduler
          (`FlowMatchEulerDiscreteScheduler`)

      Inputs:
          timesteps (`Tensor`):
              The timesteps to use for the denoising process.
          num_inference_steps (`int`):
              The number of inference steps to use for the denoising process.
          latents (`Tensor`):
              The image latent tokens to denoise. Shape: (B, seq_len, C)
          **denoiser_input_fields (`None`, *optional*):
              conditional model inputs for the denoiser: e.g. prompt_embeds, negative_prompt_embeds, etc.

      Outputs:
          latents (`Tensor`):
              The denoised latents
    """

    block_classes = [Flux3LoopDenoiser(), Flux3LoopAfterDenoiser()]
    block_names = ["denoiser", "after_denoiser"]

    @property
    def description(self) -> str:
        return (
            "Denoise step that iteratively denoises the latents for FLUX.3. \n"
            "Its loop logic is defined in `Flux3DenoiseLoopWrapper.__call__` method \n"
            "At each iteration, it runs blocks defined in `sub_blocks` sequentially:\n"
            " - `Flux3LoopDenoiser`\n"
            " - `Flux3LoopAfterDenoiser`\n"
            "This block supports text-to-image and image-to-image generation."
        )


# auto_docstring
class Flux3DistilledDenoiseStep(Flux3DenoiseLoopWrapper):
    """
    Denoise step that iteratively denoises the latents for the step-distilled (flow-map) FLUX.3 model.
      Its loop logic is defined in `Flux3DenoiseLoopWrapper.__call__` method At each iteration, it runs blocks defined
      in `sub_blocks` sequentially:
       - `Flux3DistilledLoopDenoiser`
       - `Flux3LoopAfterDenoiser`
      This block supports text-to-image and image-to-image generation, without classifier-free guidance.

      Components:
          transformer (`Flux3Transformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`)

      Inputs:
          timesteps (`Tensor`):
              The timesteps to use for the denoising process.
          num_inference_steps (`int`):
              The number of inference steps to use for the denoising process.
          latents (`Tensor`):
              The image latent tokens to denoise. Shape: (B, seq_len, C)
          **denoiser_input_fields (`None`, *optional*):
              conditional model inputs for the denoiser: e.g. prompt_embeds, negative_prompt_embeds, etc.

      Outputs:
          latents (`Tensor`):
              The denoised latents
    """

    model_name = "flux3-distilled"
    """
    Denoise step that iteratively denoises the latents for the step-distilled (flow-map) FLUX.3 model.
      Its loop logic is defined in `Flux3DenoiseLoopWrapper.__call__` method At each iteration, it runs blocks defined
      in `sub_blocks` sequentially:
       - `Flux3DistilledLoopDenoiser`
       - `Flux3LoopAfterDenoiser`
      This block supports text-to-image and image-to-image generation, without classifier-free guidance.

      Components:
          transformer (`Flux3Transformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`)

      Inputs:
          timesteps (`Tensor`):
              The timesteps to use for the denoising process.
          num_inference_steps (`int`):
              The number of inference steps to use for the denoising process.
          latents (`Tensor`):
              The image latent tokens to denoise. Shape: (B, seq_len, C)
          **denoiser_input_fields (`None`, *optional*):
              conditional model inputs for the denoiser: e.g. prompt_embeds, negative_prompt_embeds, etc.

      Outputs:
          latents (`Tensor`):
              The denoised latents
    """

    block_classes = [Flux3DistilledLoopDenoiser(), Flux3LoopAfterDenoiser()]
    block_names = ["denoiser", "after_denoiser"]

    @property
    def description(self) -> str:
        return (
            "Denoise step that iteratively denoises the latents for the step-distilled (flow-map) FLUX.3 model. \n"
            "Its loop logic is defined in `Flux3DenoiseLoopWrapper.__call__` method \n"
            "At each iteration, it runs blocks defined in `sub_blocks` sequentially:\n"
            " - `Flux3DistilledLoopDenoiser`\n"
            " - `Flux3LoopAfterDenoiser`\n"
            "This block supports text-to-image and image-to-image generation, without classifier-free guidance."
        )