modules/patch.py

import torch
import comfy.model_base
import comfy.ldm.modules.diffusionmodules.openaimodel
import comfy.samplers
import modules.anisotropic as anisotropic

from comfy.samplers import model_management, lcm, math
from comfy.ldm.modules.diffusionmodules.openaimodel import timestep_embedding, forward_timestep_embed


sharpness = 2.0


def sampling_function_patched(model_function, x, timestep, uncond, cond, cond_scale, cond_concat=None, model_options={},
                      seed=None):
    def get_area_and_mult(cond, x_in, cond_concat_in, timestep_in):
        area = (x_in.shape[2], x_in.shape[3], 0, 0)
        strength = 1.0
        if 'timestep_start' in cond[1]:
            timestep_start = cond[1]['timestep_start']
            if timestep_in[0] > timestep_start:
                return None
        if 'timestep_end' in cond[1]:
            timestep_end = cond[1]['timestep_end']
            if timestep_in[0] < timestep_end:
                return None
        if 'area' in cond[1]:
            area = cond[1]['area']
        if 'strength' in cond[1]:
            strength = cond[1]['strength']

        adm_cond = None
        if 'adm_encoded' in cond[1]:
            adm_cond = cond[1]['adm_encoded']

        input_x = x_in[:, :, area[2]:area[0] + area[2], area[3]:area[1] + area[3]]
        if 'mask' in cond[1]:
            # Scale the mask to the size of the input
            # The mask should have been resized as we began the sampling process
            mask_strength = 1.0
            if "mask_strength" in cond[1]:
                mask_strength = cond[1]["mask_strength"]
            mask = cond[1]['mask']
            assert (mask.shape[1] == x_in.shape[2])
            assert (mask.shape[2] == x_in.shape[3])
            mask = mask[:, area[2]:area[0] + area[2], area[3]:area[1] + area[3]] * mask_strength
            mask = mask.unsqueeze(1).repeat(input_x.shape[0] // mask.shape[0], input_x.shape[1], 1, 1)
        else:
            mask = torch.ones_like(input_x)
        mult = mask * strength

        if 'mask' not in cond[1]:
            rr = 8
            if area[2] != 0:
                for t in range(rr):
                    mult[:, :, t:1 + t, :] *= ((1.0 / rr) * (t + 1))
            if (area[0] + area[2]) < x_in.shape[2]:
                for t in range(rr):
                    mult[:, :, area[0] - 1 - t:area[0] - t, :] *= ((1.0 / rr) * (t + 1))
            if area[3] != 0:
                for t in range(rr):
                    mult[:, :, :, t:1 + t] *= ((1.0 / rr) * (t + 1))
            if (area[1] + area[3]) < x_in.shape[3]:
                for t in range(rr):
                    mult[:, :, :, area[1] - 1 - t:area[1] - t] *= ((1.0 / rr) * (t + 1))

        conditionning = {}
        conditionning['c_crossattn'] = cond[0]
        if cond_concat_in is not None and len(cond_concat_in) > 0:
            cropped = []
            for x in cond_concat_in:
                cr = x[:, :, area[2]:area[0] + area[2], area[3]:area[1] + area[3]]
                cropped.append(cr)
            conditionning['c_concat'] = torch.cat(cropped, dim=1)

        if adm_cond is not None:
            conditionning['c_adm'] = adm_cond

        control = None
        if 'control' in cond[1]:
            control = cond[1]['control']

        patches = None
        if 'gligen' in cond[1]:
            gligen = cond[1]['gligen']
            patches = {}
            gligen_type = gligen[0]
            gligen_model = gligen[1]
            if gligen_type == "position":
                gligen_patch = gligen_model.set_position(input_x.shape, gligen[2], input_x.device)
            else:
                gligen_patch = gligen_model.set_empty(input_x.shape, input_x.device)

            patches['middle_patch'] = [gligen_patch]

        return (input_x, mult, conditionning, area, control, patches)

    def cond_equal_size(c1, c2):
        if c1 is c2:
            return True
        if c1.keys() != c2.keys():
            return False
        if 'c_crossattn' in c1:
            s1 = c1['c_crossattn'].shape
            s2 = c2['c_crossattn'].shape
            if s1 != s2:
                if s1[0] != s2[0] or s1[2] != s2[2]:  # these 2 cases should not happen
                    return False

                mult_min = lcm(s1[1], s2[1])
                diff = mult_min // min(s1[1], s2[1])
                if diff > 4:  # arbitrary limit on the padding because it's probably going to impact performance negatively if it's too much
                    return False
        if 'c_concat' in c1:
            if c1['c_concat'].shape != c2['c_concat'].shape:
                return False
        if 'c_adm' in c1:
            if c1['c_adm'].shape != c2['c_adm'].shape:
                return False
        return True

    def can_concat_cond(c1, c2):
        if c1[0].shape != c2[0].shape:
            return False

        # control
        if (c1[4] is None) != (c2[4] is None):
            return False
        if c1[4] is not None:
            if c1[4] is not c2[4]:
                return False

        # patches
        if (c1[5] is None) != (c2[5] is None):
            return False
        if (c1[5] is not None):
            if c1[5] is not c2[5]:
                return False

        return cond_equal_size(c1[2], c2[2])

    def cond_cat(c_list):
        c_crossattn = []
        c_concat = []
        c_adm = []
        crossattn_max_len = 0
        for x in c_list:
            if 'c_crossattn' in x:
                c = x['c_crossattn']
                if crossattn_max_len == 0:
                    crossattn_max_len = c.shape[1]
                else:
                    crossattn_max_len = lcm(crossattn_max_len, c.shape[1])
                c_crossattn.append(c)
            if 'c_concat' in x:
                c_concat.append(x['c_concat'])
            if 'c_adm' in x:
                c_adm.append(x['c_adm'])
        out = {}
        c_crossattn_out = []
        for c in c_crossattn:
            if c.shape[1] < crossattn_max_len:
                c = c.repeat(1, crossattn_max_len // c.shape[1], 1)  # padding with repeat doesn't change result
            c_crossattn_out.append(c)

        if len(c_crossattn_out) > 0:
            out['c_crossattn'] = [torch.cat(c_crossattn_out)]
        if len(c_concat) > 0:
            out['c_concat'] = [torch.cat(c_concat)]
        if len(c_adm) > 0:
            out['c_adm'] = torch.cat(c_adm)
        return out

    def calc_cond_uncond_batch(model_function, cond, uncond, x_in, timestep, max_total_area, cond_concat_in,
                               model_options):
        out_cond = torch.zeros_like(x_in)
        out_count = torch.ones_like(x_in) / 100000.0

        out_uncond = torch.zeros_like(x_in)
        out_uncond_count = torch.ones_like(x_in) / 100000.0

        COND = 0
        UNCOND = 1

        to_run = []
        for x in cond:
            p = get_area_and_mult(x, x_in, cond_concat_in, timestep)
            if p is None:
                continue

            to_run += [(p, COND)]
        if uncond is not None:
            for x in uncond:
                p = get_area_and_mult(x, x_in, cond_concat_in, timestep)
                if p is None:
                    continue

                to_run += [(p, UNCOND)]

        while len(to_run) > 0:
            first = to_run[0]
            first_shape = first[0][0].shape
            to_batch_temp = []
            for x in range(len(to_run)):
                if can_concat_cond(to_run[x][0], first[0]):
                    to_batch_temp += [x]

            to_batch_temp.reverse()
            to_batch = to_batch_temp[:1]

            for i in range(1, len(to_batch_temp) + 1):
                batch_amount = to_batch_temp[:len(to_batch_temp) // i]
                if (len(batch_amount) * first_shape[0] * first_shape[2] * first_shape[3] < max_total_area):
                    to_batch = batch_amount
                    break

            input_x = []
            mult = []
            c = []
            cond_or_uncond = []
            area = []
            control = None
            patches = None
            for x in to_batch:
                o = to_run.pop(x)
                p = o[0]
                input_x += [p[0]]
                mult += [p[1]]
                c += [p[2]]
                area += [p[3]]
                cond_or_uncond += [o[1]]
                control = p[4]
                patches = p[5]

            batch_chunks = len(cond_or_uncond)
            input_x = torch.cat(input_x)
            c = cond_cat(c)
            timestep_ = torch.cat([timestep] * batch_chunks)

            if control is not None:
                c['control'] = control.get_control(input_x, timestep_, c, len(cond_or_uncond))

            transformer_options = {}
            if 'transformer_options' in model_options:
                transformer_options = model_options['transformer_options'].copy()

            if patches is not None:
                if "patches" in transformer_options:
                    cur_patches = transformer_options["patches"].copy()
                    for p in patches:
                        if p in cur_patches:
                            cur_patches[p] = cur_patches[p] + patches[p]
                        else:
                            cur_patches[p] = patches[p]
                else:
                    transformer_options["patches"] = patches

            c['transformer_options'] = transformer_options

            transformer_options['uc_mask'] = torch.Tensor(cond_or_uncond).to(input_x).float()[:, None, None, None]

            if 'model_function_wrapper' in model_options:
                output = model_options['model_function_wrapper'](model_function,
                                                                 {"input": input_x, "timestep": timestep_, "c": c,
                                                                  "cond_or_uncond": cond_or_uncond}).chunk(batch_chunks)
            else:
                output = model_function(input_x, timestep_, **c).chunk(batch_chunks)
            del input_x

            model_management.throw_exception_if_processing_interrupted()

            for o in range(batch_chunks):
                if cond_or_uncond[o] == COND:
                    out_cond[:, :, area[o][2]:area[o][0] + area[o][2], area[o][3]:area[o][1] + area[o][3]] += output[
                                                                                                                  o] * \
                                                                                                              mult[o]
                    out_count[:, :, area[o][2]:area[o][0] + area[o][2], area[o][3]:area[o][1] + area[o][3]] += mult[o]
                else:
                    out_uncond[:, :, area[o][2]:area[o][0] + area[o][2], area[o][3]:area[o][1] + area[o][3]] += output[
                                                                                                                    o] * \
                                                                                                                mult[o]
                    out_uncond_count[:, :, area[o][2]:area[o][0] + area[o][2], area[o][3]:area[o][1] + area[o][3]] += \
                    mult[o]
            del mult

        out_cond /= out_count
        del out_count
        out_uncond /= out_uncond_count
        del out_uncond_count

        return out_cond, out_uncond

    max_total_area = model_management.maximum_batch_area()
    if math.isclose(cond_scale, 1.0):
        uncond = None

    cond, uncond = calc_cond_uncond_batch(model_function, cond, uncond, x, timestep, max_total_area, cond_concat,
                                          model_options)
    if "sampler_cfg_function" in model_options:
        args = {"cond": cond, "uncond": uncond, "cond_scale": cond_scale, "timestep": timestep}
        return model_options["sampler_cfg_function"](args)
    else:
        return uncond + (cond - uncond) * cond_scale


def unet_forward_patched(self, x, timesteps=None, context=None, y=None, control=None, transformer_options={}, **kwargs):
    uc_mask = transformer_options['uc_mask']
    transformer_options["original_shape"] = list(x.shape)
    transformer_options["current_index"] = 0

    hs = []
    t_emb = timestep_embedding(timesteps, self.model_channels, repeat_only=False).to(self.dtype)
    emb = self.time_embed(t_emb)

    if self.num_classes is not None:
        assert y.shape[0] == x.shape[0]
        emb = emb + self.label_emb(y)

    h = x.type(self.dtype)
    for id, module in enumerate(self.input_blocks):
        transformer_options["block"] = ("input", id)
        h = forward_timestep_embed(module, h, emb, context, transformer_options)
        if control is not None and 'input' in control and len(control['input']) > 0:
            ctrl = control['input'].pop()
            if ctrl is not None:
                h += ctrl
        hs.append(h)
    transformer_options["block"] = ("middle", 0)
    h = forward_timestep_embed(self.middle_block, h, emb, context, transformer_options)
    if control is not None and 'middle' in control and len(control['middle']) > 0:
        h += control['middle'].pop()

    for id, module in enumerate(self.output_blocks):
        transformer_options["block"] = ("output", id)
        hsp = hs.pop()
        if control is not None and 'output' in control and len(control['output']) > 0:
            ctrl = control['output'].pop()
            if ctrl is not None:
                hsp += ctrl

        h = torch.cat([h, hsp], dim=1)
        del hsp
        if len(hs) > 0:
            output_shape = hs[-1].shape
        else:
            output_shape = None
        h = forward_timestep_embed(module, h, emb, context, transformer_options, output_shape)
    h = h.type(x.dtype)
    x0 = self.out(h)

    alpha = 1.0 - (timesteps / 999.0)[:, None, None, None].clone()
    alpha *= 0.001 * sharpness
    degraded_x0 = anisotropic.bilateral_blur(x0) * alpha + x0 * (1.0 - alpha)

    x0 = x0 * uc_mask + degraded_x0 * (1.0 - uc_mask)

    return x0


def sdxl_encode_adm_patched(self, **kwargs):
    clip_pooled = kwargs["pooled_output"]
    width = kwargs.get("width", 768)
    height = kwargs.get("height", 768)
    crop_w = kwargs.get("crop_w", 0)
    crop_h = kwargs.get("crop_h", 0)
    target_width = kwargs.get("target_width", width)
    target_height = kwargs.get("target_height", height)

    if kwargs.get("prompt_type", "") == "negative":
        width *= 0.8
        height *= 0.8
    elif kwargs.get("prompt_type", "") == "positive":
        width *= 1.5
        height *= 1.5

    out = []
    out.append(self.embedder(torch.Tensor([height])))
    out.append(self.embedder(torch.Tensor([width])))
    out.append(self.embedder(torch.Tensor([crop_h])))
    out.append(self.embedder(torch.Tensor([crop_w])))
    out.append(self.embedder(torch.Tensor([target_height])))
    out.append(self.embedder(torch.Tensor([target_width])))
    flat = torch.flatten(torch.cat(out))[None, ]
    return torch.cat((clip_pooled.to(flat.device), flat), dim=1)


def patch_all():
    comfy.samplers.sampling_function = sampling_function_patched
    comfy.model_base.SDXL.encode_adm = sdxl_encode_adm_patched
    comfy.ldm.modules.diffusionmodules.openaimodel.UNetModel.forward = unet_forward_patched