cowpuncher's picture
download
raw
40.7 kB
"""ComfyUI nodes for the Anima reference adapter (inference).
The adapter reads a reference image as an extra latent frame:
* the reference frame is fed clean (timestep 0) next to the noisy target frame;
* its RoPE (h, w) coordinates are shifted ref_offset_h target heights down, so it is not
copied pixel-aligned (no pose copying);
* a LoRA, active only while the reference is attached, teaches the model to read it.
Two-reference adapters (LoRA metadata "slots") read a character frame and a style frame, each with
its own RoPE t index; single-reference adapters read one frame that carries both.
Adapters with metadata "arch": "asym" use asymmetric attention instead of a joint frame: reference
tokens attend only to themselves and skip cross-attention, target tokens attend to both. The
reference stream then does not depend on the step, the prompt or the target, so its per-block keys
and values are computed once and reused by every sampling step and CFG branch (~2x faster steps),
and the reference may have any size.
Nodes
Anima Ref Adapter MODEL + adapter LoRA + reference IMAGE -> MODEL for KSampler or any sampler;
reference guidance runs as a post-CFG patch
Anima Ref Adapter Loader MODEL + adapter LoRA -> MODEL that understands the reference frame
Anima Ref Latent IMAGE -> letterboxed reference LATENT (+ empty target LATENT)
Anima Ref Guider positive / negative / reference with separate text and reference
guidance, plus APG or CFG-rescale correction
"""
from __future__ import annotations
import json
import logging
import os
import numpy as np
import torch
import torch.nn.functional as F
import huggingface_hub
from PIL import Image
from safetensors import safe_open
import comfy.conds
import comfy.ldm.common_dit
from comfy.ldm.anima.model import Anima as AnimaModel
import comfy.model_management
import comfy.patcher_extension
import comfy.samplers
import comfy.utils
import folder_paths
import node_helpers
REF_KEY = "anima_ref_latent"
REF_HALF_KEY = "anima_ref_latent_half" # half-size encode for SPEED's low-resolution stage
STYLE_KEY = "anima_ref_style_latent"
STYLE_HALF_KEY = "anima_ref_style_latent_half"
SLOT_KEYS = {"character": (REF_KEY, REF_HALF_KEY), "style": (STYLE_KEY, STYLE_HALF_KEY)}
SINGLE_SLOTS = {"character": {"t": 1, "offset_h": 1.0, "offset_w": 0}} # adapters trained before slots
HF_REPO = "ElectronGirl/anima-ref-adapter"
HF_FILES = ("anima_ref_adapter_v2_r64.safetensors", "anima_ref_adapter_pilot_r32.safetensors") # downloaded on first use
OPTIONS_REF_KEY ="anima_ref_latents" # transformer_options: reference latents of Anima Ref Adapter
NO_REF_KEY = "anima_ref_off" # transformer_options: run this call without the reference
WRAPPER_KEY = "anima_ref_adapter"
HOOKS_ATTR = "_anima_ref_adapter_hooks" # on each hooked module: {adapter key: hook handle}
_adapters: dict[tuple, "_Adapter"] = {} # adapter key -> loaded weights, shared by every loader run
_active = {"key": None} # key of the adapter whose LoRA the hooks apply right now
_warned_sizes: set = set()
_warned_slots: set = set()
# Anima-2.9B inserts these blocks into the 28-block base, shifting its original layers.
_INSERTED_29B = frozenset((2, 5, 8, 11, 14, 17, 21, 24, 27, 30, 33, 36))
def _adapter_block_map(indices, model) -> tuple[int, ...]:
indices = set(indices)
if not indices:
raise ValueError("Adapter has no diffusion block weights")
model_blocks = len(model.blocks)
if max(indices) + 1 == model_blocks:
return tuple(range(model_blocks))
if isinstance(model, AnimaModel) and model_blocks == 40 and indices == set(range(28)):
return tuple(i for i in range(40) if i not in _INSERTED_29B)
raise ValueError(f"Adapter covers {max(indices) + 1} blocks but the model has {model_blocks}. "
"Only native layouts and the 28-to-40-block Anima-2.9B expansion are supported.")
def _lora_names() -> list[str]:
"""Local LoRAs, with the released adapter files listed first even before they are downloaded."""
local = folder_paths.get_filename_list("loras")
return [name for name in HF_FILES if name not in local] + local
def _lora_path(name: str) -> str:
"""Full path of a LoRA; a released adapter file not on disk yet is downloaded into models/loras first."""
path = folder_paths.get_full_path("loras", name)
if path is None and name in HF_FILES:
logging.info(f"[Anima Ref Adapter] downloading {name} from huggingface.co/{HF_REPO}")
path = huggingface_hub.hf_hub_download(HF_REPO, name, local_dir=folder_paths.get_folder_paths("loras")[0])
if path is None:
raise FileNotFoundError(f"LoRA not found: {name}")
return path
def _adapter_key(path: str, strength: float) -> tuple:
return os.path.abspath(path), os.path.getmtime(path), float(strength)
class _Adapter:
"""LoRA weights applied through forward hooks, only while this adapter's key is active.
Hooks are tracked on the modules themselves and match by key, not by object identity, so a
loader node that runs again (e.g. after the upstream model chain changed) keeps its LoRA working.
"""
def __init__(self, path: str, strength: float):
self.key = _adapter_key(path, strength)
tensors = comfy.utils.load_torch_file(path, safe_load=True)
self.weights = {}
for key, down in tensors.items():
if not key.endswith(".lora_down.weight"):
continue
name = key[len("diffusion_model."):-len(".lora_down.weight")]
up = tensors[key.replace("lora_down", "lora_up")]
alpha = float(tensors.get(key.replace("lora_down.weight", "alpha"), torch.tensor(down.shape[0])))
self.weights[name] = [down, up, strength * alpha / down.shape[0]]
if not self.weights:
raise ValueError(f"{path} has no lora_down/lora_up weights")
self.block_indices = {int(n.split(".")[1]) for n in self.weights if n.startswith("blocks.")}
metadata = _read_metadata(path)
self.slots = metadata.get("slots") or SINGLE_SLOTS
self.asym = metadata.get("arch") == "asym"
self.ref_lora = metadata.get("ref_lora", True)
self.ref_cache: dict = {} # reference fingerprint -> per-block (k, v), asym adapters only
self.on_device: dict = {} # name -> device copies of (down, up); self.weights stays on the CPU
def delta(self, name: str, x: torch.Tensor) -> torch.Tensor:
weights = self.on_device.get(name)
if weights is None or weights[0].device != x.device or weights[0].dtype != x.dtype:
down, up, scale = self.weights[name]
weights = self.on_device[name] = (down.to(device=x.device, dtype=x.dtype),
up.to(device=x.device, dtype=x.dtype), scale)
return F.linear(F.linear(x, weights[0]), weights[1]) * weights[2]
def release(self) -> None:
"""Drop the reference K/V and the device copies of the LoRA (ComfyUI's memory manager tracks neither)."""
self.ref_cache.clear()
self.on_device.clear()
@classmethod
def get(cls, path: str, strength: float) -> "_Adapter":
key = _adapter_key(path, strength)
if key not in _adapters:
_adapters[key] = cls(path, strength)
return _adapters[key]
def install(self, diffusion_model) -> None:
block_map = _adapter_block_map(self.block_indices, diffusion_model)
# Resolve everything before installing hooks, so a missing target cannot leave a partial install.
targets = []
for name in self.weights:
path = name
if name.startswith("blocks."):
_, index, suffix = name.split(".", 2)
path = f"blocks.{block_map[int(index)]}.{suffix}"
targets.append((diffusion_model.get_submodule(path), name))
for module, name in targets:
installed = getattr(module, HOOKS_ATTR, None)
if installed is None:
installed = {}
setattr(module, HOOKS_ATTR, installed)
if self.key in installed:
continue
def hook(module, inputs, output, name=name):
if _active["key"] != self.key:
return output
return output + self.delta(name, inputs[0])
installed[self.key] = module.register_forward_hook(hook)
def _read_metadata(path: str) -> dict:
try:
with safe_open(path, "pt") as handle:
return json.loads((handle.metadata() or {}).get("anima_ref_adapter", "{}"))
except Exception: # not a safetensors file, or no metadata: a single-reference adapter
return {}
def _frame_positions(t_index, height, width, h0, w0, device):
hh, ww = torch.meshgrid(torch.arange(height, device=device) + h0, torch.arange(width, device=device) + w0,
indexing="ij")
return torch.full_like(hh, t_index).flatten(), hh.flatten(), ww.flatten()
def _rope_from_positions(pe, t, h, w):
"""VideoRopePosition3DEmb.generate_embeddings for explicit per-token (t, h, w) positions."""
device = t.device
h_freqs = 1.0 / ((10000.0 * pe.h_ntk_factor) ** pe.dim_spatial_range.to(device))
w_freqs = 1.0 / ((10000.0 * pe.w_ntk_factor) ** pe.dim_spatial_range.to(device))
t_freqs = 1.0 / ((10000.0 * pe.t_ntk_factor) ** pe.dim_temporal_range.to(device))
def rotations(pos, freqs):
angle = torch.outer(pos.float(), freqs)
return torch.stack([angle.cos(), -angle.sin(), angle.sin(), angle.cos()], dim=-1)
emb = torch.cat([rotations(t, t_freqs), rotations(h, h_freqs), rotations(w, w_freqs)], dim=-2)
return emb.reshape(*emb.shape[:-1], 2, 2).float()
def _pick_ref(refs, size):
"""The reference latent for a target latent of this (h, w): an exact-size encode if there is one
(Anima Ref Latent also encodes a half-size reference for SPEED's low-resolution stage),
else the closest one resized in latent space."""
refs = [r.unsqueeze(2) if r.ndim == 4 else r for r in refs]
for ref in refs:
if tuple(ref.shape[-2:]) == size:
return ref
ref = min(refs, key=lambda r: abs(r.shape[-2] - size[0]) + abs(r.shape[-1] - size[1]))
if size not in _warned_sizes:
_warned_sizes.add(size)
logging.warning(f"[Anima Ref Adapter] no reference encoded at latent size {size}; "
f"resizing the {tuple(ref.shape[-2:])} one (lower fidelity)")
shrink = ref.shape[-2] > size[0]
return F.interpolate(ref[:, :, 0].float(), size=size, mode="area" if shrink else "bilinear").unsqueeze(2)
def _ref_forward(dm, x, timesteps, context, slot_refs, slots, offset_ratio, transformer_options):
"""MiniTrainDIT._forward with each filled reference slot as an extra frame (timestep 0, the slot's
RoPE t index, (h, w) shifted offset_h * offset_ratio target heights down)."""
orig_shape = list(x.shape)
patch = (dm.patch_temporal, dm.patch_spatial, dm.patch_spatial)
x = comfy.ldm.common_dit.pad_to_patch_size(x, patch)
refs = [(_pick_ref(slot_refs[slot], tuple(x.shape[-2:])).to(device=x.device, dtype=x.dtype), spec)
for slot, spec in slots.items() if slot_refs.get(slot)]
frames = torch.cat([x[:, :, :1]] + [ref[:, :, :1] for ref, _ in refs], dim=2)
x_emb, _, extra = dm.prepare_embedded_sequence(frames, fps=None, padding_mask=None)
_, _, hp, wp, _ = x_emb.shape
positions = [_frame_positions(0, hp, wp, 0, 0, x.device)]
for _, spec in refs:
positions.append(_frame_positions(int(spec["t"]), hp, wp, round(spec["offset_h"] * offset_ratio * hp),
int(spec.get("offset_w", 0)), x.device))
rope = _rope_from_positions(dm.pos_embedder, *(torch.cat(axis) for axis in zip(*positions)))
timesteps = timesteps.reshape(x.shape[0], -1)[:, :1]
timesteps = torch.cat([timesteps] + [torch.zeros_like(timesteps)] * len(refs), dim=1)
t_emb, adaln = dm.t_embedder[1](dm.t_embedder[0](timesteps).to(x_emb.dtype))
t_emb = dm.t_embedding_norm(t_emb)
kwargs = {"rope_emb_L_1_1_D": rope.unsqueeze(1).unsqueeze(0), "adaln_lora_B_T_3D": adaln,
"extra_per_block_pos_emb": extra, "transformer_options": transformer_options}
if x_emb.dtype == torch.float16:
x_emb = x_emb.float() # residual stream in fp32, as MiniTrainDIT._forward does for fp16
for index, block in enumerate(dm.blocks):
transformer_options["block_index"] = index
x_emb = block(x_emb, t_emb, context, **kwargs)
out = dm.final_layer(x_emb.to(context.dtype), t_emb, adaln_lora_B_T_3D=adaln)
out = dm.unpatchify(out)[:, :, :1]
return out[:, :, :orig_shape[-3], :orig_shape[-2], :orig_shape[-1]]
def _block_mods(block, emb, adaln):
"""(shift, scale, gate) [B, T, 1, 1, D] for self-attention, cross-attention and MLP (Block.forward)."""
out = []
for modulation in (block.adaln_modulation_self_attn, block.adaln_modulation_cross_attn, block.adaln_modulation_mlp):
m = modulation(emb) + adaln if block.use_adaln_lora else modulation(emb)
out.append(tuple(t[:, :, None, None, :] for t in m.chunk(3, dim=-1)))
return out
def _norm(x, layer_norm, scale, shift):
return layer_norm(x) * (1 + scale) + shift
def _asym_block(block, xt, xr, emb_t, emb_r, adaln_t, adaln_r, context, rope_t, rope_r, ref_kv=None,
transformer_options=None):
"""Block.forward with asymmetric attention (train/anima_env.asym_block). Returns (xt, xr, (k_r, v_r))."""
attn = block.self_attn
k_r = v_r = None
if xr is not None:
sa, _, mlp = _block_mods(block, emb_r, adaln_r)
residual_dtype, compute_dtype = xr.dtype, emb_r.dtype
B, T, H, W, _ = xr.shape
n = _norm(xr, block.layer_norm_self_attn, sa[1], sa[0]).to(compute_dtype).flatten(1, 3)
q_r, k_r, v_r = attn.compute_qkv(n, None, rope_emb=rope_r)
out = attn.compute_attention(q_r, k_r, v_r).reshape(B, T, H, W, -1)
xr = torch.addcmul(xr, sa[2].to(residual_dtype), out.to(residual_dtype))
n = _norm(xr, block.layer_norm_mlp, mlp[1], mlp[0]).to(compute_dtype)
xr = torch.addcmul(xr, mlp[2].to(residual_dtype), block.mlp(n).to(residual_dtype))
elif ref_kv is not None:
k_r, v_r = ref_kv
if xt is not None:
options = transformer_options or {}
sa, ca, mlp = _block_mods(block, emb_t, adaln_t)
residual_dtype, compute_dtype = xt.dtype, emb_t.dtype
B, T, H, W, _ = xt.shape
n = _norm(xt, block.layer_norm_self_attn, sa[1], sa[0]).to(compute_dtype).flatten(1, 3)
q_t, k_t, v_t = attn.compute_qkv(n, None, rope_emb=rope_t)
n_q = q_t.shape[1]
if k_r is not None:
k_t, v_t = torch.cat([k_t, k_r.to(k_t)], dim=1), torch.cat([v_t, v_r.to(v_t)], dim=1)
if k_t.shape[1] != n_q and "compute_attention" in attn.__dict__:
# Another node replaced this block's attention (e.g. Anima Safe PAG, which blends each query's
# output with its own value, so it assumes as many queries as keys). Pad the queries with zeros
# to the key length and keep the target's rows: queries are independent, and a query's "own"
# value is then exactly its target token's.
q_t = torch.cat([q_t, q_t.new_zeros(q_t.shape[0], k_t.shape[1] - n_q, *q_t.shape[2:])], dim=1)
out = attn.compute_attention(q_t, k_t, v_t, transformer_options=options)[:, :n_q].reshape(B, T, H, W, -1)
xt = torch.addcmul(xt, sa[2].to(residual_dtype), out.to(residual_dtype))
n = _norm(xt, block.layer_norm_cross_attn, ca[1], ca[0]).to(compute_dtype).flatten(1, 3)
out = block.cross_attn(n, context, rope_emb=None, transformer_options=options).reshape(B, T, H, W, -1)
xt = torch.addcmul(xt, ca[2].to(residual_dtype), out.to(residual_dtype))
n = _norm(xt, block.layer_norm_mlp, mlp[1], mlp[0]).to(compute_dtype)
xt = torch.addcmul(xt, mlp[2].to(residual_dtype), block.mlp(n).to(residual_dtype))
return xt, xr, (k_r, v_r)
def _frame_embed(dm, latent, t_index, h0, w0, sigma):
"""Patch embedding, RoPE and timestep embedding of one frame [B, C, 1, H, W]."""
x_emb, _, _ = dm.prepare_embedded_sequence(latent, fps=None, padding_mask=None)
_, _, hp, wp, _ = x_emb.shape
rope = _rope_from_positions(dm.pos_embedder, *_frame_positions(t_index, hp, wp, h0, w0, latent.device))
# the timestep embedding sets the compute dtype, so take it before the residual stream goes to fp32
emb, adaln = dm.t_embedder[1](dm.t_embedder[0](sigma.reshape(-1, 1).float()).to(x_emb.dtype))
if x_emb.dtype == torch.float16:
x_emb = x_emb.float() # residual stream in fp32, as MiniTrainDIT._forward does for fp16
return x_emb, rope.unsqueeze(1).unsqueeze(0), dm.t_embedding_norm(emb), adaln
def _encode_ref_asym(dm, ref, ref_offset):
"""Per-block reference keys/values: the reference stream run once."""
xr, rope_r, emb_r, adaln_r = _frame_embed(dm, ref, 1, *ref_offset, torch.zeros(ref.shape[0], device=ref.device))
cache = []
for block in dm.blocks:
_, xr, kv = _asym_block(block, None, xr, None, emb_r, None, adaln_r, None, None, rope_r)
cache.append(kv)
return cache
def _asym_forward(dm, x, timesteps, context, ref_kv, transformer_options):
"""MiniTrainDIT._forward for the target frame alone, attending to cached reference keys/values."""
orig_shape = list(x.shape)
x = comfy.ldm.common_dit.pad_to_patch_size(x, (dm.patch_temporal, dm.patch_spatial, dm.patch_spatial))
sigma = timesteps.reshape(x.shape[0], -1)[:, 0]
xt, rope_t, emb_t, adaln_t = _frame_embed(dm, x[:, :, :1], 0, 0, 0, sigma)
batch = xt.shape[0]
for index, block in enumerate(dm.blocks):
transformer_options["block_index"] = index
k, v = ref_kv[index]
kv = (k.expand(batch, *k.shape[1:]), v.expand(batch, *v.shape[1:])) if k.shape[0] != batch else (k, v)
xt, _, _ = _asym_block(block, xt, None, emb_t, None, adaln_t, None, context, rope_t, None, kv, transformer_options)
out = dm.unpatchify(dm.final_layer(xt.to(context.dtype), emb_t, adaln_lora_B_T_3D=adaln_t))[:, :, :1]
return out[:, :, :orig_shape[-3], :orig_shape[-2], :orig_shape[-1]]
def _asym_ref_kv(adapter: _Adapter, dm, refs, x, offset_ratio):
"""Cached reference keys/values for this reference and target height (the RoPE shift depends on it)."""
ref = max(refs, key=lambda r: r.shape[-1] * r.shape[-2]) # the full-size encode; any size works here
ref = ref.unsqueeze(2) if ref.ndim == 4 else ref
ref = comfy.ldm.common_dit.pad_to_patch_size(ref, (dm.patch_temporal, dm.patch_spatial, dm.patch_spatial))
if ref.shape[0] > 1 and torch.equal(ref, ref[:1].expand_as(ref)):
ref = ref[:1] # the same reference in every batch item (the usual case): encode it once
hp = -(-x.shape[-2] // dm.patch_spatial)
offset = (round(adapter.slots["character"]["offset_h"] * offset_ratio * hp), int(adapter.slots["character"].get("offset_w", 0)))
key = (id(dm), tuple(ref.shape), offset, x.dtype, x.device, float(ref.float().sum()), float(ref.flatten()[::997].float().sum()))
if key not in adapter.ref_cache:
# ~1 GB of keys/values per reference at 1 MP, invisible to ComfyUI's memory manager: keep only the
# current one (the cache is also emptied when each sampling run ends, see _make_outer_sample_wrapper).
adapter.ref_cache.clear()
previous = _active["key"]
_active["key"] = adapter.key if adapter.ref_lora else None # the reference stream with or without the LoRA
try:
adapter.ref_cache[key] = _encode_ref_asym(dm, ref.to(device=x.device, dtype=x.dtype), offset)
finally:
_active["key"] = previous
return adapter.ref_cache[key]
def _make_wrapper(adapter: _Adapter, offset_ratio: float):
def wrapper(executor, x, timesteps, context, fps=None, padding_mask=None, **kwargs):
slot_refs = {slot: [r for r in (kwargs.pop(full, None), kwargs.pop(half, None)) if r is not None]
for slot, (full, half) in SLOT_KEYS.items()}
options = kwargs.get("transformer_options", {})
if not slot_refs["character"] and not options.get(NO_REF_KEY):
slot_refs["character"] = options.get(OPTIONS_REF_KEY, []) # set by Anima Ref Adapter
if "style" not in adapter.slots and slot_refs["style"]:
if not slot_refs["character"]: # a single-reference adapter reads the style image as its reference
slot_refs["character"] = slot_refs["style"]
if adapter.key not in _warned_slots:
_warned_slots.add(adapter.key)
logging.warning("[Anima Ref Adapter] this adapter has one reference slot; "
"style_latent is used only when ref_latent is not connected")
if not any(slot_refs[slot] for slot in adapter.slots):
return executor(x, timesteps, context, fps, padding_mask, **kwargs)
adapter.install(executor.class_obj) # no-op once hooked; covers a model reloaded since the loader ran
dm = executor.class_obj
if adapter.asym:
with torch.no_grad():
ref_kv = _asym_ref_kv(adapter, dm, slot_refs["character"], x, offset_ratio)
previous = _active["key"]
_active["key"] = adapter.key
try:
if adapter.asym:
return _asym_forward(dm, x, timesteps, context, ref_kv, kwargs.get("transformer_options", {}))
return _ref_forward(dm, x, timesteps, context, slot_refs, adapter.slots, offset_ratio,
kwargs.get("transformer_options", {}))
finally:
_active["key"] = previous
return wrapper
def _make_outer_sample_wrapper(adapter: _Adapter):
"""Free the reference cache and the LoRA's device copies when a sampling run ends, so their VRAM is not
held between runs or between the chains of a workflow (as comfy_extras/nodes_easycache.py does for its cache)."""
def wrapper(executor, *args, **kwargs):
try:
return executor(*args, **kwargs)
finally:
adapter.release()
return wrapper
def _make_extra_conds(extra_conds, process_latent_in):
# Object patches stay applied on the shared model, so a re-run loader may see its own earlier
# patch here; wrap the original function, not the patch.
extra_conds = getattr(extra_conds, "_anima_ref_inner", extra_conds)
def patched(**kwargs):
out = extra_conds(**kwargs)
for key in (REF_KEY, REF_HALF_KEY, STYLE_KEY, STYLE_HALF_KEY):
if kwargs.get(key) is not None:
out[key] = comfy.conds.CONDRegular(process_latent_in(kwargs[key]))
return out
patched._anima_ref_inner = extra_conds
return patched
class AnimaRefAdapterLoader:
@classmethod
def INPUT_TYPES(cls):
return {"required": {
"model": ("MODEL",),
"lora_name": (_lora_names(), {"default": HF_FILES[0], "tooltip": f"the adapter; the released files download from huggingface.co/{HF_REPO} into models/loras on first use"}),
"strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.05}),
"ref_offset_h": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 4.0, "step": 0.05,
"tooltip": "reference frame RoPE shift in target heights (training used 1.0)"}),
}}
RETURN_TYPES = ("MODEL",)
FUNCTION = "load"
CATEGORY = "Anima Ref Adapter"
def load(self, model, lora_name, strength, ref_offset_h):
adapter = _Adapter.get(_lora_path(lora_name), strength)
adapter.install(model.get_model_object("diffusion_model"))
m = model.clone()
m.add_object_patch("extra_conds", _make_extra_conds(m.get_model_object("extra_conds"),
m.get_model_object("process_latent_in")))
m.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, WRAPPER_KEY,
_make_wrapper(adapter, ref_offset_h))
m.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.OUTER_SAMPLE, WRAPPER_KEY,
_make_outer_sample_wrapper(adapter))
return (m,)
class AnimaRefLatent:
@classmethod
def INPUT_TYPES(cls):
return {"required": {
"image": ("IMAGE",),
"vae": ("VAE",),
"width": ("INT", {"default": 896, "min": 256, "max": 2048, "step": 16}),
"height": ("INT", {"default": 1152, "min": 256, "max": 2048, "step": 16}),
"fill": ("INT", {"default": 255, "min": 0, "max": 255, "tooltip": "letterbox padding grey level"}),
}}
RETURN_TYPES = ("LATENT", "LATENT", "IMAGE")
RETURN_NAMES = ("ref_latent", "empty_latent", "ref_image")
FUNCTION = "encode"
CATEGORY = "Anima Ref Adapter"
@staticmethod
def _letterbox(src, width, height, fill):
scale = min(width / src.width, height / src.height)
size = (max(1, round(src.width * scale)), max(1, round(src.height * scale)))
canvas = Image.new("RGB", (width, height), (fill, fill, fill))
canvas.paste(src.resize(size, Image.LANCZOS), ((width - size[0]) // 2, (height - size[1]) // 2))
return torch.from_numpy(np.asarray(canvas, dtype=np.float32) / 255.0).unsqueeze(0)
def encode(self, image, vae, width, height, fill):
src = Image.fromarray((image[0].clamp(0, 1).cpu().numpy() * 255).round().astype(np.uint8))
pixels = self._letterbox(src, width, height, fill)
ref = vae.encode(pixels) # one image: the Wan VAE reads a batch as video frames
# Half-size encode for samplers with a half-resolution stage (SPEED): shrinking the full latent
# gives off-manifold latents that the adapter cannot read.
half = vae.encode(self._letterbox(src, width // 16 * 8, height // 16 * 8, fill))
empty = torch.zeros([1, 16, height // 8, width // 8], device=comfy.model_management.intermediate_device())
return ({"samples": ref, "samples_half": half}, {"samples": empty}, pixels)
def _flat_dims(x):
return tuple(range(1, x.ndim))
def _correct(guided, pos, correction, rescale_phi):
"""APG (drop the guidance component parallel to the conditional prediction) or CFG rescale, in x0 space."""
if correction == "apg":
update = (guided - pos).double()
unit = F.normalize(pos.double().flatten(1), dim=1).view_as(pos)
parallel = (update * unit).sum(dim=_flat_dims(pos), keepdim=True) * unit
return pos + (update - parallel).to(pos.dtype)
if correction == "rescale":
ratio = pos.std(dim=_flat_dims(pos), keepdim=True) / guided.std(dim=_flat_dims(guided), keepdim=True)
return rescale_phi * guided * ratio + (1 - rescale_phi) * guided
return guided
def _in_window(model_sampling, sigma, start, end) -> bool:
"""Whether a step lies in [start, end] of the schedule (fractions of the denoising, as in ComfyUI's
ControlNet start/end percent; sigma falls as sampling goes on)."""
if start <= 0.0 and end >= 1.0:
return True
sigma = float(sigma.flatten()[0])
return model_sampling.percent_to_sigma(end) - 1e-5 <= sigma <= model_sampling.percent_to_sigma(start) + 1e-5
class AnimaRefAdapter:
"""Loader, reference encode and reference guidance in one MODEL patch, for KSampler and any other sampler."""
@classmethod
def INPUT_TYPES(cls):
return {"required": {
"model": ("MODEL",),
"vae": ("VAE",),
"image": ("IMAGE", {"tooltip": "the reference. For two characters, stitch two images side by side and say "
"in the prompt who is on the left and who is on the right"}),
"lora_name": (_lora_names(), {"default": HF_FILES[0], "tooltip": f"the adapter; the released files download from huggingface.co/{HF_REPO} into models/loras on first use"}),
"ref_guidance": ("FLOAT", {"default": 4.5, "min": 0.0, "max": 20.0, "step": 0.1,
"tooltip": "1 = reference in both CFG branches (no extra model call); higher pushes "
"toward the reference at one extra model call per step"}),
"correction": (["apg", "rescale", "none"], {"tooltip": "keeps colors in check at high cfg and ref_guidance"}),
}, "optional": {
"strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.05,
"tooltip": "pilot/v2 adapters: LoRA strength. SnapRef adapters use ref_weight instead"}),
"ref_end": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01,
"tooltip": "last step with ref_guidance, as a fraction of the denoising; the reference "
"stays in after it. ~0.6 saves time with nearly the same image"}),
"ref_megapixels": ("FLOAT", {"default": 1.0, "min": 0.25, "max": 2.0, "step": 0.05,
"tooltip": "the reference is resized to this area, keeping its aspect ratio"}),
"clip": ("CLIP", {"tooltip": "required only for SnapRef adapters; use the Anima text encoder"}),
"snapref_mode": (["cs", "char"], {"tooltip": "SnapRef only: character and style, or character only"}),
"ref_caption": ("STRING", {"default": "", "multiline": True,
"tooltip": "SnapRef only: optional reference appearance tags; empty works"}),
"snapref_guidance": ("BOOLEAN", {"default": False, "tooltip": "SnapRef only: enable ref_guidance, correction "
"and ref_end. Off keeps the trained positive-only reference behavior"}),
"ref_weight": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.05,
"tooltip": "SnapRef only: weight of the reference in attention (1 = trained, "
"0 = adapter off). Its LoRA always runs at its trained strength"}),
}}
RETURN_TYPES = ("MODEL",)
FUNCTION = "apply"
CATEGORY = "Anima Ref Adapter"
def apply(self, model, vae, image, lora_name, ref_guidance, correction, strength=1.0, ref_end=1.0, ref_megapixels=1.0,
clip=None, snapref_mode="cs", ref_caption="", snapref_guidance=False, ref_weight=1.0):
"""The adapter file's format picks the backend and the inputs that apply; the frontend extension
(js/anima_ref_adapter.js) shows only those."""
from . import snapref
path = _lora_path(lora_name)
if snapref.metadata(path) is None:
return self._apply_legacy(model, vae, image, path, ref_guidance, correction, strength, ref_end, ref_megapixels)
(m,) = snapref.apply(model, clip, vae, image, path, snapref_mode, ref_caption, ref_weight,
ref_megapixels, snapref_guidance)
if m is not model and snapref_guidance:
m.set_model_sampler_post_cfg_function(self._post_cfg(ref_guidance, correction, ref_end))
return (m,)
def _apply_legacy(self, model, vae, image, path, ref_guidance, correction, strength, ref_end, ref_megapixels):
adapter = _Adapter.get(path, strength)
adapter.install(model.get_model_object("diffusion_model"))
pixels = image[:1].movedim(-1, 1)
scale = (ref_megapixels * 1024 * 1024 / (pixels.shape[-1] * pixels.shape[-2])) ** 0.5
width, height = (max(16, round(d * scale / 16) * 16) for d in (pixels.shape[-1], pixels.shape[-2]))
process_latent_in = model.get_model_object("process_latent_in")
# a half-size encode too, for samplers with a half-resolution stage (SPEED)
refs = [process_latent_in(vae.encode(comfy.utils.common_upscale(pixels, w, h, "lanczos", "disabled").movedim(1, -1)))
for w, h in ((width, height), (width // 16 * 8, height // 16 * 8))]
m = model.clone()
m.model_options["transformer_options"][OPTIONS_REF_KEY] = refs
m.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, WRAPPER_KEY, _make_wrapper(adapter, 1.0))
m.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.OUTER_SAMPLE, WRAPPER_KEY, _make_outer_sample_wrapper(adapter))
m.set_model_sampler_post_cfg_function(self._post_cfg(ref_guidance, correction, ref_end))
return (m,)
@staticmethod
def _post_cfg(ref_guidance, correction, ref_end):
def post_cfg(args):
guided, pos = args["denoised"], args["cond_denoised"]
if ref_guidance != 1.0 and _in_window(args["model"].model_sampling, args["sigma"], 0.0, ref_end):
# reference effect = (a branch with the reference) - (the same branch without it); the uncond branch
# as in Anima Ref Guider, or the cond branch when cfg is 1 and there is no uncond
cond, ref = (args["uncond"], args["uncond_denoised"]) if args["uncond"] is not None else (args["cond"], pos)
options = args["model_options"].copy()
options["transformer_options"] = {**options["transformer_options"], NO_REF_KEY: True}
(base,) = comfy.samplers.calc_cond_batch(args["model"], [cond], args["input"], args["sigma"], options)
guided = guided + (ref_guidance - 1.0) * (ref - base)
return _correct(guided, pos, correction, 0.5)
return post_cfg
class AnimaRefGuiderImpl(comfy.samplers.CFGGuider):
def set_settings(self, cfg, ref_guidance, correction, rescale_phi, ref_start=0.0, ref_end=1.0, outside="reference, no extra guidance"):
self.cfg, self.ref_guidance, self.correction, self.rescale_phi = cfg, ref_guidance, correction, rescale_phi
self.ref_start, self.ref_end, self.outside = ref_start, ref_end, outside
def set_ref_conds(self, positive, negative, ref_latent=None, style_latent=None):
refs = {}
for latent, (full, half) in ((ref_latent, SLOT_KEYS["character"]), (style_latent, SLOT_KEYS["style"])):
if latent is not None:
refs[full] = latent["samples"]
if latent.get("samples_half") is not None:
refs[half] = latent["samples_half"]
negative = node_helpers.conditioning_set_values(negative, {"prompt_type": "negative"})
self.inner_set_conds({
"positive": node_helpers.conditioning_set_values(positive, refs),
"negative": node_helpers.conditioning_set_values(negative, refs),
"negative_base": negative,
"positive_base": positive,
})
def predict_noise(self, x, timestep, model_options={}, seed=None):
in_window = _in_window(self.inner_model.model_sampling, timestep, self.ref_start, self.ref_end)
if in_window or self.outside != "no reference":
conds = [self.conds.get("positive"), self.conds.get("negative")]
else: # outside the window without the reference: plain text CFG on the base model
conds = [self.conds.get("positive_base"), self.conds.get("negative_base")]
if in_window and self.ref_guidance != 1.0: # at 1 the reference is in both branches: plain CFG, no base branch
conds.append(self.conds.get("negative_base"))
out = comfy.samplers.calc_cond_batch(self.inner_model, conds, x, timestep, model_options)
pos, neg = out[0], out[1]
base = out[2] if len(out) > 2 else neg
# x0 space: base + ref_guidance * (reference effect) + cfg * (text effect given the reference)
guided = _correct(base + self.ref_guidance * (neg - base) + self.cfg * (pos - neg), pos, self.correction, self.rescale_phi)
# Post-CFG model patches (e.g. Anima Safe PAG), called as comfy.samplers.cfg_function does.
for fn in model_options.get("sampler_post_cfg_function", []):
guided = fn({"denoised": guided, "cond": conds[0], "uncond": conds[1], "cond_scale": self.cfg,
"model": self.inner_model, "uncond_denoised": neg, "cond_denoised": pos, "sigma": timestep,
"model_options": model_options, "input": x})
return guided
class AnimaRefGuider:
@classmethod
def INPUT_TYPES(cls):
return {"required": {
"model": ("MODEL",),
"positive": ("CONDITIONING",),
"negative": ("CONDITIONING",),
"cfg": ("FLOAT", {"default": 4.5, "min": 0.0, "max": 20.0, "step": 0.1, "tooltip": "text guidance"}),
"ref_guidance": ("FLOAT", {"default": 4.5, "min": 0.0, "max": 20.0, "step": 0.1,
"tooltip": "1 = reference in both CFG branches; higher pushes toward the reference"}),
"correction": (["apg", "rescale", "none"],),
"rescale_phi": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.05}),
}, "optional": {
"ref_latent": ("LATENT", {"tooltip": "the reference (single-reference adapter), or the character "
"reference of a two-reference adapter"}),
"style_latent": ("LATENT", {"tooltip": "style reference (two-reference adapter); connect one latent to "
"both inputs to take character and style from the same image"}),
"ref_start": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01,
"tooltip": "start of the reference guidance, as a fraction of the denoising (0 = first step)"}),
"ref_end": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01,
"tooltip": "end of the reference guidance (1 = last step). Identity and style are set early; "
"ending at ~0.6 saves one model call per remaining step"}),
"outside": (["reference, no extra guidance", "no reference"], {
"tooltip": "steps outside [ref_start, ref_end]: keep the reference in both CFG branches without "
"ref_guidance (2 calls per step), or drop it (plain text CFG)"}),
}}
RETURN_TYPES = ("GUIDER",)
FUNCTION = "get_guider"
CATEGORY = "Anima Ref Adapter"
def get_guider(self, model, positive, negative, cfg, ref_guidance, correction, rescale_phi, ref_latent=None,
style_latent=None, ref_start=0.0, ref_end=1.0, outside="reference, no extra guidance"):
if ref_latent is None and style_latent is None:
raise ValueError("Anima Ref Guider needs ref_latent, style_latent, or both")
if ref_start > ref_end:
raise ValueError(f"Anima Ref Guider: ref_start ({ref_start}) is after ref_end ({ref_end})")
guider = AnimaRefGuiderImpl(model)
guider.set_ref_conds(positive, negative, ref_latent, style_latent)
guider.set_settings(cfg, ref_guidance, correction, rescale_phi, ref_start, ref_end, outside)
return (guider,)
NODE_CLASS_MAPPINGS = {
"AnimaRefAdapter": AnimaRefAdapter,
"AnimaRefAdapterLoader": AnimaRefAdapterLoader,
"AnimaRefLatent": AnimaRefLatent,
"AnimaRefGuider": AnimaRefGuider,
}
WEB_DIRECTORY = "./js"
def _adapter_format(name: str) -> str | None:
""""snapref" or "legacy" for a lora_name, without downloading; None when the file cannot be read."""
path = folder_paths.get_full_path("loras", name)
if path is None:
return "legacy" if name in HF_FILES else None # released files not downloaded yet are pilot/v2 adapters
from . import snapref
try:
return "legacy" if snapref.metadata(path) is None else "snapref"
except Exception:
return None
try:
from aiohttp import web
from server import PromptServer
@PromptServer.instance.routes.get("/anima_ref_adapter/format")
async def _format_route(request):
return web.json_response({"format": _adapter_format(request.query.get("name", ""))})
except Exception: # no server (tests, scripts)
pass
NODE_DISPLAY_NAME_MAPPINGS = {
"AnimaRefAdapter": "Anima Ref Adapter",
"AnimaRefAdapterLoader": "Anima Ref Adapter Loader",
"AnimaRefLatent": "Anima Ref Latent",
"AnimaRefGuider": "Anima Ref Guider",
}

Xet Storage Details

Size:
40.7 kB
·
Xet hash:
7330c64e2afe476ec94864c34ff28a2a14c00cd97b34a283f3a89bd3c28746a4

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.