Xabsurd commited on
Commit
5487b64
·
verified ·
1 Parent(s): 63e7e66

Add ComfyUI custom node source files

Browse files
ComfyUI-Moss-Tile-Control/README.md ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # MOSS Tile Control
2
+
3
+ Standalone residual tile control for MOSS-Image, trained in diffusion-pipe with
4
+ `control_adapter = 'tile_residual'` and adapter `type = 'moss_tile'`.
5
+ No additional Python packages are required beyond ComfyUI's dependencies.
6
+
7
+ 1. Restart ComfyUI after installing this directory under `custom_nodes`.
8
+ 2. Put a trained residual adapter in `models/model_patches`.
9
+ 3. Use **Load MOSS Tile Control** → **Apply MOSS Tile Control**.
10
+ 4. Connect MOSS MODEL, Flux2 VAE and the tile image; send the output MODEL to the sampler.
11
+
12
+ Strength scales every control residual. At 0 the input model is returned unchanged.
13
+ Start/end percentages restrict the control to part of sampling. Text conditioning
14
+ connects directly to the sampler. Other LoRAs may be loaded before or after this node.
15
+
16
+ Remove legacy concat tile LoRAs from the input MODEL chain before applying this
17
+ adapter; pending input-projection replacements are checked before VAE encoding.
18
+ Compatibility checks use the current ModelPatcher's original weights and patches,
19
+ so switching from a previous legacy tile run does not reject a compatible model.
20
+ Control residual allocations run outside the backbone's aimdo allocation scope,
21
+ so dynamic VRAM and the Comfy memory compiler can remain enabled.
22
+
23
+ Legacy MOSS concat tile LoRAs and Anima LLLite weights cannot be loaded here. These
24
+ weights require a fresh training run. Training and inference should use the same
25
+ Base/SFT family. Establish quality with ordinary sampling before testing accelerated
26
+ LoRAs or quantized backbones.
27
+
28
+ The architecture in `model.py` matches diffusion-pipe's `models/moss_tile_control.py`.
29
+ The node uses ModelPatcher wrappers, block patches and additional-model loading;
30
+ it does not replace backbone modules or merge control weights into the backbone.
ComfyUI-Moss-Tile-Control/__init__.py ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from comfy_api.latest import ComfyExtension
2
+
3
+ from .nodes import MossTileControlLoader, MossTileControlApply
4
+
5
+
6
+ class MossTileExtension(ComfyExtension):
7
+ async def get_node_list(self):
8
+ return [MossTileControlLoader, MossTileControlApply]
9
+
10
+
11
+ async def comfy_entrypoint():
12
+ return MossTileExtension()
ComfyUI-Moss-Tile-Control/model.py ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """MOSS tile residual architecture, shared with diffusion-pipe/models/moss_tile_control.py."""
2
+
3
+ import torch
4
+ from torch import nn
5
+ import torch.nn.functional as F
6
+
7
+
8
+ FORMAT = "moss_tile_residual"
9
+ VERSION = "1"
10
+ PREFIX = "moss_tile."
11
+
12
+
13
+ class MossTileConditioner(nn.Module):
14
+ def __init__(self, latent_channels, condition_dim, *, operations=nn, device=None, dtype=None):
15
+ super().__init__()
16
+ self.conv1 = operations.Conv2d(latent_channels, condition_dim, 3, padding=1, device=device, dtype=dtype)
17
+ self.conv2 = operations.Conv2d(condition_dim, condition_dim, 3, padding=1, device=device, dtype=dtype)
18
+ self.proj = operations.Conv2d(condition_dim, condition_dim, 1, device=device, dtype=dtype)
19
+
20
+ def forward(self, latent):
21
+ x = F.silu(self.conv1(latent))
22
+ x = self.proj(F.silu(self.conv2(x)))
23
+ return x.flatten(2).transpose(1, 2)
24
+
25
+
26
+ class MossTileResidual(nn.Module):
27
+ def __init__(self, model_dim, condition_dim, rank, *, operations=nn, device=None, dtype=None):
28
+ super().__init__()
29
+ self.down = operations.Linear(model_dim, rank, device=device, dtype=dtype)
30
+ self.mid = operations.Linear(rank, rank, device=device, dtype=dtype)
31
+ self.cond_to_film = operations.Linear(condition_dim, 2 * rank, device=device, dtype=dtype)
32
+ self.up = operations.Linear(rank, model_dim, device=device, dtype=dtype)
33
+
34
+ def forward(self, x, condition, strength=1.0):
35
+ count = condition.shape[1]
36
+ image = x[:, :count]
37
+ hidden = F.silu(self.down(F.layer_norm(image, (image.shape[-1],), eps=1e-6)))
38
+ gamma, beta = self.cond_to_film(condition).chunk(2, dim=-1)
39
+ hidden = F.silu(self.mid(hidden) * (1 + gamma) + beta)
40
+ image = image + self.up(hidden) * strength
41
+ return torch.cat((image, x[:, count:]), dim=1)
42
+
43
+
44
+ class MossTileControl(nn.Module):
45
+ def __init__(self, model_dim, num_blocks, rank, condition_dim=128, latent_channels=128, *, operations=nn, device=None, dtype=None):
46
+ super().__init__()
47
+ self.config = dict(model_dim=model_dim, num_blocks=num_blocks, rank=rank,
48
+ condition_dim=condition_dim, latent_channels=latent_channels)
49
+ if any(type(value) is not int or value <= 0 for value in self.config.values()):
50
+ raise ValueError("MOSS tile dimensions must be positive integers")
51
+ self.conditioner = MossTileConditioner(latent_channels, condition_dim, operations=operations, device=device, dtype=dtype)
52
+ self.blocks = nn.ModuleList([
53
+ MossTileResidual(model_dim, condition_dim, rank, operations=operations, device=device, dtype=dtype)
54
+ for _ in range(num_blocks)
55
+ ])
56
+
57
+ def metadata(self):
58
+ return {"format": FORMAT, "moss_tile.version": VERSION,
59
+ **{f"moss_tile.{key}": str(value) for key, value in self.config.items()}}
60
+
61
+
62
+ def config_from_metadata(metadata):
63
+ if metadata is None or metadata.get("format") != FORMAT or metadata.get("moss_tile.version") != VERSION:
64
+ raise ValueError("Expected a MOSS tile residual v1 checkpoint; legacy concat/LoRA weights cannot be loaded here")
65
+ return {key: int(metadata[f"moss_tile.{key}"])
66
+ for key in ("model_dim", "num_blocks", "rank", "condition_dim", "latent_channels")}
ComfyUI-Moss-Tile-Control/nodes.py ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+ import folder_paths
4
+ import comfy.ldm.moss_image.model
5
+ import comfy.lora
6
+ import comfy.model_management
7
+ import comfy.model_patcher
8
+ import comfy.model_prefetch
9
+ import comfy.ops
10
+ import comfy.patcher_extension
11
+ import comfy.utils
12
+ from comfy_api.latest import io
13
+
14
+ from .model import MossTileControl, PREFIX, config_from_metadata
15
+
16
+
17
+ TileControl = io.Custom("MOSS_TILE_CONTROL")
18
+ CONTROL_KEY = "moss_tile_residual"
19
+
20
+
21
+ class MossTileControlLoader(io.ComfyNode):
22
+ @classmethod
23
+ def define_schema(cls):
24
+ return io.Schema(
25
+ node_id="MossTileControlLoader", display_name="Load MOSS Tile Control",
26
+ category="model/loaders",
27
+ inputs=[io.Combo.Input("name", options=folder_paths.get_filename_list("model_patches"))],
28
+ outputs=[TileControl.Output()],
29
+ )
30
+
31
+ @classmethod
32
+ def execute(cls, name):
33
+ path = folder_paths.get_full_path_or_raise("model_patches", name)
34
+ state, metadata = comfy.utils.load_torch_file(path, safe_load=True, return_metadata=True)
35
+ config = config_from_metadata(metadata)
36
+ control = MossTileControl(
37
+ **config, device=comfy.model_management.unet_offload_device(),
38
+ dtype=comfy.utils.weight_dtype(state), operations=comfy.ops.manual_cast,
39
+ )
40
+ if any(not key.startswith(PREFIX) for key in state):
41
+ raise ValueError("MOSS tile checkpoint contains non-control weights")
42
+ control.load_state_dict({key[len(PREFIX):]: value for key, value in state.items()}, strict=True)
43
+ patcher = comfy.model_patcher.ModelPatcher(
44
+ control, load_device=comfy.model_management.get_torch_device(),
45
+ offload_device=comfy.model_management.unet_offload_device(),
46
+ )
47
+ return io.NodeOutput(patcher)
48
+
49
+
50
+ class MossTilePatch:
51
+ def __init__(self, control, latent, strength, sigma_start, sigma_end):
52
+ self.control = control
53
+ self.latent = latent
54
+ self.strength = strength
55
+ self.sigma_start = sigma_start
56
+ self.sigma_end = sigma_end
57
+
58
+ def __call__(self, executor, x, timestep, context, transformer_options={}, **kwargs):
59
+ if x.shape[1] != self.control.model.config["latent_channels"]:
60
+ raise ValueError("Remove the legacy concat tile LoRA from the MODEL chain before applying MOSS Tile Control")
61
+ sigmas = transformer_options.get("sigmas")
62
+ if sigmas is not None and not self.sigma_end <= float(sigmas.max()) <= self.sigma_start:
63
+ return executor(x, timestep, context, transformer_options, **kwargs)
64
+ latent = comfy.utils.common_upscale(self.latent.to(device=x.device, dtype=x.dtype), x.shape[-1], x.shape[-2], "bilinear", "center")
65
+ latent = comfy.utils.repeat_to_batch_size(latent, x.shape[0])
66
+ condition = self.control.model.conditioner(latent)
67
+ options = transformer_options.copy()
68
+ options[CONTROL_KEY] = {**options.get(CONTROL_KEY, {}), self: condition}
69
+ return executor(x, timestep, context, options, **kwargs)
70
+
71
+
72
+ class MossTileBlockPatch:
73
+ def __init__(self, patch, index, previous):
74
+ self.patch = patch
75
+ self.index = index
76
+ self.previous = previous
77
+
78
+ def __call__(self, args, extra):
79
+ out = self.previous(args, extra) if self.previous is not None else extra["original_block"](args)
80
+ condition = args["transformer_options"].get(CONTROL_KEY, {}).get(self.patch)
81
+ if condition is not None:
82
+ # Adapter tensors outlive the backbone block's allocation scope.
83
+ with comfy.model_prefetch.pause_malloc_graph():
84
+ out["img"] = self.patch.control.model.blocks[self.index](out["img"], condition, self.patch.strength)
85
+ return out
86
+
87
+ def to(self, device_or_dtype):
88
+ return self
89
+
90
+
91
+ class MossTileControlApply(io.ComfyNode):
92
+ @classmethod
93
+ def define_schema(cls):
94
+ return io.Schema(
95
+ node_id="MossTileControlApply", display_name="Apply MOSS Tile Control",
96
+ category="model/patch/moss_image",
97
+ description="Controls MOSS structure through a separate residual adapter. Strength 0 returns the input model unchanged.",
98
+ inputs=[
99
+ io.Model.Input("model"), TileControl.Input("control"), io.Vae.Input("vae"),
100
+ io.Image.Input("image"),
101
+ io.Float.Input("strength", default=1.0, min=0.0, max=10.0, step=0.01),
102
+ io.Float.Input("start_percent", default=0.0, min=0.0, max=1.0, step=0.001),
103
+ io.Float.Input("end_percent", default=1.0, min=0.0, max=1.0, step=0.001),
104
+ ],
105
+ outputs=[io.Model.Output()],
106
+ )
107
+
108
+ @classmethod
109
+ def execute(cls, model, control, vae, image, strength=1.0, start_percent=0.0, end_percent=1.0):
110
+ if strength == 0.0 or start_percent >= end_percent:
111
+ return io.NodeOutput(model)
112
+ backbone = model.get_model_object("diffusion_model")
113
+ if not isinstance(backbone, comfy.ldm.moss_image.model.MOSSImage):
114
+ raise ValueError("MOSS Tile Control requires a MOSS-Image model")
115
+ config = control.model.config
116
+ for _strength, weight_patch, *_rest in model.patches.get("diffusion_model.video_embedder.weight", []):
117
+ if (isinstance(weight_patch, tuple) and len(weight_patch) == 2 and weight_patch[0] == "set"
118
+ and weight_patch[1][0].shape[1] != config["latent_channels"]):
119
+ raise ValueError("Remove the legacy concat tile LoRA from the MODEL chain before applying MOSS Tile Control")
120
+ key = "diffusion_model.video_embedder.weight"
121
+ # Clones share live weights; use the original weight and this patcher's patches.
122
+ projection = model.get_key_patches(key)[key]
123
+ projection_shape = comfy.lora.calculate_shape(projection[1:], projection[0][0], key)
124
+ if (projection_shape != (config["model_dim"], config["latent_channels"])
125
+ or len(backbone.layers) != config["num_blocks"]):
126
+ raise ValueError(f"MOSS tile adapter expects projection {(config['model_dim'], config['latent_channels'])} and {config['num_blocks']} blocks; got {tuple(projection_shape)} and {len(backbone.layers)} blocks")
127
+ latent = vae.encode(image[..., :3])
128
+ if latent.ndim != 4 or latent.shape[1] != config["latent_channels"]:
129
+ raise ValueError("MOSS Tile Control requires the Flux2 VAE (128 latent channels)")
130
+ sampling = model.get_model_object("model_sampling")
131
+ patch = MossTilePatch(control, latent, strength,
132
+ float(sampling.percent_to_sigma(start_percent)), float(sampling.percent_to_sigma(end_percent)))
133
+ result = model.clone()
134
+ result.add_wrapper(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, patch)
135
+ existing = result.model_options["transformer_options"].get("patches_replace", {}).get("dit", {})
136
+ for index in range(config["num_blocks"]):
137
+ previous = existing.get(("single_block", index))
138
+ result.set_model_patch_replace(MossTileBlockPatch(patch, index, previous), "dit", "single_block", index)
139
+ result.set_additional_models(CONTROL_KEY, result.get_additional_models_with_key(CONTROL_KEY) + [control])
140
+ return io.NodeOutput(result)